lnst-ctl

#! /usr/bin/env python2
"""
Net test controller

Copyright 2011 Red Hat, Inc.
Licensed under the GNU General Public License, version 2 as
published by the Free Software Foundation; see COPYING for details.
"""

__author__ = """
jpirko@redhat.com (Jiri Pirko)
"""

import getopt
import sys
import logging
import os
import re
import datetime
from lnst.Common.Logs import LoggingCtl, log_exc_traceback
from lnst.Common.Config import lnst_config
from lnst.Common.Colours import load_presets_from_config
from lnst.Common.Utils import mkdir_p
from lnst.Controller.NetTestController import NetTestController, NetTestError
from lnst.Controller.NetTestController import NoMatchError
from lnst.Controller.NetTestResultSerializer import NetTestResultSerializer
from lnst.Controller.SlavePool import SlavePool
from lnst.Controller.XmlProcessing import XmlProcessingError

RETVAL_PASS = 0
RETVAL_FAIL = 1
RETVAL_ERR = 2
def usage(retval=0):
    """
    Print usage of this app
    """
    print "Usage: %s [OPTIONS...] ACTION [RECIPES...]" % sys.argv[0]
    print ""
    print "ACTION = [ run | config_only | deconfigure | match_setup | " \
                     "list_pools]"
    print ""
    print "OPTIONS"
    print "  -A, --override-alias name=value define top-level alias that " \
              "will override any other definitions in the recipe"
    print "  -a, --define-alias name=value   define top-level alias"
    print "  -c, --config=FILE               load additional config file"
    print "  -C, --config-override=FILE      reset config defaults and load " \
              "the following config file"
    print "  -d, --debug                     emit debugging messages"
    print "      --dump-config               dumps the join of all loaded " \
              "configuration files on stdout and exits"
    print "  -v, --verbose                   verbose version of list_pools " \
              "command"
    print "  -h, --help                      print this message"
    print "  -m, --no-colours                disable coloured terminal output"
    print "  -o, --disable-pool-checks       don't check the availability of " \
              "machines in the pool"
    print "  -p, --packet-capture            capture and log all ongoing " \
              "network communication during the test"
    print "      --pools=NAME[,...]          restricts which pools to use "\
              "for matching, value can be a comma separated list of values or"
    print "      --pools=PATH                a single path to a pool directory"
    print "  -r, --reduce-sync               reduces resource synchronization "\
              "for python tasks, see documentation"
    print "  -s, --xslt-url=URL              URL to a XSLT document that will "\
              "be used when transforming the xml result file, only useful "\
              "when -t is used as well"
    print "  -t, --html=FILE                 generate a formatted result html"
    print "  -u, --multi-match               run each recipe with every "\
              "pool match possible"
    print "  -x, --result=FILE               file to write xml_result"
    sys.exit(retval)

def list_pools(restrict_pools, verbose):
    conf_pools = lnst_config.get_pools()
    pools = {}
    if len(restrict_pools) > 0:
        for pool_name in restrict_pools:
            if pool_name in conf_pools:
                pools[pool_name] = conf_pools[pool_name]
            elif len(restrict_pools) == 1 and os.path.isdir(pool_name):
                pools = {"cmd_line_pool": pool_name}
            else:
                raise NetTestError("Pool %s does not exist!" % pool_name)
    else:
        pools = conf_pools

    sp = SlavePool(pools, pool_checks=False)

    out = ""
    # iterate over all pools
    sp_pools = sp.get_pools()
    for pool_name, pool_content in sp_pools.iteritems():
        out += "Pool: %s (%s)\n" % (pool_name, pools[pool_name])
        # verbose output
        if verbose:
        # iterate over all slave machine cfgs
            for filename, pool in pool_content.iteritems():
                # print in human-readable format
                out += 3*" " + filename + ".xml\n"
                out += 5*" " + "params:\n"
                for key in pool['params']:
                    out += 7*" " + key + " : " +\
                            str(pool['params'][key]) + "\n"
                if pool['interfaces']:
                    out += 5*" " + "interfaces:\n"
                for key in sorted(pool['interfaces']):
                    out += 7*" " + "id : " + key + "\tlabel : " +\
                           str(pool['interfaces'][key]['network']) + "\n"
                    for param in pool['interfaces'][key]['params']:
                        out += 9*" " + param + " : " +\
                           str(pool['interfaces'][key]['params'][param]) + "\n"
                out += "\n"
    # print wihout newlines on the end of string
    if verbose:
        print out[:-2]
    else:
        print out[:-1]


def store_alias(alias_def, aliases_dict):
    try:
        name, value = alias_def.split("=")
    except:
        msg = "The alias definition '%s' not supported. The proper" + \
              " format is alias_name=alias_value."
        raise Exception(msg)

    if name in aliases_dict:
        msg = "The same alias %s was defined multiple times through CLI."
        logging.warning(msg)

    aliases_dict[name] = value

def exec_action(action, nettestctl):
    if action == "run":
        return nettestctl.run_recipe()
    elif action == "config_only":
        return nettestctl.config_only_recipe()
    elif action == "match_setup":
        return nettestctl.match_setup()

def get_recipe_result(action, file_path, log_ctl, res_serializer,
                      pool_checks, packet_capture,
                      defined_aliases, overriden_aliases,
                      reduce_sync, multi_match, pools):
    retval = RETVAL_PASS

    matches = 1
    no_match = False

    log_ctl.set_recipe(file_path, prepend=True, expand="match_%d" % matches)
    log_dir = log_ctl.get_recipe_log_path()
    recipe_head_log_entry(file_path, log_dir, matches)
    res_serializer.add_recipe(file_path, matches)

    res = {}
    try:
        nettestctl = NetTestController(file_path, log_ctl,
                                       res_serializer=res_serializer,
                                       pool_checks=pool_checks,
                                       packet_capture=packet_capture,
                                       defined_aliases=defined_aliases,
                                       overriden_aliases=overriden_aliases,
                                       reduce_sync=reduce_sync,
                                       restrict_pools=pools)
    except XmlProcessingError as err:
        log_exc_traceback()
        logging.error(err)
        res["passed"] = False
        res["err_msg"] = str(err)
        retval = RETVAL_ERR
        res_serializer.set_recipe_result(res)
        return retval

    while True:
        res = {}
        if matches == 1:
            try:
                nettestctl.provision_machines()
                nettestctl.print_match_description()
                res = exec_action(action, nettestctl)
            except Exception as err:
                no_match = True
                log_exc_traceback()
                logging.error(err)
                res["passed"] = False
                res["err_msg"] = str(err)
                retval = RETVAL_ERR
        elif matches > 1:
            try:
                nettestctl.provision_machines()
                log_ctl.set_recipe(file_path, prepend=True, expand="match_%d" % matches)
                log_dir = log_ctl.get_recipe_log_path()
                recipe_head_log_entry(file_path, log_dir, matches)
                res_serializer.add_recipe(file_path, matches)
                nettestctl.print_match_description()
                res = exec_action(action, nettestctl)
            except NoMatchError as err:
                no_match = True
                log_ctl.unset_recipe()
                logging.warning("Match %d not possible." % matches)
            except Exception as err:
                no_match = True
                log_exc_traceback()
                logging.error(err)
                res["passed"] = False
                res["err_msg"] = str(err)
                retval = RETVAL_ERR

        if no_match and matches > 1:
            break

        res_serializer.set_recipe_pool_match(nettestctl.get_pool_match())
        res_serializer.set_recipe_result(res)

        # The test failed, but don't override erro
        if not res["passed"] and retval < RETVAL_FAIL:
            retval = RETVAL_FAIL

        if not multi_match or no_match:
            break
        matches += 1

    return retval

def recipe_head_log_entry(filename, log_dir, match_num=1):
    head_str = "\nTrying recipe file \"%s\" match %d\n" % (filename,
                                                           match_num)
    log_dir_str = "Logs for this recipe will be stored in '%s'\n" % log_dir
    dash_count = max(len(head_str.strip()), len(log_dir_str.strip()))
    logging.info("-" * dash_count
                 + head_str
                 + log_dir_str
                 + "-" * dash_count)

def main():
    """
    Main function
    """
    try:
        opts, args = getopt.getopt(
            sys.argv[1:],
            "A:a:c:C:dhmoprs:t:ux:v",
            [
             "override_alias=",
             "define_alias=",
             "config=",
             "config-override=",
             "debug",
             "dump-config",
             "help",
             "no-colours",
             "disable-pool-checks",
             "packet-capture",
             "reduce-sync",
             "xslt-url=",
             "html=",
             "multi-match",
             "result=",
             "pools=",
             "verbose"
            ]
        )
    except getopt.GetoptError as err:
        print str(err)
        usage(RETVAL_ERR)

    lnst_config.controller_init()
    dirname = os.path.dirname(sys.argv[0])
    gitcfg = os.path.join(dirname, "lnst-ctl.conf")
    if os.path.isfile(gitcfg):
        lnst_config.load_config(gitcfg)
    else:
        lnst_config.load_config('/etc/lnst-ctl.conf')

    usr_cfg = os.path.expanduser('~/.lnst/lnst-ctl.conf')
    if os.path.isfile(usr_cfg):
        lnst_config.load_config(usr_cfg)
    else:
        usr_cfg_dir = os.path.dirname(usr_cfg)
        pool_dir = usr_cfg_dir + "/pool"
        mkdir_p(pool_dir)
        global_pools = lnst_config.get_section("pools")
        if (len(global_pools) == 0):
            lnst_config.add_pool("default", pool_dir, "./")
        with open(usr_cfg, 'w') as f:
            f.write(lnst_config.dump_config())

    debug = 0
    result_path = None
    html_result_path = None
    xslt_url = None
    packet_capture = False
    pool_checks = True
    coloured_output = True
    defined_aliases = {}
    overriden_aliases = {}
    reduce_sync = False
    multi_match = False
    dump_config = False
    verbose = False
    pools = []
    for opt, arg in opts:
        if opt in ("-d", "--debug"):
            debug += 1
        elif opt in ("-h", "--help"):
            usage(RETVAL_PASS)
        elif opt in ("-c", "--config"):
            if not os.path.isfile(arg):
                print "File '%s' doesn't exist!" % arg
                usage(RETVAL_ERR)
            else:
                lnst_config.load_config(arg)
        elif opt in ("-C", "--config-override"):
            if not os.path.isfile(arg):
                print "File '%s' doesn't exist!" % arg
                usage(RETVAL_ERR)
            else:
                print >> sys.stderr, "Reloading config defaults!"
                lnst_config.controller_init()
                lnst_config.load_config(arg)
        elif opt in ("-x", "--result"):
            result_path = arg
        elif opt in ("-t", "--html"):
            html_result_path = arg
        elif opt in ("-p", "--packet-capture"):
            packet_capture = True
        elif opt in ("-o", "--disable-pool-checks"):
            pool_checks = False
        elif opt in ("-m", "--no-colours"):
            coloured_output = False
        elif opt in ("-a", "--define-alias"):
            store_alias(arg, defined_aliases)
        elif opt in ("-A", "--override-alias"):
            store_alias(arg, overriden_aliases)
        elif opt in ("-r", "--reduce-sync"):
            reduce_sync = True
        elif opt in ("-s", "--xslt-url"):
            xslt_url = arg
        elif opt in ("-u", "--multi-match"):
            multi_match = True
        elif opt in ("--dump-config"):
            dump_config = True
        elif opt in ("--pools"):
            pools.extend(arg.split(","))
        elif opt in ("-v", "--verbose"):
            verbose= True

    if xslt_url != None:
        lnst_config.set_option("environment", "xslt_url", xslt_url)

    if dump_config:
        print lnst_config.dump_config()
        return RETVAL_PASS

    if coloured_output:
        coloured_output = not lnst_config.get_option("colours",
                                                     "disable_colours")

    load_presets_from_config(lnst_config)

    date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
    log_ctl = LoggingCtl(debug,
                     log_dir=lnst_config.get_option('environment', 'log_dir'),
                     log_subdir=date, colours=coloured_output)

    if len(args) <= 0:
        logging.error("No action specified")
        usage(RETVAL_ERR)

    action = args[0]
    recipes = args[1:]
    if not action in ['run', 'config_only', 'deconfigure', 'match_setup',
                      'list_pools']:
        logging.error("Action '%s' not recognised" % action)
        usage(RETVAL_ERR)
    if action == 'deconfigure':
        NetTestController.remove_saved_machine_config()
        return 0
    elif action == 'list_pools':
        log_ctl.unset_formatter()
        logging.disable(logging.INFO)
        list_pools(pools, verbose)
        return RETVAL_PASS

    if recipes == []:
        logging.error("No recipe specified!")
        usage(RETVAL_ERR)

    recipe_files = []
    for recipe_path in recipes:
        if os.path.isdir(recipe_path):
            all_files = []
            for root, dirs, files in os.walk(recipe_path):
                dirs[:] = [] # do not walk subdirs
                all_files += files

            all_files.sort()
            for f in all_files:
                recipe_file = os.path.join(recipe_path, f)
                if re.match(r'^.*\.xml$', recipe_file):
                    recipe_files.append(recipe_file)
        else:
            recipe_files.append(recipe_path)

    retval = RETVAL_PASS
    res_serializer = NetTestResultSerializer()
    for recipe_file in recipe_files:
        rv = get_recipe_result(action, recipe_file, log_ctl, res_serializer,
                               pool_checks, packet_capture,
                               defined_aliases, overriden_aliases,
                               reduce_sync, multi_match, pools)
        if rv > retval:
            retval = rv

    log_ctl.set_recipe("", clean=False)

    res_serializer.print_summary()

    log_ctl.print_log_dir()

    if result_path:
        result_path = os.path.expanduser(result_path)
        handle = open(result_path, "w")
        handle.write(res_serializer.get_result_xml())
        handle.close()
    if html_result_path:
        html_result_path = os.path.expanduser(html_result_path)
        handle = open(html_result_path, "w")
        handle.write(res_serializer.get_result_html())
        handle.close()

    return retval

if __name__ == "__main__":
    sys.exit(main())