2.5 test-ready -> master (#145)

* Get min max and average into collectinfo summary

* revert when timeout is applied to socket

* add option  to use data agent in live cluster mode
Add caching to license_usage request

* Add tests

* WIP manage roster and manage revive

* Add show roster command to live and collectinfo mode

* Update version

* hack fix healthcheck test

* fix test

* Add tests

* Add dynamic diff to show roster

* fix tests

* Add manage roster/revive tests

* Add warn modes to manage roster.
Turn on warn by default for manage truncate

* Update help documentation

* Update lib/live_cluster/manage_controller.py

* fix test

* WIP Add job info calls to node

* TOOLS-1811 wip fix sheets bug

* WIP manage jobs

* WIP manage jobs

* Add collectinfo show jobs, add warn

* Add manage jobs tests

* Fix test

* Add view test and some light refactoring

* WIP show racks

* show racks

* fix ui

* fix test, help info, and warnings

* fix jobs

* make show controller match for live and collect

* Fix help docs
Convert Show Job entires
Remove kill all sindex-builder

* fix tests

* Revert info_racks api for health_check backwards compatibility.

* Review changes

* TOOLS-1847

* Turn off --warn by default for `manage roster stage observed`

* Add ability to create a more complex aggregator

* Add weighted-avg aggregator and sum to show_lateny

* Change comment per review

* Added prole-tombstones to Total Records displayed under info namespace

* Abort creating collectinfo when error occurs in making .json.
Add file to store collectinfo stdout/err for easier debugging

* Standardize getter return schema
Get effective rack-id for info_namespace

* Add some comments

* Fix error message when authentication failes.
TOOLS-1858

* Make it so login error is not called once per node

* remove debug print

* Add pmem to summary command and rename Disk.

* Add set index column to info set.  Change sindex state to be cleaer

* Fix issue with aerospike.conf not stored in collectinfo

* Fix merge error

* docs: Add a more descriptive comment about info namespace effective rack-id

* Fix merge errors

* refactor: Remove float conversions required for python3

* refactor: Change name of Drives back to Devices

* refactor: TOOLS-1849 change name of Features row in summary command

* docs: Change manage quiesce help output

* fix: "sindex not in sync with primary" health-check does not check the correct sync condition

* fix: TOOLS-1500 Filter out exceptions before it gets to collectinfo json serializer

* fix roster stored into collectinfo

* fix:  Issue with term color after collectinfo.  Errors for --agent-host are now correctly displayed

asadm.py

@@ -21,8 +21,6 @@
 import re
 import shlex
 import sys
-import logging
-import traceback
 
 if sys.version_info[0] < 3:
     raise Exception(
@@ -42,110 +40,14 @@
 
 # Setup logger before anything
 
-
-class BaseLogger(logging.Logger, object):
-    execute_only_mode = False
-
-    def __init__(self, name, level=logging.WARNING):
-        return super(BaseLogger, self).__init__(name, level=level)
-
-    def _handle_exception(self, msg):
-        if (
-            isinstance(msg, Exception)
-            and not isinstance(msg, ShellException)
-            and not isinstance(msg, info.ASProtocolError)
-            and not isinstance(msg, ASInfoError)
-        ):
-            traceback.print_exc()
-
-    def _print_message(self, msg, level, red_color=False, *args, **kwargs):
-        try:
-            message = str(msg).format(*args, **kwargs)
-        except Exception:
-            message = str(msg)
-
-        message = level + ": " + message
-
-        if red_color:
-            message = terminal.fg_red() + message + terminal.fg_clear()
-
-        print(message)
-
-    def debug(self, msg, *args, **kwargs):
-        if self.level <= logging.DEBUG:
-            self._log(self.level, msg, args, **kwargs)
-
-    def info(self, msg, *args, **kwargs):
-        if self.level <= logging.INFO:
-            self._print_message(msg, "INFO", False, *args, **kwargs)
-
-    def warning(self, msg, *args, **kwargs):
-        if self.level <= logging.WARNING:
-            self._print_message(msg, "WARNING", True, *args, **kwargs)
-
-    def error(self, msg, *args, **kwargs):
-        if self.level <= logging.ERROR:
-            self._print_message(msg, "ERROR", True, *args, **kwargs)
-            self._handle_exception(msg)
-
-            if self.execute_only_mode:
-                exit(2)
-
-    def critical(self, msg, *args, **kwargs):
-        if self.level <= logging.CRITICAL:
-            self._print_message(msg, "ERROR", True, *args, **kwargs)
-            self._handle_exception(msg)
-        exit(1)
-
-
-class LogFormatter(logging.Formatter):
-    def __init__(self, fmt="%(levelno)s: %(msg)s"):
-        super().__init__(fmt=fmt, datefmt=None, style="%")
-
-    def format(self, record):
-        original_fmt = self._style._fmt
-
-        if record.levelno == logging.DEBUG:
-            path_split = record.pathname.split("lib")
-
-            if len(path_split) > 1:
-                record.pathname = "lib" + record.pathname.split("lib")[1]
-                self._style._fmt = (
-                    "{%(pathname)s:%(lineno)d} %(levelname)s - %(message)s"
-                )
-            else:
-                self._style._fmt = (
-                    "{%(filename)s:%(lineno)d} %(levelname)s - %(message)s"
-                )
-
-        formatter = logging.Formatter(self._style._fmt)
-        result = formatter.format(record)
-
-        self._style._fmt = original_fmt
-
-        return result
-
-
-logging.setLoggerClass(BaseLogger)
-logging.basicConfig(level=logging.WARNING)
-logger = logging.getLogger("asadm")
-logger.propagate = False
-logger.setLevel(logging.INFO)
-logging_handler = logging.StreamHandler()
-logging_handler.setLevel(logging.INFO)
-logging_handler.setFormatter(LogFormatter())
-logger.addHandler(logging_handler)
-
-
+from lib.utils.logger import logger
 from lib.collectinfo_analyzer.collectinfo_root_controller import (
     CollectinfoRootController,
 )
-from lib.base_controller import ShellException
 from lib.live_cluster.live_cluster_root_controller import LiveClusterRootController
 from lib.live_cluster.client import info
 from lib.live_cluster.client.assocket import ASSocket
 from lib.live_cluster.client.ssl_context import SSLContext
-from lib.live_cluster.client.node import ASInfoError
 from lib.log_analyzer.log_analyzer_root_controller import LogAnalyzerRootController
 from lib.utils import common, util, conf
 from lib.utils.constants import ADMIN_HOME, AdminMode, AuthMode
@@ -261,7 +163,7 @@ def __init__(
                             "Not able to connect any cluster with " + str(seeds) + "."
                         )
 
-                self.set_prompt(DEFAULT_PROMPT)
+                self.set_default_prompt()
                 self.intro = ""
                 if not execute_only_mode:
                     self.intro += str(self.ctrl.cluster) + "\n"
@@ -309,14 +211,23 @@ def __init__(
             if command != "help":
                 self.commands.add(command)
 
-    def set_prompt(self, prompt):
+    def set_prompt(self, prompt, color="green"):
         self.prompt = prompt
 
         if self.use_rawinput:
+            if color == "green":
+                color_func = terminal.fg_green
+            elif color == "red":
+                color_func = terminal.fg_red
+            else:
+
+                def color_func():
+                    return ""
+
             self.prompt = (
                 "\001"
                 + terminal.bold()
-                + terminal.fg_red()
+                + color_func()
                 + "\002"
                 + self.prompt
                 + "\001"
@@ -325,6 +236,12 @@ def set_prompt(self, prompt):
                 + "\002"
             )
 
+    def set_default_prompt(self):
+        self.set_prompt(DEFAULT_PROMPT, "green")
+
+    def set_privaliged_prompt(self):
+        self.set_prompt(PRIVILEGED_PROMPT, "red")
+
     def clean_line(self, line):
         # get rid of extra whitespace
         lexer = shlex.shlex(line)
@@ -334,7 +251,7 @@ def clean_line(self, line):
 
         command = []
         build_token = ""
-        lexer.wordchars += ".*-:/_{}"
+        lexer.wordchars += ".*-:/_{}@"
         for token in lexer:
             build_token += token
             if token == "-":
@@ -397,10 +314,10 @@ def precmd(
                     return "exit"
 
                 elif response == "ENABLE":
-                    self.set_prompt(PRIVILEGED_PROMPT)
+                    self.set_privaliged_prompt()
 
                 elif response == "DISABLE":
-                    self.set_prompt(DEFAULT_PROMPT)
+                    self.set_default_prompt()
 
             except Exception as e:
                 logger.error(e)
@@ -662,7 +579,7 @@ def main():
 
     if cli_args.execute is not None:
         execute_only_mode = True
-        BaseLogger.execute_only_mode = True
+        logger.execute_only_mode = True
 
     cli_args, seeds = conf.loadconfig(cli_args, logger)
 
@@ -725,7 +642,7 @@ def main():
     use_yappi = False
     if cli_args.profile:
         try:
-            import yappi
+            import yappi  # noqa F401
 
             use_yappi = True
         except Exception as a:

lib/base_controller.py

@@ -23,6 +23,9 @@
 
 DEFAULT = "_do_default"
 
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.CRITICAL)
+
 
 class CommandHelp:
 
@@ -86,6 +89,28 @@ def is_hidden(func):
             return False
 
 
+class CommandName:
+    def __init__(self, name):
+        self._assigned_name = name
+
+    def __call__(self, func):
+        func._assigned_name = self._assigned_name
+        return func
+
+    @staticmethod
+    def name(func):
+        return func._assigned_name
+
+    @staticmethod
+    def has_name(func):
+        try:
+            if func._assigned_name:
+                return True
+            return False
+        except Exception:
+            return False
+
+
 class DisableAutoComplete:
     """Decorator to disable tab completion and auto completion. e.g. if ShowController
     was decoracted it would not allow 'sho' **enter** to resolve to 'show'.
@@ -168,7 +193,11 @@ def _init_commands(self):
         self.commands = PrefixDict()
 
         for command in commands:
-            self.commands.add(command[1], getattr(self, command[0]))
+            func = getattr(self, command[0])
+            if CommandName.has_name(func):
+                self.commands.add(CommandName.name(func), func)
+            else:
+                self.commands.add(command[1], func)
 
         for command, controller in self.controller_map.items():
             if self.context:
@@ -189,24 +218,24 @@ def complete(self, line):
         command = line.pop(0) if line else ""
         commands = self.commands.get_key(command)
 
-        self.logger.debug("Auto-complete: command {}".format(command))
+        logger.debug("Auto-complete: command {}".format(command))
 
         if not DisableAutoComplete.has_auto_complete(self.commands[commands[0]][0]):
             return []
 
         if command != commands[0] and len(line) == 0:
             # if user has full command name and is hitting tab,
             # the user probably wants the next level
-            self.logger.debug("Auto-complete: results {}".format(commands))
+            logger.debug("Auto-complete: results {}".format(commands))
             return sorted(commands)
 
         try:
-            self.logger.debug(
+            logger.debug(
                 "Auto-complete: going to next controller {}".format(commands[0])
             )
             return self.commands.get(commands[0])[0].complete(line)
         except Exception as e:
-            self.logger.debug(
+            logger.debug(
                 "Auto-complete: command is ambiguous or exact match {}, error: {}".format(
                     commands[0], e
                 )
@@ -248,9 +277,9 @@ def _find_method(self, line):
                 return method
 
         try:
-            self.logger.debug("Looking for {} in command_map".format(command))
+            logger.debug("Looking for {} in command_map".format(command))
             method = self.commands[command]
-            self.logger.debug("Found method {} in command_map".format(method))
+            logger.debug("Found method {} in command_map".format(method))
 
             if len(method) > 1:
                 # handle ambiguos commands
@@ -462,7 +491,7 @@ def parse_modifiers(self, line, duplicates_in_line_allowed=False):
 
     def _check_required_modifiers(self, line):
         for mod in self.required_modifiers:
-            if not len(self.mods[mod]):
+            if not self.mods[mod]:
                 self.execute_help(line)
                 if mod == "line":
                     raise IOError("Missing required argument")

lib/collectinfo_analyzer/collectinfo_handler/collectinfo_log.py

@@ -60,6 +60,7 @@ def __init__(self, cluster_name, timestamp, cinfo_data, cinfo_file):
         self.timestamp = timestamp
         self.nodes = {}
         self.node_names = {}
+        self.node_ids = {}
         self.cinfo_data = self.ns_name_fault_check(cinfo_data)
         self.cinfo_file = cinfo_file
         self.node_lookup = LookupDict()
@@ -152,6 +153,16 @@ def get_node_names(self, nodes=None):
                 self.node_names[node_name] = node_name
         return copy.deepcopy(self.node_names)
 
+    def get_node_ids(self, nodes=None):
+        if not self.node_ids:
+            if not self.nodes:
+                return {}
+
+            for key, node in self.nodes.items():
+                self.node_ids[key] = node.node_id
+
+        return copy.deepcopy(self.node_ids)
+
     def get_data(self, type="", stanza=""):
         data = {}
 
@@ -413,7 +424,10 @@ def __init__(self, cinfo_path, files, reader):
         self.reader = reader
         self.snapshots = {}
         self.data = {}
-        full_parser.parse_info_all(files, self.data, True)
+        self.license_data_usage = {}
+        full_parser.parse_collectinfo_files(
+            files, self.data, self.license_data_usage, True
+        )
 
         if self.data:
             for ts in sorted(self.data.keys(), reverse=True):