diff --git a/squad/plugins/linux_log_parser.py b/squad/plugins/linux_log_parser.py index 6aa81d05..44341b9e 100644 --- a/squad/plugins/linux_log_parser.py +++ b/squad/plugins/linux_log_parser.py @@ -4,26 +4,28 @@ from collections import defaultdict from squad.plugins import Plugin as BasePlugin from squad.core.models import SuiteMetadata +from django.template.defaultfilters import slugify logger = logging.getLogger() REGEX_NAME = 0 REGEX_BODY = 1 +REGEX_EXTRACT_NAME = 2 MULTILINERS = [ - ('check-kernel-exception', r'-+\[? cut here \]?-+.*?-+\[? end trace \w* \]?-+'), - ('check-kernel-kasan', r'=+\n\[[\s\.\d]+\]\s+BUG: KASAN:.*?=+'), - ('check-kernel-kfence', r'=+\n\[[\s\.\d]+\]\s+BUG: KFENCE:.*?=+'), + ('check-kernel-exception', r'-+\[? cut here \]?-+.*?-+\[? end trace \w* \]?-+', r"\d][^\+\n]*"), + ('check-kernel-kasan', r'=+\n\[[\s\.\d]+\]\s+BUG: KASAN:.*?=+', r"BUG: KASAN:[^\+\n]*"), + ('check-kernel-kfence', r'=+\n\[[\s\.\d]+\]\s+BUG: KFENCE:.*?=+', r"BUG: KFENCE:[^\+\n]*"), ] ONELINERS = [ - ('check-kernel-oops', r'^[^\n]+Oops(?: -|:).*?$'), - ('check-kernel-fault', r'^[^\n]+Unhandled fault.*?$'), - ('check-kernel-warning', r'^[^\n]+WARNING:.*?$'), - ('check-kernel-bug', r'^[^\n]+(?: kernel BUG at|BUG:).*?$'), - ('check-kernel-invalid-opcode', r'^[^\n]+invalid opcode:.*?$'), - ('check-kernel-panic', r'Kernel panic - not syncing.*?$'), + ('check-kernel-oops', r'^[^\n]+Oops(?: -|:).*?$', r"Oops[^\+\n]*"), + ('check-kernel-fault', r'^[^\n]+Unhandled fault.*?$', r"Unhandled [^\+\n]*"), + ('check-kernel-warning', r'^[^\n]+WARNING:.*?$', r"WARNING: [^\+\n]*"), + ('check-kernel-bug', r'^[^\n]+(?: kernel BUG at|BUG:).*?$', r"BUG: [^\+\n]*"), + ('check-kernel-invalid-opcode', r'^[^\n]+invalid opcode:.*?$', r"invalid opcode: [^\+\n]*"), + ('check-kernel-panic', r'Kernel panic - not syncing.*?$', r"Kernel [^\+\n]*"), ] # Tip: broader regexes should come first @@ -70,7 +72,7 @@ def __join_matches(self, matches, regexes): snippets[regex_id].append(match[regex_id]) return snippets - def __create_tests(self, testrun, suite, test_name, lines): + def __create_tests(self, testrun, suite, test_name, lines, test_regex=None): """ There will be at least one test per regex. If there were any match for a given regex, then a new test will be generated using test_name + shasum. This helps @@ -92,6 +94,9 @@ def __create_tests(self, testrun, suite, test_name, lines): shas = defaultdict(set) for line in lines: sha = self.__create_shasum(line) + name = self.__create_name(line, test_regex) + if name: + sha = f"{name}-{sha}" shas[sha].add(line) for sha, lines in shas.items(): @@ -106,11 +111,27 @@ def __create_tests(self, testrun, suite, test_name, lines): environment=testrun.environment, ) + def __remove_numbers_and_time(self, snippet): + without_numbers = re.sub(r"(0x[a-f0-9]+|[<\[][0-9a-f]+?[>\]]|\d+)", "", snippet) + without_time = re.sub(r"^\[[^\]]+\]", "", without_numbers) + + return without_time + + def __create_name(self, snippet, regex=None): + matches = None + if regex: + matches = regex.findall(snippet) + if not matches: + return None + snippet = matches[0] + without_numbers_and_time = self.__remove_numbers_and_time(snippet) + + return slugify(without_numbers_and_time)[:191] + def __create_shasum(self, snippet): sha = hashlib.sha256() - without_numbers = re.sub(r'(0x[a-f0-9]+|[<\[][0-9a-f]+?[>\]]|\d+)', '', snippet) - without_time = re.sub(r'^\[[^\]]+\]', '', without_numbers) - sha.update(without_time.encode()) + without_numbers_and_time = self.__remove_numbers_and_time(snippet) + sha.update(without_numbers_and_time.encode()) return sha.hexdigest() def postprocess_testrun(self, testrun): @@ -133,4 +154,8 @@ def postprocess_testrun(self, testrun): for regex_id in range(len(REGEXES)): test_name = REGEXES[regex_id][REGEX_NAME] - self.__create_tests(testrun, suite, test_name, snippets[regex_id]) + regex_pattern = REGEXES[regex_id][REGEX_EXTRACT_NAME] + test_name_regex = None + if regex_pattern: + test_name_regex = re.compile(regex_pattern, re.S | re.M) + self.__create_tests(testrun, suite, test_name, snippets[regex_id], test_name_regex) diff --git a/test/plugins/test_linux_log_parser.py b/test/plugins/test_linux_log_parser.py index e0565044..0b72ac41 100644 --- a/test/plugins/test_linux_log_parser.py +++ b/test/plugins/test_linux_log_parser.py @@ -259,7 +259,7 @@ def test_sha_name(self): self.assertNotIn('Kernel panic', test.log) # Now check if a test with sha digest in the name - test = testrun.tests.get(suite__slug='log-parser-boot', metadata__name='check-kernel-oops-a1acf2f0467782c9c2f6aeadb1d1d3cec136642b13d7231824a66ef63ee62220') + test = testrun.tests.get(suite__slug='log-parser-boot', metadata__name='check-kernel-oops-oops-bug-preempt-smp-a1acf2f0467782c9c2f6aeadb1d1d3cec136642b13d7231824a66ef63ee62220') self.assertFalse(test.result) self.assertIsNotNone(test.log) self.assertIn('Internal error: Oops - BUG: 0 [#0] PREEMPT SMP', test.log)