diff --git a/Dockerfile b/Dockerfile index 94cd2cca..4b4fb13f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,7 +44,7 @@ WORKDIR /backend/ COPY --from=builder /backend ./ COPY --from=builder /venv /venv COPY --from=builder /usr/share/nltk_data /usr/share/nltk_data/ -RUN dnf -y upgrade && dnf -y install libgomp pcre-devel \ +RUN dnf -y upgrade && dnf -y install pcre-devel \ && dnf -y remove emacs-filesystem libjpeg-turbo libtiff libpng wget \ && dnf -y autoremove \ && dnf clean all \ diff --git a/app/commons/clusterizer.py b/app/commons/clusterizer.py index aede0eca..30bea243 100644 --- a/app/commons/clusterizer.py +++ b/app/commons/clusterizer.py @@ -31,7 +31,7 @@ class Clusterizer: def __init__(self): pass - def calculate_hashes(self, messages, n_gram=2, n_permutations=64): + def calculate_hashes(self, messages: list[str], n_gram: int = 2, n_permutations: int = 64) -> list[list[str]]: hashes = [] for message in messages: words = message.split() @@ -39,11 +39,12 @@ def calculate_hashes(self, messages, n_gram=2, n_permutations=64): len_words = (len(words) - n_gram) if len(words) > n_gram else len(words) for i in range(len_words): hash_print.add(hashlib.md5(" ".join(words[i:i + n_gram]).encode("utf-8")).hexdigest()) - hash_print = list(heapq.nlargest(n_permutations, hash_print)) - hashes.append(hash_print) + hashes.append(list(heapq.nlargest(n_permutations, hash_print))) return hashes - def find_groups_by_similarity(self, messages, groups_to_check, threshold=0.95): + def find_groups_by_similarity( + self, messages: list[str], groups_to_check: dict[int, list[int]], + threshold: float = 0.95) -> dict[int, list[int]]: if len(messages) == 0: return {} rearranged_groups = {} @@ -67,7 +68,9 @@ def find_groups_by_similarity(self, messages, groups_to_check, threshold=0.95): logger.debug("Time for finding groups: %.2f s", time() - start_time) return rearranged_groups - def similarity_groupping(self, hash_prints, block_size=1000, for_text=True, threshold=0.95): + def similarity_groupping( + self, hash_prints: list[list[str]] | list[str], block_size: int = 1000, for_text: bool = True, + threshold: float = 0.95) -> dict[int, int]: num_of_blocks = int(np.ceil(len(hash_prints) / block_size)) hash_groups = {} global_ind = 0 @@ -105,7 +108,7 @@ def similarity_groupping(self, hash_prints, block_size=1000, for_text=True, thre hash_groups[j] = hash_groups[i] return hash_groups - def unite_groups_by_hashes(self, messages, threshold=0.95): + def unite_groups_by_hashes(self, messages: list[str], threshold: float = 0.95) -> dict[int, list[int]]: start_time = time() hash_prints = self.calculate_hashes(messages) has_no_empty = False @@ -125,7 +128,7 @@ def unite_groups_by_hashes(self, messages, threshold=0.95): logger.debug("Time for finding hash groups: %.2f s", time() - start_time) return rearranged_groups - def perform_light_deduplication(self, messages): + def perform_light_deduplication(self, messages: list[str]) -> tuple[list[str], dict[int, list[int]]]: text_messages_set = {} messages_to_cluster = [] ids_with_duplicates = {} @@ -142,11 +145,10 @@ def perform_light_deduplication(self, messages): ids_with_duplicates[text_messages_set[text_message_normalized]].append(idx) return messages_to_cluster, ids_with_duplicates - def find_clusters(self, messages, threshold=0.95): + def find_clusters(self, messages: list[str], threshold: float = 0.95) -> dict[int, list[int]]: messages_to_cluster, ids_with_duplicates = self.perform_light_deduplication(messages) hash_groups = self.unite_groups_by_hashes(messages_to_cluster, threshold=threshold) - groups = self.find_groups_by_similarity( - messages_to_cluster, hash_groups, threshold=threshold) + groups = self.find_groups_by_similarity(messages_to_cluster, hash_groups, threshold=threshold) new_groups = {} for cluster in groups: new_log_ids = [] diff --git a/app/commons/log_merger.py b/app/commons/log_merger.py index 8a81b363..01513985 100644 --- a/app/commons/log_merger.py +++ b/app/commons/log_merger.py @@ -13,6 +13,7 @@ # limitations under the License. import copy +from typing import Any, Optional from app.utils import text_processing @@ -34,9 +35,11 @@ def __init__(self): "paths", "message_params", "detected_message_without_params_extended", "whole_message"] - def merge_big_and_small_logs(self, logs: list[dict], log_level_ids_to_add: dict, log_level_messages: dict, - log_level_ids_merged, logs_ids_in_merged_logs) -> tuple[list[dict], dict]: - """Merge big message logs with small ones""" + def merge_big_and_small_logs( + self, logs: list[dict[str, Any]], log_level_ids_to_add: dict[int, list[int]], + log_level_messages: dict[str, dict[int, str]], log_level_ids_merged: dict[int, dict[str, Any]], + logs_ids_in_merged_logs: dict[int, list[int]]) -> tuple[list[dict[str, Any]], dict[str, list[int]]]: + """Merge big message logs with small ones.""" new_logs = [] for log in logs: if not log["_source"]["message"].strip(): @@ -50,9 +53,7 @@ def merge_big_and_small_logs(self, logs: list[dict], log_level_ids_to_add: dict, log_ids_for_merged_logs = {} for log_level in log_level_messages["message"]: - - if not log_level_ids_to_add[log_level] and \ - log_level_messages["message"][log_level].strip(): + if not log_level_ids_to_add[log_level] and log_level_messages["message"][log_level].strip(): log = log_level_ids_merged[log_level] merged_logs_id = str(log["_id"]) + "_m" new_log = self.prepare_new_log( @@ -73,8 +74,9 @@ def merge_big_and_small_logs(self, logs: list[dict], log_level_ids_to_add: dict, new_logs.append(new_log) return new_logs, log_ids_for_merged_logs - def decompose_logs_merged_and_without_duplicates(self, logs: list[dict]) -> tuple[list[dict], dict]: - """Merge big logs with small ones without duplicates""" + def decompose_logs_merged_and_without_duplicates( + self, logs: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], dict[str, list[int]]]: + """Merge big logs with small ones without duplicates.""" log_level_messages = {} for field in self.fields_to_merge: log_level_messages[field] = {} @@ -127,8 +129,8 @@ def decompose_logs_merged_and_without_duplicates(self, logs: list[dict]) -> tupl return self.merge_big_and_small_logs( logs, log_level_ids_to_add, log_level_messages, log_level_ids_merged, logs_ids_in_merged_logs) - def prepare_new_log(self, old_log: dict, new_id, is_merged: bool, merged_small_logs: str, - fields_to_clean: list[str] | None = None) -> dict: + def prepare_new_log(self, old_log: dict[str, Any], new_id, is_merged: bool, merged_small_logs: str, + fields_to_clean: Optional[list[str]] = None) -> dict[str, Any]: """Prepare updated log""" merged_log = copy.deepcopy(old_log) merged_log["_source"]["is_merged"] = is_merged diff --git a/app/commons/log_requests.py b/app/commons/log_requests.py index e8172e4d..3744f388 100644 --- a/app/commons/log_requests.py +++ b/app/commons/log_requests.py @@ -13,6 +13,7 @@ # limitations under the License. from datetime import datetime +from typing import Any from app.commons.model.launch_objects import Launch, TestItem, Log, TestItemInfo from app.commons.log_merger import LogMerger @@ -21,47 +22,50 @@ from app.utils.log_preparation import basic_prepare +def create_log_template() -> dict: + return { + "_id": "", + "_index": "", + "_source": { + "launch_id": "", + "launch_name": "", + "launch_number": 0, + "launch_start_time": "", + "test_item": "", + "test_item_name": "", + "unique_id": "", + "cluster_id": "", + "cluster_message": "", + "test_case_hash": 0, + "is_auto_analyzed": False, + "issue_type": "", + "log_time": "", + "log_level": 0, + 'original_message': '', + "original_message_lines": 0, + "original_message_words_number": 0, + "message": "", + "is_merged": False, + "start_time": "", + "merged_small_logs": "", + "detected_message": "", + "detected_message_with_numbers": "", + "stacktrace": "", + "only_numbers": "", + "found_exceptions": "", + "whole_message": "", + "potential_status_codes": "", + "found_tests_and_methods": "", + "cluster_with_numbers": False + } + } + + class LogRequests: def __init__(self): self.log_merger = LogMerger() - @staticmethod - def _create_log_template() -> dict: - return { - "_id": "", - "_index": "", - "_source": { - "launch_id": "", - "launch_name": "", - "launch_number": 0, - "launch_start_time": "", - "test_item": "", - "test_item_name": "", - "unique_id": "", - "cluster_id": "", - "cluster_message": "", - "test_case_hash": 0, - "is_auto_analyzed": False, - "issue_type": "", - "log_time": "", - "log_level": 0, - "original_message_lines": 0, - "original_message_words_number": 0, - "message": "", - "is_merged": False, - "start_time": "", - "merged_small_logs": "", - "detected_message": "", - "detected_message_with_numbers": "", - "stacktrace": "", - "only_numbers": "", - "found_exceptions": "", - "whole_message": "", - "potential_status_codes": "", - "found_tests_and_methods": "", - "cluster_with_numbers": False}} - @staticmethod def transform_issue_type_into_lowercase(issue_type): return issue_type[:2].lower() + issue_type[2:] @@ -79,14 +83,12 @@ def _fill_launch_test_item_fields(log_template: dict, launch: Launch, test_item: log_template["_source"]["test_case_hash"] = test_item.testCaseHash log_template["_source"]["is_auto_analyzed"] = test_item.isAutoAnalyzed log_template["_source"]["test_item_name"] = text_processing.preprocess_test_item_name(test_item.testItemName) - log_template["_source"]["issue_type"] = LogRequests.transform_issue_type_into_lowercase( - test_item.issueType) - log_template["_source"]["start_time"] = datetime( - *test_item.startTime[:6]).strftime("%Y-%m-%d %H:%M:%S") + log_template["_source"]["issue_type"] = LogRequests.transform_issue_type_into_lowercase(test_item.issueType) + log_template["_source"]["start_time"] = datetime(*test_item.startTime[:6]).strftime("%Y-%m-%d %H:%M:%S") return log_template @staticmethod - def _fill_log_fields(log_template: dict, log: Log, number_of_lines: int): + def _fill_log_fields(log_template: dict, log: Log, number_of_lines: int) -> dict[str, Any]: prepared_log = PreparedLogMessage(log.message, number_of_lines) log_template["_id"] = log.logId log_template["_source"]["log_time"] = datetime(*log.logTime[:6]).strftime("%Y-%m-%d %H:%M:%S") @@ -94,6 +96,7 @@ def _fill_log_fields(log_template: dict, log: Log, number_of_lines: int): log_template["_source"]["cluster_message"] = log.clusterMessage log_template["_source"]["cluster_with_numbers"] = utils.extract_clustering_setting(log.clusterId) log_template["_source"]["log_level"] = log.logLevel + log_template["_source"]['original_message'] = log.message log_template["_source"]["original_message_lines"] = text_processing.calculate_line_number( prepared_log.clean_message) log_template["_source"]["original_message_words_number"] = len( @@ -137,13 +140,13 @@ def _fill_log_fields(log_template: dict, log: Log, number_of_lines: int): @staticmethod def _prepare_log(launch: Launch, test_item: TestItem, log: Log, project: str) -> dict: - log_template = LogRequests._create_log_template() + log_template = create_log_template() log_template = LogRequests._fill_launch_test_item_fields(log_template, launch, test_item, project) log_template = LogRequests._fill_log_fields(log_template, log, launch.analyzerConfig.numberOfLogLines) return log_template @staticmethod - def _fill_test_item_info_fields(log_template: dict, test_item_info: TestItemInfo, project: str) -> dict: + def _fill_test_item_info_fields(log_template: dict, test_item_info: TestItemInfo, project: str) -> dict[str, Any]: log_template["_index"] = project log_template["_source"]["launch_id"] = test_item_info.launchId log_template["_source"]["launch_name"] = test_item_info.launchName @@ -160,7 +163,7 @@ def _fill_test_item_info_fields(log_template: dict, test_item_info: TestItemInfo @staticmethod def _prepare_log_for_suggests(test_item_info: TestItemInfo, log: Log, project: str) -> dict: - log_template = LogRequests._create_log_template() + log_template = create_log_template() log_template = LogRequests._fill_test_item_info_fields(log_template, test_item_info, project) log_template = LogRequests._fill_log_fields( log_template, log, test_item_info.analyzerConfig.numberOfLogLines) @@ -184,14 +187,15 @@ def prepare_log_words(launches: list[Launch]) -> tuple[dict[str, int], int]: return log_words, project @staticmethod - def prepare_log_clustering_light(launch: Launch, test_item: TestItem, log: Log, project: str): - log_template = LogRequests._create_log_template() + def prepare_log_clustering_light(launch: Launch, test_item: TestItem, log: Log, project: str) -> dict[str, Any]: + log_template = create_log_template() log_template = LogRequests._fill_launch_test_item_fields(log_template, launch, test_item, project) prepared_log = PreparedLogMessage(log.message, -1) log_template["_id"] = log.logId log_template["_source"]["cluster_id"] = str(log.clusterId) log_template["_source"]["cluster_message"] = log.clusterMessage log_template["_source"]["log_level"] = log.logLevel + log_template["_source"]['original_message'] = log.message log_template["_source"]["original_message_lines"] = text_processing.calculate_line_number( prepared_log.clean_message) log_template["_source"]["original_message_words_number"] = len( @@ -206,7 +210,8 @@ def prepare_log_clustering_light(launch: Launch, test_item: TestItem, log: Log, + prepared_log.stacktrace) return log_template - def prepare_logs_for_clustering(self, launch: Launch, number_of_lines: int, clean_numbers: bool, project: str): + def prepare_logs_for_clustering(self, launch: Launch, number_of_lines: int, clean_numbers: bool, + project: str) -> tuple[list[str], dict[int, dict[str, Any]], dict[str, list[int]]]: log_messages = [] log_dict = {} ind = 0 @@ -219,8 +224,8 @@ def prepare_logs_for_clustering(self, launch: Launch, number_of_lines: int, clea prepared_logs.append(LogRequests.prepare_log_clustering_light(launch, test_item, log, project)) merged_logs, log_ids_for_merged_logs = self.log_merger.decompose_logs_merged_and_without_duplicates( prepared_logs) - for _id in log_ids_for_merged_logs: - full_log_ids_for_merged_logs[_id] = log_ids_for_merged_logs[_id] + for _id, merged_list in log_ids_for_merged_logs.items(): + full_log_ids_for_merged_logs[_id] = merged_list for log in merged_logs: number_of_log_lines = number_of_lines if log["_source"]["is_merged"]: diff --git a/app/service/cluster_service.py b/app/service/cluster_service.py index 72aa9887..8da7ffc4 100644 --- a/app/service/cluster_service.py +++ b/app/service/cluster_service.py @@ -14,8 +14,10 @@ import hashlib import json +from collections import defaultdict from datetime import datetime from time import time +from typing import Any import numpy as np from sklearn.feature_extraction.text import CountVectorizer @@ -46,7 +48,35 @@ def __init__(self, app_config: ApplicationConfig, search_cfg: SearchConfig): self.log_requests = LogRequests() self.log_merger = LogMerger() - def build_search_similar_items_query(self, queried_log, message, launch_info, min_should_match="95%"): + def add_query_with_start_time_decay(self, main_query: dict[str, Any]) -> dict[str, Any]: + return { + "size": main_query["size"], + "query": { + "function_score": { + "query": main_query["query"], + "functions": [ + { + "exp": { + "start_time": { + "origin": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "scale": "7d", + "offset": "1d", + "decay": self.search_cfg.TimeWeightDecay + } + } + }, + { + "script_score": {"script": {"source": "0.2"}} + }], + "score_mode": "max", + "boost_mode": "multiply" + } + } + } + + def build_search_similar_items_query( + self, queried_log: dict[str, Any], message: str, launch_info: LaunchInfoForClustering, + min_should_match: str = "95%") -> dict[str, Any]: """Build search query""" query = { "_source": ["whole_message", "test_item", "is_merged", @@ -71,7 +101,10 @@ def build_search_similar_items_query(self, queried_log, message, launch_info, mi field_name="whole_message", boost=1.0, override_min_should_match=None, max_query_terms=self.search_cfg.MaxQueryTerms) - ]}}} + ] + } + } + } if launch_info.forUpdate: query["query"]["bool"]["should"].append( {"term": {"launch_id": queried_log["_source"]["launch_id"]}}) @@ -91,41 +124,16 @@ def build_search_similar_items_query(self, queried_log, message, launch_info, mi queried_log["_source"]["found_exceptions"], field_name="found_exceptions", boost=1.0, override_min_should_match="1", - max_query_terms=self.search_cfg.MaxQueryTerms)) - utils.append_potential_status_codes(query, queried_log, boost=1.0, - max_query_terms=self.search_cfg.MaxQueryTerms) + max_query_terms=self.search_cfg.MaxQueryTerms + ) + ) + utils.append_potential_status_codes( + query, queried_log, boost=1.0, max_query_terms=self.search_cfg.MaxQueryTerms) return self.add_query_with_start_time_decay(query) - def add_query_with_start_time_decay(self, main_query): - return { - "size": main_query["size"], - "query": { - "function_score": { - "query": main_query["query"], - "functions": [ - { - "exp": { - "start_time": { - "origin": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), - "scale": "7d", - "offset": "1d", - "decay": self.search_cfg.TimeWeightDecay - } - } - }, - { - "script_score": {"script": {"source": "0.2"}} - }], - "score_mode": "max", - "boost_mode": "multiply" - } - } - } - - def find_similar_items_from_es( - self, groups, log_dict, - log_messages, log_ids, launch_info, - additional_results, unique_errors_min_should_match): + def find_similar_items_from_es(self, groups: dict[int, list[int]], log_dict: dict[int, dict[str, Any]], + log_messages: list[str], log_ids: set[str], launch_info: LaunchInfoForClustering, + unique_errors_min_should_match: float) -> dict[int, ClusterInfo]: new_clusters = {} _clusterizer = clusterizer.Clusterizer() for global_group in groups: @@ -134,14 +142,9 @@ def find_similar_items_from_es( log_messages[first_item_ind], unique_errors_min_should_match) query = self.build_search_similar_items_query( - log_dict[first_item_ind], - log_messages[first_item_ind], - launch_info, - min_should_match=text_processing.prepare_es_min_should_match( - min_should_match)) - search_results = self.es_client.es_client.search( - index=log_dict[first_item_ind]["_index"], - body=query) + log_dict[first_item_ind], log_messages[first_item_ind], launch_info, + min_should_match=text_processing.prepare_es_min_should_match(min_should_match)) + search_results = self.es_client.es_client.search(index=log_dict[first_item_ind]["_index"], body=query) log_messages_part = [log_messages[first_item_ind]] log_dict_part = {0: log_dict[first_item_ind]} ind = 1 @@ -208,6 +211,7 @@ def find_similar_items_from_es( break if new_group: new_clusters[global_group] = new_group + additional_results = {} for group in new_clusters: if group in additional_results: additional_results[group].logIds.extend(new_clusters[group].logIds) @@ -216,7 +220,46 @@ def find_similar_items_from_es( additional_results[group] = new_clusters[group] return additional_results - def calculate_hash(self, group_ids, log_dict, log_messages, launch_info): + def regroup_by_error_and_status_codes( + self, log_messages: list[str], log_dict: dict[int, dict[str, Any]]) -> dict[tuple[str, str], list[int]]: + regroupped_by_error = defaultdict(list) + for i in range(len(log_messages)): + found_exceptions = " ".join( + sorted(log_dict[i]["_source"]["found_exceptions"].split())) + potential_status_codes = " ".join( + sorted(log_dict[i]["_source"]["potential_status_codes"].split())) + group_key = (found_exceptions, potential_status_codes) + regroupped_by_error[group_key].append(i) + return regroupped_by_error + + def cluster_messages_with_grouping_by_error( + self, log_messages: list[str], log_dict: dict[int, dict[str, Any]], + unique_errors_min_should_match: float) -> dict[int, list[int]]: + regroupped_by_error = self.regroup_by_error_and_status_codes( + log_messages, log_dict) + _clusterizer = clusterizer.Clusterizer() + all_groups = {} + start_group_id = 0 + for group in regroupped_by_error.values(): + log_messages_part = [] + log_messages_idx_dict = {} + for i, idx in enumerate(group): + log_messages_part.append(log_messages[idx]) + log_messages_idx_dict[i] = idx + groups = _clusterizer.find_clusters(log_messages_part, threshold=unique_errors_min_should_match) + max_group_id = max(groups.keys()) + for group_id in groups: + global_idx = start_group_id + group_id + if global_idx not in all_groups: + all_groups[global_idx] = [] + for i in groups[group_id]: + all_groups[global_idx].append(log_messages_idx_dict[i]) + start_group_id = start_group_id + max_group_id + 1 + return all_groups + + def calculate_hash( + self, group_ids: list[int], log_dict: dict[int, dict[str, Any]], log_messages: list[str], + launch_info: LaunchInfoForClustering) -> tuple[int, str]: group_logs = [] log_message = "" for i in range(min(100, len(group_ids))): @@ -244,8 +287,10 @@ def calculate_hash(self, group_ids, log_dict, log_messages, launch_info): return hash_message, log_message def gather_cluster_results( - self, groups, additional_results, log_dict, log_messages, - log_ids_for_merged_logs, launch_info): + self, groups: dict[int, list[int]], additional_results: dict[int, ClusterInfo], + log_dict: dict[int, dict[str, Any]], log_messages: list[str], + log_ids_for_merged_logs: dict[str, list[int]], + launch_info: LaunchInfoForClustering) -> tuple[list[ClusterInfo], int, dict[str, tuple[int, str]]]: merged_logs_to_update = {} clusters_found = {} cluster_message_by_id = {} @@ -293,44 +338,6 @@ def gather_cluster_results( itemIds=list(set(clusters_found[cluster_id][1])))) return results_to_return, len(results_to_return), merged_logs_to_update - def regroup_by_error_ans_status_codes(self, log_messages, log_dict): - regroupped_by_error = {} - for i in range(len(log_messages)): - found_exceptions = " ".join( - sorted(log_dict[i]["_source"]["found_exceptions"].split())) - potential_status_codes = " ".join( - sorted(log_dict[i]["_source"]["potential_status_codes"].split())) - group_key = (found_exceptions, potential_status_codes) - if group_key not in regroupped_by_error: - regroupped_by_error[group_key] = [] - regroupped_by_error[group_key].append(i) - return regroupped_by_error - - def cluster_messages_with_grouping_by_error(self, log_messages, log_dict, unique_errors_min_should_match): - regroupped_by_error = self.regroup_by_error_ans_status_codes( - log_messages, log_dict) - _clusterizer = clusterizer.Clusterizer() - all_groups = {} - start_group_id = 0 - for group_key in regroupped_by_error: - log_messages_part = [] - log_messages_idx_dict = {} - for i, idx in enumerate(regroupped_by_error[group_key]): - log_messages_part.append(log_messages[idx]) - log_messages_idx_dict[i] = idx - groups = _clusterizer.find_clusters( - log_messages_part, - threshold=unique_errors_min_should_match) - max_group_id = max(groups.keys()) - for group_id in groups: - global_idx = start_group_id + group_id - if global_idx not in all_groups: - all_groups[global_idx] = [] - for i in groups[group_id]: - all_groups[global_idx].append(log_messages_idx_dict[i]) - start_group_id = start_group_id + max_group_id + 1 - return all_groups - @utils.ignore_warnings def find_clusters(self, launch_info: LaunchInfoForClustering): logger.info("Started clusterizing logs") @@ -347,29 +354,23 @@ def find_clusters(self, launch_info: LaunchInfoForClustering): errors_count = 0 cluster_num = 0 clusters = [] - log_ids = [] + log_ids = {} try: - unique_errors_min_should_match = launch_info.launch.analyzerConfig.uniqueErrorsMinShouldMatch / 100 # noqa + unique_errors_min_should_match = launch_info.launch.analyzerConfig.uniqueErrorsMinShouldMatch / 100.0 # noqa log_messages, log_dict, log_ids_for_merged_logs = self.log_requests.prepare_logs_for_clustering( # noqa launch_info.launch, launch_info.numberOfLogLines, launch_info.cleanNumbers, index_name) log_ids = set([str(log["_id"]) for log in log_dict.values()]) - groups = self.cluster_messages_with_grouping_by_error( - log_messages, log_dict, - unique_errors_min_should_match) + log_messages, log_dict, unique_errors_min_should_match) logger.debug("Groups: %s", groups) additional_results = self.find_similar_items_from_es( - groups, log_dict, log_messages, - log_ids, launch_info, - {}, unique_errors_min_should_match) + groups, log_dict, log_messages, log_ids, launch_info, unique_errors_min_should_match) clusters, cluster_num, merged_logs_to_update = self.gather_cluster_results( - groups, additional_results, log_dict, log_messages, - log_ids_for_merged_logs, launch_info) + groups, additional_results, log_dict, log_messages, log_ids_for_merged_logs, launch_info) if clusters: bodies = [] for result in clusters: - logger.debug("Cluster Id: %s, Cluster message: %s", - result.clusterId, result.clusterMessage) + logger.debug("Cluster Id: %s, Cluster message: %s", result.clusterId, result.clusterMessage) logger.debug("Cluster Ids: %s", result.logIds) for log_id in result.logIds: bodies.append({ @@ -413,6 +414,14 @@ def find_clusters(self, launch_info: LaunchInfoForClustering): logger.debug("Stats info %s", results_to_share) logger.info("Processed the launch. It took %.2f sec.", time() - t_start) logger.info("Finished clustering for the launch with %d clusters.", cluster_num) + for cluster in clusters: + # Set original messages for clusters to show in UI + log_ids = set(cluster.logIds) + for test_item in launch_info.launch.testItems: + for log in test_item.logs: + if log.logId in log_ids: + cluster.clusterMessage = log.message + break return ClusterResult( project=launch_info.project, launchId=launch_info.launch.launchId, diff --git a/app/service/search_service.py b/app/service/search_service.py index 0dee7565..ede9f862 100644 --- a/app/service/search_service.py +++ b/app/service/search_service.py @@ -21,7 +21,7 @@ from app.commons.esclient import EsClient from app.commons.model.launch_objects import SearchLogInfo, Log, SearchConfig, ApplicationConfig from app.commons.log_merger import LogMerger -from app.commons.log_requests import LogRequests +from app.commons.log_requests import LogRequests, create_log_template from app.machine_learning.models.weighted_similarity_calculator import WeightedSimilarityCalculator from app.utils import utils, text_processing @@ -161,7 +161,7 @@ def prepare_messages_for_queries(self, search_req): if not message.strip(): continue - queried_log = LogRequests._create_log_template() + queried_log = create_log_template() queried_log = LogRequests._fill_log_fields(queried_log, Log(logId=global_id, message=message), search_req.logLines) diff --git a/app/utils/text_processing.py b/app/utils/text_processing.py index da00a5b8..2c7c446b 100644 --- a/app/utils/text_processing.py +++ b/app/utils/text_processing.py @@ -324,8 +324,8 @@ def first_lines(log_str: str, n_lines: int) -> str: return '\n'.join((log_str.split('\n')[:n_lines])) if n_lines >= 0 else log_str -def prepare_message_for_clustering(message, number_of_log_lines, clean_numbers, - leave_log_structure=False): +def prepare_message_for_clustering(message: str, number_of_log_lines: int, clean_numbers: bool, + leave_log_structure: bool = False) -> str: potential_status_codes = get_potential_status_codes(message) message = remove_starting_datetime(message) if clean_numbers: @@ -746,7 +746,7 @@ def replace_text_pieces(text: str, text_pieces: Iterable[str]) -> str: return result -def prepare_es_min_should_match(min_should_match): +def prepare_es_min_should_match(min_should_match: float) -> str: return str(int(min_should_match * 100)) + "%" diff --git a/app/utils/utils.py b/app/utils/utils.py index cf11fb16..846118cd 100644 --- a/app/utils/utils.py +++ b/app/utils/utils.py @@ -313,7 +313,8 @@ def build_more_like_this_query(min_should_match: str, log_message, "boost": boost}} -def append_potential_status_codes(query, log, *, boost=8.0, max_query_terms=50): +def append_potential_status_codes( + query: dict[str, Any], log: dict[str, Any], *, boost: float = 8.0, max_query_terms: int = 50) -> None: potential_status_codes = log["_source"]["potential_status_codes"].strip() if potential_status_codes: number_of_status_codes = str(len(set(potential_status_codes.split()))) diff --git a/test/mock_service.py b/test/mock_service.py index 8aa24595..53ce7b28 100644 --- a/test/mock_service.py +++ b/test/mock_service.py @@ -256,7 +256,16 @@ def shutdown_server(test_calls): if "rq" in expected_test_call: expected_body = expected_test_call["rq"] real_body = test_call.parse_request_body(test_call.body) + json_rq = False if type(expected_body) is str and type(real_body) is not str: expected_body = json.loads(expected_body) - assert expected_body == real_body, f'Error in request {i}' + json_rq = True + if expected_body != real_body: + print(f'Error in request {i}') + if json_rq: + expected_body = json.dumps(expected_body) + real_body = json.dumps(real_body) + print(f'Expected: {expected_body}') + print(f'Actual: {real_body}') + raise AssertionError(f'Error in request {i}') httpretty.disable() diff --git a/test/service/test_cluster_service.py b/test/service/test_cluster_service.py index d2aaf55b..0fb37d9f 100644 --- a/test/service/test_cluster_service.py +++ b/test/service/test_cluster_service.py @@ -121,12 +121,12 @@ def test_find_clusters(self): clusters=[ launch_objects.ClusterInfo( clusterId=21874152824769751, - clusterMessage="error occurred\nerror found\nerror mined", + clusterMessage="error occurred \n error found \n error mined", logIds=[4, 5], itemIds=[2, 5]), launch_objects.ClusterInfo( clusterId=44972330576749361, - clusterMessage="error occurred\nerror found\nassert query", + clusterMessage="error occurred \n error found \n assert query", logIds=[9], itemIds=[6]) ]) @@ -163,7 +163,7 @@ def test_find_clusters(self): clusters=[ launch_objects.ClusterInfo( clusterId="48859729558090231", - clusterMessage="error occurred\nerror found", + clusterMessage="error occurred \n error found \n assert query", logIds=[4, 5, 9], itemIds=[2, 5, 6]) ]) @@ -206,12 +206,12 @@ def test_find_clusters(self): clusters=[ launch_objects.ClusterInfo( clusterId="21874152824769751", - clusterMessage="error occurred\nerror found\nerror mined", + clusterMessage="error occurred \n error found \n error mined", logIds=[4, 5], itemIds=[2, 5]), launch_objects.ClusterInfo( clusterId="44972330576749361", - clusterMessage="error occurred\nerror found\nassert query", + clusterMessage="error occurred \n error found \n assert query", logIds=[9], itemIds=[6]), ]) @@ -260,7 +260,7 @@ def test_find_clusters(self): itemIds=[2, 5]), launch_objects.ClusterInfo( clusterId="44972330576749361", - clusterMessage="error occurred\nerror found\nassert query", + clusterMessage="error occurred \n error found \n assert query", logIds=[9], itemIds=[6]) ]) @@ -297,7 +297,7 @@ def test_find_clusters(self): clusters=[ launch_objects.ClusterInfo( clusterId="48859729558090231", - clusterMessage="error occurred\nerror found", + clusterMessage="error occurred \n error found \n assert query", logIds=[4, 5, 9], itemIds=[2, 5, 6]) ]) @@ -335,7 +335,7 @@ def test_find_clusters(self): clusters=[ launch_objects.ClusterInfo( clusterId="48859729558090231", - clusterMessage="error occurred\nerror found", + clusterMessage="error occurred \n error found \n assert query", logIds=[4, 5, 9], itemIds=[2, 5, 6]) ]) @@ -388,17 +388,18 @@ def test_find_clusters(self): clusters=[ launch_objects.ClusterInfo( clusterId="37711525315085941", - clusterMessage="AssertionError error occurred\nerror found", + clusterMessage="AssertionError error occurred \n error found \n error mined", logIds=[4], itemIds=[2]), launch_objects.ClusterInfo( clusterId="48851059259117511", - clusterMessage="AssertionError status code SPECIALNUMBER error occurred\nerror found", + clusterMessage="AssertionError status code: 500 error occurred \n error found \n error " + "mined", logIds=[5], itemIds=[5]), launch_objects.ClusterInfo( clusterId="90988898127574211", - clusterMessage="NoSuchElementException error occurred\nerror found", + clusterMessage="NoSuchElementException error occurred \n error found \n assert query", logIds=[9], itemIds=[6]), ]) @@ -449,18 +450,18 @@ def test_find_clusters(self): clusters=[ launch_objects.ClusterInfo( clusterId="60604459849884091", - clusterMessage="error occurred twice\nAssertionError error occurred\nerror found", + clusterMessage="error occurred twice", # noqa logIds=[3, 4], itemIds=[2]), launch_objects.ClusterInfo( clusterId="9398573272102061", - clusterMessage="AssertionError status code SPECIALNUMBER error occurred", + clusterMessage="AssertionError status code: 500 error occurred", logIds=[5], itemIds=[5]), launch_objects.ClusterInfo( clusterId="86465058569810291", - clusterMessage="NoSuchElementException error occurred\nerror found\nassert query", + clusterMessage="NoSuchElementException error occurred \n error found \n assert query", # noqa logIds=[9], itemIds=[6]), diff --git a/test_res/fixtures/index_logs_rq_big_messages.json b/test_res/fixtures/index_logs_rq_big_messages.json index 2e1f9597..eae5fbd0 100644 --- a/test_res/fixtures/index_logs_rq_big_messages.json +++ b/test_res/fixtures/index_logs_rq_big_messages.json @@ -1,4 +1,4 @@ {"index":{"_index":"2","_id":1}} -{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"0","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message_lines":5,"original_message_words_number":9,"message":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER","detected_message_with_numbers":"Message 1\n Message 2\n Message 3 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.1","stacktrace":"","only_numbers":"1 2 3","found_exceptions":"java.lang.NoClassDefFoundError","whole_message":"Message SPECIALNUMBER\nMessage SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER\n","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":false,"urls":"","paths":"/src/prod/results.html","message_params":"prod_en","found_exceptions_extended":"java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError","detected_message_extended":"Message 1\n Message 2\n Message 3 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError \n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.1","detected_message_without_params_extended":"Message SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","stacktrace_extended":"","message_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError \n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER MandatoryAttributesValidator.SPECIALNUMBER SPECIALNUMBER ","message_without_params_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER prod en src prod results html\n java lang NoClassDefFoundError\n de hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","detected_message_without_params_and_brackets":"Message SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","message_without_params_and_brackets":"Message SPECIALNUMBER\n Message SPECIALNUMBER prod en src prod results html\n java lang NoClassDefFoundError\n de hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER"} +{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"0","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message":"Message 1 \n Message 2 \n Message 3 'prod_en' /src/prod/results.html \n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator$1@31ca8ab4 \n ca.canadiantire.steps.hybris.ws.WebserviceHybrisCustomerAndCartSteps$$EnhancerByCGLIB$$84837ae7.CGLIB$add_products_to_cart$16()","original_message_lines":5,"original_message_words_number":9,"message":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER","detected_message_with_numbers":"Message 1\n Message 2\n Message 3 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.1","stacktrace":"","only_numbers":"1 2 3","found_exceptions":"java.lang.NoClassDefFoundError","whole_message":"Message SPECIALNUMBER\nMessage SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER\n","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":false,"urls":"","paths":"/src/prod/results.html","message_params":"prod_en","found_exceptions_extended":"java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError","detected_message_extended":"Message 1\n Message 2\n Message 3 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError \n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.1","detected_message_without_params_extended":"Message SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","stacktrace_extended":"","message_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError \n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER MandatoryAttributesValidator.SPECIALNUMBER SPECIALNUMBER ","message_without_params_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER prod en src prod results html\n java lang NoClassDefFoundError\n de hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","detected_message_without_params_and_brackets":"Message SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","message_without_params_and_brackets":"Message SPECIALNUMBER\n Message SPECIALNUMBER prod en src prod results html\n java lang NoClassDefFoundError\n de hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER"} {"index":{"_index":"2","_id":2}} -{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"0","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message_lines":4,"original_message_words_number":7,"message":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError","detected_message_with_numbers":"Message 2\n Message 4\n Message 5 http : localhost/admin\n java.lang.NoClassDefFoundError","stacktrace":"","only_numbers":"2 4 5","found_exceptions":"java.lang.NoClassDefFoundError","whole_message":"Message SPECIALNUMBER\nMessage SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError\n","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":false,"urls":"","paths":"","message_params":"","found_exceptions_extended":"java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError","detected_message_extended":"Message 2\n Message 4\n Message 5 http : localhost/admin\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError ","detected_message_without_params_extended":"Message SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError","stacktrace_extended":"","message_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError ","message_without_params_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER http localhost admin\n java lang NoClassDefFoundError","detected_message_without_params_and_brackets":"Message SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError","message_without_params_and_brackets":"Message SPECIALNUMBER\n Message SPECIALNUMBER http localhost admin\n java lang NoClassDefFoundError"} +{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"0","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message":"Message 2 \n Message 4 \n Message 5 http:localhost/admin \n java.lang.NoClassDefFoundError\n For documentation on this error, please visit https://www.seleniumhq.org/exceptions/stale_element_reference.html","original_message_lines":4,"original_message_words_number":7,"message":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError","detected_message_with_numbers":"Message 2\n Message 4\n Message 5 http : localhost/admin\n java.lang.NoClassDefFoundError","stacktrace":"","only_numbers":"2 4 5","found_exceptions":"java.lang.NoClassDefFoundError","whole_message":"Message SPECIALNUMBER\nMessage SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError\n","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":false,"urls":"","paths":"","message_params":"","found_exceptions_extended":"java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError","detected_message_extended":"Message 2\n Message 4\n Message 5 http : localhost/admin\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError ","detected_message_without_params_extended":"Message SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError","stacktrace_extended":"","message_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError ","message_without_params_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER http localhost admin\n java lang NoClassDefFoundError","detected_message_without_params_and_brackets":"Message SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError","message_without_params_and_brackets":"Message SPECIALNUMBER\n Message SPECIALNUMBER http localhost admin\n java lang NoClassDefFoundError"} diff --git a/test_res/fixtures/index_logs_rq_big_messages_with_clusters.json b/test_res/fixtures/index_logs_rq_big_messages_with_clusters.json index 79e97af2..8c6c5f09 100644 --- a/test_res/fixtures/index_logs_rq_big_messages_with_clusters.json +++ b/test_res/fixtures/index_logs_rq_big_messages_with_clusters.json @@ -1,4 +1,4 @@ {"index":{"_index":"2","_id":1}} -{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"2727777272727727721","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message_lines":5,"original_message_words_number":9,"message":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER","detected_message_with_numbers":"Message 1\n Message 2\n Message 3 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.1","stacktrace":"","only_numbers":"1 2 3","found_exceptions":"java.lang.NoClassDefFoundError","whole_message":"Message SPECIALNUMBER\nMessage SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER\n","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":true,"urls":"","paths":"/src/prod/results.html","message_params":"prod_en","found_exceptions_extended":"java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError","detected_message_extended":"Message 1\n Message 2\n Message 3 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError \n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.1","detected_message_without_params_extended":"Message SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","stacktrace_extended":"","message_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError \n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER MandatoryAttributesValidator.SPECIALNUMBER SPECIALNUMBER ","message_without_params_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER prod en src prod results html\n java lang NoClassDefFoundError\n de hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","detected_message_without_params_and_brackets":"Message SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","message_without_params_and_brackets":"Message SPECIALNUMBER\n Message SPECIALNUMBER prod en src prod results html\n java lang NoClassDefFoundError\n de hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER"} +{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"2727777272727727721","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message":"Message 1 \n Message 2 \n Message 3 'prod_en' /src/prod/results.html \n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator$1@31ca8ab4 \n ca.canadiantire.steps.hybris.ws.WebserviceHybrisCustomerAndCartSteps$$EnhancerByCGLIB$$84837ae7.CGLIB$add_products_to_cart$16()","original_message_lines":5,"original_message_words_number":9,"message":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER","detected_message_with_numbers":"Message 1\n Message 2\n Message 3 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError\n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.1","stacktrace":"","only_numbers":"1 2 3","found_exceptions":"java.lang.NoClassDefFoundError","whole_message":"Message SPECIALNUMBER\nMessage SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER\n","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":true,"urls":"","paths":"/src/prod/results.html","message_params":"prod_en","found_exceptions_extended":"java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError","detected_message_extended":"Message 1\n Message 2\n Message 3 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError \n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.1","detected_message_without_params_extended":"Message SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","stacktrace_extended":"","message_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER 'prod_en' /src/prod/results.html\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError \n de.hybris.platform.servicelayer.interceptor.impl.MandatoryAttributesValidator.SPECIALNUMBER MandatoryAttributesValidator.SPECIALNUMBER SPECIALNUMBER ","message_without_params_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER prod en src prod results html\n java lang NoClassDefFoundError\n de hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","detected_message_without_params_and_brackets":"Message SPECIALNUMBER\nMessage SPECIALNUMBER prod en src prod results html\njava lang NoClassDefFoundError\nde hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER","message_without_params_and_brackets":"Message SPECIALNUMBER\n Message SPECIALNUMBER prod en src prod results html\n java lang NoClassDefFoundError\n de hybris platform servicelayer interceptor impl MandatoryAttributesValidator SPECIALNUMBER"} {"index":{"_index":"2","_id":2}} -{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"0","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message_lines":4,"original_message_words_number":7,"message":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError","detected_message_with_numbers":"Message 2\n Message 4\n Message 5 http : localhost/admin\n java.lang.NoClassDefFoundError","stacktrace":"","only_numbers":"2 4 5","found_exceptions":"java.lang.NoClassDefFoundError","whole_message":"Message SPECIALNUMBER\nMessage SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError\n","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":false,"urls":"","paths":"","message_params":"","found_exceptions_extended":"java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError","detected_message_extended":"Message 2\n Message 4\n Message 5 http : localhost/admin\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError ","detected_message_without_params_extended":"Message SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError","stacktrace_extended":"","message_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError ","message_without_params_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER http localhost admin\n java lang NoClassDefFoundError","detected_message_without_params_and_brackets":"Message SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError","message_without_params_and_brackets":"Message SPECIALNUMBER\n Message SPECIALNUMBER http localhost admin\n java lang NoClassDefFoundError"} +{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"0","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message":"Message 2 \n Message 4 \n Message 5 http:localhost/admin \n java.lang.NoClassDefFoundError\n For documentation on this error, please visit https://www.seleniumhq.org/exceptions/stale_element_reference.html","original_message_lines":4,"original_message_words_number":7,"message":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError","detected_message_with_numbers":"Message 2\n Message 4\n Message 5 http : localhost/admin\n java.lang.NoClassDefFoundError","stacktrace":"","only_numbers":"2 4 5","found_exceptions":"java.lang.NoClassDefFoundError","whole_message":"Message SPECIALNUMBER\nMessage SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError\n","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":false,"urls":"","paths":"","message_params":"","found_exceptions_extended":"java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError","detected_message_extended":"Message 2\n Message 4\n Message 5 http : localhost/admin\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError ","detected_message_without_params_extended":"Message SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError","stacktrace_extended":"","message_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER http : localhost/admin\n java.lang.NoClassDefFoundError lang.NoClassDefFoundError NoClassDefFoundError ","message_without_params_extended":"Message SPECIALNUMBER\n Message SPECIALNUMBER http localhost admin\n java lang NoClassDefFoundError","detected_message_without_params_and_brackets":"Message SPECIALNUMBER\nMessage SPECIALNUMBER http localhost admin\njava lang NoClassDefFoundError","message_without_params_and_brackets":"Message SPECIALNUMBER\n Message SPECIALNUMBER http localhost admin\n java lang NoClassDefFoundError"} diff --git a/test_res/fixtures/index_logs_rq_different_log_level.json b/test_res/fixtures/index_logs_rq_different_log_level.json index a5be7c06..688ee101 100644 --- a/test_res/fixtures/index_logs_rq_different_log_level.json +++ b/test_res/fixtures/index_logs_rq_different_log_level.json @@ -1,2 +1,2 @@ {"index":{"_index":"2","_id":1}} -{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"0","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message_lines":3,"original_message_words_number":8,"message":" Message SPECIALNUMBER \n java.lang.reflect.Method.invoke(Method.java : SPECIALNUMBER)\n message error caused by exception","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":" Message SPECIALNUMBER \n message error caused by exception","detected_message_with_numbers":" Message 1 \n message error caused by exception","stacktrace":" java.lang.reflect.Method.invoke","only_numbers":"1","found_exceptions":"","whole_message":" Message SPECIALNUMBER\nmessage error caused by exception\n java.lang.reflect.Method.invoke","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":false,"urls":"","paths":"","message_params":"","found_exceptions_extended":"","detected_message_extended":" Message 1 \n message error caused by exception","detected_message_without_params_extended":" Message SPECIALNUMBER\nmessage error caused by exception","stacktrace_extended":" java.lang.reflect.Method.invoke Method.invoke invoke ","message_extended":" Message SPECIALNUMBER \n java.lang.reflect.Method.invoke Method.invoke invoke (Method.java : SPECIALNUMBER)\n message error caused by exception","message_without_params_extended":" Message SPECIALNUMBER \n java lang reflect Method invoke Method java SPECIALNUMBER \n message error caused by exception","detected_message_without_params_and_brackets":" Message SPECIALNUMBER\nmessage error caused by exception","message_without_params_and_brackets":" Message SPECIALNUMBER \n java lang reflect Method invoke Method java SPECIALNUMBER \n message error caused by exception"} +{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"0","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message":"Message 1 \n java.lang.reflect.Method.invoke(Method.java:498) \n message error caused by exception\n ... 34 more","original_message_lines":3,"original_message_words_number":8,"message":" Message SPECIALNUMBER \n java.lang.reflect.Method.invoke(Method.java : SPECIALNUMBER)\n message error caused by exception","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":" Message SPECIALNUMBER \n message error caused by exception","detected_message_with_numbers":" Message 1 \n message error caused by exception","stacktrace":" java.lang.reflect.Method.invoke","only_numbers":"1","found_exceptions":"","whole_message":" Message SPECIALNUMBER\nmessage error caused by exception\n java.lang.reflect.Method.invoke","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":false,"urls":"","paths":"","message_params":"","found_exceptions_extended":"","detected_message_extended":" Message 1 \n message error caused by exception","detected_message_without_params_extended":" Message SPECIALNUMBER\nmessage error caused by exception","stacktrace_extended":" java.lang.reflect.Method.invoke Method.invoke invoke ","message_extended":" Message SPECIALNUMBER \n java.lang.reflect.Method.invoke Method.invoke invoke (Method.java : SPECIALNUMBER)\n message error caused by exception","message_without_params_extended":" Message SPECIALNUMBER \n java lang reflect Method invoke Method java SPECIALNUMBER \n message error caused by exception","detected_message_without_params_and_brackets":" Message SPECIALNUMBER\nmessage error caused by exception","message_without_params_and_brackets":" Message SPECIALNUMBER \n java lang reflect Method invoke Method java SPECIALNUMBER \n message error caused by exception"} diff --git a/test_res/fixtures/index_logs_rq_different_log_level_with_prefix.json b/test_res/fixtures/index_logs_rq_different_log_level_with_prefix.json index 0e7ad1cf..56e9ac11 100644 --- a/test_res/fixtures/index_logs_rq_different_log_level_with_prefix.json +++ b/test_res/fixtures/index_logs_rq_different_log_level_with_prefix.json @@ -1,2 +1,2 @@ {"index":{"_index":"rp_2","_id":1}} -{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"0","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message_lines":3,"original_message_words_number":8,"message":" Message SPECIALNUMBER \n java.lang.reflect.Method.invoke(Method.java : SPECIALNUMBER)\n message error caused by exception","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":" Message SPECIALNUMBER \n message error caused by exception","detected_message_with_numbers":" Message 1 \n message error caused by exception","stacktrace":" java.lang.reflect.Method.invoke","only_numbers":"1","found_exceptions":"","whole_message":" Message SPECIALNUMBER\nmessage error caused by exception\n java.lang.reflect.Method.invoke","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":false,"urls":"","paths":"","message_params":"","found_exceptions_extended":"","detected_message_extended":" Message 1 \n message error caused by exception","detected_message_without_params_extended":" Message SPECIALNUMBER\nmessage error caused by exception","stacktrace_extended":" java.lang.reflect.Method.invoke Method.invoke invoke ","message_extended":" Message SPECIALNUMBER \n java.lang.reflect.Method.invoke Method.invoke invoke (Method.java : SPECIALNUMBER)\n message error caused by exception","message_without_params_extended":" Message SPECIALNUMBER \n java lang reflect Method invoke Method java SPECIALNUMBER \n message error caused by exception","detected_message_without_params_and_brackets":" Message SPECIALNUMBER\nmessage error caused by exception","message_without_params_and_brackets":" Message SPECIALNUMBER \n java lang reflect Method invoke Method java SPECIALNUMBER \n message error caused by exception"} +{"launch_id":1234567892,"launch_name":"Launch with test items with logs","launch_number":0,"launch_start_time":"2020-01-15 10:57:43","test_item":1,"test_item_name":"first test","unique_id":"unique1","cluster_id":"0","cluster_message":"","test_case_hash":-1126886180,"is_auto_analyzed":false,"issue_type":"ti001","log_time":"2020-01-15 10:57:43","log_level":40000,"original_message":"Message 1 \n java.lang.reflect.Method.invoke(Method.java:498) \n message error caused by exception\n ... 34 more","original_message_lines":3,"original_message_words_number":8,"message":" Message SPECIALNUMBER \n java.lang.reflect.Method.invoke(Method.java : SPECIALNUMBER)\n message error caused by exception","is_merged":false,"start_time":"2020-01-15 10:57:43","merged_small_logs":"","detected_message":" Message SPECIALNUMBER \n message error caused by exception","detected_message_with_numbers":" Message 1 \n message error caused by exception","stacktrace":" java.lang.reflect.Method.invoke","only_numbers":"1","found_exceptions":"","whole_message":" Message SPECIALNUMBER\nmessage error caused by exception\n java.lang.reflect.Method.invoke","potential_status_codes":"","found_tests_and_methods":"","cluster_with_numbers":false,"urls":"","paths":"","message_params":"","found_exceptions_extended":"","detected_message_extended":" Message 1 \n message error caused by exception","detected_message_without_params_extended":" Message SPECIALNUMBER\nmessage error caused by exception","stacktrace_extended":" java.lang.reflect.Method.invoke Method.invoke invoke ","message_extended":" Message SPECIALNUMBER \n java.lang.reflect.Method.invoke Method.invoke invoke (Method.java : SPECIALNUMBER)\n message error caused by exception","message_without_params_extended":" Message SPECIALNUMBER \n java lang reflect Method invoke Method java SPECIALNUMBER \n message error caused by exception","detected_message_without_params_and_brackets":" Message SPECIALNUMBER\nmessage error caused by exception","message_without_params_and_brackets":" Message SPECIALNUMBER \n java lang reflect Method invoke Method java SPECIALNUMBER \n message error caused by exception"}