Skip to content

Commit

Permalink
EPMRPP-88420 - update query logic
Browse files Browse the repository at this point in the history
  • Loading branch information
@HardNorth
HardNorth committed Nov 30, 2023
1 parent 3044787 commit 8f2f813
Show file tree
Hide file tree
Showing 4 changed files with 102 additions and 51 deletions.
102 changes: 76 additions & 26 deletions app/service/analyzer_service.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import logging
import re
from typing import Any

from app.boosting_decision_making import weighted_similarity_calculator
from app.commons.log_merger import LogMerger
Expand All @@ -39,32 +40,83 @@ def find_min_should_match_threshold(self, analyzer_config):
return analyzer_config.minShouldMatch if analyzer_config.minShouldMatch > 0 else \
int(re.search(r"\d+", self.search_cfg["MinShouldMatch"]).group(0))

def add_constraints_for_launches_into_query(self, query: dict, launch, *, suggest: bool = False) -> dict:
def create_path(self, query: dict, path: tuple[str, ...], value: Any) -> Any:
path_length = len(path)
last_element = path[path_length - 1]
current_node = query
for i in range(path_length - 1):
element = path[i]
if element not in current_node:
current_node[element] = {}
current_node = current_node[element]
if last_element not in current_node:
current_node[last_element] = value
return current_node[last_element]

def _add_launch_name_and_id_boost(self, query: dict, launch_name: str, launch_id: int) -> None:
launch_boost = abs(self.search_cfg['BoostLaunch'])
should = self.create_path(query, ('query', 'bool', 'should'), [])
should.append({'term': {'launch_id': {'value': launch_id, 'boost': launch_boost}}})
should.append({'term': {'launch_name': {'value': launch_name, 'boost': launch_boost}}})

def add_constraints_for_launches_into_query(self, query: dict, launch) -> dict:
previous_launch_id = getattr(launch, 'previousLaunchId', 0) or 0
previous_launch_id = int(previous_launch_id)
analyzer_mode = launch.analyzerConfig.analyzerMode
if analyzer_mode in {'LAUNCH_NAME', 'CURRENT_AND_THE_SAME_NAME'}:
query['query']['bool']['must'].append({'term': {'launch_name': {'value': launch.launchName}}})
# Boost current launch to address cases when previous launch issue types are more probable than current
query['query']['bool']['should'].append(
{'term': {'launch_id': {'value': launch.launchId, 'boost': abs(self.search_cfg['BoostLaunch'])}}})
launch_name = launch.launchName
launch_id = launch.launchId
launch_boost = abs(self.search_cfg['BoostLaunch'])
if analyzer_mode == 'LAUNCH_NAME':
# Previous launches with the same name
must = self.create_path(query, ('query', 'bool', 'must'), [])
must_not = self.create_path(query, ('query', 'bool', 'must_not'), [])
must.append({'term': {'launch_name': {'value': launch_name}}})
must_not.append({'term': {'launch_id': {'value': launch_id}}})
elif analyzer_mode == 'CURRENT_AND_THE_SAME_NAME':
# All launches with the same name
must = self.create_path(query, ('query', 'bool', 'must'), [])
should = self.create_path(query, ('query', 'bool', 'should'), [])
must.append({'term': {'launch_name': {'value': launch_name}}})
should.append({'term': {'launch_id': {'value': launch_id, 'boost': launch_boost}}})
elif analyzer_mode == 'CURRENT_LAUNCH':
if suggest:
query['query']['bool']['should'].append(
{'term': {'launch_id': {'value': launch.launchId, 'boost': abs(self.search_cfg['BoostLaunch'])}}})
else:
query['query']['bool']['must'].append({'term': {'launch_id': {'value': launch.launchId}}})
# Just current launch
must = self.create_path(query, ('query', 'bool', 'must'), [])
must.append({'term': {'launch_id': {'value': launch_id}}})
elif analyzer_mode == 'PREVIOUS_LAUNCH':
# Just previous launch
if previous_launch_id:
if suggest:
query['query']['bool']['should'].append(
{'term': {
'launch_id': {'value': previous_launch_id, 'boost': abs(self.search_cfg['BoostLaunch'])}}})
else:
query['query']['bool']['must'].append({'term': {'launch_id': {'value': previous_launch_id}}})
must = self.create_path(query, ('query', 'bool', 'must'), [])
must.append({'term': {'launch_id': {'value': previous_launch_id}}})
else:
query['query']['bool']['should'].append(
{'term': {'launch_name': {'value': launch.launchName, 'boost': abs(self.search_cfg['BoostLaunch'])}}})
# Just all launches
# Boost launches with the same name, but do not ignore any
self._add_launch_name_and_id_boost(query, launch_name, launch_id)
return query

def add_constraints_for_launches_into_query_suggest(self, query: dict, test_item_info) -> dict:
previous_launch_id = getattr(test_item_info, 'previousLaunchId', 0) or 0
previous_launch_id = int(previous_launch_id)
analyzer_mode = test_item_info.analyzerConfig.analyzerMode
launch_name = test_item_info.launchName
launch_id = test_item_info.launchId
launch_boost = abs(self.search_cfg['BoostLaunch'])
if analyzer_mode == 'LAUNCH_NAME':
# Previous launches with the same name
should = self.create_path(query, ('query', 'bool', 'should'), [])
should.append({'term': {'launch_name': {'value': launch_name, 'boost': launch_boost}}})
should.append({'term': {'launch_id': {'value': launch_id, 'boost': 1 / launch_boost}}})
elif analyzer_mode == 'PREVIOUS_LAUNCH':
# Just previous launch
should = self.create_path(query, ('query', 'bool', 'should'), [])
if previous_launch_id:
should.append({'term': {'launch_id': {'value': previous_launch_id, 'boost': launch_boost}}})
else:
# For:
# * CURRENT_LAUNCH
# * CURRENT_AND_THE_SAME_NAME
# * All launches
# Boost launches with the same name, but do not ignore any
self._add_launch_name_and_id_boost(query, launch_name, launch_id)
return query

def build_more_like_this_query(self,
Expand Down Expand Up @@ -92,26 +144,24 @@ def build_common_query(self, log, size=10, filter_no_defect=True):
issue_type_conditions = self.prepare_restrictions_by_issue_type(
filter_no_defect=filter_no_defect)
return {"size": size,
"sort": ["_score",
{"start_time": "desc"}, ],
"sort": ["_score", {"start_time": "desc"}, ],
"query": {
"bool": {
"filter": [
{"range": {"log_level": {"gte": utils.ERROR_LOGGING_LEVEL}}},
{"exists": {"field": "issue_type"}},
],
"must_not": issue_type_conditions + [
{"term": {"test_item": log["_source"]["test_item"]}}
],
"must": [],
"must_not": issue_type_conditions + [{"term": {"test_item": log["_source"]["test_item"]}}],
"should": [
{"term": {"test_case_hash": {
"value": log["_source"]["test_case_hash"],
"boost": abs(self.search_cfg["BoostTestCaseHash"])}}},
{"term": {"is_auto_analyzed": {
"value": str(self.search_cfg["BoostAA"] > 0).lower(),
"boost": abs(self.search_cfg["BoostAA"]), }}},
]}}}
]
}
}}

def add_query_with_start_time_decay(self, main_query, start_time):
return {
Expand Down
15 changes: 8 additions & 7 deletions app/service/auto_analyzer_service.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,11 +119,11 @@ def build_analyze_query(self, launch, log, size=10):
else:
query["query"]["bool"]["filter"].append({"term": {"is_merged": True}})
query["query"]["bool"]["must_not"].append({"wildcard": {"message": "*"}})
query["query"]["bool"]["must"].append(
self.build_more_like_this_query(min_should_match,
log["_source"]["merged_small_logs"],
field_name="merged_small_logs",
boost=2.0))
must = self.create_path(query, ("query", "bool", "must"), [])
must.append(self.build_more_like_this_query(min_should_match,
log["_source"]["merged_small_logs"],
field_name="merged_small_logs",
boost=2.0))

if log["_source"]["found_exceptions"].strip():
query["query"]["bool"]["must"].append(
Expand All @@ -137,7 +137,8 @@ def build_analyze_query(self, launch, log, size=10):
("only_numbers", 2.0), ("potential_status_codes", 8.0),
("found_tests_and_methods", 2), ("test_item_name", 2.0)]:
if log["_source"][field].strip():
query["query"]["bool"]["should"].append(
should = self.create_path(query, ('query', 'bool', 'should'), [])
should.append(
self.build_more_like_this_query("1",
log["_source"][field],
field_name=field,
Expand Down Expand Up @@ -365,7 +366,7 @@ def _query_elasticsearch(self, launches, max_batch_size=30):

except Exception as err:
logger.error("Error in ES query")
logger.error(err)
logger.exception(err)
self.finished_queue.put("Finished")
logger.info("Es queries finished %.2f s.", time() - t_start)

Expand Down
12 changes: 6 additions & 6 deletions app/service/suggest_service.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ def build_suggest_query(self, test_item_info, log, size=10,
log_lines = test_item_info.analyzerConfig.numberOfLogLines

query = self.build_common_query(log, size=size, filter_no_defect=False)
query = self.add_constraints_for_launches_into_query(query, test_item_info, suggest=True)
query = self.add_constraints_for_launches_into_query_suggest(query, test_item_info)

if log["_source"]["message"].strip():
query["query"]["bool"]["filter"].append({"term": {"is_merged": False}})
Expand Down Expand Up @@ -124,11 +124,11 @@ def build_suggest_query(self, test_item_info, log, size=10,
else:
query["query"]["bool"]["filter"].append({"term": {"is_merged": True}})
query["query"]["bool"]["must_not"].append({"wildcard": {"message": "*"}})
query["query"]["bool"]["must"].append(
self.build_more_like_this_query(min_should_match,
log["_source"]["merged_small_logs"],
field_name="merged_small_logs",
boost=2.0))
must = self.create_path(query, ('query', 'bool', 'must'), [])
must.append(self.build_more_like_this_query(min_should_match,
log["_source"]["merged_small_logs"],
field_name="merged_small_logs",
boost=2.0))

utils.append_potential_status_codes(query, log, max_query_terms=self.search_cfg["MaxQueryTerms"])

Expand Down
24 changes: 12 additions & 12 deletions test/unit/service/test_unit_analyzer_service.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,28 +40,28 @@ def get_empty_bool_query():


@pytest.mark.parametrize(
'previous_launch_id, launch_mode, suggest, expected_query',
'previous_launch_id, launch_mode, expected_query',
[
(2, 'LAUNCH_NAME', False, DEFAULT_LAUNCH_NAME_SEARCH),
(2, 'CURRENT_AND_THE_SAME_NAME', False, DEFAULT_LAUNCH_NAME_SEARCH),
(2, 'CURRENT_LAUNCH', False, {'must': [{'term': {'launch_id': {'value': DEFAULT_LAUNCH_ID}}}], 'should': []}),
(2, 'CURRENT_LAUNCH', True, {'must': [], 'should': [{'term': {'launch_id': {
(2, 'LAUNCH_NAME', DEFAULT_LAUNCH_NAME_SEARCH),
(2, 'CURRENT_AND_THE_SAME_NAME', DEFAULT_LAUNCH_NAME_SEARCH),
(2, 'CURRENT_LAUNCH', {'must': [{'term': {'launch_id': {'value': DEFAULT_LAUNCH_ID}}}], 'should': []}),
(2, 'CURRENT_LAUNCH', {'must': [], 'should': [{'term': {'launch_id': {
'value': DEFAULT_LAUNCH_ID, 'boost': DEFAULT_BOOST_LAUNCH}}}]}),
(2, 'PREVIOUS_LAUNCH', False, {'must': [{'term': {'launch_id': {'value': 2}}}], 'should': []}),
(None, 'PREVIOUS_LAUNCH', False, {'must': [], 'should': []}),
('3', 'PREVIOUS_LAUNCH', False, {'must': [{'term': {'launch_id': {'value': 3}}}], 'should': []}),
('3', 'PREVIOUS_LAUNCH', True, {'must': [], 'should': [{'term': {'launch_id': {
(2, 'PREVIOUS_LAUNCH', {'must': [{'term': {'launch_id': {'value': 2}}}], 'should': []}),
(None, 'PREVIOUS_LAUNCH', {'must': [], 'should': []}),
('3', 'PREVIOUS_LAUNCH', {'must': [{'term': {'launch_id': {'value': 3}}}], 'should': []}),
('3', 'PREVIOUS_LAUNCH', {'must': [], 'should': [{'term': {'launch_id': {
'value': 3, 'boost': DEFAULT_BOOST_LAUNCH}}}]}),
(2, None, False, {'must': [], 'should': [{'term': {'launch_name': {
(2, None, {'must': [], 'should': [{'term': {'launch_name': {
'value': DEFAULT_LAUNCH_NAME, 'boost': DEFAULT_BOOST_LAUNCH}}}]})
]
)
def test_add_constraints_for_launches_into_query(previous_launch_id, launch_mode, suggest, expected_query):
def test_add_constraints_for_launches_into_query(previous_launch_id, launch_mode, expected_query):
launch = mock.Mock()
launch.launchId = DEFAULT_LAUNCH_ID
launch.previousLaunchId = previous_launch_id
launch.launchName = DEFAULT_LAUNCH_NAME
launch.analyzerConfig.analyzerMode = launch_mode
analyzer = AnalyzerService(None, DEFAULT_SEARCH_CONFIG)
result = analyzer.add_constraints_for_launches_into_query(get_empty_bool_query(), launch, suggest=suggest)
result = analyzer.add_constraints_for_launches_into_query(get_empty_bool_query(), launch)
assert result['query']['bool'] == expected_query

0 comments on commit 8f2f813

Please sign in to comment.