diff --git a/schema/check_test_output.py b/schema/check_test_output.py index fb5fadf6..68a6a5da 100644 --- a/schema/check_test_output.py +++ b/schema/check_test_output.py @@ -55,7 +55,7 @@ def main(args): test_type = schema_files.TEST_FILE_TO_TEST_TYPE_MAP[test_file_prefix] test_type_set.add(test_type) except BaseException as err: - logging.error('!!! %s for file %s', err, file + logging.info('No file (%s) during schema check output: %s', file, err ) for dir in icu_dirs: icu_version_set.add(os.path.basename(dir)) diff --git a/testdriver/ddtargs.py b/testdriver/ddtargs.py index 4ebdc317..6a2a07bb 100644 --- a/testdriver/ddtargs.py +++ b/testdriver/ddtargs.py @@ -60,8 +60,8 @@ def __init__(self, args): self.parser.add_argument('--custom_verifier', default=None) # self.parser.add_argument( - '--run_serially', default=None, - help='Execute tests in series rather than in parallel') + '--run_serial', default=None, + help='Set if execution should be done serially. Parallel is the default.') self.options = self.parser.parse_args(args) @@ -93,6 +93,9 @@ def __init__(self, args): self.parser.add_argument('--test_verifier', help='Flag to run in test mode', default=None) + self.parser.add_argument('--run_serial', default=None, + help='Set if execution should be done serially. Parallel is the default.') + self.options = self.parser.parse_args(args) return diff --git a/testdriver/testdriver.py b/testdriver/testdriver.py index 95e5c893..a696acfd 100644 --- a/testdriver/testdriver.py +++ b/testdriver/testdriver.py @@ -29,7 +29,7 @@ def __init__(self): self.test_plans = [] self.debug = False - self.run_serially = False # Default is to operate in parallel + self.run_serial = False # Default is to operate in parallel logging.config.fileConfig("../logging.conf") @@ -40,7 +40,7 @@ def set_args(self, arg_options): self.icuVersion = arg_options.icu_version self.cldrVersion = arg_options.cldr - self.run_serially = arg_options.run_serially + self.run_serial = arg_options.run_serial # Create "test plans" for each option for test_type in arg_options.test_type: @@ -125,7 +125,7 @@ def main(args): # print('ARGS = %s' % (args)) driver.parse_args(args[1:]) - if driver.run_serially: + if driver.run_serial: driver.run_plans() else: driver.run_plans_parallel() diff --git a/verifier/check_known_issues.py b/verifier/check_known_issues.py new file mode 100644 index 00000000..f5e59cec --- /dev/null +++ b/verifier/check_known_issues.py @@ -0,0 +1,172 @@ +# Functions to handle Known Issue category of results + + +from report_template import reportTemplate + +from collections import defaultdict +from enum import Enum + +from difflib import HtmlDiff +from difflib import Differ +from difflib import SequenceMatcher + +from datetime import datetime + +import glob +import json +import logging +import logging.config +import os +from string import Template +import sys + +sys.path.append('../testdriver') +import datasets as ddt_data + +# For information on characters and scripts +import unicodedata + +# Handle known issues database + +# Automatically compute some known issues with certain patterns of differences +# in actual output vs. expected + +# E.g., NBSP vs SP in NodeJS DateTime in ICU73, ... + +# Constants +NBSP = '\u202f' +SP = '\u0020' + + +# Global KnownIssue Info types and strings +class knownIssueType(Enum): + known_issue_nbsp_sp = 'ASCII Space instead of NBSP' + known_issue_replaced_numerals = 'Not creating non-ASCII numerals' + +# TODO! Load known issues from file of known problems rather than hardcoding the detection in each test + +# Tests for specific kinds of known issues +def diff_nbsp_vs_ascii_space(actual, expected_value): + # Returns the ID of this if the only difference in the two strings + # is Narrow Non-breaking Space (NBSP) in expected vs. ASCII space in the actual result. + # Found in datetime testing. + if not expected_value or not actual: + return None + + # If replacing all the NBSP characdters in expected gives the actual result, + # then the only differences were with this type of space in formatted output. + if expected_value.replace(NBSP, SP) == actual: + return knownIssueType.known_issue_nbsp_sp + else: + return None + + +def numerals_replaced_by_another_numbering_system(expected, actual): + # If the only difference are one type of digit + # where other digits were expected, return True + # Found in datetime testing. + # Returns an known issue ID (or string) if the the numbering system changed + + # sm_opcodes describe the change to turn expected string into the actual string + # See https://docs.python.org/3/library/difflib.html#difflib.SequenceMatcher.get_opcodes + sm = SequenceMatcher(None, expected, actual) + sm_opcodes = sm.get_opcodes() + + digit_replace = False + non_digit_replacement = False + + # sm_opcodes describe the changes to turn expected string into the actual string + # See https://docs.python.org/3/library/difflib.html#difflib.SequenceMatcher.get_opcodes + # The tuple is [tag, i1, i2, j1, j2] + # Tag indicates the type of change. + # i1:i2 is the range of the substring in expected + # j1:j2 is the range of the substring in actual + + for diff in sm_opcodes: + tag = diff[0] # 'replace', 'delete', 'insert', or 'equal' + old_val = expected[diff[1]:diff[2]] + new_val = actual[diff[3]:diff[4]] + if tag == 'replace': + # expected[i1:i2] was replaced by actual[j1:j2] + if old_val.isdigit() and new_val.isdigit(): + # TODO!! : check the value of the numeral + # If the same value, then its a numbering system difference + if unicodedata.numeric(old_val) == unicodedata.numeric(new_val): + digit_replace = True + else: + # Both were digits but different numeric values + non_digit_replacement = True + else: + # a digit was replaced with a non-digit + non_digit_replacement = True + + # Only true if the only changes were replacing digits + if digit_replace and not non_digit_replace: + return knownIssueType.known_issue_replaced_numerals + else: + return None + + +def check_datetime_known_issues(test): + # Examine a single test for date/time isses + # Returns known issues identified for this test in this category + remove_this_one = False + try: + result = test['result'] + expected = test['expected'] + is_ki = diff_nbsp_vs_ascii_space(result, expected) + if is_ki: + # Mark the test with this issue + test['known_issue'] = knownIssueType.known_issue_nbsp_sp.value + remove_this_one = True + + is_ki = numerals_replaced_by_another_numbering_system(result, expected) + if is_ki: + test['known_issue_id'] = knownIssueType.known_issue_replaced_numerals.value + remove_this_one = True + + except BaseException as err: + # Can't get the info + pass + + return remove_this_one + + +def compute_known_issues_for_single_test(test_type, test): + # Based on the type of test, check known issues against the expected vs. actual + # results + + # Returns True if this single test is an example of one or moore known issues, + known_issue_found = False + if test_type == ddt_data.testType.datetime_fmt.value: + known_issue_found = check_datetime_known_issues(test) + + # TODO: Add checks here for known issues in other test types + + return known_issue_found + +def check_issues(test_type, test_results_to_check): + # Look at the array of test result types, failure, error, unsupported + # Extract any tests from these that are known issues + # Return the list of tests that are known issues + # + known_issues_list = [] + + for category in test_results_to_check: + test_indices_with_known_issues = set() + index = 0 + + for test in category: + is_known_issue = compute_known_issues_for_single_test(test_type, test) + if is_known_issue: + known_issues_list.append(test) + test_indices_with_known_issues.add(index) + index += 1 + + # Remove those that were marked as known issues + # Reverse order to not confuse the position while deleting + rev_indices = sorted(test_indices_with_known_issues, reverse=True) + for index in rev_indices: + del category[index] + + return known_issues_list diff --git a/verifier/detail_template.html b/verifier/detail_template.html index 9c89cae4..2cad2d56 100644 --- a/verifier/detail_template.html +++ b/verifier/detail_template.html @@ -34,7 +34,7 @@ font-size:20px; } - #diff_area { + .diff_area { font-size: 24px; font-color: blue; } @@ -231,9 +231,9 @@ ['Results', 'Count', {role:'style'}], ['Passing', test_results['pass'].json.length, '#44ff77'], ['Failing', test_results['fail'].json.length, '#ff0000'], + ['Known issues', test_results['known_issue'].json.length, '#ff8200'], ['Errors', test_results['error'].json.length, '#ffdd00'], - ['Unsupported', test_results['unsupported'].json.length, '#777777'], - ['Known issues', test_results['known_issue'].json.length, '#ff8200'] + ['Unsupported', test_results['unsupported'].json.length, '#777777'] ]; const chart = new google.visualization.BarChart(chart_output_area); let chart_data = google.visualization.arrayToDataTable(input_data); @@ -347,13 +347,19 @@ fill_pagination("#characterized-pagination-container_" + item_type, "#characterized-data-container_" + item_type, selected_json_data, - "selected_" + item_type); + item_type); } // UL Template for pagination.js function simpleTemplating(data, c_type) { let possible_fields = ['label', 'expected', 'result', 'error', 'error_detail', 'options', 'input_data', 'actual_options']; + // Support output to different regions, depending on the type + // of the data, e.g., "fail", "known_issue", etc. + const diff_area_name = "diff_area_" + c_type; + const onclick_call = + '"captureInputDataOnClick(this);" onmouseover="hoverDiffText(this,' + diff_area_name + ');"'; + let table_opening = ''; @@ -377,7 +383,7 @@ } else { output = item[key]; } - html.push(''); + html.push(''); } } html.push(""); @@ -637,8 +643,9 @@ navigator.clipboard.writeText(output); } - // On hover, show the differen between expected and actual result. - function hoverDiffText(element) { + // On hover, show the difference between expected and actual result + // in the named area. + function hoverDiffText(element, diff_area) { // First, get the row of this item. const row = element.parentNode; const text1 = row.children[1].innerHTML; @@ -651,7 +658,6 @@ dmp.diff_cleanupSemantic(diff_text); // And show the difference nicely. const ds = dmp.diff_prettyHtml(diff_text); - const diff_area = document.getElementById("diff_area"); diff_area.innerHTML = ds; } @@ -672,7 +678,7 @@

$platform_info

$total_tests attempted. Pass: $passing_tests, Fail: $failing_tests, Errors: $error_count, - Unsupported: $unsupported, + Unsupported: $unsupported

, Known issues: $unsupported

@@ -717,7 +723,7 @@

Acknowledgements

Failing tests ($failing_tests) -

+

@@ -741,6 +747,32 @@

Acknowledgements

+
+ Known issues ($known_issue_count) +

+
+
+
+
+ Known issues characterized +
+
+

Filtered count = 0 + + +

+
+
+
+
+
+
+
+
+
+
+
+
Test errors ($error_count)

Test Errors ($error_count)

@@ -793,30 +825,6 @@

Test Errors ($error_count)

-
- Known issues ($known_issue_count) -
-
-
-
- Known issues characterized -
-
-

Filtered count = 0 - - -

-
-
-
-
-
-
-
-
-
-
-
diff --git a/verifier/testreport.py b/verifier/testreport.py index a8071e37..1a79eb56 100644 --- a/verifier/testreport.py +++ b/verifier/testreport.py @@ -3,6 +3,9 @@ # TODO: get templates from this module instead of local class from report_template import reportTemplate +# For identifying test cases that are known problems +from check_known_issues import check_issues + from collections import defaultdict from difflib import HtmlDiff @@ -111,6 +114,7 @@ def __init__(self, report_path, report_html_path): self.passing_tests = [] self.test_errors = [] self.unsupported_cases = [] + self.known_issues = [] self.templates = templates = reportTemplate() @@ -161,7 +165,7 @@ def __init__(self, report_path, report_html_path): self.known_issue_data = result_class_data( 'known_issues', None, - None, # ?? templates.known_issue_table_template, + self.known_issue_summary_template, templates.known_issue_table_template, self.compute_known_issue_category_summary) @@ -196,6 +200,9 @@ def record_test_error(self, test): def record_unsupported(self, test): self.unsupported_cases.append(test) + def record_known_issue(self, test): + self.known_issues.append(test) + def record_missing_verify_data(self, test): self.missing_verify_data.append(test) @@ -208,21 +215,21 @@ def compute_test_error_summary(self, test_errors, group_tag): for error in test_errors: try: label = error['label'] - details = error.get('error_detail') + details = error.get('error_detail', None) if not details: # Try getting the group_tag - details = error.get(group_tag) + details = error.get(group_tag, None) if isinstance(details, str): detail = details group = group_tag else: - detail = details.get(group_tag) + detail = details.get(group_tag, None) group = group_tag if group: - if not groups.get(group): + if not groups.get(group, None): groups[group] = {detail: []} # insert empty list - if not groups[group].get(detail): + if not groups[group].get(detail, None): groups[group][detail] = [label] else: groups[group][detail].append(label) @@ -235,19 +242,19 @@ def compute_unsupported_category_summary(self, unsupported_cases, group_tag): # For the items, count messages and arguments for each groups = {} for case in unsupported_cases: - error_detail = case.get('error_detail') + error_detail = case.get('error_detail', None) label = case['label'] if isinstance(error_detail, str): detail = error_detail else: if isinstance(error_detail, dict): - detail = error_detail.get(group_tag) + detail = error_detail.get(group_tag, None) else: detail = error_detail # Specific for unsupported options - count instances of the detail value = str(detail) - if groups.get(value): + if groups.get(value, None): groups[value].append(label) else: groups[value] = [label] @@ -278,16 +285,22 @@ def create_report(self): report['platform'] = self.platform_info report['test_environment'] = self.test_environment report['timestamp'] = self.timestamp - report['failCount'] = len(self.failing_tests) + report['passCount'] = len(self.passing_tests) report['failingTests'] = self.failing_tests - report['unsupportedTests'] = len(self.unsupported_cases) - report['missing_verify_data'] = self.missing_verify_data - report['test_error_count'] = len(self.test_errors) + report['failCount'] = len(self.failing_tests) report['test_errors'] = self.test_errors + report['test_error_count'] = len(self.test_errors) + report['unsupported'] = self.unsupported_cases + report['unsupportedTests'] = len(self.unsupported_cases) + report['known_issues'] = self.known_issues + report['known_issue_count'] = len(self.known_issues) + + report['missing_verify_data'] = self.missing_verify_data + self.report = report return json.dumps(report) @@ -361,6 +374,17 @@ def combine_same_sets_of_labels(self, label_sets): return combined_dictionary + + def add_known_issues(self): + # Call functions to identify known issues, moving things from fail, error, and unsupported + # to known_issues as needed + new_known_issues = check_issues( + self.test_type, + [self.failing_tests, self.test_errors, self.unsupported_cases]) + + if new_known_issues: + self.known_issues.extend(new_known_issues) + def create_html_report(self): # Human-readable summary of test results if self.platform_info['icuVersion'] == 'unknown': @@ -387,8 +411,8 @@ def create_html_report(self): 'failing_tests': len(self.failing_tests), 'error_count': len(self.test_errors), 'unsupported_count': len(self.unsupported_cases), - 'known_issue_count': len(self.known_issues) - + 'known_issue_count': len(self.known_issues), + 'known_issues': self.known_issues # ... } @@ -405,31 +429,38 @@ def create_html_report(self): line = self.fail_line_template.safe_substitute(fail) fail_lines.append(line) - #html_map['failure_table_lines'] = '\n'.join(fail_lines) + # Call functions to identify known issues, moving things from fail, error, and unsupported + # to known_issues as needed + new_known_issues = check_issues( + self.test_type, + [self.failing_tests, self.test_errors, self.unsupported_cases]) + + if new_known_issues: + self.known_issues.extend(new_known_issues) + # Characterize successes, too. - pass_characterized = self.characterize_failures_by_options(self.passing_tests) + pass_characterized = self.characterize_results_by_options(self.passing_tests, 'pass') flat_combined_passing = self.flatten_and_combine(pass_characterized, None) self.save_characterized_file(flat_combined_passing, "pass") # Get and save failures, errors, unsupported - error_characterized = self.characterize_failures_by_options(self.test_errors) + error_characterized = self.characterize_results_by_options(self.test_errors, 'error') flat_combined_errors = self.flatten_and_combine(error_characterized, None) self.save_characterized_file(flat_combined_errors, "error") - unsupported_characterized = self.characterize_failures_by_options(self.unsupported_cases) + unsupported_characterized = self.characterize_results_by_options(self.unsupported_cases, 'unsupported') flat_combined_unsupported = self.flatten_and_combine(unsupported_characterized, None) self.save_characterized_file(flat_combined_unsupported, "unsupported") - - known_issues_characterized = self.characterize_failures_by_options(self.known_issues) + known_issues_characterized = self.characterize_results_by_options(self.known_issues, 'known_issue') flat_combined_known_issues = self.flatten_and_combine(known_issues_characterized, None) self.save_characterized_file(flat_combined_known_issues, "known_issues") # TODO: Should we compute top 3-5 overlaps for each set? # Flatten and combine the dictionary values - fail_characterized = self.characterize_failures_by_options(self.failing_tests) - fail_simple_diffs = self.check_simple_text_diffs() + fail_characterized = self.characterize_results_by_options(self.failing_tests, 'fail') + fail_simple_diffs = self.check_simple_text_diffs(self.failing_tests, 'fail') flat_combined_dict = self.flatten_and_combine(fail_characterized, fail_simple_diffs) self.save_characterized_file(flat_combined_dict, "fail") @@ -551,45 +582,26 @@ def flatten_and_combine(self, input_dict, input_simple): flat_combined_dict = self.combine_same_sets_of_labels(flat_items) return dict(sort_dict_by_count(flat_combined_dict)) - def characterize_failures_by_options(self, failing_tests): + def characterize_results_by_options(self, test_list, category): # User self.failing_tests, looking at options results = defaultdict(lambda : defaultdict(list)) results['locale'] = {} # Dictionary of labels for each locale - for test in failing_tests: + for test in test_list: # Get input_data, if available - input_data = test.get('input_data') + input_data = test.get('input_data', None) - try: - label = test['label'] - except: - label = '' - key_list = ['locale', 'locale_label', 'option', 'options', - 'language_label', - # Collation - # 'ignorePunctuation', 'compare_result', - 'compare_type', 'test_description', 'unsupported_options', 'rules', 'test_description', - 'warning' - # Number format - 'input_data', - 'notation', 'compactDisplay', 'style', 'currency', 'unit', 'roundingMode', - # list_fmt - 'type', 'input_list', - # date/time format - 'skeleton'] - - option_keys = ['notation', 'compactDisplay', 'style', 'currency', 'unit', 'roundingMode'] + label = test.get('label', '') + key_list = ['locale', 'locale_label', 'option', 'options'] + # Special case for locales for key in key_list: - try: - locale = input_data.get('locale') - except: - locale = None + locale = input_data.get('locale', None) if locale: if locale not in results['locale']: results['locale'][locale] = set() results['locale'][locale].add(label) - options = input_data.get('options') + options = input_data.get('options', None) if options: # Get each combo of key/value for k, value in options.items(): @@ -599,43 +611,41 @@ def characterize_failures_by_options(self, failing_tests): results[k][value] = set() results[k][value].add(label) - # Try fields in lang_names - for key in ['language_label', 'locale_label']: - try: - if input_data.get(key): - value = input_data[key] - if key not in results: - results[key] = {} - if value in results[key]: - results[key][value] = set() - results[key][value].add(label) - except: - continue - - # Try fields in likely_subtags - for key in ['option', 'locale']: - try: - if input_data.get(key): - value = input_data[key] - if key not in results: - results[key] = {} - if value not in results[key]: - results[key][value] = set() - results[key][value].add(label) - except: - continue - - for key in ['language_label', 'ignorePunctuation', 'compare_result', 'compare_type', 'test_description']: - try: - if test.get(key): # For collation results - value = test[key] - if key not in results: - results[key] = {} - if value not in results[key]: - results[key][value] = set() - results[key][value] = set(label) - except: - continue + key_list = ['locale', 'locale_label', 'option', 'options', + 'language_label', + # Collation + # 'ignorePunctuation', 'compare_result', + 'compare_type', 'test_description', 'unsupported_options', 'rules', 'test_description', + 'warning' + # Number format + 'input_data', + 'notation', 'compactDisplay', 'style', 'currency', 'unit', 'roundingMode', + # list_fmt + 'type', 'input_list', + # date/time format + 'skeleton', + 'language_label', 'locale_label', # in lang_names + 'option', 'locale', # in likely_subtags + 'language_label', 'ignorePunctuation', 'compare_result', 'compare_type', 'test_description' + ] + for key in key_list: + if test.get(key, None): # For collation results + value = test[key] + if key not in results: + results[key] = {} + if value not in results[key]: + results[key][value] = set() + results[key][value].add(label) + + ki_key_list = ['known_issue', 'known_issue_id'] + for key in ki_key_list: + if test.get(key, None): # For collation results + value = test[key] + if key not in results: + results[key] = {} + if value not in results[key]: + results[key][value] = set() + results[key][value].add(label) # Look at the input_data part of the test result # TODO: Check the error_detail and error parts, too. @@ -663,11 +673,11 @@ def characterize_failures_by_options(self, failing_tests): # Special case for input_data / options. special_key = 'options' - if input_data and input_data.get(special_key): + if input_data and input_data.get(special_key, None): options = input_data[special_key] self.add_to_results_by_key(label, results, options, test, options.keys()) - error_detail = test.get('error_detail') + error_detail = test.get('error_detail', None) if error_detail: error_keys = error_detail.keys() # ['options'] self.add_to_results_by_key(label, results, error_detail, test, error_keys) @@ -676,52 +686,42 @@ def characterize_failures_by_options(self, failing_tests): # TODO: Add replacing (...) with "-" for numbers # TODO: Find the largest intersections of these sets and sort by size - pass - return results # TODO: Use the following function to update lists. def add_to_results_by_key(self, label, results, input_data, test, key_list): if input_data: for key in key_list: - try: - if input_data.get(key): # For collation results - value = input_data.get(key) - if key == 'input_list': - if 'input_size' not in results: - results['input_size'] = {} - else: - results['input_size'].add(len(value)) - if key == 'rules': - value = 'RULE' # A special case to avoid over-characterization - if key not in results: - results[key] = {} - if value in results[key]: - results[key][value].add(label) + if input_data.get(key, None): # For collation results + value = input_data.get(key, None) + if key == 'input_list': + if 'input_size' not in results: + results['input_size'] = {} else: + results['input_size'].add(len(value)) + if key == 'rules': + value = 'RULE' # A special case to avoid over-characterization + if key not in results: + results[key] = {} + try: + if not results[key].get(value, None): results[key][value] = set() - results[key][value].add(label) - except: - continue - - def check_simple_text_diffs(self): + results[key][value].add(label) + except TypeError as err: + # value may not be hashable. This should be skipped + pass + def check_simple_text_diffs(self, test_list, category): results = defaultdict(list) - results['insert'] = set() - results['delete'] = set() - results['insert_digit'] = set() - results['insert_space'] = set() - results['delete_digit'] = set() - results['delete_space'] = set() - results['replace_digit'] = set() - results['replace_dff'] = set() - results['whitespace_diff'] = set() - results['replace'] = set() - results['parens'] = set() # Substitions of brackets for parens, etc. + all_checks = ['insert', 'delete', 'insert_digit', 'insert_space', 'delete_digit', + 'delete_space', 'replace_digit', 'replace_dff', 'whitespace_diff', + 'replace', 'parens'] + for check in all_checks: + results[check] = set() - for fail in self.failing_tests: + for fail in test_list: label = fail['label'] - actual = fail.get('result') - expected = fail.get('expected') + actual = fail.get('result', None) + expected = fail.get('expected', None) if (actual is None) or (expected is None): continue # Special case for differing by a single character. @@ -954,16 +954,16 @@ def fill_templates(self, result_data, result_cases, result_table_template, summa # For filling in templates for cases of passing, failing, errors, unsupported, known_issue result_class = result_data.name section_name = '%s_section' % result_class - summary_name = '%s_summary % result_class' + summary_name = '%s_summary' % result_class # TODO: Call this for each cases instead of duplicated lines if result_cases: case_lines = [] - test_table_name = 'test_%s_' % result_class + test_table_name = 'test_%s' % result_class options_name = '%s_options' % result_class options_string = '%s options' % result_class - # Create a table of all test errors. + # Create a table of all test results in this category. for unsupported in result_cases: line = result_data.result_table_template.safe_substitute(unsupported) case_lines.append(line) @@ -981,7 +981,7 @@ def fill_templates(self, result_data, result_cases, result_table_template, summa # ??? TODO: examine if "error" is correct below for key, labels in case_summary.items(): count = len(labels) - sub = {'error': key, 'count': count} + sub = {'known_issue': key, 'count': count} summary_lines.append( result_data.summary_template.safe_substitute(sub) ) @@ -1103,6 +1103,7 @@ def summarize_reports(self): 'pass_count': int(test_json['passCount']), 'error_count': int(test_json['test_error_count']), 'unsupported_count': len(test_json['unsupported']), + 'known_issue_count': int(test_json['known_issue_count']), 'missing_verify_count': len(test_json['missing_verify_data']), 'json_file_name': filename, 'html_file_name': relative_html_path, # Relative to the report base @@ -1110,13 +1111,6 @@ def summarize_reports(self): 'icu_version': icu_version, 'platform_version': '%s %s' % (platform['platform'], platform['platformVersion']) } - - # Handle this sepparately for now as we add known issue support - try: - test_results['known_issue_count'] = len(test_json['known_issue']) - except BaseException as err: - test_results['known_issue_count'] = 0 - except BaseException as err: logging.error('SUMMARIZE REPORTS for file %s. Error: %s' % (filename, err)) diff --git a/verifier/verifier.py b/verifier/verifier.py index 7df28707..da6e890b 100644 --- a/verifier/verifier.py +++ b/verifier/verifier.py @@ -47,6 +47,8 @@ def __init__(self): self.report = None self.reports = [] + self.run_in_parallel = True + # Set of [result filepath, verify filepath, report path] self.result_timestamp = None @@ -222,15 +224,20 @@ def setup_verify_plans(self): # Verify plans in parallel def parallel_verify_data_results(self): - num_processors = mp.cpu_count() - verify_plans = self.verify_plans - logging.info('JSON validation: %s processors for %s plans', - num_processors, len(verify_plans)) - - processor_pool = mp.Pool(num_processors) - with processor_pool as p: - result = p.map(self.verify_one_plan, verify_plans) - return result + if not self.options.run_serial: + num_processors = mp.cpu_count() + verify_plans = self.verify_plans + logging.info('JSON validation: %s processors for %s plans', + num_processors, len(verify_plans)) + + processor_pool = mp.Pool(num_processors) + with processor_pool as p: + result = p.map(self.verify_one_plan, verify_plans) + return result + else: + logging.info('Running serially!') + for vplan in self.verify_plans: + self.verify_one_plan(vplan) # For one VerifyPlan, get data and run verification def verify_one_plan(self, vplan): @@ -249,16 +256,19 @@ def verify_one_plan(self, vplan): vplan.setup_report_data() result = {'compare_success': True} + + # Do more analysis on the failures and compute known issues + vplan.report.summarize_failures() + + vplan.report.add_known_issues() # Save the results + if not vplan.report.save_report(): logging.error('!!! Could not save report for (%s, %s)', vplan.test_type, vplan.exec) else: vplan.report.create_html_report() - # Do more analysis on the failures - vplan.report.summarize_failures() - logging.debug('\nTEST RESULTS in %s for %s. %d tests found', vplan.exec, vplan.test_type, len(vplan.test_results))
' + output +'' + output +'