diff --git a/genData100.sh b/genData100.sh index aa48add2..9027d752 100755 --- a/genData100.sh +++ b/genData100.sh @@ -5,15 +5,20 @@ # Save the results set -e +# Rotate log files +logrotate -s logrotate.state logrotate.conf + +########## +# Setup (generate) test data & expected values +########## + # Enable seting the version of NodeJS export NVM_DIR=$HOME/.nvm; source $NVM_DIR/nvm.sh; -# -# Setup -# +export TEST_LIMIT=100 -export TEMP_DIR=TEMP_DATA +export TEMP_DIR=TEMP_DATA_100 rm -rf $TEMP_DIR # Clear out old data, then create new directory and copy test / verify data there @@ -28,9 +33,10 @@ source_file=${1:-'run_config.json'} # Generates all new test data +source_file=${1:-'run_config.json'} pushd testgen all_icu_versions=$(jq '.[].run.icu_version' ../$source_file | jq -s '.' | jq 'unique' | jq -r 'join(" ")') -python3 testdata_gen.py --icu_versions $all_icu_versions --run_limit 100 +python3 testdata_gen.py --icu_versions $all_icu_versions --run_limit $TEST_LIMIT # And copy results to subdirectories. cp -r icu* ../$TEMP_DIR/testData popd @@ -42,21 +48,19 @@ python3 check_schemas.py $pwd python3 check_generated_data.py ../$TEMP_DIR/testData popd -all_execs_json=$(jq '.[].run.exec' $source_file | jq -s '.' | jq 'unique') +########## +# Run tests using per-platform executors +########## + # # Run test data tests through all executors # -# Compile Rust executor code for ICU4X 1.0 -if jq -e 'index("rust")' <<< $all_execs_json > /dev/null -then - pushd executors/rust/ -# cargo clean -# cargo build --release - popd -fi + +all_execs_json=$(jq '.[].run.exec' $source_file | jq -s '.' | jq 'unique') if jq -e 'index("dart_native")' <<< $all_execs_json > /dev/null then + echo "DART NATIVE COMPILE" pushd executors/dart_native/ dart pub get dart compile exe bin/executor.dart @@ -65,6 +69,7 @@ fi if jq -e 'index("dart_web")' <<< $all_execs_json > /dev/null then + echo "DART WEB COMPILE" pushd executors/dart_web/ dart pub get dart run bin/make_runnable_by_node.dart @@ -73,6 +78,7 @@ fi # Executes all tests on that new data in the new directory mkdir -p $TEMP_DIR/testOutput +echo "Created $TEMP_DIR/testOutput" # Invoke all tests on all platforms pushd testdriver @@ -80,6 +86,8 @@ pushd testdriver # Set to use NVM source "$HOME/.nvm/nvm.sh" +echo "RUNNING FROM $source_file" + jq -c '.[]' ../$source_file | while read i; do if jq -e 'has("prereq")' <<< $i > /dev/null then @@ -90,7 +98,8 @@ jq -c '.[]' ../$source_file | while read i; do exec_command=$(jq -r -c '.run.exec' <<< $i) test_type=$(jq -r -c '.run.test_type | join(" ")' <<< $i) per_execution=$(jq -r -c '.run.per_execution' <<< $i) - python3 testdriver.py --icu_version $icu_version --exec $exec_command --test_type $test_type --file_base ../$TEMP_DIR --per_execution $per_execution --run_limit 100 + ignore=$(jq -r -c '.run.ignore' <<< $i) + python3 testdriver.py --icu_version $icu_version --exec $exec_command --test_type $test_type --file_base ../$TEMP_DIR --per_execution $per_execution --ignore $ignore --run_limit $TEST_LIMIT echo $? done diff --git a/generateDataAndRun.sh b/generateDataAndRun.sh index 00f55c48..094fdde3 100755 --- a/generateDataAndRun.sh +++ b/generateDataAndRun.sh @@ -59,7 +59,7 @@ popd # Run Dart executors in a custom way # -# TODO(?): Figure out why datasets.py can't support runnign multiple CLI commands, +# TODO(?): Figure out why datasets.py can't support running multiple CLI commands, # if that is the reason why Dart needs custom handling in this end-to-end script all_execs_json=$(jq '.[].run.exec' $source_file | jq -s '.' | jq 'unique') diff --git a/verifier/detail_template.html b/verifier/detail_template.html index 9a2e6762..0bc325ac 100644 --- a/verifier/detail_template.html +++ b/verifier/detail_template.html @@ -101,34 +101,36 @@ count: 0, characterized: null, check_boxes: [], - selected_set: null, - count: 0 + box_labels: [], + selected_set: null }, 'fail': { json: null, count: 0, characterized: null, check_boxes: [], - selected_set: null, + box_labels: [], + selected_set: null }, 'error': { json: null, count: 0, characterized: null, check_boxes: [], - selected_set: null, - count: 0, + box_labels: [], + selected_set: null }, 'unsupported': { json: null, count: 0, characterized: null, check_boxes: [], + box_labels: [], selected_set: null, } }; - // Get the JSON data from the tests. let test_results['unsupported'].json; + // Get the JSON data from the tests. // Characterizations of non-passing. let fail_char; @@ -173,7 +175,6 @@ .then((data) => { test_results['unsupported'].characterized = data}), - new Promise((resolve, reject) => { $(document).ready(resolve); }) @@ -211,38 +212,31 @@ } } - const characterized_failure_labels = - $characterized_failure_labels; - let selectedSet = null; let intersection_labels = []; let accum_boxes = []; function checkboxChanged(box) { - // TODO - the class of the box, e.g., 'pass', 'fail', etc. // Update the group selected with the intersection of values const box_class = box.className; const test_data = test_results[box_class]; test_data.selected_set = null; // TODO: Finish this - //let char_items = test_data.characterized;\ - for (let index in characterized_failure_labels) { - const tag = characterized_failure_labels[index]; - const check_item = document.getElementById(tag); - if (check_item.checked) { - const label_string = check_item.value; - const str_len = label_string.length; - const labels = label_string.substring(2, str_len - 2). split("', '"); + let characterized_labels = test_data.box_labels; + for (let check_box of test_data.check_boxes) { + // For each checkbox + if (check_box.checked) { + const labels_string = check_box.value; + const str_len = labels_string.length; + const labels = labels_string.split(","); const newSet = new Set(labels); - if (! test_data.selected_set) { - // Start the set of selected items. + // Start the set of selected items. test_data.selected_set = newSet; } else { // Update by intersection with this set. test_data.selected_set = - new Set( - [...test_data.selected_set].filter((x) => newSet.has(x))); + new Set([...test_data.selected_set].filter((x) => newSet.has(x))); } } } @@ -251,32 +245,34 @@ const class_name = box_class; // get from a parent. const element_name = 'selectedCount'; const selected_count_items = document.getElementsByName(element_name); + // Spread this out into an array for filtering. const output = [...selected_count_items].filter(elem => elem.className == class_name); const newSize = test_data.selected_set == null ? 0 : test_data.selected_set.size; - // TODO: Get the characterized data from the right set of items + + // TODO: Get the characterized data from the correct set of items if (newSize == 0) { - // Turn on all the check boxes - for (let index in characterized_failure_labels) { - const tag = characterized_failure_labels[index]; - const check_div = document.getElementById(tag + "_div"); + // Turn on all the check boxes in this group + for (let index in test_data.check_boxes) { + const check_item = test_data.check_boxes[index]; + const check_div = check_item.parentElement; check_div.style.display = 'block'; - const check_box = document.getElementById(tag); - check_box.checked = false; + check_item.checked = false; check_div.attributeStyleMap.clear(); } + // Reset the count to all? } else { - // TODO: Look at all the sets that intersect with selected. + // Consider all the sets that intersect with selected. let intersection_checkboxes = []; - for (let index in characterized_failure_labels) { - const tag = characterized_failure_labels[index]; - const check_item = document.getElementById(tag); + for (let index in test_data.check_boxes) { + const check_item = test_data.check_boxes[index]; const label_string = check_item.value; const str_len = label_string.length; - const labels = label_string.substring(2, str_len - 2). split("', '"); + const labels = label_string.split(","); const newSet = new Set(labels); let intersectSet = new Set([...newSet].filter(i => test_data.selected_set.has(i))); - const div_for_checkbox = document.getElementById(tag + "_div"); + + const div_for_checkbox = check_item.parentElement; if (intersectSet.size > 0) { intersection_checkboxes.push(check_item); div_for_checkbox.style.setProperty("text-decoration", ""); @@ -294,13 +290,16 @@ } function showSelectedItems(id, item_type) { - // Get the selected failures as a JSON list + // Get the selected items as a JSON list let selected_json_data = []; - for (const failure of test_results[item_type].json) { - let label = failure['label']; + for (const item of test_results[item_type].json) { + if (test_results[item_type].selected_set) { + + let label = item['label']; if (test_results[item_type].selected_set.has(label)) { - selected_json_data.push(failure); + selected_json_data.push(item); } + } } fill_pagination("#characterized-pagination-container_" + item_type, "#characterized-data-container_" + item_type, @@ -359,7 +358,7 @@ continue; } // Get formatted count of the data - // And pu these strings in the HTML fields + // And put these strings in the HTML fields total_summary_count += data.length; test_results[c_type].count = (data.length).toLocaleString('en', {useGrouping:true}); @@ -376,8 +375,7 @@ let pagination_container_name = '#' + c_type + '-pagination-container'; - let data_container_name = '#' + c_type + - '-data-container'; + let data_container_name = '#' + c_type + '-data-container'; fill_pagination(pagination_container_name, data_container_name, @@ -395,11 +393,9 @@ /* set up areas for error and unsupported detail */ create_checkbox_area('pass'); + create_checkbox_area('fail'); create_checkbox_area('error'); create_checkbox_area('unsupported'); - - // TODO: Set this up. - // create_checkbox_area('fail'); } function fill_pagination(pagination_container_name, @@ -428,41 +424,100 @@ function create_checkbox_area(data_class) { /* Given a set of labels and named characteristics, create a set of checkboxes with - labels for selecting a subset of items. Put this in the indicated div. + labels for selecting a subset of items. Put this in the proper div. + Create "Other" category for any non-categorized tests. */ - let test_info = test_results[data_class]; - let data_set = test_info.characterized; + const test_info = test_results[data_class]; + + let leftover_labels = new Set(); + const json = test_info['json']; + for (const item of json) { + leftover_labels.add(item['label']); + } + let div_name = data_class + '_characterized'; - let container = document.getElementById(div_name); - for (let item in data_set) { - let id = div_name + "_" + item; - let name = id; - const dict_values = data_set[item]; - let count = 0; - for (let key of Object.keys(dict_values)) { - count += dict_values[key].length; - } - let box = document.createElement("INPUT"); - box.setAttribute("type", "checkbox"); - box.setAttribute("id", id); - box.setAttribute("name", name); - box.setAttribute("value", dict_values); - box.setAttribute("onclick", "checkboxChanged(this);"); - box.className = data_class; - let label = document.createElement("label"); - label.htmlFor = id; - const text = count + ":" + item; - label.appendChild(document.createTextNode(text)); - - let br = document.createElement('br'); - if (container) { - container.appendChild(box); - container.appendChild(label); - container.appendChild(br); - } - test_info.check_boxes.push(box); - } - } + let container = document.getElementById(div_name); + for (const characterization in test_info.characterized) { + let id = div_name + "_" + characterization; + const dict_values = test_info.characterized[characterization]; + for (const label of dict_values) { + leftover_labels.delete(label); + } + let count = 0; + for (let key of Object.keys(dict_values)) { + const values = dict_values[key]; + if (Array.isArray(values)) { + count += values.length; + } else { + count += 1; + } + } + if (count > 0) { + const new_box = make_characterized_box( + id, dict_values, data_class, characterization, + count, container); + test_info.box_labels.push(characterization); + test_info.check_boxes.push(new_box); + } + } + if (leftover_labels.size > 0) { + // Create an item for non-categorized results. + const characterization = "others"; + let id = div_name + "_" + characterization; + const new_box = make_characterized_box( + id, leftover_labels, data_class, characterization, + leftover_labels.size, container); + + test_info.box_labels.push(characterization); + test_info.check_boxes.push(new_box); + } + } + + /* + Create a check box with the labels for a characterization. + id: data_class plus the characterization + dict_values: the set of labels attached to this check box + data_class: one of "pass", "fail", etc. + characterization: string describing the identified attribute(s) + count: how many are in the list of labels + container: the HTML object containing the div of this checkbox + */ + function make_characterized_box( + id, dict_values, data_class, characterization, count, container) { + let box = document.createElement("INPUT"); + box.setAttribute("type", "checkbox"); + box.setAttribute("id", id); + box.setAttribute("name", id); + box.setAttribute("value", [...dict_values]); + box.setAttribute("onclick", "checkboxChanged(this);"); + box.className = data_class; + let box_label = document.createElement("label"); + box_label.htmlFor = id; + const text = count + ":" + characterization; + box_label.appendChild(document.createTextNode(text)); + + let box_div = document.createElement("div"); + box_div.setAttribute("id", "div_" + id); + box_div.appendChild(box); + box_div.appendChild(box_label); + box_div.appendChild(document.createElement('br')); + + if (container) { + container.appendChild(box_div); + } + return box; + } + + function clearSelectedItems(the_button, test_class) { + // TODO: Clear all the check boxes for this test_class. + const test_data = test_results[test_class]; + for (const box in test_data.check_boxes) { + box.checked = false; + } + + } + + @@ -486,103 +541,110 @@

Test details


-
- Passing tests ($passing_tests) - -
-
-
-
- Passing tests characterized -
-
-

Filtered count = 0 - -

-
+
+ Passing tests ($passing_tests) + +
+
+
+
+ Passing tests characterized +
+
+

Filtered count = 0 + + +

+
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
-
- -
- Failing tests ($failing_tests) -
-
-
-
- Failures characterized -
-
-

Filtered count = 0 - -

$failures_characterized -
-
-
-
-
-
-
-
-
- -
- Test errors ($error_count) -

Test Errors ($error_count)

-
-
-
-
- Errors characterized -
-
-

Filtered count = 0 - -

-
+
+ +
+ Failing tests ($failing_tests) +
+
+
+
+ Failures characterized +
+
+

Filtered count = 0 + + + +

+
+
+
+
+
+
+
+
+
+
+ +
+ Test errors ($error_count) +

Test Errors ($error_count)

+
+
+
+
+ Errors characterized +
+
+

Filtered count = 0 + + + +

+
-
-
-
-
-
-
-
-
-
- -
- Unsupported tests ($unsupported_count) -
-
-
-
- Unsupported characterized -
-
-

Filtered count = 0 - -

- +
+
+
+
+
+
+
+
+
+ +
+ Unsupported tests ($unsupported_count) +
+
+
+
+ Unsupported characterized +
+
+

Filtered count = 0 + + +

+
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+ + + - + diff --git a/verifier/testreport.py b/verifier/testreport.py index a6914d9b..6da7e1b6 100644 --- a/verifier/testreport.py +++ b/verifier/testreport.py @@ -10,6 +10,7 @@ from difflib import SequenceMatcher from datetime import datetime + import glob import json import logging @@ -349,22 +350,22 @@ def create_html_report(self): #html_map['failure_table_lines'] = '\n'.join(fail_lines) # Characterize successes, too. - pass_characterized = self.characterize_failures_by_options(self.passing_tests) + pass_characterized = self.characterize_failures_by_options(self.passing_tests, 'pass') flat_combined_passing = self.flatten_and_combine(pass_characterized, None) self.save_characterized_file(flat_combined_passing, "pass") # Get and save failures, errors, unsupported - error_characterized = self.characterize_failures_by_options(self.test_errors) + error_characterized = self.characterize_failures_by_options(self.test_errors, 'error') flat_combined_errors = self.flatten_and_combine(error_characterized, None) self.save_characterized_file(flat_combined_errors, "error") - unsupported_characterized = self.characterize_failures_by_options(self.unsupported_cases) + unsupported_characterized = self.characterize_failures_by_options(self.unsupported_cases, 'unsupported') flat_combined_unsupported = self.flatten_and_combine(unsupported_characterized, None) self.save_characterized_file(flat_combined_unsupported, "unsupported") - # TODO: SHhould we compute top 3-5 overlaps for each set? + # TODO: Should we compute top 3-5 overlaps for each set? # Flatten and combine the dictionary values - fail_characterized = self.characterize_failures_by_options(self.failing_tests) + fail_characterized = self.characterize_failures_by_options(self.failing_tests, 'fail') fail_simple_diffs = self.check_simple_text_diffs() flat_combined_dict = self.flatten_and_combine(fail_characterized, fail_simple_diffs) @@ -490,6 +491,8 @@ def flatten_and_combine(self, input_dict, input_simple): # Flatten the dictionary. flat_items = {} for key, value in all_items.items(): + if len(value) <= 0: + continue if type(value) == list: flat_items[key] = value else: @@ -500,85 +503,19 @@ def flatten_and_combine(self, input_dict, input_simple): flat_combined_dict = self.combine_same_sets_of_labels(flat_items) return flat_combined_dict - def characterize_failures_by_options(self, failing_tests): - # User self.failing_tests, looking at options + def characterize_failures_by_options(self, tests, result_type): + # Looking at options results = defaultdict(list) - results['locale'] = {} # Dictionary of labels for each locale - for test in failing_tests: - # Get input_data, if available + for test in tests: try: label = test['label'] except: label = '' - - input_data = test.get('input_data') - - if input_data: - # Look at locale + key_list = ['locale', 'locale_label', 'option', 'options', + 'language_label', 'ignorePunctuation', 'compare_result', + 'compare_type', 'test_description', 'unsupported_options'] + for key in key_list: try: - locale = input_data.get('locale') - except: - locale = None - if locale: - if locale in results['locale']: - results['locale'][locale].append(label) - else: - results['locale'][locale] = [label] - - options = input_data.get('options') - if options: - # Get each combo of key/value - for key, value in options.items(): - if key not in results: - results[key] = {} - if value in results[key]: - results[key][value].append(label) - else: - results[key][value] = [label] - - # Try fields in language_names - for key in ['language_label', 'locale_label']: - try: - if input_data.get(key): - value = input_data[key] - if key not in results: - results[key] = {} - if value in results[key]: - results[key][value].append(label) - else: - results[key][value] = [label] - except: - continue - - # Try fields in likely_subtags - for key in ['option', 'locale']: - try: - if input_data.get(key): - value = input_data[key] - if key not in results: - results[key] = {} - if value in results[key]: - results[key][value].append(label) - else: - results[key][value] = [label] - except: - continue - - options =['language_label', 'ignorePunctuation', 'compare_result', 'compare_type', 'test_description'] - for key in options: - try: - if test.get(key): # For collation results - value = test[key] - if key not in results: - results[key] = {} - if value in results[key]: - results[key][value].append(label) - else: - results[key][value] = [label] - except: - continue - - for key in ['language_label', 'ignorePunctuation', 'compare_result', 'compare_type', 'test_description']: if test.get(key): # For collation results value = test[key] if key not in results: @@ -587,27 +524,60 @@ def characterize_failures_by_options(self, failing_tests): results[key][value].append(label) else: results[key][value] = [label] - for key in ['ignorePunctuation']: - try: - if (test.get('input_data') and test['input_data'].get(key)): # For collation results - value = test['input_data'][key] - if key not in results: - results[key] = {} - if value in results[key]: - results[key][value].append(label) - else: - results[key][value] = [label] - except: - continue + except: + continue + + # Look at the input_data part of the test result + # TODO: Check the error_detail and error pars, too. + key_list = ['ignorePunctuation', 'options', 'unsupported_options', 'error_detail'] + input_data = test.get('input_data') + self.add_to_results_by_key(results, input_data, test, key_list) + error_detail = test.get('error_detail') + if error_detail: + error_keys = error_detail.keys() # ['options'] + self.add_to_results_by_key(results, error_detail, test, error_keys) + + # if input_data: + # add_to_results_by_key(results, input_data, test, key_list) + # for key in key_list: + # try: + # if (input_data.get(key)): # For collation results + # value = test['input_data'][key] + # if key not in results: + # results[key] = {} + # if value in results[key]: + # results[key][value].append(label) + # else: + # results[key][value] = [label] + # except: + # continue # TODO: Add substitution of [] for () # TODO: Add replacing (...) with "-" for numbers # TODO: Find the largest intersections of these sets and sort by size - combo_list = [(combo, len(results[combo])) for combo in results] - combo_list.sort(key=take_second, reverse=True) + + # This is not used! + combo_list = [(combo, len(results[combo])) for combo in results] + combo_list.sort(key=take_second, reverse=True) return dict(results) + # TODO: Use the following function to update lists. + def add_to_results_by_key(self, results, input_data, test, key_list): + if input_data: + for key in key_list: + try: + if (input_data.get(key)): # For collation results + value = test['input_data'][key] + if key not in results: + results[key] = {} + if value in results[key]: + results[key][value].append(label) + else: + results[key][value] = [label] + except: + continue + def check_simple_text_diffs(self): results = defaultdict(list) results['insert'] = []