Skip to content

Commit

Permalink
Add known issues result type to verification (unicode-org#253)
Browse files Browse the repository at this point in the history 
* Add plural rules to CPP

* Add the Rust datetime fmt code

* Rust datetime fmt and updates to ICU4C and NodeJS (unicode-org#240)

* Add plural rules to CPP

* Starting ICU4X datetime fmt

* DateTime format: Set default timezone explicitly

* Fix formatting

* Added ICU4X timezone computation - not working yet!

* DateTime generator updated to ISO formatted string

* DateTime updates for Node and ICU4C.

* Formatted src/datetimefmt.rs

* Updated using CustomTimeZone in ICU4X date time fmt

* Cargo clippified

* Remove unused function

* Update version to ~1.3

* Adding known issues to reports

* Fixing comment

* removing extra file

* Structured for known issues in reports

* Updated summary page with known issues color

* Fix typo

* Fixes to testreports
  • Loading branch information
@sven-oly
sven-oly authored Jul 17, 2024
1 parent b08229f commit 7b1c4db
Show file tree
Hide file tree
Showing 5 changed files with 224 additions and 65 deletions.
64 changes: 57 additions & 7 deletions verifier/detail_template.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,8 @@
.class_error tr:hover {background-color: #ffdd00;}
.class_unsupported th {background-color: #777777;}
.class_unsupported tr:hover {background-color: #dddddd;}
.class_known_issue th {background-color: #dd8200;}
.class_known_issue tr:hover {background-color: #dd8200;}


.class_selected_fails th {background-color: #0000ee;}
Expand Down Expand Up @@ -142,7 +144,18 @@
widget_label_sets: [],
box_labels: [],
selected_set: null,
},
'known_issue': {
json: null,
all_labels: new Set(),
count: 0,
characterized: null,
check_boxes: [],
widget_label_sets: [],
box_labels: [],
selected_set: null,
}

};

// Get the JSON data from the tests.
Expand All @@ -153,11 +166,13 @@
let p2 = fetch('./failing_tests.json');
let p3 = fetch('./test_errors.json');
let p4 = fetch('./unsupported.json');
let p5 = fetch('./known_issues.json');

let p_pass_char = fetch('./pass_characterized.json');
let p_fail_char = fetch('./fail_characterized.json');
let p_error_char = fetch('./error_characterized.json')
let p_unsupported_char = fetch('./unsupported_characterized.json')
let p_known_issue_char = fetch('./known_issues_characterized.json')

// Synchronize all the data loading and charts / pagination
Promise.all([
Expand All @@ -173,8 +188,12 @@
p4.then((response) => response.json())
.then((data) => {
test_results['unsupported'].json = data}),

// TODO: Make a separate promise.all for this?
p5.then((response) => response.json())
.then((data) => {
test_results['known_issue'].json = data}),


// TODO: Make a separate promise.all for this?
p_pass_char.then((response) => response.json())
.then((data) => {
test_results['pass'].characterized = data}),
Expand All @@ -187,11 +206,14 @@
p_unsupported_char.then((response) => response.json())
.then((data) => {
test_results['unsupported'].characterized = data}),
p_known_issue_char.then((response) => response.json())
.then((data) => {
test_results['known_issue'].characterized = data}),

new Promise((resolve, reject) => {
$(document).ready(resolve);
})
]).then(([p1, p2, p3, p4, _ready]) => {
]).then(([p1, p2, p3, p4, p5, _ready]) => {
// Load the Visualization API and the corechart package.
google.charts.load('current', {'packages':['corechart']});
// Set a callback to run when the Google Visualization API is loaded.
Expand All @@ -210,8 +232,9 @@
['Passing', test_results['pass'].json.length, '#44ff77'],
['Failing', test_results['fail'].json.length, '#ff0000'],
['Errors', test_results['error'].json.length, '#ffdd00'],
['Unsupported', test_results['unsupported'].json.length, '#777777']
];
['Unsupported', test_results['unsupported'].json.length, '#777777'],
['Known issues', test_results['known_issue'].json.length, '#ff8200']
];
const chart = new google.visualization.BarChart(chart_output_area);
let chart_data = google.visualization.arrayToDataTable(input_data);
if (chart && chart_data) {
Expand Down Expand Up @@ -368,7 +391,7 @@
function onloadFn() {
// Set up for pagination of each set of results
// Do this for each class of results.
const container_types = ['pass', 'fail', 'error', 'unsupported'];
const container_types = ['pass', 'fail', 'error', 'unsupported', 'known_issue'];
let total_summary_count = 0;
for (c_type of container_types) {
let container_type = c_type;
Expand Down Expand Up @@ -416,6 +439,7 @@
create_widget_area('fail', create_tristate_area);
create_widget_area('error', create_tristate_area);
create_widget_area('unsupported', create_tristate_area);
create_widget_area('known_issue', create_tristate_area);
}

function fill_pagination(pagination_container_name,
Expand Down Expand Up @@ -648,7 +672,8 @@ <h2>$platform_info</h2>
<p><span id="total_summary_count">$total_tests</span> attempted. Pass: <span id="pass_summary_count">$passing_tests</span>,
Fail: <span id="fail_summary_count">$failing_tests</span>,
Errors: <span id="error_summary_count">$error_count</span>,
Unsupported: <span id="unsupported_summary_count">$unsupported</span></p>
Unsupported: <span id="unsupported_summary_count">$unsupported</span>,
Known issues: <span id="known_issue_summary_count">$unsupported</span></p>
<div id='chart_div'>
</div>
</div>
Expand Down Expand Up @@ -767,6 +792,31 @@ <h2 id='testErrors'>Test Errors ($error_count)</h2>
</details>
</div>
</details>

<details>
<summary>Known issues <span id='known_issue_count'>($known_issue_count)</span></summary>
<div id="known_issue-pagination-container"></div>
<div id="known_issue-data-container"></div>
<div id='testKnownIssueCharacterized'>
<details>
<summary class="known_issue">Known issues characterized</summary>
<div bp="grid">
<div bp="3">
<p>Filtered count = <span id='selectedCount_known_issue' name='selectedCount' class='known_issue'>0</span>
<button id="showSelected_known_issue" onclick="showSelectedItems(this, 'known_issue');">Update display</button>
<button id="clearSelected_known_issue" onclick="clearSelectedItems(this, 'known_issue');">Clear</button>
</p>
<div id='known_issue_characterized'>
</div>
</div> <!-- end of checkbox div -->
<div bp="9">
<div id="characterized-pagination-container_known_issue"></div>
<div id="characterized-data-container_known_issue"></div>
</div>
</div> <!-- grid end -->
</details>
</div>
</details>
</div>

</body>
Expand Down
16 changes: 13 additions & 3 deletions verifier/report_template.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,12 +45,20 @@ def __init__(self):
self.unsupported_table_template = Template(
""" <table id='test_unsupported_table'>
<tr><th width="10%">Label</th><th width="20%">Unsupported message</th><th>Details</th><th>Input data</th></tr>
<!-- For each failing test, output row with columns
<!-- For each unsupported test, output row with columns
label, expected, actual, difference -->
$test_unsupported_table
</table>
""")

self.known_issue_table_template = Template(
""" <table id='test_known_issue_table'>
<tr><th width="10%">Label</th><th width="20%">known_issue message</th><th>Details</th><th>Input data</th></tr>
<!-- For each known issue, output row with columns
label, expected, actual, difference -->
$test_unknown_issue_table
</table>
""")
self.fail_line_template = Template(
'<tr id="$label"><td>$label</td><td>$expected</td><td>$result</td><td>$input_data</td></tr>'
)
Expand All @@ -66,8 +74,6 @@ def __init__(self):
self.summary_table_template = Template(
""" <table id='$type_summary_table'">
<tr><th width="10%">$type</th><th width="20%">Count</th></tr>
<!-- For each failing test, output row with columns
item, count -->
$table_content
</table>
""")
Expand All @@ -76,6 +82,10 @@ def __init__(self):
'<tr id="$label"><td>$label</td><td>$unsupported</$unsupported><td>$error_detail</td><td>$input_data</td></tr>'
)

self.test_known_issue_template = Template(
'<tr id="$label"><td>$label</td><td>known_issue</$known_issue><td>$error_detail</td><td>$input_data</td></tr>'
)

# Template for test failures - will be replaced by html generated in detail_template.html
self.checkbox_option_template = Template(
'<div id="$id_div"><input type=checkbox class="fail" id="$id" name="$name" value="$value" onclick="checkboxChanged(this);"</input><label for="$id">$count $id</label></div>'
Expand Down
8 changes: 4 additions & 4 deletions verifier/summary_template.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@
let table = document.getElementById('exec_summary_table')

let data_groups = ['Result', 'Pass', 'Fail', 'Error', 'Unsupported',
'known_issue',
{role: 'annotation'}];

// Get all the types of tests available
Expand All @@ -155,7 +156,6 @@
const tests = exec_summary_json[test_type];
for (const node_version of tests) {
exec_set.add(node_version['version']['platform'])
// exec_set.add(node_version['exec']);
}
}

Expand Down Expand Up @@ -187,19 +187,18 @@
let details = [];
for (const report of tests) {
if (report['version']['platform'] == exec) {
// if (report['exec'] == exec) {
reports.push(report);
// Add data for this report.
const version_label = report['version']['platformVersion'] +
'\n' +
report['version']['icuVersion'];
const report_data = [
version_label,
// report['exec_version'],
report['pass_count'],
report['fail_count'],
report['error_count'],
report['unsupported_count'],
report['known_issue_count'],
''
];
data.push(report_data);
Expand All @@ -224,7 +223,8 @@
0: {color:'#00dd77'}, // Passing
1: {color:'#dd0000'}, // Failing
2: {color:'#ffdd00'}, // Error
3: {color: '#777777'} //
3: {color: '#777777'}, // Unsupported
4: {color: '#ff8200'} // Known issue (UTK orange)
},
};
// Set up options and links
Expand Down
Loading

0 comments on commit 7b1c4db

Please sign in to comment.