Update test report with better characterization tools (unicode-org#139)

* Clean up test report characterization code

* Clean up test report characterization code

* Update characterization of pass, fail, etc.

* Updating characterization UI in detail pages

* Fix merge typos

* Update verifier/detail_template.html

Co-authored-by: Elango Cheran <[email protected]>

* Better detail_template for characterization

* Fix HTML typo

* Format HTML

* Update Javascript comments and assignments

---------

Co-authored-by: Elango Cheran <[email protected]>

genData100.sh

@@ -5,15 +5,20 @@
 # Save the results
 set -e
 
+# Rotate log files
+logrotate -s logrotate.state logrotate.conf
+
+##########
+# Setup (generate) test data & expected values
+##########
+
 # Enable seting the version of NodeJS
 export NVM_DIR=$HOME/.nvm;
 source $NVM_DIR/nvm.sh;
 
-#
-# Setup
-#
+export TEST_LIMIT=100
 
-export TEMP_DIR=TEMP_DATA
+export TEMP_DIR=TEMP_DATA_100
 rm -rf $TEMP_DIR
 
 # Clear out old data, then create new directory and copy test / verify data there
@@ -28,9 +33,10 @@ source_file=${1:-'run_config.json'}
 
 
 # Generates all new test data
+source_file=${1:-'run_config.json'}
 pushd testgen
 all_icu_versions=$(jq '.[].run.icu_version' ../$source_file | jq -s '.' | jq 'unique' | jq -r 'join(" ")')
-python3 testdata_gen.py  --icu_versions $all_icu_versions --run_limit 100
+python3 testdata_gen.py  --icu_versions $all_icu_versions --run_limit $TEST_LIMIT
 # And copy results to subdirectories.
 cp -r icu* ../$TEMP_DIR/testData
 popd
@@ -42,21 +48,19 @@ python3 check_schemas.py $pwd
 python3 check_generated_data.py ../$TEMP_DIR/testData
 popd
 
-all_execs_json=$(jq '.[].run.exec' $source_file | jq -s '.' | jq 'unique')
+##########
+# Run tests using per-platform executors
+##########
+
 #
 # Run test data tests through all executors
 #
-# Compile Rust executor code for ICU4X 1.0
-if jq -e 'index("rust")' <<< $all_execs_json > /dev/null
-then
-    pushd executors/rust/
-#    cargo clean
-#    cargo build --release
-    popd
-fi
+
+all_execs_json=$(jq '.[].run.exec' $source_file | jq -s '.' | jq 'unique')
 
 if jq -e 'index("dart_native")' <<< $all_execs_json > /dev/null
 then
+    echo "DART NATIVE COMPILE"
     pushd executors/dart_native/
     dart pub get
     dart compile exe bin/executor.dart
@@ -65,6 +69,7 @@ fi
 
 if jq -e 'index("dart_web")' <<< $all_execs_json > /dev/null
 then
+    echo "DART WEB COMPILE"
     pushd executors/dart_web/
     dart pub get
     dart run bin/make_runnable_by_node.dart
@@ -73,13 +78,16 @@ fi
 
 # Executes all tests on that new data in the new directory
 mkdir -p $TEMP_DIR/testOutput
+echo "Created $TEMP_DIR/testOutput"
 
 # Invoke all tests on all platforms
 pushd testdriver
 
 # Set to use NVM
 source "$HOME/.nvm/nvm.sh"
 
+echo "RUNNING FROM $source_file"
+
 jq -c '.[]' ../$source_file | while read i; do
     if jq -e 'has("prereq")' <<< $i > /dev/null
     then
@@ -90,7 +98,8 @@ jq -c '.[]' ../$source_file | while read i; do
     exec_command=$(jq -r -c '.run.exec' <<< $i)
     test_type=$(jq -r -c '.run.test_type | join(" ")'  <<< $i)
     per_execution=$(jq -r -c '.run.per_execution' <<< $i)
-    python3 testdriver.py --icu_version $icu_version --exec $exec_command --test_type $test_type --file_base ../$TEMP_DIR --per_execution $per_execution --run_limit 100
+    ignore=$(jq -r -c '.run.ignore' <<< $i)
+    python3 testdriver.py --icu_version $icu_version --exec $exec_command --test_type $test_type --file_base ../$TEMP_DIR --per_execution $per_execution --ignore $ignore --run_limit $TEST_LIMIT
     echo $?
 done
 

generateDataAndRun.sh

@@ -59,7 +59,7 @@ popd
 # Run Dart executors in a custom way
 #
 
-# TODO(?): Figure out why datasets.py can't support runnign multiple CLI commands,
+# TODO(?): Figure out why datasets.py can't support running multiple CLI commands,
 # if that is the reason why Dart needs custom handling in this end-to-end script
 
 all_execs_json=$(jq '.[].run.exec' $source_file | jq -s '.' | jq 'unique')