diff --git a/executors/cpp/likely_subtags.cpp b/executors/cpp/likely_subtags.cpp index 825edc50..5b0828c6 100644 --- a/executors/cpp/likely_subtags.cpp +++ b/executors/cpp/likely_subtags.cpp @@ -1,7 +1,3 @@ -/******************************************************************** - * Comments and license as needed - ************************************ - /****** * testing likely subtags for locales */ diff --git a/executors/cpp/main.cpp b/executors/cpp/main.cpp index 2d5a22ef..368f50b0 100644 --- a/executors/cpp/main.cpp +++ b/executors/cpp/main.cpp @@ -44,12 +44,11 @@ extern const string test_langnames(json_object *json_in); extern const string test_likely_subtags(json_object *json_in); extern const string test_numfmt(json_object *json_in); -// extern const string test_numfmt(json_object *json_in); -std::string supported_tests[6] = { - "collation_short" +std::string supported_tests[4] = { + "collation_short", + "likely_subtags", "lang_names", - "likely_subtags" - "number_fmt", + "number_fmt" }; @@ -111,9 +110,6 @@ int main(int argc, const char** argv) else if (test_type == "lang_names") { outputLine = test_langnames(json_input); } - else if (test_type == "likely_subtags") { - // !!! outputLine = test_likely_subtags(json_input); - } else { outputLine = "# BAD TEST " + test_type; // "{\"error\": \"unknown test type\"," + @@ -121,8 +117,6 @@ int main(int argc, const char** argv) // "\"unsupported_test:\"" + test_type + "}"; } - // TODO: Instead of a string, get JSON output and stringify it. - // cout << "# GOT FROM TEST: " << outputLine << endl; cout << outputLine << endl; } diff --git a/executors/cpp/number_fmt.cpp b/executors/cpp/number_fmt.cpp index d97f2f6f..749b5f73 100644 --- a/executors/cpp/number_fmt.cpp +++ b/executors/cpp/number_fmt.cpp @@ -12,6 +12,7 @@ #include "unicode/utypes.h" //#include "unicode/appendable.h" #include "unicode/bytestream.h" +#include "unicode/compactdecimalformat.h" #include "unicode/currunit.h" #include "unicode/dcfmtsym.h" #include "unicode/fieldpos.h" @@ -22,8 +23,12 @@ //#include "unicode/parseerr.h" #include "unicode/plurrule.h" #include "unicode/ucurr.h" + +#include "unicode/stringpiece.h" + #include "unicode/unum.h" #include "unicode/unumberformatter.h" +// TODO: NEEDED? #include "unicode/unumberoptions.h" #include "unicode/uobject.h" #include "unicode/unistr.h" @@ -32,6 +37,7 @@ #include "unicode/numfmt.h" #include "unicode/numberrangeformatter.h" +#include "unicode/numsys.h" #include "util.h" #include @@ -47,7 +53,20 @@ using std::cout; using std::endl; using std::string; +using icu::NumberingSystem; + using icu::number::NumberFormatter; +using icu::number::NumberFormatterSettings; +using icu::number::Notation; +using icu::number::Precision; +using icu::number::IntegerWidth; +using icu::number::Scale; + +using icu::CurrencyUnit; +using icu::MeasureUnit; + +using number::impl::UFormattedNumberData; + using icu::number::LocalizedNumberFormatter; using icu::number::FormattedNumber; using icu::number::impl::UFormattedNumberData; @@ -55,6 +74,161 @@ using icu::number::impl::UFormattedNumberData; const string error_message = "error"; +// Get the integer value of a settting +int16_t get_integer_setting(string key_value_string) { + int16_t return_val = -1; + size_t colon_pos = key_value_string.find(":"); + if (colon_pos >= 0) { + string num_string = key_value_string.substr(colon_pos +1); + return_val = std::stoi(num_string); + } + return return_val; +} + +// Get the double value of a settting +double get_double_setting(string key_value_string) { + double return_val = -1.0; + size_t colon_pos = key_value_string.find(":"); + if (colon_pos >= 0) { + string num_string = key_value_string.substr(colon_pos +1); + return_val = std::stod(num_string); + } + return return_val; + +} + +// Get integer width settings +IntegerWidth set_integerWidth(json_object* options_obj) { + IntegerWidth integerWidth_setting = IntegerWidth::zeroFillTo(1); + if (!options_obj) { + return integerWidth_setting; + } + string int_width_string; + + json_object* integer_obj_min = json_object_object_get( + options_obj, "minimumIntegerDigits"); + json_object* integer_obj_max = json_object_object_get( + options_obj, "maximumIntegerDigits"); + if (integer_obj_min && integer_obj_max) { + int_width_string = json_object_get_string(integer_obj_min); + int32_t val_min32 = get_integer_setting(int_width_string); + int_width_string = json_object_get_string(integer_obj_max); + int32_t val_max32 = get_integer_setting(int_width_string); + integerWidth_setting = IntegerWidth::zeroFillTo(val_min32).truncateAt(val_max32); + } + else if (integer_obj_min && !integer_obj_max) { + int_width_string = json_object_get_string(integer_obj_min); + int32_t val_min32 = get_integer_setting(int_width_string); + int_width_string = json_object_get_string(integer_obj_min); + integerWidth_setting = IntegerWidth::zeroFillTo(val_min32); + } + else if (!integer_obj_min && integer_obj_max) { + int_width_string = json_object_get_string(integer_obj_max); + int32_t val_max32 = get_integer_setting(int_width_string); + integerWidth_setting = IntegerWidth::zeroFillTo(1).truncateAt(val_max32); + } + return integerWidth_setting; +} + +// Get fraction and siginfication digits settings +Precision set_precision_digits(json_object* options_obj, Precision previous_setting) { + Precision precision_setting = previous_setting; + if (!options_obj) { + return precision_setting; + } + + // First, consider fraction digits. + json_object* precision_obj_max = + json_object_object_get(options_obj, "maximumFractionDigits"); + json_object* precision_obj_min = + json_object_object_get(options_obj, "minimumFractionDigits"); + + string precision_string; + + int16_t val_max = 0; + int16_t val_min = 0; + if (precision_obj_max) { + val_max = get_integer_setting( + json_object_get_string(precision_obj_max)); + } + if (precision_obj_min) { + val_min = get_integer_setting( + json_object_get_string(precision_obj_min)); + } + if (precision_obj_max && precision_obj_min) { + // Both are set + precision_setting = Precision::minMaxFraction(val_min, val_max); + } + else if (!precision_obj_max && precision_obj_min) { + precision_setting = Precision::minFraction(val_min); + } else if (precision_obj_max && ! precision_obj_min) { + precision_setting = Precision::maxFraction(val_max); + } + + // Now handle significant digits + precision_obj_max = + json_object_object_get(options_obj, "maximumSignificantDigits"); + precision_obj_min = + json_object_object_get(options_obj, "minimumSignificantDigits"); + + if (precision_obj_max) { + val_max = get_integer_setting( + json_object_get_string(precision_obj_max)); + } + if (precision_obj_min) { + val_min = get_integer_setting( + json_object_get_string(precision_obj_min)); + } + if (precision_obj_max && precision_obj_min) { + // Both are set + precision_setting = Precision::minMaxSignificantDigits(val_min, val_max); + } + else if (!precision_obj_max && precision_obj_min) { + precision_setting = Precision::minSignificantDigits(val_min); + } else if (precision_obj_max && !precision_obj_min) { + precision_setting = Precision::maxSignificantDigits(val_max); + } + + return precision_setting; +} + +UNumberSignDisplay set_sign_display(json_object* options_obj) { + UNumberSignDisplay signDisplay_setting = UNUM_SIGN_AUTO; + + if (!options_obj) { + return signDisplay_setting; + } + + json_object* signDisplay_obj = + json_object_object_get(options_obj, "signDisplay"); + if (signDisplay_obj) { + string signDisplay_string = json_object_get_string(signDisplay_obj); + + if (signDisplay_string == "exceptZero") { + signDisplay_setting = UNUM_SIGN_EXCEPT_ZERO; + } + else if (signDisplay_string == "always") { + signDisplay_setting = UNUM_SIGN_ALWAYS; + } + else if (signDisplay_string == "never") { + signDisplay_setting = UNUM_SIGN_NEVER; + } + else if (signDisplay_string == "negative") { + signDisplay_setting = UNUM_SIGN_NEGATIVE; + } + else if (signDisplay_string == "accounting") { + signDisplay_setting = UNUM_SIGN_ACCOUNTING; + } + else if (signDisplay_string == "accounting_exceptZero") { + signDisplay_setting = UNUM_SIGN_ACCOUNTING_EXCEPT_ZERO; + } + else if (signDisplay_string == "accounting_negative") { + signDisplay_setting = UNUM_SIGN_ACCOUNTING_NEGATIVE; + } + } + return signDisplay_setting; +} + const string test_numfmt(json_object *json_in) { UErrorCode status = U_ZERO_ERROR; @@ -62,6 +236,10 @@ const string test_numfmt(json_object *json_in) { json_object *label_obj = json_object_object_get(json_in, "label"); string label_string = json_object_get_string(label_obj); + // Other parts of the input. + json_object *options_obj = json_object_object_get(json_in, "options"); + json_object *skeleton_obj = json_object_object_get(json_in, "skeleton"); + json_object *pattern_obj = json_object_object_get(json_in, "pattern"); // The locale for numbers json_object *locale_label_obj = json_object_object_get(json_in, "locale"); @@ -69,17 +247,199 @@ const string test_numfmt(json_object *json_in) { if (locale_label_obj) { locale_string = json_object_get_string(locale_label_obj); } - Locale displayLocale(locale_string.c_str()); - std:: cout<< "Locale = " << locale_string << std::endl; + const Locale displayLocale(locale_string.c_str()); + + // Get options + json_object *notation_obj; + json_object *unit_obj; + json_object *unitDisplay_obj; + json_object *style_obj; + json_object *currencyDisplay_obj; + json_object *group_obj; + json_object *precision_obj_min; + json_object *precision_obj_max; + json_object *min_integer_digits_obj; + json_object *max_integer_digits_obj; + json_object *roundingMode_obj; + json_object *compactDisplay_obj; + json_object *currency_obj; + json_object *signDisplay_obj; + + string notation_string = ""; + string precision_string = ""; + string unit_string = ""; + string unitDisplay_string = ""; + string style_string = ""; + string compactDisplay_string = ""; + + // Defaults for settings. + CurrencyUnit currency_unit_setting = CurrencyUnit(); + IntegerWidth integerWidth_setting = IntegerWidth::zeroFillTo(1); + MeasureUnit unit_setting = NoUnit::base(); + Notation notation_setting = Notation::simple(); + Precision precision_setting = Precision::unlimited(); + Scale scale_setting = Scale::none(); + UNumberSignDisplay signDisplay_setting = UNUM_SIGN_AUTO; + UNumberFormatRoundingMode rounding_setting = UNUM_ROUND_HALFEVEN; + UNumberDecimalSeparatorDisplay separator_setting = UNUM_DECIMAL_SEPARATOR_AUTO; + UNumberGroupingStrategy grouping_setting = UNUM_GROUPING_AUTO; + UNumberUnitWidth unit_width_setting = + UNumberUnitWidth::UNUM_UNIT_WIDTH_NARROW; + + NumberingSystem* numbering_system = NumberingSystem::createInstance(displayLocale, status); + + // Check all the options + if (options_obj) { + signDisplay_setting = set_sign_display(options_obj); + + notation_obj = json_object_object_get(options_obj, "notation"); + if (notation_obj) { + notation_string = json_object_get_string(notation_obj); + } + + // TODO: Initialize setting based on this string. + unit_obj = json_object_object_get(options_obj, "unit"); + if (unit_obj) { + unit_string = json_object_get_string(unit_obj); + if (unit_string == "percent") { + unit_setting= NoUnit::percent(); + } + else if (unit_string == "permille") { + unit_setting= NoUnit::permille(); + } + else if (unit_string == "furlong") { + unit_setting= MeasureUnit::getFurlong(); + } + } + + unitDisplay_obj = json_object_object_get(options_obj, "unitDisplay"); + if (unitDisplay_obj) { + unitDisplay_string = json_object_get_string(unitDisplay_obj); + if (unitDisplay_string == "narrow") { + unit_width_setting = UNumberUnitWidth::UNUM_UNIT_WIDTH_NARROW; + } + else if (unitDisplay_string == "long") { + unit_width_setting = UNumberUnitWidth::UNUM_UNIT_WIDTH_FULL_NAME; + } + } + + style_obj = json_object_object_get(options_obj, "style"); + if (style_obj) { + style_string = json_object_get_string(style_obj); + } + + compactDisplay_obj = json_object_object_get(options_obj, "compactDisplay"); + if (compactDisplay_obj) { + compactDisplay_string = json_object_get_string(compactDisplay_obj); + if (compactDisplay_string == "short") { + notation_setting = Notation::compactShort(); + } else { + notation_setting = Notation::compactLong(); + } + } + + currency_obj = json_object_object_get(options_obj, "currency"); + if (currency_obj) { + string currency_string = json_object_get_string(currency_obj); + // Set the unit to a currency value + unit_setting = CurrencyUnit(icu::StringPiece(currency_string), status); + } + + // TODO: make a function + currencyDisplay_obj = json_object_object_get(options_obj, "currencyDisplay"); + if (currencyDisplay_obj) { + string currencyDisplay_string = json_object_get_string(currencyDisplay_obj); + unit_width_setting = UNumberUnitWidth::UNUM_UNIT_WIDTH_NARROW; + if (currencyDisplay_string == "narrowSymbol") { + unit_width_setting = UNumberUnitWidth::UNUM_UNIT_WIDTH_NARROW; + } + else if (currencyDisplay_string == "symbol") { + unit_width_setting = UNumberUnitWidth::UNUM_UNIT_WIDTH_SHORT; + } + else if (currencyDisplay_string == "name") { + unit_width_setting = UNumberUnitWidth::UNUM_UNIT_WIDTH_FULL_NAME; + } + } + + // TODO: Make this a function rather than inline. + roundingMode_obj = json_object_object_get(options_obj, "roundingMode"); + if (roundingMode_obj) { + string roundingMode_string = json_object_get_string(roundingMode_obj); + if (roundingMode_string == "floor") { + rounding_setting = UNUM_ROUND_FLOOR; + } else if (roundingMode_string == "ceil") { + rounding_setting = UNUM_ROUND_CEILING; + } else if (roundingMode_string == "halfEven") { + rounding_setting = UNUM_ROUND_HALFEVEN; + } else if (roundingMode_string == "halfTrunc") { + rounding_setting = UNUM_ROUND_HALFDOWN; + } else if (roundingMode_string == "halfExpand") { + rounding_setting = UNUM_ROUND_HALFUP; + } else if (roundingMode_string == "trunc") { + rounding_setting = UNUM_ROUND_DOWN; + } else if (roundingMode_string == "expand") { + rounding_setting = UNUM_ROUND_UP; + } + // TODO: Finish this + // UNUM_ROUND_HALFEVEN , UNUM_FOUND_HALFEVEN = UNUM_ROUND_HALFEVEN , UNUM_ROUND_HALFDOWN = UNUM_ROUND_HALFEVEN + 1 , UNUM_ROUND_HALFUP , + // UNUM_ROUND_UNNECESSARY , UNUM_ROUND_HALF_ODD , UNUM_ROUND_HALF_CEILING , UNUM_ROUND_HALF_FLOOR + } + + // TODO: make a function + group_obj = json_object_object_get(options_obj, "useGrouping"); + if (group_obj) { + string group_string = json_object_get_string(group_obj); + if (group_string == "false") { + grouping_setting = UNUM_GROUPING_OFF; + } + else if (group_string == "true") { + grouping_setting = UNUM_GROUPING_AUTO; + } + else if (group_string == "on_aligned") { + grouping_setting = UNUM_GROUPING_ON_ALIGNED; + } + // TODO: FINISH - could be OFF, MIN2, AUTO, ON_ALIGNED, THOUSANDS + } + + // Need to avoid resetting when not options are specifierd. + precision_setting = set_precision_digits(options_obj, precision_setting); + + // Minimum integer digits + integerWidth_setting = set_integerWidth(options_obj); + + // Check on scaling the value + json_object* scale_obj = json_object_object_get(options_obj, "conformanceScale"); + if (scale_obj) { + string scale_string = json_object_get_string(scale_obj); + double scale_val = get_double_setting(scale_string); + scale_setting = Scale::byDouble(scale_val); + } + + // Other settings... + // NumberFormatter::with().symbols(DecimalFormatSymbols(Locale("de_CH"), status)) + + json_object* numbering_system_obj = json_object_object_get(options_obj, "numberingSystem"); + if (numbering_system_obj) { + string numbering_system_string = json_object_get_string(numbering_system_obj); + numbering_system = NumberingSystem::createInstanceByName(numbering_system_string.c_str(), status); + } + + // Handling decimal point + json_object* decimal_always_obj = json_object_object_get(options_obj, "conformanceDecimalAlways"); + if (decimal_always_obj) { + string separator_string = json_object_get_string( + decimal_always_obj); + if (separator_string == "true") { + separator_setting = UNUM_DECIMAL_SEPARATOR_ALWAYS; + } + } + } + // Additional parameters and values json_object *input_obj = json_object_object_get(json_in, "input"); string input_string = json_object_get_string(input_obj); - json_object *options_obj = json_object_object_get(json_in, "options"); - json_object *rounding_mode_obj = json_object_object_get(json_in, "roundingMode"); - json_object *skeleton_obj = json_object_object_get(json_in, "skeleton"); - json_object *pattern_obj = json_object_object_get(json_in, "pattern"); // Start using these things @@ -91,40 +451,76 @@ const string test_numfmt(json_object *json_in) { // Get the numeric value double input_double = std::stod(input_string); - std:: cout<< "num_double = " << input_double << std::endl; - // TEMPORARY - just return the input. - test_result = input_string; + LocalizedNumberFormatter nf; + if (notation_string == "scientific") { + notation_setting = Notation::scientific(); + if (options_obj) { + json_object* conformanceExponent_obj = + json_object_object_get(options_obj, "conformanceExponent"); + if (conformanceExponent_obj) { + // Check for the number of digits and sign + string confExp_string = json_object_get_string(conformanceExponent_obj); + // https://unicode-org.github.io/icu-docs/apidoc/released/icu4c/unumberformatter_8h.html#a18092ae1533c9c260f01c9dbf25589c9 + // TODO: Parse to find the number of values and the sign setting + if (confExp_string == "+ee") { + notation_setting = Notation::scientific().withMinExponentDigits(2) + .withExponentSignDisplay(UNUM_SIGN_ALWAYS); + } + } + } + } - // TODO!!! - // SimpleNumberFormatter snf = SimpleNumberFormatter::forLocale(displayLocale, status); + // Use settings to initialize the formatter + nf = NumberFormatter::withLocale(displayLocale) + .notation(notation_setting) + .decimal(separator_setting) + .precision(precision_setting) + .integerWidth(integerWidth_setting) + .grouping(grouping_setting) + .adoptSymbols(numbering_system) + .roundingMode(rounding_setting) + .scale(scale_setting) + .sign(signDisplay_setting) + .unit(unit_setting) + .unitWidth(unit_width_setting); - LocalizedNumberFormatter nf = LocalizedNumberFormatter(); // (displayLocale); + if (U_FAILURE(status)) { + test_result = error_message.c_str(); + // TODO: report the error in creating the instance + } - FormattedNumber fnum = nf.formatDouble(input_double, status); + UnicodeString number_result; + FormattedNumber fmt_number = nf.formatDouble(input_double, status); + number_result = fmt_number.toString(status); if (U_FAILURE(status)) { test_result = error_message.c_str(); + // TODO: report the error } // Get the resulting value as a string - UnicodeString number_result = fnum.toString(status); char test_result_string[1000] = ""; int32_t chars_out = number_result.extract(test_result_string, 1000, nullptr, status); test_result = test_result_string; if (U_FAILURE(status)) { - test_result = error_message.c_str(); - } - - if (U_FAILURE(status)) { + // Report a failure + test_result = error_message.c_str(); json_object_object_add(return_json, "error", json_object_new_string("langnames extract error")); } else { + // It worked! json_object_object_add(return_json, "result", json_object_new_string(test_result.c_str())); } + // To see what was actually used. + UnicodeString u_skeleton_out = nf.toSkeleton(status); + chars_out = u_skeleton_out.extract(test_result_string, 1000, nullptr, status); + json_object_object_add(return_json, + "actual_skeleton", + json_object_new_string(test_result_string)); string return_string = json_object_to_json_string(return_json); return return_string; diff --git a/executors/cpp/set_icu4c_binary.sh b/executors/cpp/set_icu4c_binary.sh index a9f7e72c..29fb4d43 100755 --- a/executors/cpp/set_icu4c_binary.sh +++ b/executors/cpp/set_icu4c_binary.sh @@ -23,7 +23,6 @@ ls -ltra pushd $TMP -# curl -L $ICU_PATH | tar xvfpz - tar xvfz *.tgz rm *.tgz diff --git a/executors/rust/src/numberfmt.rs b/executors/rust/src/numberfmt.rs new file mode 100644 index 00000000..bc33531d --- /dev/null +++ b/executors/rust/src/numberfmt.rs @@ -0,0 +1,88 @@ +/* + * Executor provides tests for NumberFormat and DecimalFormat. + */ + +use serde_json::{json, Value}; +use writeable::Writeable; + +use icu::decimal::options; +use icu::decimal::FixedDecimalFormatter; + +use fixed_decimal::FixedDecimal; + +use icu::locid::{locale, Locale}; +use icu_provider::DataLocale; + +use std::panic; +use std::str::FromStr; + +// Support options - update when ICU4X adds support +static SUPPORTED_OPTIONS: [&str; 5] = [ + "minimumIntegerDigits", + "maximumIntegerDigits", + "minimumFractionDigits", + "maximumFractionDigits", + "RoundingMode", +]; + +// Runs decimal and number formatting given patterns or skeletons. +pub fn run_numberformat_test(json_obj: &Value) -> Result { + let provider = icu_testdata::unstable(); + + // TODO: Handle errors of missing JSON fields + let label = &json_obj["label"].as_str().unwrap(); + + // Default locale if not specified. + let langid = if json_obj.get("locale") != None { + let locale_name = &json_obj["locale"].as_str().unwrap(); + Locale::from_str(locale_name).unwrap() + } else { + locale!("und") + }; + let data_locale = DataLocale::from(langid); + + let input = &json_obj["input"].as_str().unwrap(); + + // TODO: Get the options from JSON. If there are unsupported values, return + // "unsupported" rather than an error. + let options = &json_obj["options"]; // This will be an array. + + let mut unsupported_options: Vec<&str> = Vec::new(); + // If any option is not yet supported, + for (option, _setting) in options.as_object().unwrap() { + if !SUPPORTED_OPTIONS.contains(&option.as_str()) { + unsupported_options.push(&option); + } + } + if unsupported_options.len() > 0 { + let json_result = json!({ + "label": label, + "unsupported": label, + "unsupported_options": unsupported_options + }); + return Ok(json_result); + } + + let mut options: options::FixedDecimalFormatterOptions = Default::default(); + // TODO: Use options to call operations including pad and trunc with rounding. + + // Iterator over options, applying the + + // !! A test. More options to consider! + options.grouping_strategy = options::GroupingStrategy::Min2; + + // Can this fail with invalid options? + let fdf = FixedDecimalFormatter::try_new_unstable(&provider, &data_locale, options) + .expect("Data should load successfully"); + + // Returns error if parsing the number string fails. + let input_num = input.parse::().map_err(|e| e.to_string())?; + // TODO: Can this fail? + let result_string = fdf.format(&input_num); + + // Result to stdout. + let json_result = json!({ + "label": label, + "result": result_string.write_to_string()}); + Ok(json_result) +} diff --git a/executors/test_strings.txt b/executors/test_strings.txt index 63f0ee58..0c8aec46 100644 --- a/executors/test_strings.txt +++ b/executors/test_strings.txt @@ -1,4 +1,49 @@ # LANGUAGE DISPLAY NAMES +{"test_type": "lang_names", "label": "01", "language_label": "en", "locale_label": "af"} +{"test_type": "lang_names", "label": "01", "language_label": "", "locale_label": "fr"} +{"test_type": "lang_names", "label": "01", "language_label": "de", "locale_label": "fr"} +{"test_type": "lang_names", "label": "02", "language_label": "fr", "locale_label": "de"} +{"test_type": "lang_names", "label": "LANG_ABC", "language_label": "en", "locale_label": "ja"} +{"test_type": "lang_names", "label": "LANG_ABC", "language_label": "en", "locale_label": "pt-PT"} +{"test_type": "lang_names", "label": "LANG_ABC", "language_label": "en", "locale_label": "zh-CN"} +{"test_type": "lang_names", "label": "LANG_ABC", "language_label": "en", "locale_label": "es"} +{"test_type": "lang_names", "label": "LANG_af_NA", "language_label": "en", "locale_label": "af_NA"} +{"label":"188691","language_label":"zh_MO","locale_label":"en_150","test_type":"lang_names"} +{"label":"188691","language_label":"zh-TW","locale_label":"en","test_type":"lang_names"} +{"label":"188691","language_label":"zh","locale_label":"en","test_type":"lang_names"} +{"label":"zh zh","language_label":"zh","locale_label":"zh","test_type":"lang_names"} +{"label":"en zh","language_label":"en","locale_label":"zh-CN","test_type":"lang_names"} +{"label":"188691","language_label":"zh-CN","locale_label":"en","test_type":"lang_names"} +{"label":"zh en-150","language_label":"zh","locale_label":"en_150","test_type":"lang_names"} + +# COLLATION +{"test_type": "collation_short", "label": "COLL_ABC1", "s1": "de", "s2" : "da"} +{"test_type": "collation_short", "label": "COLL_ABC2", "s1": "de", "s2" : "df"} +{"test_type": "collation_short", "label":"000009","s1":"\\u000d?","s2":"…!"} +{"test_type": "collation_short", "label":"000017","s1":" !","s2":" !"} + +# Equal comparison +{"test_type": "collation_short", "label":"xxxx","s1":"aa","s2":"ab", "compare_type":"="} +{"test_type": "collation_short", "label":"xxxx","s1":"aa","s2":"ab", "compare_type":"<"} + {"test_type": "collation_short", "label":"xxxx","s1":"aa","s2":"xx", "compare_type":"="} + {"test_type": "collation_short", "label":"aa_aa","s1":"aa","s2":"aa", "compare_type":"="} + +# COLLATION Non-ignorable +{"test_type": "collation_short", "label": "COLL_ABC1", "s1": "de", "s2" : "da"} +{"test_type": "collation_short", "label": "COLL_ABC2", "s1": "de", "s2" : "df"} +{"test_type": "collation_short", "label":"0001377","s1":"\\u0009?","s2":"\\u000a!","line":12,"ignorePunctuation":true} + {"test_type": "collation_short", "label":"0001379","s1":"\\u000a?","s2":"\\u000b!","line":14,"ignorePunctuation":true} + + # COLLATION CPP fails: +{"test_type": "collation_short", "label":"0003744","s1":"a\\u0301","s2":"a\\u0302!","line":2379,"ignorePunctuation":true} +{"test_type": "collation_short", "label":"0003744","s1":"꣎?","s2":"꣏!","line":2379,"ignorePunctuation":true} +{"test_type": "collation_short", "label":"0003744","s1":"꣎?","s2":"꣏!","line":2379,"ignorePunctuation":true} +{"test_type": "collation_short", "label":"0001377_single","s1":"\u000a?","s2":"\u0009!","line":12,"ignorePunctuation":true} +{"test_type": "collation_short", "label":"0001377_rev","s1":"\\u000a?","s2":"\\u0009!","line":12,"ignorePunctuation":true} +{"test_type": "collation_short", "label":"0001377","s1":"\\u0009?","s2":"\\u000a!","line":12,"ignorePunctuation":true} +{"test_type": "collation_short", "label":"00008","s1":"ä","s2":"ä","compare_type":"=","test_description":" simple CEs & expansions","rules":"&\\x01\n<<<\\u0300\n&9<\\x00\n&\\uA00A\\uA00B=\\uA002\n&\\uA00A\\uA00B\\u00050005=\\uA003"} + +======= {"test_type": "display_names", "label": "01", "language_label": "en", "locale_label": "af"} {"test_type": "display_names", "label": "01", "language_label": "", "locale_label": "fr"} {"test_type": "display_names", "label": "01", "language_label": "de", "locale_label": "fr"} @@ -18,10 +63,14 @@ {"label":"zh en-150","language_label":"zh","locale_label":"en_150","test_type":"language_display_name"} # COLLATION -{"test_type": "collation_short", "label": "COLL_ABC1", "s1": "de", "s2" : "da"} -{"test_type": "collation_short", "label": "COLL_ABC2", "s1": "de", "s2" : "df"} -{"test_type": "collation_short", "label":"000009","s1":"\\u000d?","s2":"…!"} -{"test_type": "collation_short", "label":"000017","s1":" !","s2":" !"} +{"test_type": "coll_shift_short", "label": "COLL_ABC1", "string1": "de", "string2" : "da"} +{"test_type": "coll_shift_short", "label": "COLL_ABC2", "string1": "de", "string2" : "df"} +{"test_type": "coll_shift_short", "label":"000009","string1":"\\u000d?","string2":"…!"} +{"test_type": "coll_shift_short", "label":"000017","string1":" !","string2":" !"} + +# COLLATION Non-ignorable +{"test_type": "coll_nonignorable_short", "label": "COLL_ABC1", "string1": "de", "string2" : "da"} +{"test_type": "coll_nonignorable_short", "label": "COLL_ABC2", "string1": "de", "string2" : "df"} # Equal comparison {"test_type": "collation_short", "label":"xxxx","s1":"aa","s2":"ab", "compare_type":"="} @@ -44,28 +93,71 @@ {"test_type": "collation_short", "label":"0001377","s1":"\\u0009?","s2":"\\u000a!","line":12,"ignorePunctuation":true} {"test_type": "collation_short", "label":"00008","s1":"ä","s2":"ä","compare_type":"=","test_description":" simple CEs & expansions","rules":"&\\x01\n<<<\\u0300\n&9<\\x00\n&\\uA00A\\uA00B=\\uA002\n&\\uA00A\\uA00B\\u00050005=\\uA003"} - +# NUMBER FORMAT {"test_type": "number_fmt", "label":"NUM_2345", "input": "23.45","locale":"it","options":{"maximumFractionDigits":3}} {"test_type": "number_fmt", "label":"NUM_2345", "input": "23.4567","locale":"fr","options":{"maximumFractionDigits":"1"}} +{"test_type": "number_fmt", "label":"NUM_2345_scale", "input": "23.4567","locale":"fr","options":{"conformanceScale":"0.5", "maximumFractionDigits":"1"}} +{"test_type": "number_fmt", "label":"NUM_2345_scale", "input": "23.4567","locale":"fr","options":{"conformanceScale":"7", "maximumFractionDigits":"1"}} + +{"test_type": "number_fmt","label": "0379", "locale": "es-MX", "skeleton": "compact-short percent integer-width/##00", "input": "91827.3645", "options": {"notation": "compact", "compactDisplay": "short", "style": "unit", "unit": "percent", "maximumIntegerDigits": 4, "minimumIntegerDigits": 2, "maximumFractionDigits": 2}} + +{"test_type": "number_fmt", "label":"permille_2345", "input": "23.4567","locale":"fr","options":{"maximumFractionDigits":"1", "unit": "permille"}} + {"test_type": "number_fmt", "label": "0000", "locale": "es-MX", "skeleton": "compact-short percent unit-width-narrow", "input": "123456", "options": { "notation": "compact", "compactDisplay": "short"}} {"test_type": "number_fmt", "label": "0000", "locale": "en-GB", "skeleton": "compact-short percent unit-width-narrow", "input": "123456", "options": { "notation": "compact", "compactDisplay": "short"}} {"test_type": "number_fmt", "label": "0000", "locale": "bn", "skeleton": "compact-short percent unit-width-narrow", "input": "123456", "options": { "notation": "compact", "compactDisplay": "short"}} {"label": "7333", "op": "format", "skeleton": "0.0", "input": "1.00", "options": {"roundingMode": "halfExpand", "minimumIntegerDigits": 1, "minimumFractionDigits": 1, "maximumFractionDigits": 1, "useGrouping": false}, "test_type": "number_fmt"} +{"label": "7333_3min", "op": "format", "skeleton": "0.0", "input": "1.50", "options": {"roundingMode": "halfExpand", "minimumIntegerDigits": 3, "minimumFractionDigits": 1, "maximumFractionDigits": 1, "useGrouping": false}, "test_type": "number_fmt"} +{"label": "7333_3min", "op": "format", "skeleton": "0.0", "input": "1.50", "options": {"roundingMode": "floor", "minimumIntegerDigits": 3, "minimumFractionDigits": 1, "maximumFractionDigits": 1, "useGrouping": false}, "test_type": "number_fmt"} - +{"label": "7333_sig", "op": "format", "skeleton": "0.0", "input": "1.00", "options": {"roundingMode": "halfExpand", "minimumIntegerDigits": 1, "minimumFractionDigits": 1, "maximumSignificantDigits": 4, "useGrouping": false}, "test_type": "number_fmt"} +{"label": "7333_sig", "op": "format", "skeleton": "0.0", "input": "1.00", "options": {"roundingMode": "halfExpand", "minimumIntegerDigits": 1, "minimumFractionDigits": 1, "minimumSignificantDigits": 2, "useGrouping": false}, "test_type": "number_fmt"} +{"label": "7333_sig", "op": "format", "skeleton": "0.0", "input": "1.00", "options": {"roundingMode": "halfExpand", "minimumIntegerDigits": 1, "minimumFractionDigits": 1, "maximumSignificantDigits": 3, "maximumSignificantDigits": 5, "useGrouping": false}, "test_type": "number_fmt"} + {"label":"0598","locale":"es","skeleton":"compact-short percent sign-accounting-except-zero","input":"91827.3645","options":{"notation":"compact","compactDisplay":"short","style":"unit","unit":"percent","signDisplay":"exceptZero","currencySign":"accounting"}, "test_type": "number_fmt"} {"label":"0598","locale":"zh-TW","skeleton":"compact-short percent sign-accounting-except-zero","input":"91827.3645","options":{"notation":"compact","compactDisplay":"short","style":"unit","unit":"percent","signDisplay":"exceptZero","currencySign":"accounting"}, "test_type": "number_fmt"} {"label":"0598","locale":"zh-TW","skeleton":"compact-short percent sign-accounting-except-zero","input":"9182734567890","options":{"notation":"compact","compactDisplay":"short","style":"unit","unit":"percent","signDisplay":"exceptZero","currencySign":"accounting"}, "test_type": "number_fmt"} +>>>>>>> 8eca805e8c4cd69552153c7137eeea111144d528 + +{"label":"0598","locale":"zh-TW","skeleton":"compact-short percent sign-accounting-except-zero","input":"91827.3645","options":{"notation":"compact","compactDisplay":"short","style":"unit","unit":"percent","signDisplay":"exceptZero","currencySign":"accounting"}, "test_type": "number_fmt"} +{"label":"0598_L","locale":"zh-TW","skeleton":"compact-long percent sign-accounting-except-zero","input":"91827.3645","options":{"notation":"compact","compactDisplay":"long","style":"unit","unit":"percent","signDisplay":"exceptZero","currencySign":"accounting"}, "test_type": "number_fmt"} +{"label":"0598","locale":"zh-TW","skeleton":"compact-short percent sign-accounting-except-zero","input":"9182734567890","options":{"notation":"compact","compactDisplay":"short","style":"unit","unit":"percent","signDisplay":"exceptZero","currencySign":"accounting"}, "test_type": "number_fmt"} +{"test_type": "number_fmt", "label":"2062","locale":"es-MX","skeleton":"currency/EUR unit-width-narrow .000","input":"91827.3645","options":{"style":"currency","currencyDisplay":"narrowSymbol","currency":"EUR","unitDisplay":"narrow","maximumFractionDigits":3,"minimumFractionDigits":3}} +{"test_type": "number_fmt", "label":"2062","locale":"es-MX","skeleton":"currency/EUR unit-width-narrow .000","input":"91827.3645","options":{"style":"currency","currencyDisplay":"narrowSymbol","currency":"EUR","unitDisplay":"ymbolnarrow","maximumFractionDigits":3,"minimumFractionDigits":3}} +{"test_type": "number_fmt", "label":"2062_long","locale":"es-MX","skeleton":"currency/EUR unit-width-long .000","input":"91827.3645","options":{"style":"currency","currencyDisplay":"narrowSymbol","currency":"EUR","unitDisplay":"long","maximumFractionDigits":3,"minimumFractionDigits":3}} +{"test_type": "number_fmt", "label": "0219", "locale": "zh-TW", "skeleton": "scientific/+ee/sign-always percent precision-integer", "input": "0", "options": {"notation": "scientific", "style": "unit", "unit": "percent", "maximumFractionDigits": 0, "minimumFractionDigits": 0, "roundingType": "fractionDigits"}} + +{"test_type": "number_fmt","label": "0054", "locale": "es-MX", "skeleton": "scientific/+ee/sign-always percent unit-width-narrow", "input": "0", "options": {"notation": "scientific", "conformanceExponent": "+ee", "conformanceSign": "always", "style": "unit", "unit": "percent", "unitDisplay": "narrow", "currencyDisplay": "narrowSymbol", "maximumFractionDigits": 6}} + +// Check significant digits +{"test_type": "number_fmt", "label": "2701", "locale": "es-MX", "skeleton": "percent .##/@@@+ integer-width/##00", "input": "91827.3645", "options": {"style": "unit", "unit": "percent", "maximumFractionDigits": 2, "maximumSignificantDigits": 3, "roundingPriority": "morePrecision", "maximumIntegerDigits": 4, "minimumIntegerDigits": 2}} +{"test_type": "number_fmt", "label": "2701", "locale": "es-MX", "skeleton": "percent .##/@@@+ integer-width/##00", "input": "91827.3645", "options": {"style": "unit", "unit": "percent", "maximumFractionDigits": 2, "maximumSignificantDigits": 3, "roundingPriority": "morePrecision"}} + +{"test_type": "number_fmt", "label": "0543", "locale": "bn-BD", "skeleton": "compact-short percent latin", "input": "0", "options": {"notation": "compact", "compactDisplay": "short", "style": "unit", "unit": "percent", "numberingSystem": "latn", "maximumFractionDigits": 2}} +{"test_type": "number_fmt", "label": "0543", "locale": "ar-EG", "skeleton": "compact-short percent latin", "input": "1234.5678", "options": {"notation": "compact", "compactDisplay": "short", "style": "unit", "unit": "percent", "numberingSystem": "arab", "maximumFractionDigits": 2}} + +{"test_type": "number_fmt", "label": "0648", "locale": "es-MX", "skeleton": "compact-short percent decimal-always", "input": "0", "options": {"notation": "compact", "compactDisplay": "short", "style": "unit", "unit": "percent", "conformanceDecimalAlways": true, "maximumFractionDigits": 2}} + +{"test_type": "number_fmt", "label": "0292_furlong", "locale": "en-AU", "skeleton": "scientific/+ee/sign-always measure-unit/length-furlong precision-integer", "input": "91827.3645", "options": {"notation": "scientific", "style": "unit", "unit": "furlong", "maximumFractionDigits": 0, "minimumFractionDigits": 0, "roundingType": "fractionDigits"}, "test_type": "number_fmt"} +{"test_type": "number_fmt", "label": "0292_furlong", "locale": "zh-TW", "skeleton": "scientific/+ee/sign-always measure-unit/length-furlong precision-integer", "input": "91827.3645", "options": {"notation": "scientific", "style": "unit", "unit": "furlong", "maximumFractionDigits": 0, "minimumFractionDigits": 0, "roundingType": "fractionDigits"}, "test_type": "number_fmt"} # FOR TESTING REJECTION OF UNKNOWN TYPE {"test_type": "bogus_fmt", "label":"NUM_2345", "input": "23.45","locale":"it","options":{"maximumFractionDigits":3}} # LOCALE_INFO {"test_type":"likely_subtags", "option":"maximize", "locale":"en", "label":"en_max"} {"test_type":"likely_subtags", "option":"minimize", "locale":"en-US", "label":"en_min"} + +{"label":"0598","locale":"zh-TW","skeleton":"compact-short percent sign-accounting-except-zero","input":"91827.3645","options":{"notation":"compact","compactDisplay":"short","style":"unit","unit":"percent","signDisplay":"exceptZero","currencySign":"accounting"}, "test_type": "number_fmt"} +{"label":"0598","locale":"zh-TW","skeleton":"compact-short percent sign-accounting-except-zero","input":"9182734567890","options":{"notation":"compact","compactDisplay":"short","style":"unit","unit":"percent","signDisplay":"exceptZero","currencySign":"accounting"}, "test_type": "number_fmt"} + +# For testing rejection of unknown type +{"test_type": "bogus_fmt", "label":"NUM_2345", "input": "23.45","locale":"it","options":{"maximumFractionDigits":3}} + +# Local_info {"test_type":"likely_subtags", "option":"maximize", "locale":"fr", "label":"1"} {"test_type":"likely_subtags", "option":"maximize", "locale":"fr-Latn-FR", "label":"1"} {"test_type":"likely_subtags", "option":"minimize", "locale":"fr-Latn-FR", "label":"1"} @@ -77,8 +169,8 @@ {"test_type":"likely_subtags", "label": "0007", "locale": "af", "option": "minimizeFavorScript"} {"test_type":"likely_subtags", "label": "bad option", "locale": "af", "option": "who_knows"} - {"test_type": "collation_short", "label": "00689", "s1": "101a", "compare_type": "<2", "test_description": " simple locale data test", "s2": "\u00c3\u00a4", "locale": "de"} + {"test_type": "collation_short", "label": "0000000", "s1": "\\u0009!", "s2": "\\u0009?", "ignorePunctuation": true} {"test_type": "collation_short", "label": "do_fail", "s1": "abc", "s2": "aba", "ignorePunctuation": true} {"test_type": "collation_short", "label": "dont_fail", "s1": "abc", "s2": "abe", "ignorePunctuation": true} @@ -86,5 +178,5 @@ {"test_type": "collation_short", "label": "0178423", "s1": "\udbff\udff8A", "s2": "\udbff\udff8b"}, {"test_type": "collation_short", "label": "0178424", "s1": "\udbff\udff8b", "s2": "\udbff\udff9!"} -// + {"test_type": "number_fmt", "label":"2062","locale":"es-MX","skeleton":"currency/EUR unit-width-narrow .000","input":"91827.3645","options":{"style":"currency","currencyDisplay":"narrowSymbol","currency":"EUR","unitDisplay":"narrow","maximumFractionDigits":3,"minimumFractionDigits":3}} diff --git a/genData100.sh b/genData100.sh index f1c2d1a9..b2c6c23d 100644 --- a/genData100.sh +++ b/genData100.sh @@ -12,7 +12,6 @@ logrotate -s logrotate.state logrotate.conf # Setup (generate) test data & expected values ########## -# Enable seting the version of NodeJS export NVM_DIR=$HOME/.nvm; source $NVM_DIR/nvm.sh; @@ -112,7 +111,6 @@ jq -c '.[]' ../$source_file | while read i; do test_type=$(jq -r -c '.run.test_type | join(" ")' <<< $i) per_execution=$(jq -r -c '.run.per_execution' <<< $i) - ignore=$(jq -r -c '.run.ignore' <<< $i) python3 testdriver.py --icu_version $icu_version --exec $exec_command --test_type $test_type --file_base ../$TEMP_DIR --per_execution $per_execution --ignore $ignore --run_limit $TEST_LIMIT echo $? diff --git a/schema/check_test_output.py b/schema/check_test_output.py index 8e856251..5d1014bb 100644 --- a/schema/check_test_output.py +++ b/schema/check_test_output.py @@ -92,15 +92,18 @@ def main(args): failed_validations.append(result) # Create .json - summary_json = { - 'validation_type': 'output of test executors', - 'description': 'Validation of test execution outputs vs. schema', - 'when_processed': datetime.now().strftime('%Y-%m-%d T%H%M%S.%f'), - 'validations': { - 'failed': failed_validations, - 'passed': passed_validations + try: + summary_json = { + 'validation_type': 'output of test executors', + 'description': 'Validation of test execution outputs vs. schema', + 'when_processed': datetime.now().strftime('%Y-%m-%d T%H%M%S.%f'), + 'validations': { + 'failed': failed_validations, + 'passed': passed_validations + } } - } + except BaseException as error: + summary_json = {} try: summary_data = json.dumps(summary_json) @@ -113,7 +116,7 @@ def main(args): file_out.write(summary_data) file_out.close() except BaseException as error: - logging.warning('Error: %s. Cannot save validation summary in file %s', err, output_filename) + logging.warning('Error: %s. Cannot save validation summary in file %s', error, output_filename) if schema_errors: diff --git a/schema/number_format/result_schema.json b/schema/number_format/result_schema.json index 5dcbde7f..9f75660d 100644 --- a/schema/number_format/result_schema.json +++ b/schema/number_format/result_schema.json @@ -14,7 +14,7 @@ "type": "array", "items": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "label": { "description": "A numeric ID, unique for the set of tests", @@ -144,6 +144,10 @@ "description": "Should we group the outputs?", "enum": [false, true, "always", "auto", "min2"], "default": "auto" + }, + "input_data": { + "type": "string", + "description": "Test data received" } }, "digits": { diff --git a/schema/number_format/test_schema.json b/schema/number_format/test_schema.json index 187f2bbe..d413a9fd 100644 --- a/schema/number_format/test_schema.json +++ b/schema/number_format/test_schema.json @@ -54,6 +54,10 @@ "description": "string representation of the number to be formatted", "type": "string" }, + "op": { + "description": "operation on the data, e.g., 'format", + "type": "string" + }, "options": { "description": "Formatting paramters for the number", "type": "object", diff --git a/testgen/testdata_gen.py b/testgen/testdata_gen.py index 32cbdfd7..5cc59aa9 100644 --- a/testgen/testdata_gen.py +++ b/testgen/testdata_gen.py @@ -159,6 +159,7 @@ def processNumberFmtTestData(self): def processLangNameTestData(self): json_test = {'test_type': 'lang_names'} json_verify = {'test_type': 'lang_names'} + languageNameDescr(json_test, json_verify) filename = 'languageNameTable.txt' rawlangnametestdata = readFile(filename, self.icu_version) @@ -364,7 +365,7 @@ def parseDcmlFmtTestData(rawtestdata): def mapFmtSkeletonToECMA402(options): ecma402_map = { "compact-short": {"notation": "compact", "compactDisplay": "short"}, - "scientific/+ee/sign-always": {"notation": "scientific"}, + "scientific/+ee/sign-always": {"notation": "scientific", "conformanceExponent":"+ee", "conformanceSign": "always"}, # Percent with word "percent": "percent": {"style": "unit", "unit": "percent"}, # "style": "percent", "currency/EUR": {"style": "currency", "currencyDisplay": "symbol", "currency": "EUR"}, @@ -377,7 +378,9 @@ def mapFmtSkeletonToECMA402(options): ".000": {"maximumFractionDigits": 3, "minimumFractionDigits": 3}, # Use maximumFractionDigits: 2, maximumSignificantDigits: 3, roundingPriority: "morePrecision" - ".##/@@@+": {"maximumFractionDigits": 2, "maximumSignificantDigits": 3,"roundingPriority": "morePrecision"}, + ".##/@@@+": {"maximumFractionDigits": 2, + "maximumSignificantDigits": 3, + "roundingPriority": "morePrecision"}, "@@": {"maximumSignificantDigits": 2, "minimumSignificantDigits": 2}, "rounding-mode-floor": {"roundingMode": "floor"}, "integer-width/##00": {"maximumIntegerDigits": 4, "minimumIntegerDigits":2}, @@ -550,6 +553,10 @@ def resolveOptions(raw_options, skeleton_list): ('style' not in resolved or resolved['style'] != 'currency')): resolved['maximumFractionDigits'] = 6 + if ('maximumFractionDigits' not in resolved and + ('notation' in resolved and resolved['notation'] == 'compact')): + resolved['maximumFractionDigits'] = 2 + if skeleton_list and 'percent' in skeleton_list: resolved['style'] = 'unit' resolved['unit'] = 'percent' @@ -581,7 +588,7 @@ def generateDcmlFmtTestDataObjects(rawtestdata, count=0): pattern, round_mode, test_input, expected = parseDcmlFmtTestData(item) rounding_mode = mapRoundingToECMA402(round_mode) label = str(count).rjust(max_digits, '0') - entry = {'label': label, 'skeleton': pattern , 'input': test_input, 'options': {} } + entry = {'label': label, 'op': 'format', 'skeleton': pattern , 'input': test_input, 'options': {} } json_part = mapFmtSkeletonToECMA402([pattern]) @@ -775,6 +782,7 @@ def generateCollTestData2(filename, test_case['compare_type'] = compare_type if test_description: test_case['test_description'] = test_description + if compare_comment: test_case['compare_comment'] = compare_comment if rules: @@ -784,6 +792,7 @@ def generateCollTestData2(filename, test_list.append(test_case) # We always expect True as the result + verify_list.append({ 'label': label, 'verify': True @@ -926,10 +935,6 @@ def languageNameDescr(tests_json, verify_json): 'source_version': version } } - - verify_json = {'test_type': test_id, - 'Test scenario': test_id - } return diff --git a/verifier/detail_template.html b/verifier/detail_template.html index 0bc325ac..1f5553e4 100644 --- a/verifier/detail_template.html +++ b/verifier/detail_template.html @@ -511,9 +511,9 @@ function clearSelectedItems(the_button, test_class) { // TODO: Clear all the check boxes for this test_class. const test_data = test_results[test_class]; - for (const box in test_data.check_boxes) { - box.checked = false; - } + test_data.check_boxes.forEach((box) => { + box.checked = false; + }); } @@ -579,7 +579,7 @@

Test details

Filtered count = 0 - +

@@ -605,7 +605,7 @@

Test Errors ($error_count)

Filtered count = 0 - +

@@ -631,7 +631,7 @@

Test Errors ($error_count)

Filtered count = 0 - +

diff --git a/verifier/testreport.py b/verifier/testreport.py index 5f4c5e48..211a850a 100644 --- a/verifier/testreport.py +++ b/verifier/testreport.py @@ -38,7 +38,7 @@ def dict_to_html(dict_data): def sort_dict_by_count(dict_data): return sorted(dict_data.items(), - key=lambda item: item[1], reverse=True) + key=lambda item: len(item[1]), reverse=True) class DiffSummary: @@ -77,8 +77,7 @@ def __init__(self, report_path, report_html_path): self.report = None self.simple_results = None self.failure_summaries = None - self.debug = 1 - + self.debug = 0 self.verifier_obj = None @@ -501,8 +500,9 @@ def flatten_and_combine(self, input_dict, input_simple): key_new = str(key) + '.' + str(key2) flat_items[key_new] = value2 + # Sort in reverse order by length of item flat_combined_dict = self.combine_same_sets_of_labels(flat_items) - return flat_combined_dict + return dict(sort_dict_by_count(flat_combined_dict)) def characterize_failures_by_options(self, tests, result_type): # Looking at options @@ -519,7 +519,10 @@ def characterize_failures_by_options(self, tests, result_type): 'compare_type', 'test_description', 'unsupported_options', 'rules', 'test_description', 'warning' # Number format + 'input_data', 'notation', 'compactDisplay', 'style', 'currency', 'unit', 'roundingMode', ] + option_keys = ['notation', 'compactDisplay', 'style', 'currency', 'unit', 'roundingMode'] + for key in key_list: try: locale = input_data.get('locale') @@ -586,6 +589,7 @@ def characterize_failures_by_options(self, tests, result_type): for key in ['language_label', 'ignorePunctuation', 'compare_result', 'compare_type', 'test_description']: if test.get(key): # For collation results value = test[key] + # TODO: Check for option_keys in input_data if key not in results: results[key] = {} if value in results[key]: @@ -600,6 +604,13 @@ def characterize_failures_by_options(self, tests, result_type): ] input_data = test.get('input_data') self.add_to_results_by_key(label, results, input_data, test, key_list) + + # Special case for input_data / options. + special_key = 'options' + if input_data.get(special_key): + options = input_data[special_key] + self.add_to_results_by_key(label, results, options, test, options.keys()) + error_detail = test.get('error_detail') if error_detail: error_keys = error_detail.keys() # ['options'] @@ -609,11 +620,9 @@ def characterize_failures_by_options(self, tests, result_type): # TODO: Add replacing (...) with "-" for numbers # TODO: Find the largest intersections of these sets and sort by size - # This is not used! - combo_list = [(combo, len(results[combo])) for combo in results] - combo_list.sort(key=take_second, reverse=True) + pass - return dict(results) + return results # TODO: Use the following function to update lists. def add_to_results_by_key(self, label, results, input_data, test, key_list): @@ -621,7 +630,7 @@ def add_to_results_by_key(self, label, results, input_data, test, key_list): for key in key_list: try: if (input_data.get(key)): # For collation results - value = test['input_data'][key] + value = input_data.get(key) if key == 'rules': value = 'RULE' # A special case to avoid over-characterization if key not in results: @@ -640,6 +649,7 @@ def check_simple_text_diffs(self): results['insert_digit'] = [] results['insert_space'] = [] results['delete_digit'] = [] + results['delete_space'] = [] results['replace_digit'] = [] results['exponent_diff'] = [] results['replace'] = [] @@ -681,12 +691,16 @@ def check_simple_text_diffs(self): elif kind == "delete": if old_val.isdigit(): results['delete_digit'].append(label) + elif old_val.isspace(): + results['delete_space'].append(label) else: results['delete'].append(label) elif kind == "insert": if new_val.isdigit(): results['insert_digit'].append(label) + elif old_val.isspace(): + results['insert_space'].append(label) else: results['insert'].append(label) else: @@ -733,12 +747,15 @@ def check_simple_text_diffs(self): return dict(results) def save_characterized_file(self, characterized_data, characterized_type): - json_data = json.dumps(characterized_data) - file_name = characterized_type + "_characterized.json" - character_file_path = os.path.join(self.report_directory, file_name) - file = open(character_file_path, mode='w', encoding='utf-8') - file.write(json_data) - file.close() + try: + json_data = json.dumps(characterized_data) + file_name = characterized_type + "_characterized.json" + character_file_path = os.path.join(self.report_directory, file_name) + file = open(character_file_path, mode='w', encoding='utf-8') + file.write(json_data) + file.close() + except BaseException as error: + logging.error("CANNOT WRITE CHARACTERIZE FILE FOR %s", characterized_type) return def create_html_diff_report(self): diff --git a/verifier/verifier.py b/verifier/verifier.py index 2d68f5e2..f4ff5cd5 100644 --- a/verifier/verifier.py +++ b/verifier/verifier.py @@ -228,7 +228,7 @@ def setup_verify_plans(self): if os.path.isfile(new_verify_plan.result_path): self.verify_plans.append(new_verify_plan) else: - logging.warning('** No results for %s, %s, %s', executor, test_type, result_version) + logging.debug('** No results for %s, %s, %s', executor, test_type, result_version) def verify_data_results(self): # For each pair of files in the test plan, compare with expected