From 369f1e3a6042e8e6146ce108ba12bd599b3f8100 Mon Sep 17 00:00:00 2001 From: Craig Cornelius Date: Mon, 19 Aug 2024 14:29:21 -0700 Subject: [PATCH] relative date time format: add numeric options and use in format testing (#280) * Update relative date time data for both auto and always options * Add numberic option auto to test data for RDFT * Update CPP RDTF to handle numeric options * Respond to suggestions * Picky change --- executors/cpp/relativedatetime_fmt.cpp | 151 ++++++------------ .../RelativeDateFormatNumeric.java | 16 ++ .../RelativeDateTimeFormatInputJson.java | 2 + .../RelativeDateTimeFormatTester.java | 18 ++- .../icu74/RelativeDateTimeFormatTest.java | 23 +++ testgen/generators/rdt_fmt_gen.js | 149 ++++++++++++----- 6 files changed, 217 insertions(+), 142 deletions(-) create mode 100644 executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/relativedatetimeformat/RelativeDateFormatNumeric.java diff --git a/executors/cpp/relativedatetime_fmt.cpp b/executors/cpp/relativedatetime_fmt.cpp index df425f57..7110c403 100644 --- a/executors/cpp/relativedatetime_fmt.cpp +++ b/executors/cpp/relativedatetime_fmt.cpp @@ -1,33 +1,37 @@ -/******************************************************************** +/* * testing icu4c relative datetime format */ +#include + #include -#include +#include +#include +#include +#include + #include #include -#include #include #include - -#include "unicode/utypes.h" -#include "unicode/unistr.h" -#include "unicode/locid.h" -#include "unicode/numfmt.h" -#include "unicode/reldatefmt.h" -#include "unicode/udisplaycontext.h" +#include using icu::Locale; using icu::NumberFormat; using icu::RelativeDateTimeFormatter; using icu::UnicodeString; -using std::cout; -using std::endl; using std::string; +/* + * Check for ICU errors and add to output if needed. + */ +extern const bool check_icu_error(UErrorCode error_code, + json_object *return_json, + string message_to_add_if_error); + UDateRelativeDateTimeFormatterStyle StringToStyleEnum(string style_string) { if (style_string == "long") return UDAT_STYLE_LONG; if (style_string == "short") return UDAT_STYLE_SHORT; @@ -35,25 +39,27 @@ UDateRelativeDateTimeFormatterStyle StringToStyleEnum(string style_string) { return UDAT_STYLE_LONG; // Default } -UDateRelativeUnit StringToRelativeUnitEnum(string unit_string) { - UDateRelativeUnit rel_unit; +URelativeDateTimeUnit StringToRelativeUnitEnum(string unit_string) { + URelativeDateTimeUnit rel_unit; if (unit_string == "day") { - return UDAT_RELATIVE_DAYS; + return UDAT_REL_UNIT_DAY; } else if (unit_string == "hour") { - return UDAT_RELATIVE_HOURS; + return UDAT_REL_UNIT_HOUR; } else if (unit_string == "minute") { - return UDAT_RELATIVE_MINUTES; + return UDAT_REL_UNIT_MINUTE; } else if (unit_string == "month") { - return UDAT_RELATIVE_MONTHS; + return UDAT_REL_UNIT_MONTH; } else if (unit_string == "second") { - return UDAT_RELATIVE_SECONDS; + return UDAT_REL_UNIT_SECOND; } else if (unit_string == "week") { - return UDAT_RELATIVE_WEEKS; + return UDAT_REL_UNIT_WEEK; + } else if (unit_string == "quarter") { + return UDAT_REL_UNIT_QUARTER; } else if (unit_string == "year") { - return UDAT_RELATIVE_YEARS; + return UDAT_REL_UNIT_YEAR; } // A default - return UDAT_RELATIVE_DAYS; + return UDAT_REL_UNIT_DAY; } const string TestRelativeDateTimeFmt(json_object *json_in) { @@ -89,53 +95,28 @@ const string TestRelativeDateTimeFmt(json_object *json_in) { string style_string = "long"; // Default string numbering_system_string = ""; // Default + string numeric_option = ""; if (options_obj) { json_object *style_obj = json_object_object_get(options_obj, "style"); if (style_obj) { style_string = json_object_get_string(style_obj); } - json_object *ns_obj = json_object_object_get(options_obj, "numberingSystem"); + json_object *ns_obj = json_object_object_get( + options_obj, "numberingSystem"); if (ns_obj) { numbering_system_string = json_object_get_string(ns_obj); } + json_object *numeric_obj = json_object_object_get(options_obj, "numeric"); + if (numeric_obj) { + numeric_option = json_object_get_string(numeric_obj); + } } UDateRelativeDateTimeFormatterStyle rdtf_style = StringToStyleEnum(style_string); - UDateRelativeUnit rel_unit; - if (unit_string != "quarter") { - rel_unit = StringToRelativeUnitEnum(unit_string); - } else { - // This is not supported. - json_object_object_add( - return_json, - "error", - json_object_new_string("unsupported unit")); - json_object_object_add( - return_json, - "error_type", - json_object_new_string("unsupported")); - json_object_object_add( - return_json, - "unsupported", - json_object_new_string("Bad relative date time unit")); - - json_object *unit_name_obj = - json_object_new_string(unit_string.c_str()); - - // Include details about the failure - json_object *detail_obj = json_object_new_object(); - json_object_object_add(detail_obj, "unsupported_unit", - unit_name_obj); - json_object_object_add( - return_json, - "error_detail", - detail_obj); - - // This can't be processed so return now. - return json_object_to_json_string(return_json); - } + URelativeDateTimeUnit rel_unit; + rel_unit = StringToRelativeUnitEnum(unit_string); // Add variants to the locale. string locale_selection_string = locale_string; @@ -150,58 +131,32 @@ const string TestRelativeDateTimeFmt(json_object *json_in) { RelativeDateTimeFormatter rdtf(display_locale, nf, rdtf_style, cap_context, status); - if (U_FAILURE(status)) { - json_object_object_add( - return_json, - "error", - json_object_new_string("Error creating RelativeDateTimeFormatter")); - json_object_object_add( - return_json, - "error_detail", - json_object_new_string("no detail available")); + if (check_icu_error(status, return_json, "Constructor")) { return json_object_to_json_string(return_json); } - UDateDirection direction = UDAT_DIRECTION_NEXT; - if (quantity < 0.0) { - direction = UDAT_DIRECTION_LAST; - } else if (quantity > 0.0) { - direction = UDAT_DIRECTION_NEXT; - } - // The output of the formatting UnicodeString formatted_result; - rdtf.format(fabs(quantity), direction, rel_unit, formatted_result, status); + if (numeric_option == "auto") { + rdtf.format(quantity, rel_unit, formatted_result, status); + } else { + // Default is "always" + rdtf.formatNumeric(quantity, rel_unit, formatted_result, status); + } - if (U_FAILURE(status)) { - json_object_object_add( - return_json, - "error", - json_object_new_string("Error calling rdtf.format")); + if (check_icu_error(status, return_json, "In format or formatNumeric")) { return json_object_to_json_string(return_json); } - // Get the resulting value as a string - char test_result_string[1000] = ""; - formatted_result.extract( - test_result_string, 1000, nullptr, status); // ignore return value - - if (U_FAILURE(status)) { - json_object_object_add( - return_json, - "error", - json_object_new_string("Failed extracting test result")); - json_object_object_add( - return_json, - "error_detail", - json_object_new_string("no detail available")); - } else { - // Good calls all around. Send the result! - json_object_object_add(return_json, - "result", - json_object_new_string(test_result_string)); - } + // Get the resulting value as a string for output + string test_result_string; + formatted_result.toUTF8String(test_result_string); + + // Good calls all around. Send the result! + json_object_object_add(return_json, + "result", + json_object_new_string(test_result_string.c_str())); return json_object_to_json_string(return_json); } diff --git a/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/relativedatetimeformat/RelativeDateFormatNumeric.java b/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/relativedatetimeformat/RelativeDateFormatNumeric.java new file mode 100644 index 00000000..79096f00 --- /dev/null +++ b/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/relativedatetimeformat/RelativeDateFormatNumeric.java @@ -0,0 +1,16 @@ +package org.unicode.conformance.testtype.relativedatetimeformat; + +public enum RelativeDateFormatNumeric { + ALWAYS, + AUTO; + + public static org.unicode.conformance.testtype.relativedatetimeformat.RelativeDateFormatNumeric DEFAULT = ALWAYS; + + public static org.unicode.conformance.testtype.relativedatetimeformat.RelativeDateFormatNumeric getFromString(String s) { + try { + return org.unicode.conformance.testtype.relativedatetimeformat.RelativeDateFormatNumeric.valueOf(s.toUpperCase()); + } catch (Exception e){ + return DEFAULT; + } + } +} diff --git a/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/relativedatetimeformat/RelativeDateTimeFormatInputJson.java b/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/relativedatetimeformat/RelativeDateTimeFormatInputJson.java index 5f9328bb..2c31bae0 100644 --- a/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/relativedatetimeformat/RelativeDateTimeFormatInputJson.java +++ b/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/relativedatetimeformat/RelativeDateTimeFormatInputJson.java @@ -12,6 +12,8 @@ public class RelativeDateTimeFormatInputJson implements ITestTypeInputJson { public String numberingSystem; + public RelativeDateFormatNumeric numeric;// always (default) or auto + public String count; public RelativeDateTimeFormatUnits unit; diff --git a/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/relativedatetimeformat/RelativeDateTimeFormatTester.java b/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/relativedatetimeformat/RelativeDateTimeFormatTester.java index 553ce97d..06273850 100644 --- a/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/relativedatetimeformat/RelativeDateTimeFormatTester.java +++ b/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/relativedatetimeformat/RelativeDateTimeFormatTester.java @@ -37,9 +37,13 @@ public ITestTypeInputJson inputMapToJson(Map inputMapData) { } } - result.style = RelativeDateTimeFormatStyle.getFromString( - "" + inputOptions.get("style") - ); + if (inputOptions != null) { + result.style = RelativeDateTimeFormatStyle.getFromString( + "" + inputOptions.get("style")); + + result.numeric = RelativeDateFormatNumeric.getFromString( + "" + inputOptions.get("numeric")); + } String unitInput = (String) inputMapData.get("unit", "0"); result.unit = RelativeDateTimeFormatUnits.getFromString( @@ -136,6 +140,12 @@ public String getRelativeDateTimeFormatResultString(RelativeDateTimeFormatInputJ RelativeDateTimeFormatter rdtf = RelativeDateTimeFormatter.getInstance(locale, nf, style, dc); - return rdtf.format(input.quantity, unit); + String result; + if (input.numeric == RelativeDateFormatNumeric.ALWAYS) { + result = rdtf.formatNumeric(input.quantity, unit); + } else { + result = rdtf.format(input.quantity, unit); + } + return result; } } diff --git a/executors/icu4j/74/executor-icu4j/src/test/java/org/unicode/conformance/relativedatetimeformat/icu74/RelativeDateTimeFormatTest.java b/executors/icu4j/74/executor-icu4j/src/test/java/org/unicode/conformance/relativedatetimeformat/icu74/RelativeDateTimeFormatTest.java index b03b4c93..a886a833 100644 --- a/executors/icu4j/74/executor-icu4j/src/test/java/org/unicode/conformance/relativedatetimeformat/icu74/RelativeDateTimeFormatTest.java +++ b/executors/icu4j/74/executor-icu4j/src/test/java/org/unicode/conformance/relativedatetimeformat/icu74/RelativeDateTimeFormatTest.java @@ -69,6 +69,29 @@ public void testEn100SAgo() { assertEquals("100s ago", output.result); } + @Test + public void testNumericAutoTomorrow() { + String testInput = + "\t{\"unit\":\"day\",\"count\":\"1\",\"locale\":\"en-US\",\"options\":{\"style\":\"narrow\", \"numeric\":\"auto\"},\"hexhash\":\"a57aac792\",\"label\":\"0\"}"; + + RelativeDateTimeFormatOutputJson output = + (RelativeDateTimeFormatOutputJson) RelativeDateTimeFormatTester.INSTANCE.getStructuredOutputFromInputStr( + testInput); + + assertEquals("tomorrow", output.result); + } + + @Test + public void testNumericAlwaysInOneDay() { + String testInput = + "\t{\"unit\":\"day\",\"count\":\"1\",\"locale\":\"en-US\",\"options\":{\"style\":\"long\", \"numeric\":\"always\"},\"hexhash\":\"a57aac792\",\"label\":\"0\"}"; + + RelativeDateTimeFormatOutputJson output = + (RelativeDateTimeFormatOutputJson) RelativeDateTimeFormatTester.INSTANCE.getStructuredOutputFromInputStr( + testInput); + + assertEquals("in 1 day", output.result); + } @Ignore // This doesn't yet handle non-ASCII numbering systems // https://github.com/unicode-org/conformance/issues/261 diff --git a/testgen/generators/rdt_fmt_gen.js b/testgen/generators/rdt_fmt_gen.js index 5dd9af28..3d319445 100644 --- a/testgen/generators/rdt_fmt_gen.js +++ b/testgen/generators/rdt_fmt_gen.js @@ -6,6 +6,7 @@ 4. numeric 5. unit, e.g., 'day' 6. value: in the range of min to max + 7. numeric: 'always' and 'auto' unless the resutls are equal */ // Set up Node version to generate data specific to ICU/CLDR version @@ -46,6 +47,64 @@ const numeric = ['auto', 'always']; const counts = [-100, -4, -2, -1, 0, 1, 1.3, 2, 3, 4, 10]; +function sample_tests(all_tests, run_limit) { + // Gets a sampling of the data based on total and the expected number. + + if (run_limit < 0 || all_tests.length <= run_limit) { + return all_tests; + } + + let size_all = all_tests.length; + let increment = Math.floor(size_all / run_limit); + let samples = []; + for (let index = 0; index < size_all; index += increment) { + samples.push(all_tests[index]); + } + return samples; +} + +// Create the test and verify JSON data for this case. +function save_test(unit, count, locale, all_options, result, label_num, + test_cases, verify_cases) { + const label_string = String(label_num); + // Without label + let test_case = { + 'unit': unit, + 'count': String(count), + }; + + if (locale != '') { + test_case["locale"] = locale; + } + + if (all_options != null) { + test_case["options"] = {...all_options}; + } + + if (debug) { + console.log("TEST CASE :", test_case); + } + + gen_hash.generate_hash_for_test(test_case); + test_case['label'] = label_string; + + test_cases.push(test_case); + + // Generate what we get. + try { + verify_cases.push({'label': label_string, + 'verify': result}); + if (debug) { + console.log(' expected = ', result); + } + } catch (error) { + console.log('!!! error ', error, ' in label ', label_num); + } + +} + + + function generateAll() { let test_obj = { @@ -68,6 +127,8 @@ function generateAll() { } let verify_cases = []; + // How many are different between numeric auto vs. always + let diff_count = 0; let label_num = 0; const expected_count = locales.length * @@ -95,12 +156,27 @@ function generateAll() { all_options['numberingSystem'] = number_system; } - let formatter; + // Create to sets of options + let all_options_numeric_always = {...all_options, 'numeric': 'always'}; + let all_options_numeric_auto = {...all_options, 'numeric': 'auto'}; + + let formatter_numeric_always; try { - formatter = new Intl.RelativeTimeFormat(locale, all_options); + formatter_numeric_always = + new Intl.RelativeTimeFormat(locale, all_options_numeric_always); } catch (error) { console.log(error, ' with locale ', - locale, ' and options: ', all_options); + locale, ' and options: ', all_options_numeric_always); + continue; + } + + let formatter_numeric_auto; + try { + formatter_numeric_auto = + new Intl.RelativeTimeFormat(locale, all_options_numeric_auto); + } catch (error) { + console.log(error, ' with locale ', + locale, ' and options: ', all_options_numeric_auto); continue; } @@ -111,49 +187,33 @@ function generateAll() { for (const unit of units) { for (const count of counts) { + let result_always; + let result_auto; + try { - result = formatter.format(count, unit); + result_always = formatter_numeric_always.format(count, unit); } catch (error) { console.log('FORMATTER CREATION FAILS! ', error); } - const label_string = String(label_num); - - // Without label - let test_case = { - 'unit': unit, - 'count': String(count), - }; - - if (locale != '') { - test_case["locale"] = locale; - } - - if (all_options != null) { - test_case["options"] = {...all_options}; - } - - if (debug) { - console.log("TEST CASE :", test_case); + try { + result_auto = formatter_numeric_auto.format(count, unit); + } catch (error) { + console.log('FORMATTER CREATION FAILS! ', error); } - gen_hash.generate_hash_for_test(test_case); - test_case['label'] = label_string; + // Always save the "auto" form. + save_test(unit, count, locale, all_options_numeric_auto, result_auto, label_num, test_cases, verify_cases); + label_num ++; - test_cases.push(test_case); + if (result_always != result_auto) { + // Since results are different, save the "always" form, too. + diff_count += 1; + // console.log(' DIFFERENT RESULTS: %s vs %s', result_always, result_auto); - // Generate what we get. - try { - verify_cases.push({'label': label_string, - 'verify': result}); - if (debug) { - console.log(' expected = ', result); - } - } catch (error) { - console.log('!!! error ', error, ' in label ', label_num, - ' for date = ', d); + save_test(unit, count, locale, all_options_numeric_always, result_always, label_num, test_cases, verify_cases); + label_num ++; } - label_num ++; } } } @@ -163,16 +223,17 @@ function generateAll() { console.log('Number of relative date/time tests generated for ', process.versions.icu, ': ', label_num); + console.log(' %d tests are different between numeric auto and always', diff_count); - test_obj['tests'] = test_cases; + test_obj['tests'] = sample_tests(test_cases, run_limit); try { - fs.writeFileSync('rdt_fmt_test.json', JSON.stringify(test_obj, null)); + fs.writeFileSync('rdt_fmt_test.json', JSON.stringify(test_obj, null, 2)); // file written successfully } catch (err) { console.error(err); } - verify_obj['verifications'] = verify_cases; + verify_obj['verifications'] = sample_tests(verify_cases, run_limit); try { fs.writeFileSync('rdt_fmt_verify.json', JSON.stringify(verify_obj, null, 2)); // file written successfully @@ -182,4 +243,12 @@ function generateAll() { } /* Call the generator */ -generateAll(); +let run_limit = -1; +if (process.argv.length >= 5) { + if (process.argv[3] == '-run_limit') { + run_limit = Number(process.argv[4]); + } +} + +/* Call the generator */ +generateAll(run_limit);