From 919ee65efe5eb55030b6480b5c28c989d96d2fb0 Mon Sep 17 00:00:00 2001 From: Craig Cornelius Date: Thu, 19 Sep 2024 15:12:37 -0700 Subject: [PATCH] Fix date time format in ICU4J, ICU4C, ICU4X (#297) * Basic lint applied * Remove unneeded include * Updating to use Z as timezone in generator and in Java * ICU4J date time format - add calendar, fixing many issues * Fix date/time generator to use instant. Remove 'und' locale * Fix ICU4C DateTime parsing of 'Z', resolving many test failures * Fix Rust date/time to accept new UTC instant data. Still not handling timezones correctly * formatted .rs file * Removing debug printing from datatime_fmt.cpp * Add tz_offset_secs to date/time test data so ICU4X can use timezone * Trying to fix timezone setting in ICU4X datetime * Fixing about 1/2 of the ICU4X test failures * Update based on feedback * Moving code lines as suggested --- executors/cpp/datetime_fmt.cpp | 77 ++++++----- .../DateTimeFormatterDateStyle.java | 5 +- .../DateTimeFormatterInputJson.java | 17 ++- .../DateTimeFormatterTester.java | 75 +++++----- .../DateTimeFormatterTimeStyle.java | 5 +- .../DateTimeFormatterTest.java | 95 +++++++++++-- executors/rust/1.3/Cargo.toml | 2 +- executors/rust/1.3/src/datetimefmt.rs | 129 +++++++++++------- executors/rust/1.4/Cargo.toml | 2 +- schema/datetime_fmt/test_schema.json | 4 + testgen/generators/datetime_gen.js | 82 +++++------ verifier/testreport.py | 4 + 12 files changed, 305 insertions(+), 192 deletions(-) diff --git a/executors/cpp/datetime_fmt.cpp b/executors/cpp/datetime_fmt.cpp index 4cab9ab9..acd72e84 100644 --- a/executors/cpp/datetime_fmt.cpp +++ b/executors/cpp/datetime_fmt.cpp @@ -48,7 +48,9 @@ const string TestDatetimeFmt(json_object *json_in) { string label_string = json_object_get_string(label_obj); Calendar *cal = nullptr; - TimeZone *tz = nullptr; + + UnicodeString u_tz_utc("UTC"); + TimeZone *tz = nullptr; // TimeZone::createTimeZone(u_tz_utc); // The locale for formatted output json_object *locale_label_obj = json_object_object_get(json_in, "locale"); @@ -65,46 +67,55 @@ const string TestDatetimeFmt(json_object *json_in) { json_object *return_json = json_object_new_object(); json_object_object_add(return_json, "label", label_obj); - string calendar_str; + string calendar_str = "gregory"; // Get fields out of the options if present json_object* options_obj = json_object_object_get(json_in, "options"); if (options_obj) { - json_object* cal_item = json_object_object_get(options_obj, "calendar"); + // Check for timezone and calendar + json_object* option_item = + json_object_object_get(options_obj, "timeZone"); + if (option_item) { + string timezone_str = json_object_get_string(option_item); + UnicodeString u_tz(timezone_str.c_str()); + tz = TimeZone::createTimeZone(u_tz); + } + + json_object* cal_item = + json_object_object_get(options_obj, "calendar"); if (cal_item) { calendar_str = json_object_get_string(cal_item); - - // Add '@calendar=' + calendar_string to locale - locale_string = locale_string + "@calendar=" + calendar_str; - display_locale = locale_string.c_str(); - - if (tz) { - cal = Calendar::createInstance(*tz, display_locale, status); - } else { - cal = Calendar::createInstance(display_locale, status); - } - if (U_FAILURE(status)) { - json_object_object_add( - return_json, - "error", - json_object_new_string("Error in createInstance for calendar")); - return json_object_to_json_string(return_json); - } } } + // Add '@calendar=' + calendar_string to locale + locale_string = locale_string + "@calendar=" + calendar_str; + display_locale = locale_string.c_str(); + + if (tz) { + cal = Calendar::createInstance(tz, display_locale, status); + } else { + cal = Calendar::createInstance(display_locale, status); + } + if (U_FAILURE(status)) { + json_object_object_add( + return_json, + "error", + json_object_new_string("Error in createInstance for calendar")); + return json_object_to_json_string(return_json); + } + DateFormat* df; // Get the input data as a date object. // Types of input: - // "input_string" parsable ISO formatted string such as - // "2020-03-02 10:15:17 -08:00" + // "input_string" parsable ISO formatted string of an instant + // "2020-03-02 10:15:17Z string dateStyle_str; string timeStyle_str; - string timezone_str; // Expected values if neither dateStyle nor timeStyle is given explicitly. icu::DateFormat::EStyle date_style = icu::DateFormat::EStyle::kNone; @@ -126,17 +137,6 @@ const string TestDatetimeFmt(json_object *json_in) { timeStyle_str = json_object_get_string(option_item); time_style = StringToEStyle(timeStyle_str); } - - option_item = json_object_object_get(options_obj, "timeZone"); - if (option_item) { - timezone_str = json_object_get_string(option_item); - UnicodeString u_tz(timezone_str.c_str()); - tz = TimeZone::createTimeZone(u_tz); - } else { - // Default is UTC - UnicodeString u_tz("UTC"); - tz = TimeZone::createTimeZone(u_tz); - } } json_object *date_skeleton_obj = @@ -183,10 +183,9 @@ const string TestDatetimeFmt(json_object *json_in) { } // !!! IS OFFSET ALREADY CONSIDERED? - // if (tz) { - // df->setTimeZone(*tz); - // } - + if (tz) { + df->setTimeZone(*tz); + } // Use ISO string form of the date/time. json_object *input_string_obj = @@ -219,7 +218,7 @@ const string TestDatetimeFmt(json_object *json_in) { UnicodeString date_ustring(input_date_string.c_str()); // TODO: handles the offset +/- - SimpleDateFormat iso_date_fmt(u"y-M-d'T'h:m:s", und_locale, status); + SimpleDateFormat iso_date_fmt(u"y-M-d'T'h:m:sZ", und_locale, status); if (U_FAILURE(status)) { string error_name = u_errorName(status); string error_message = diff --git a/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterDateStyle.java b/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterDateStyle.java index 0a78718a..bbdea657 100644 --- a/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterDateStyle.java +++ b/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterDateStyle.java @@ -4,9 +4,10 @@ public enum DateTimeFormatterDateStyle { FULL, LONG, MEDIUM, - SHORT; + SHORT, + UNDEFINED; - public static org.unicode.conformance.testtype.datetimeformatter.DateTimeFormatterDateStyle DEFAULT = MEDIUM; + public static org.unicode.conformance.testtype.datetimeformatter.DateTimeFormatterDateStyle DEFAULT = UNDEFINED; public static org.unicode.conformance.testtype.datetimeformatter.DateTimeFormatterDateStyle getFromString( String s) { diff --git a/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterInputJson.java b/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterInputJson.java index 46f9e408..4060b732 100644 --- a/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterInputJson.java +++ b/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterInputJson.java @@ -1,7 +1,12 @@ package org.unicode.conformance.testtype.datetimeformatter; +import java.time.Instant; + import java.util.Date; import com.ibm.icu.util.Calendar; +import com.ibm.icu.util.TimeZone; + +import java.util.Locale; import org.unicode.conformance.testtype.ITestTypeInputJson; @@ -11,11 +16,13 @@ public class DateTimeFormatterInputJson implements ITestTypeInputJson { public String label; - public String locale; + public String locale_string; + public Locale locale_with_calendar; - // UTC formatted time + // UTC formatted instant in time public String inputString; + public Instant time_instant; public Date myDate; public String skeleton; @@ -25,13 +32,13 @@ public class DateTimeFormatterInputJson implements ITestTypeInputJson { public DateTimeFormatterTimeStyle timeStyle; // TODO!!! - public String calendarString; - // Set calendar from calendarString! + public String calendar_string; + // Set calendar from calendar_string! public Calendar calendar; public String numberingSystem; - public String timeZone; + public TimeZone timeZone; public String timeZoneName; } diff --git a/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterTester.java b/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterTester.java index 953403bb..0f2873b5 100644 --- a/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterTester.java +++ b/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterTester.java @@ -1,26 +1,26 @@ package org.unicode.conformance.testtype.datetimeformatter; -import java.text.ParseException; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; - import java.util.Date; +import java.time.Instant; + +import java.util.Locale; +import java.util.Locale.Builder; import com.ibm.icu.util.Calendar; import com.ibm.icu.text.DateFormat; +import com.ibm.icu.util.TimeZone; import com.ibm.icu.util.ULocale; import io.lacuna.bifurcan.IMap; import io.lacuna.bifurcan.Map; - import org.unicode.conformance.ExecutorUtils; import org.unicode.conformance.testtype.ITestType; import org.unicode.conformance.testtype.ITestTypeInputJson; import org.unicode.conformance.testtype.ITestTypeOutputJson; public class DateTimeFormatterTester implements ITestType { + private static final int UNDEFINED_DATETIME_STYLE = -1; public static DateTimeFormatterTester INSTANCE = new DateTimeFormatterTester(); @@ -29,32 +29,27 @@ public ITestTypeInputJson inputMapToJson(Map inputMapData) { DateTimeFormatterInputJson result = new DateTimeFormatterInputJson(); result.label = (String) inputMapData.get("label", null); - result.locale = (String) inputMapData.get("locale", null); + result.locale_string = (String) inputMapData.get("locale", null); result.skeleton = (String) inputMapData.get("skeleton", null); + // The instant in UTC time. result.inputString = (String) inputMapData.get("input_string", null); java.util.Map inputOptions = (java.util.Map) inputMapData.get("options", null); - result.timeZone = (String) inputOptions.get("timeZone"); - ZoneId thisZoneId; - if (result.timeZone == null) { - thisZoneId = ZoneId.systemDefault(); + result.timeZoneName = (String) inputOptions.get("timeZone"); + if (result.timeZoneName == null) { + result.timeZone = TimeZone.GMT_ZONE; } else { - thisZoneId = ZoneId.of(result.timeZone); + result.timeZone = TimeZone.getTimeZone(result.timeZoneName); } // Extract ISO part of the input string to parse. - String inputStringDateTime = result.inputString.substring(0, 25); + result.time_instant = Instant.parse(result.inputString); + result.myDate = Date.from(result.time_instant); // For parsing the input string and converting to java.util.date - LocalDateTime parsedLocalDateTime = - LocalDateTime.parse(inputStringDateTime, DateTimeFormatter.ISO_OFFSET_DATE_TIME); - result.myDate = - java.util.Date.from(parsedLocalDateTime.atZone(thisZoneId) - .toInstant()); - result.dateStyle = DateTimeFormatterDateStyle.getFromString( "" + inputOptions.get("dateStyle") ); @@ -63,15 +58,15 @@ public ITestTypeInputJson inputMapToJson(Map inputMapData) { "" + inputOptions.get("timeStyle") ); - result.calendarString = (String) inputOptions.get("calendar"); + result.calendar_string = (String) inputOptions.get("calendar"); - // TODO!!! Get calendar object. Depends on timezone and locale. - // Just a placeholder for now. - result.calendar = Calendar.getInstance(); + result.locale_with_calendar = new Builder().setLanguageTag(result.locale_string) + .setUnicodeLocaleKeyword("ca", result.calendar_string) + .build(); - result.numberingSystem = (String) inputOptions.get("numberingSystem"); + result.calendar = Calendar.getInstance(result.locale_with_calendar); - result.timeZoneName = (String) inputOptions.get("timeZoneName"); + result.numberingSystem = (String) inputOptions.get("numberingSystem"); return result; } @@ -114,9 +109,9 @@ public String formatOutputJson(ITestTypeOutputJson outputJson) { public String getDateTimeFormatterResultString(DateTimeFormatterInputJson input) { - ULocale locale = ULocale.forLanguageTag(input.locale); + ULocale locale = ULocale.forLanguageTag(input.locale_string); - int dateStyle; + int dateStyle = UNDEFINED_DATETIME_STYLE; switch (input.dateStyle) { case FULL: dateStyle = DateFormat.FULL; @@ -124,16 +119,18 @@ public String getDateTimeFormatterResultString(DateTimeFormatterInputJson input) case LONG: dateStyle = DateFormat.LONG; break; - default: case MEDIUM: dateStyle = DateFormat.MEDIUM; break; case SHORT: dateStyle = DateFormat.SHORT; break; + default: + dateStyle = UNDEFINED_DATETIME_STYLE; // Undefined + break; } - int timeStyle; + int timeStyle = UNDEFINED_DATETIME_STYLE; switch (input.timeStyle) { case FULL: timeStyle = DateFormat.FULL; @@ -141,13 +138,15 @@ public String getDateTimeFormatterResultString(DateTimeFormatterInputJson input) case LONG: timeStyle = DateFormat.LONG; break; - default: case MEDIUM: timeStyle = DateFormat.MEDIUM; break; case SHORT: timeStyle = DateFormat.SHORT; break; + default: + timeStyle = UNDEFINED_DATETIME_STYLE; // Undefined + break; } @@ -156,10 +155,22 @@ public String getDateTimeFormatterResultString(DateTimeFormatterInputJson input) DateFormat dtf; if (input.skeleton != null) { - dtf = DateFormat.getInstanceForSkeleton(cal, input.skeleton, locale); + dtf = DateFormat.getInstanceForSkeleton(cal, input.skeleton, input.locale_with_calendar); } else { - dtf = DateFormat.getDateTimeInstance(cal, dateStyle, timeStyle, locale); + if (dateStyle != UNDEFINED_DATETIME_STYLE && timeStyle != UNDEFINED_DATETIME_STYLE) { + dtf = DateFormat.getDateTimeInstance(cal, dateStyle, timeStyle, input.locale_with_calendar); + } else + if (dateStyle != UNDEFINED_DATETIME_STYLE) { + dtf = DateFormat.getDateInstance(cal, dateStyle,input.locale_with_calendar); + } else + if (timeStyle != UNDEFINED_DATETIME_STYLE) { + dtf = DateFormat.getTimeInstance(cal, timeStyle, input.locale_with_calendar); + } else { + dtf = DateFormat.getInstance(cal, input.locale_with_calendar); + } } + dtf.setCalendar(input.calendar); + dtf.setTimeZone(input.timeZone); return dtf.format(input.myDate); } diff --git a/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterTimeStyle.java b/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterTimeStyle.java index 50842fe9..fda5aa49 100644 --- a/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterTimeStyle.java +++ b/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterTimeStyle.java @@ -4,9 +4,10 @@ public enum DateTimeFormatterTimeStyle { FULL, LONG, MEDIUM, - SHORT; + SHORT, + UNDEFINED; - public static org.unicode.conformance.testtype.datetimeformatter.DateTimeFormatterTimeStyle DEFAULT = MEDIUM; + public static org.unicode.conformance.testtype.datetimeformatter.DateTimeFormatterTimeStyle DEFAULT = UNDEFINED; public static org.unicode.conformance.testtype.datetimeformatter.DateTimeFormatterTimeStyle getFromString( String s) { diff --git a/executors/icu4j/74/executor-icu4j/src/test/java/org/unicode/conformance/datetimeformatter/DateTimeFormatterTest.java b/executors/icu4j/74/executor-icu4j/src/test/java/org/unicode/conformance/datetimeformatter/DateTimeFormatterTest.java index 07f12318..647057c0 100644 --- a/executors/icu4j/74/executor-icu4j/src/test/java/org/unicode/conformance/datetimeformatter/DateTimeFormatterTest.java +++ b/executors/icu4j/74/executor-icu4j/src/test/java/org/unicode/conformance/datetimeformatter/DateTimeFormatterTest.java @@ -12,7 +12,11 @@ public class DateTimeFormatterTest { @Test public void TestDateTime49() { String testInput = - "{\"test_type\": \"datetime_fmt\", \"input_string\":\"2024-03-07T00:00:01-08:00[America/Los_Angeles][u-ca=gregory]\",\"skeleton\":\"j\",\"locale\":\"en-US\",\"options\":{\"hour\":\"numeric\",\"calendar\":\"gregory\",\"timeZone\":\"America/Los_Angeles\",\"numberingSystem\":\"latn\"},\"hexhash\":\"30c5191c8041eb6d8afa05aab80f811753bc082f\",\"label\":\"49\"}"; + "{\"test_type\": \"datetime_fmt\", \"input_string\":\"2024-03-07T00:00:01.00Z\"," + + + "\"skeleton\":\"j\",\"locale\":\"en-US\",\"options\":{\"hour\":\"numeric\",\"calendar\":\"gregory\"," + + + "\"timeZone\":\"America/Los_Angeles\",\"numberingSystem\":\"latn\"},\"hexhash\":\"30c5191c8041eb6d8afa05aab80f811753bc082f\",\"label\":\"49\"}"; DateTimeFormatterOutputJson output = (DateTimeFormatterOutputJson) DateTimeFormatterTester.INSTANCE.getStructuredOutputFromInputStr( @@ -23,7 +27,12 @@ public void TestDateTime49() { @Test public void TestDateTime15455() { String testInput = - "{\"test_type\": \"datetime_fmt\", \"input_string\":\"2001-09-09T01:46:40-07:00[America/Los_Angeles]\",\"skeleton\":\"vvvv\",\"locale\":\"zu\",\"options\":{\"timeZoneName\":\"longGeneric\",\"calendar\":\"persian\",\"timeZone\":\"America/Los_Angeles\",\"numberingSystem\":\"latn\"},\"hexhash\":\"d4cfde2db66f8d4aec9254fc66ef2db298d7a0ba\",\"label\":\"15455\"}"; + "{\"test_type\": \"datetime_fmt\", \"input_string\":\"2001-09-09T01:46:40.00Z\"," + + "\"skeleton\":\"vvvv\",\"locale\":\"zu\",\"options\":{\"timeZoneName\":\"longGeneric\"," + + + "\"calendar\":\"persian\",\"timeZone\":\"America/Los_Angeles\",\"numberingSystem\":\"latn\"}," + + + "\"hexhash\":\"d4cfde2db66f8d4aec9254fc66ef2db298d7a0ba\",\"label\":\"15455\"}"; DateTimeFormatterOutputJson output = (DateTimeFormatterOutputJson) DateTimeFormatterTester.INSTANCE.getStructuredOutputFromInputStr( @@ -31,11 +40,14 @@ public void TestDateTime15455() { assertEquals("Isikhathi sase-North American Pacific", output.result); } - @Ignore @Test - public void testDateTime0() { + public void testDateTime0x() { String testInput = - "{\"test_type\":\"datetime_fmt\", \"input_string\":\"2024-03-07T00:00:01+00:00[UTC][u-ca=gregory]\",\"locale\":\"en-US\",\"options\":{\"dateStyle\":\"short\",\"timeStyle\":\"short\",\"calendar\":\"gregory\",\"timeZone\":\"UTC\",\"numberingSystem\":\"latn\"},\"hexhash\":\"048d17f248ef4f6835d6b9b3bcbfdc934f3fcad5\",\"label\":\"0\"}"; + "{\"test_type\":\"datetime_fmt\", \"input_string\":\"2024-03-07T00:00:01.00Z\"," + + + "\"locale\":\"en-US\",\"options\":{\"dateStyle\":\"short\",\"timeStyle\":\"short\"," + + "\"calendar\":\"gregory\",\"timeZone\":\"UTC\",\"numberingSystem\":\"latn\"}," + + "\"hexhash\":\"048d17f248ef4f6835d6b9b3bcbfdc934f3fcad5\",\"label\":\"0\"}"; DateTimeFormatterOutputJson output = (DateTimeFormatterOutputJson) DateTimeFormatterTester.INSTANCE.getStructuredOutputFromInputStr( @@ -44,11 +56,14 @@ public void testDateTime0() { assertEquals("3/7/24, 12:00 AM", output.result); } - @Ignore @Test public void testDateTime3864() { String testInput = - "{\"test_type\":\"datetime_fmt\",\"input_string\":\"2024-03-07T00:00:01+00:00[UTC][u-ca=gregory]\",\"locale\":\"zh-TW\",\"options\":{\"dateStyle\":\"short\",\"timeStyle\":\"short\",\"calendar\":\"gregory\",\"timeZone\":\"UTC\",\"numberingSystem\":\"latn\"},\"hexhash\":\"2f22cb2c0656fd092d12c17b2545cec4ca9f23b3\",\"label\":\"3864\"}"; + "{\"test_type\":\"datetime_fmt\",\"input_string\":\"2024-03-07T00:00:12.00Z\"," + + + "\"locale\":\"zh-TW\",\"options\":{\"dateStyle\":\"short\",\"timeStyle\":\"short\"," + + "\"calendar\":\"gregory\",\"timeZone\":\"UTC\",\"numberingSystem\":\"latn\"}," + + "\"hexhash\":\"2f22cb2c0656fd092d12c17b2545cec4ca9f23b3\",\"label\":\"3864\"}"; DateTimeFormatterOutputJson output = (DateTimeFormatterOutputJson) DateTimeFormatterTester.INSTANCE.getStructuredOutputFromInputStr( @@ -57,11 +72,43 @@ public void testDateTime3864() { assertEquals("2024/3/7 凌晨12:00", output.result); } - @Ignore @Test public void testDateTime17387() { + String testInput = + "\t{\"test_type\":\"datetime_fmt\",\"input_string\":\"2001-09-09T01:46:40.01Z\"," + + "\"skeleton\":\"vvvv\",\"locale\":\"en\",\"options\":{\"timeZoneName\":\"longGeneric\"," + + + "\"calendar\":\"persian\",\"timeZone\":\"Australia/Brisbane\",\"numberingSystem\":\"latn\"}," + + + "\"hexhash\":\"8a39dcac98f0487ead82dedd3447bdf393b35081\",\"label\":\"17387\"}"; + + DateTimeFormatterOutputJson output = + (DateTimeFormatterOutputJson) DateTimeFormatterTester.INSTANCE.getStructuredOutputFromInputStr( + testInput); + + assertEquals("Australian Eastern Standard Time", output.result); + } + + @Test + public void testDateTime5126() { + String testInput = + "\t{\"test_type\": \"datetime_fmt\", \"input_string\":\"1984-05-29T07:53:00.01Z\"," + + "\"skeleton\": \"OOOO\",\"locale\":\"zh-TW\",\"options\":{\"timeZoneName\":\"longOffset\"," + + "\"calendar\":\"japanese\",\"timeZone\":\"Europe/Kiev\",\"numberingSystem\":\"latn\"}," + + "\"hexhash\":\"f44d2a93e473d0ead6b008a33aad3d2a93a2aa3c\",\"label\":\"5126\"}"; - String testInput = "\t{\"test_type\":\"datetime_fmt\",\"input_string\":\"2001-09-09T01:46:40+10:00[Australia/Brisbane]\",\"skeleton\":\"vvvv\",\"locale\":\"und\",\"options\":{\"timeZoneName\":\"longGeneric\",\"calendar\":\"persian\",\"timeZone\":\"Australia/Brisbane\",\"numberingSystem\":\"latn\"},\"hexhash\":\"8a39dcac98f0487ead82dedd3447bdf393b35081\",\"label\":\"17387\"}"; + DateTimeFormatterOutputJson output = + (DateTimeFormatterOutputJson) DateTimeFormatterTester.INSTANCE.getStructuredOutputFromInputStr( + testInput); + + assertEquals("GMT+04:00", output.result); + } + + @Test + public void testDateTime0() { + String testInput = "\t{\"test_type\": \"datetime_fmt\", \"input_string\":\"2024-03-07T00:00:01.00Z\"," + + "\"locale\":\"en-US\",\"options\":{\"dateStyle\":\"short\",\"timeStyle\":\"short\",\"calendar\":\"gregory\"," + + "\"timeZone\":\"UTC\",\"numberingSystem\":\"latn\"},\"hexhash\":\"048d17f248ef4f6835d6b9b3bcbfdc934f3fcad5\",\"label\":\"0\"}"; DateTimeFormatterOutputJson output = (DateTimeFormatterOutputJson) DateTimeFormatterTester.INSTANCE.getStructuredOutputFromInputStr( @@ -70,4 +117,34 @@ public void testDateTime17387() { assertEquals("3/7/24, 12:00 AM", output.result); } + @Test + public void testDate35() { + String testInput = + "\t{\"test_type\": \"datetime_fmt\"," + + "\"input_string\":\"2024-03-07T00:00:01.00Z\"," + + "\"locale\":\"en-US\"," + + "\"options\":{\"dateStyle\":\"long\",\"calendar\":\"gregory\",\"timeZone\":\"Asia/Tehran\",\"numberingSystem\":\"latn\"}," + + "\"hexhash\":\"27b3476384c651ce0812edd378f127c1b3eb1dae\",\"label\":\"35\"}"; + + DateTimeFormatterOutputJson output = + (DateTimeFormatterOutputJson) DateTimeFormatterTester.INSTANCE.getStructuredOutputFromInputStr( + testInput); + + assertEquals("3/7/24", output.result); + } + + @Test + public void testDate217() { + String testInput = + "\t{\"test_type\": \"datetime_fmt\", \"input_string\":\"2024-03-07T00:00:01.00Z\"," + + "\"locale\":\"en-US\"," + + "\"options\":{\"dateStyle\":\"short\",\"timeStyle\":\"short\",\"calendar\":\"buddhist\",\"timeZone\":\"America/Los_Angeles\",\"numberingSystem\":\"latn\"}," + + "\"hexhash\":\"d099e6a6a45df0ce89f1e60d9d5caa85fc64da28\",\"label\":\"217\"}"; + + DateTimeFormatterOutputJson output = + (DateTimeFormatterOutputJson) DateTimeFormatterTester.INSTANCE.getStructuredOutputFromInputStr( + testInput); + + assertEquals("3/6/2567 BE, 4:00 PM", output.result); + } } \ No newline at end of file diff --git a/executors/rust/1.3/Cargo.toml b/executors/rust/1.3/Cargo.toml index f97e4d87..859f5282 100644 --- a/executors/rust/1.3/Cargo.toml +++ b/executors/rust/1.3/Cargo.toml @@ -15,7 +15,7 @@ serde = "1.0.171" serde_json = "1.0.100" rustc_version_runtime = "0.1.*" -icu = { version = "~1.3", features = ["serde", "icu_compactdecimal", "icu_displaynames", "compiled_data", "icu_relativetime"] } +icu = { version = "~1.3", features = ["serde", "icu_compactdecimal", "icu_displaynames", "compiled_data", "icu_relativetime", "icu_datetime_experimental"] } fixed_decimal = "=0.5.5" writeable = "0.5.2" diff --git a/executors/rust/1.3/src/datetimefmt.rs b/executors/rust/1.3/src/datetimefmt.rs index 7f216776..7e195772 100644 --- a/executors/rust/1.3/src/datetimefmt.rs +++ b/executors/rust/1.3/src/datetimefmt.rs @@ -3,7 +3,11 @@ // https://docs.rs/ixdtf/latest/ixdtf/ use icu::calendar::DateTime; -use icu::datetime::{options::length, ZonedDateTimeFormatter}; +use icu::datetime::{ + options::components, options::length, options::DateTimeFormatterOptions, pattern::reference, + pattern::runtime, ZonedDateTimeFormatter, +}; + use icu::locid::Locale; // https://docs.rs/icu/latest/icu/timezone/struct.CustomTimeZone.html#method.maybe_calculate_metazone @@ -23,12 +27,22 @@ use serde_json::{json, Value}; #[derive(Deserialize, Serialize, Debug)] #[serde(rename_all = "camelCase")] struct DateTimeFormatOptions { + calendar: Option, date_style: Option, + numbering_system: Option, time_style: Option, time_zone: Option, + era: Option, - calendar: Option, - numbering_system: Option, + year: Option, + month: Option, + week: Option, + day: Option, + weekday: Option, + hour: Option, + minute: Option, + second: Option, + fractional_second: Option, } pub fn run_datetimeformat_test(json_obj: &Value) -> Result { @@ -64,70 +78,87 @@ pub fn run_datetimeformat_test(json_obj: &Value) -> Result { let mut _unsupported_options: Vec<&str> = Vec::new(); - // TODO: Get and use timeZone - let option_struct: DateTimeFormatOptions = serde_json::from_str(&options.to_string()).unwrap(); let date_style_str = &option_struct.date_style; - let date_style = if date_style_str == &Some("full".to_string()) { - length::Date::Full - } else if date_style_str == &Some("long".to_string()) { - length::Date::Long - } else if date_style_str == &Some("short".to_string()) { - length::Date::Short - } else if date_style_str == &Some("medium".to_string()) { - length::Date::Medium - } else { - length::Date::Full + let date_style = match date_style_str.as_deref() { + Some("full") => Some(length::Date::Full), + Some("long") => Some(length::Date::Long), + Some("medium") => Some(length::Date::Medium), + Some("short") => Some(length::Date::Short), + _ => None, }; // TimeStyle long or full requires that you use ZonedDateTimeFormatter. // long known issue that is documented and has been filed several times. // Is fixed in 2.0 let time_style_str = &option_struct.time_style; - let time_style = if time_style_str == &Some("full".to_string()) { - length::Time::Full - } else if time_style_str == &Some("long".to_string()) { - length::Time::Long - } else if time_style_str == &Some("short".to_string()) { - length::Time::Short - } else if time_style_str == &Some("medium".to_string()) { - length::Time::Medium - } else { - // !!! SET TO UNDEFINED - length::Time::Full + let time_style = match time_style_str.as_deref() { + Some("full") => Some(length::Time::Full), + Some("long") => Some(length::Time::Long), + Some("medium") => Some(length::Time::Medium), + Some("short") => Some(length::Time::Short), + _ => None, }; // Set up DT option if either is set - let dt_options = if date_style_str.is_some() && time_style_str.is_some() { - length::Bag::from_date_time_style(date_style, time_style) - } else if date_style_str.is_none() && time_style_str.is_some() { - length::Bag::from_time_style(time_style) - } else if date_style_str.is_some() && time_style_str.is_none() { - length::Bag::from_date_style(date_style) + let mut dt_length_options = length::Bag::empty(); + dt_length_options.date = date_style; + dt_length_options.time = time_style; + + let dt_options = if dt_length_options != length::Bag::empty() { + DateTimeFormatterOptions::Length(dt_length_options) } else { - length::Bag::default() + // For versions 1.X, but not in 2.X. + // This is using an interal feature. + let skeleton_str = &json_obj["skeleton"].as_str().unwrap(); + let parsed_skeleton = skeleton_str.parse::().unwrap(); + let mut components_bag = components::Bag::from(&runtime::PatternPlurals::SinglePattern( + runtime::Pattern::from(&parsed_skeleton), + )); + + let option_struct: DateTimeFormatOptions = + serde_json::from_str(&options.to_string()).unwrap(); + + components_bag.hour = match option_struct.hour.as_deref() { + Some("numeric") => Some(components::Numeric::Numeric), + Some("2-digit") => Some(components::Numeric::TwoDigit), + _ => None, + }; + DateTimeFormatterOptions::Components(components_bag) }; - // Get ISO input string including offset and time zone + // Get ISO instant in UTC time zone let input_iso = &json_obj["input_string"].as_str().unwrap(); - // let input_iso: String = input_time_string.to_string() + "[-00:00]"; let dt_iso = IxdtfParser::new(input_iso).parse().unwrap(); let date = dt_iso.date.unwrap(); let time = dt_iso.time.unwrap(); - let tz_offset = dt_iso.offset.unwrap(); - let _tz_annotation = dt_iso.tz.unwrap(); - let datetime_iso = DateTime::try_new_iso_datetime( + // Compute the seconds for the timezone's offset + let offset_seconds: i32 = json_obj["tz_offset_secs"] + .as_i64() + .unwrap() + .try_into() + .unwrap(); + + let gmt_offset_seconds = GmtOffset::try_from_offset_seconds(offset_seconds).ok(); + + let mut datetime_iso = DateTime::try_new_iso_datetime( date.year, date.month, date.day, time.hour, time.minute, - time.second, + 0, // Seconds added below. ) .expect("Failed to initialize ISO DateTime instance."); + let mut dt_integer_minutes = datetime_iso.minutes_since_local_unix_epoch(); + dt_integer_minutes += offset_seconds / 60; + + datetime_iso = DateTime::from_minutes_since_local_unix_epoch(dt_integer_minutes); + datetime_iso.time.second = time.second.try_into().unwrap(); + let any_datetime = datetime_iso.to_any(); // Testing with a default timezone @@ -141,15 +172,9 @@ pub fn run_datetimeformat_test(json_obj: &Value) -> Result { let mzc = MetazoneCalculator::new(); let my_metazone_id = mzc.compute_metazone_from_time_zone(mapped_tz.unwrap(), &datetime_iso); - // Compute the seconds for the - let offset_seconds = GmtOffset::try_from_offset_seconds( - tz_offset.sign as i32 * (tz_offset.hour as i32 * 3600 + tz_offset.minute as i32 * 60), - ) - .ok(); - let time_zone = if timezone_str.is_some() { CustomTimeZone { - gmt_offset: offset_seconds, + gmt_offset: gmt_offset_seconds, time_zone_id: mapped_tz, metazone_id: my_metazone_id, zone_variant: None, @@ -161,8 +186,11 @@ pub fn run_datetimeformat_test(json_obj: &Value) -> Result { // The constructor is called with the given options // The default parameter is time zone formatter options. Not used yet. - let dtf_result = - ZonedDateTimeFormatter::try_new(&data_locale, dt_options.into(), Default::default()); + let dtf_result = ZonedDateTimeFormatter::try_new_experimental( + &data_locale, + dt_options.clone(), + Default::default(), + ); let datetime_formatter = match dtf_result { Ok(dtf) => dtf, @@ -175,9 +203,6 @@ pub fn run_datetimeformat_test(json_obj: &Value) -> Result { } }; - // Note: A "classical" skeleton is used in most cases, but this version - // of the executor does not use it. - let formatted_dt = datetime_formatter .format(&any_datetime, &time_zone) .expect("should work"); @@ -187,6 +212,6 @@ pub fn run_datetimeformat_test(json_obj: &Value) -> Result { "label": label, "result": result_string, "actual_options": - format!("{tz_offset:?}, {dt_options:?}, {time_zone:?}"), // , {dt_iso:?}"), + format!("{dt_options:?}, {time_zone:?}"), })) } diff --git a/executors/rust/1.4/Cargo.toml b/executors/rust/1.4/Cargo.toml index f242f105..c78b152e 100644 --- a/executors/rust/1.4/Cargo.toml +++ b/executors/rust/1.4/Cargo.toml @@ -15,7 +15,7 @@ serde = "1.0.171" serde_json = "1.0.100" rustc_version_runtime = "0.1.*" -icu = { version = "~1.4", features = ["serde", "icu_compactdecimal", "icu_displaynames", "icu_relativetime"] } +icu = { version = "~1.4", features = ["serde", "icu_compactdecimal", "icu_displaynames", "compiled_data", "icu_relativetime", "icu_datetime_experimental"] } fixed_decimal = "=0.5.5" writeable = "0.5.2" diff --git a/schema/datetime_fmt/test_schema.json b/schema/datetime_fmt/test_schema.json index 67cb5159..3c55996c 100644 --- a/schema/datetime_fmt/test_schema.json +++ b/schema/datetime_fmt/test_schema.json @@ -32,6 +32,10 @@ "description": "String in ISO 8601 form, e.g. YYYY-MM-DD hh:mm:ss.sss", "type": "string" }, + "tz_offset_secs": { + "description": "Offset in timezone from UTC", + "type": "number" + }, "datetime_skeleton": { "description": "Skeleton for date/time format: https://unicode-org.github.io/icu/userguide/format_parse/datetime/", "type": "string" diff --git a/testgen/generators/datetime_gen.js b/testgen/generators/datetime_gen.js index abd10ace..f7e806c0 100644 --- a/testgen/generators/datetime_gen.js +++ b/testgen/generators/datetime_gen.js @@ -27,7 +27,7 @@ const skip_things = false; // To limit some options for generating tests let use_milliseconds = false; // Add numbering system to the test options -// Don't test these across all other options, however. +// Do not test these across all other options, however. const numbering_systems = ['latn', 'arab', 'beng'] // ICU4X locales, maybe 20 @@ -126,7 +126,7 @@ const dates = [ let temporal_dates = [ { - timeZone: 'America/Los_Angeles', + timeZone: 'UTC', year: 2024, month: 3, day: 7, @@ -139,7 +139,7 @@ let temporal_dates = [ calendar: "gregory" }, { - timeZone: 'America/Los_Angeles', + timeZone: 'UTC', year: 2001, month: 7, day: 2, @@ -151,7 +151,7 @@ let temporal_dates = [ nanosecond: 0 }, { - timeZone: 'America/Los_Angeles', + timeZone: 'UTC', year: 1984, month: 5, day: 29, @@ -163,7 +163,7 @@ let temporal_dates = [ nanosecond: 0 }, { - timeZone: 'America/Los_Angeles', + timeZone: 'UTC', year: 2030, month: 5, day: 29, @@ -175,7 +175,7 @@ let temporal_dates = [ nanosecond: 0 }, { - timeZone: 'America/Los_Angeles', + timeZone: 'UTC', year: 1969, month: 7, day: 16, @@ -186,8 +186,8 @@ let temporal_dates = [ microsecond: 0, nanosecond: 0 }, - { // 1e9 - timeZone: 'America/Los_Angeles', + { // Approximately 1e9 milliseconds, 1e6 seconds + timeZone: 'UTC', year: 1970, month: 1, day: 12, @@ -198,8 +198,8 @@ let temporal_dates = [ microsecond: 0, nanosecond: 0 }, - { // 1e12 - timeZone: 'America/Los_Angeles', + { // Approximately 1e12 milliseconds, 1e9 seconds + timeZone: 'UTC', year: 2001, month: 9, day: 9, @@ -426,45 +426,28 @@ function generateAll(run_limit) { for (const date_index in dates) { label_num ++; - let this_date = dates[date_index]; - - // Get the temporal representation, including TZ and calendar + let zone_temporal_date = temporal_dates[date_index]; + zone_temporal_date['timeZone'] = timezone; + let zdt_zoned = Temporal.ZonedDateTime.from(zone_temporal_date); + const zoned_input_string = zdt_zoned.toString(); + const offset_part = zoned_input_string.substring(19,25); + const hours = offset_part.substring(0,3); + const minutes = offset_part.substring(4,6) + const tz_offset_secs = + 3600 * Number(hours)+ 60 * Number(minutes); + + // Get the ISO string with 'Z'. + // Set up the instant in UTC. + // Get the temporal representation, let temporal_date = temporal_dates[date_index]; - temporal_date['timeZone'] = timezone; - - try { - let vanilla_locale = 'en-US'; - let vanilla_calendar = 'gregory'; - // temporal_date['calendar'] = vanilla_calendar; - - // For computing the string with Date - let zdt_vanilla = Temporal.ZonedDateTime.from(temporal_date); - try { - this_date = new Date(zdt_vanilla.epochMilliseconds); - } catch (error) { - console.log('new Date fail %s with %s on %s', error, - temporal_date); - continue; - } - if (! this_date) { - console.log('$$$$ %s no date from zdt_to_date. %s, %s %s %s', - label_num, zdt_vanilla, temporal_date, - vanilla_locale, vanilla_calendar); - continue; - } - } catch (error) { - console.log(' SKIPPING %s temporal.from %s: input: %s', - label_num, error, temporal_date); - continue; - } + temporal_date['timeZone'] = 'UTC'; + let zdt = Temporal.ZonedDateTime.from(temporal_date); + let temporal_instant = zdt.toInstant(); + let input_string = temporal_instant.toString(); - // Get the ISO string with - // temporal_date['calendar'] = calendar; - let zdt_full = Temporal.ZonedDateTime.from(temporal_date); - input_string = zdt_full.toString(); + let this_date = new Date(temporal_instant.epochMilliseconds); - // !! TEMPORARY !! - // console.log(' TEMPORAL %s %s %s', label_num, input_string, this_date); + const full_input_string = zdt.toString(); let result; let parts; @@ -531,8 +514,9 @@ function generateAll(run_limit) { const label_string = String(label_num); let test_case = { - 'input_string': input_string - }; + 'input_string': input_string, + 'tz_offset_secs': tz_offset_secs + } if (skeleton) { test_case['skeleton'] = skeleton; @@ -584,7 +568,7 @@ function generateAll(run_limit) { test_obj['tests'] = sample_tests(test_cases, run_limit); try { - fs.writeFileSync('datetime_fmt_test.json', JSON.stringify(test_obj, null)); + fs.writeFileSync('datetime_fmt_test.json', JSON.stringify(test_obj, null, 2)); // file written successfully } catch (err) { console.error(err); diff --git a/verifier/testreport.py b/verifier/testreport.py index 2b513b7b..33a3953f 100644 --- a/verifier/testreport.py +++ b/verifier/testreport.py @@ -593,6 +593,10 @@ def characterize_results_by_options(self, test_list, category): # Why no data? continue + if not input_data: + # Why no data? + continue + label = test.get('label', '') key_list = ['locale', 'locale_label', 'option', 'options']