From 57ed0a2a53cc1ed5ed61bec6d0cbbc139e1b4542 Mon Sep 17 00:00:00 2001 From: Tim Chevalier Date: Thu, 20 Jun 2024 16:02:50 -0700 Subject: [PATCH] ICU-22794 MF2: Move .json files for tests into top-level testdata/ directory Modify ICU4C and ICU4J test readers to handle all tests Add `ignoreJava` and `ignoreCpp` properties to tests where needed Includes parser bug fixes: ICU4J: require a complex-body after declarations ICU4J: Correctly parse the complex body after an unsupported statement ICU4J: Handle date params in tests and remove default params for tests ICU4J: Handle decimal params in tests ICU4J: Require whitespace before variable/literal in reserved annotation ICU4J: Require whitespace between options ICU4J: Require a variable-expression in an .input declaration ICU4J: don't require space between last key and pattern in variant ICU4J: don't require space between selectors ICU4J: allow whitespace after '=' in option ICU4J: parse escape sequences in quoted literals according to grammar ICU4J: allow whitespace within markup after attributes list --- icu4c/source/config/dist.mk | 2 + icu4c/source/test/intltest/intltest.cpp | 55 +++ icu4c/source/test/intltest/intltest.h | 2 + .../intltest/messageformat2test_read_json.cpp | 45 +- .../source/test/testdata/message2/README.txt | 40 -- .../testdata/message2/invalid-options.json | 10 - .../testdata/message2/reserved-syntax.json | 39 -- .../testdata/message2/resolution-errors.json | 8 - .../message2/spec/test-functions.json | 306 ------------- .../message2/syntax-errors-diagnostics.json | 352 --------------- .../dev/test/message2/Mf2FeaturesTest.java | 8 +- .../java/com/ibm/icu/message2/MFParser.java | 129 ++++-- .../ibm/icu/dev/test/message2/CoreTest.java | 23 +- .../message2/CustomFormatterPersonTest.java | 2 +- .../test/message2/DataModelErrorsTest.java | 25 +- .../icu/dev/test/message2/FunctionsTest.java | 17 +- .../dev/test/message2/IcuFunctionsTest.java | 9 +- .../dev/test/message2/MessageFormat2Test.java | 24 +- .../dev/test/message2/SerializationTest.java | 6 +- .../dev/test/message2/SyntaxErrorsTest.java | 21 +- .../ibm/icu/dev/test/message2/TestUtils.java | 60 ++- .../com/ibm/icu/dev/test/message2/Unit.java | 8 +- .../dev/test/message2/data-model-errors.json | 32 -- .../dev/test/message2/icu-parser-tests.json | 61 --- .../icu/dev/test/message2/syntax-errors.json | 56 --- .../ibm/icu/dev/test/message2/test-core.json | 212 --------- icu4j/pom.xml | 1 + testdata/message2/README.txt | 88 ++++ .../message2/alias-selector-annotations.json | 0 .../message2/duplicate-declarations.json | 0 .../message2}/icu-parser-tests.json | 4 +- .../message2/icu-test-functions.json | 71 +++- .../message2/icu-test-previous-release.json | 141 ++---- .../message2/icu-test-selectors.json | 4 +- .../invalid-number-literals-diagnostics.json | 0 testdata/message2/invalid-options.json | 34 ++ .../message2/markup.json | 0 .../message2/matches-whitespace.json | 18 +- .../message2/more-data-model-errors.json | 4 - .../message2/more-functions.json | 14 +- testdata/message2/more-syntax-errors.json | 5 + testdata/message2/reserved-syntax.json | 40 ++ testdata/message2/resolution-errors.json | 14 + .../message2/runtime-errors.json | 12 +- .../message2/spec/data-model-errors.json | 0 .../message2/spec/syntax-errors.json | 0 .../message2/spec/test-core.json | 21 +- .../message2/spec}/test-functions.json | 68 +-- .../syntax-errors-diagnostics-multiline.json | 0 .../message2/syntax-errors-diagnostics.json | 402 ++++++++++++++++++ .../message2/syntax-errors-end-of-input.json | 0 .../message2/tricky-declarations.json | 0 .../message2/valid-tests.json | 70 ++- 53 files changed, 1119 insertions(+), 1444 deletions(-) delete mode 100644 icu4c/source/test/testdata/message2/README.txt delete mode 100644 icu4c/source/test/testdata/message2/invalid-options.json delete mode 100644 icu4c/source/test/testdata/message2/reserved-syntax.json delete mode 100644 icu4c/source/test/testdata/message2/resolution-errors.json delete mode 100644 icu4c/source/test/testdata/message2/spec/test-functions.json delete mode 100644 icu4c/source/test/testdata/message2/syntax-errors-diagnostics.json delete mode 100644 icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/data-model-errors.json delete mode 100644 icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-parser-tests.json delete mode 100644 icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/syntax-errors.json delete mode 100644 icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/test-core.json create mode 100644 testdata/message2/README.txt rename {icu4c/source/test/testdata => testdata}/message2/alias-selector-annotations.json (100%) rename {icu4c/source/test/testdata => testdata}/message2/duplicate-declarations.json (100%) rename {icu4c/source/test/testdata/message2/icu4j => testdata/message2}/icu-parser-tests.json (98%) rename {icu4j/main/core/src/test/resources/com/ibm/icu/dev/test => testdata}/message2/icu-test-functions.json (66%) rename {icu4j/main/core/src/test/resources/com/ibm/icu/dev/test => testdata}/message2/icu-test-previous-release.json (70%) rename {icu4j/main/core/src/test/resources/com/ibm/icu/dev/test => testdata}/message2/icu-test-selectors.json (98%) rename {icu4c/source/test/testdata => testdata}/message2/invalid-number-literals-diagnostics.json (100%) create mode 100644 testdata/message2/invalid-options.json rename {icu4c/source/test/testdata => testdata}/message2/markup.json (100%) rename {icu4c/source/test/testdata => testdata}/message2/matches-whitespace.json (82%) rename {icu4c/source/test/testdata => testdata}/message2/more-data-model-errors.json (94%) rename {icu4c/source/test/testdata => testdata}/message2/more-functions.json (87%) create mode 100644 testdata/message2/more-syntax-errors.json create mode 100644 testdata/message2/reserved-syntax.json create mode 100644 testdata/message2/resolution-errors.json rename {icu4c/source/test/testdata => testdata}/message2/runtime-errors.json (61%) rename {icu4c/source/test/testdata => testdata}/message2/spec/data-model-errors.json (100%) rename {icu4c/source/test/testdata => testdata}/message2/spec/syntax-errors.json (100%) rename {icu4c/source/test/testdata => testdata}/message2/spec/test-core.json (83%) rename {icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2 => testdata/message2/spec}/test-functions.json (77%) rename {icu4c/source/test/testdata => testdata}/message2/syntax-errors-diagnostics-multiline.json (100%) create mode 100644 testdata/message2/syntax-errors-diagnostics.json rename {icu4c/source/test/testdata => testdata}/message2/syntax-errors-end-of-input.json (100%) rename {icu4c/source/test/testdata => testdata}/message2/tricky-declarations.json (100%) rename {icu4c/source/test/testdata => testdata}/message2/valid-tests.json (68%) diff --git a/icu4c/source/config/dist.mk b/icu4c/source/config/dist.mk index 6558c16477e5..a2750f99e8c3 100644 --- a/icu4c/source/config/dist.mk +++ b/icu4c/source/config/dist.mk @@ -68,6 +68,8 @@ $(DISTY_FILE_TGZ) $(DISTY_FILE_ZIP) $(DISTY_DATA_ZIP): $(DISTY_DAT) $(DISTY_TMP ( cd $(ICU4CTOP)/.. && git archive --format=tar --prefix=icu/ HEAD:icu4c/ ) | ( cd "$(DISTY_TMP)" && tar xf - ) # special handling for LICENSE file. The symlinks will be included as files by tar and zip. cp -fv $(ICU4CTOP)/LICENSE "$(DISTY_TMP)/LICENSE" + # Copy top-level testdata directory so it's a sibling of the source/ directory + cp -R $(ICU4CTOP)/../testdata $(DISTY_TMP)/icu ( cd $(DISTY_TMP)/icu/source ; zip -rlq $(DISTY_DATA_ZIP) data ) $(MKINSTALLDIRS) $(DISTY_IN) echo DISTY_DAT=$(DISTY_DAT) diff --git a/icu4c/source/test/intltest/intltest.cpp b/icu4c/source/test/intltest/intltest.cpp index 868dcbbd52d5..e46eaafca327 100644 --- a/icu4c/source/test/intltest/intltest.cpp +++ b/icu4c/source/test/intltest/intltest.cpp @@ -1679,6 +1679,61 @@ const char *IntlTest::getSourceTestData(UErrorCode& /*err*/) { return srcDataDir; } +static bool fileExists(const char* fileName) { + // Test for `srcDataDir` existing by checking for `srcDataDir`/message2/valid-tests.json + U_ASSERT(fileName != nullptr); + FILE *f = fopen(fileName, "r"); + if (f) { + fclose(f); + return true; + } + return false; +} + +/** + * Returns the path to icu/testdata/ + */ +const char *IntlTest::getSharedTestData(UErrorCode& err) { +#define SOURCE_TARBALL_TOP U_TOPSRCDIR U_FILE_SEP_STRING ".." U_FILE_SEP_STRING +#define REPO_TOP SOURCE_TARBALL_TOP ".." U_FILE_SEP_STRING +#define FILE_NAME U_FILE_SEP_STRING "message2" U_FILE_SEP_STRING "valid-tests.json" + const char *srcDataDir = nullptr; + const char *testFile = nullptr; + if (U_SUCCESS(err)) { +#ifdef U_TOPSRCDIR + // Try U_TOPSRCDIR/../testdata (source tarball) + srcDataDir = SOURCE_TARBALL_TOP "testdata" U_FILE_SEP_STRING; + testFile = SOURCE_TARBALL_TOP "testdata" FILE_NAME; + if (!fileExists(testFile)) { + // If that doesn't exist, try U_TOPSRCDIR/../../testdata (in-repo) + srcDataDir = REPO_TOP "testdata" U_FILE_SEP_STRING; + testFile = REPO_TOP "testdata" FILE_NAME; + if (!fileExists(testFile)) { + // If neither exists, return null + err = U_FILE_ACCESS_ERROR; + srcDataDir = nullptr; + } + } +#else + // Try ../../../../testdata (if we're in icu/source/test/intltest) + // and ../../../../../../testdata (if we're in icu/source/test/intltest/Platform/(Debug|Release) +#define TOP ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING +#define TOP_TOP ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING TOP + srcDataDir = TOP "testdata" U_FILE_SEP_STRING; + testFile = TOP "testdata" FILE_NAME; + if (!fileExists(testFile)) { + srcDataDir = TOP_TOP "testdata" U_FILE_SEP_STRING; + testFile = TOP_TOP "testdata" FILE_NAME; + if (!fileExists(testFile)) { + err = U_FILE_ACCESS_ERROR; + srcDataDir = nullptr; + } + } +#endif + } + return srcDataDir; +} + char *IntlTest::getUnidataPath(char path[]) { const int kUnicodeDataTxtLength = 15; // strlen("UnicodeData.txt") diff --git a/icu4c/source/test/intltest/intltest.h b/icu4c/source/test/intltest/intltest.h index 146b7421bb5b..04cdb5c41c2b 100644 --- a/icu4c/source/test/intltest/intltest.h +++ b/icu4c/source/test/intltest/intltest.h @@ -420,6 +420,8 @@ class IntlTest : public TestLog { static const char* loadTestData(UErrorCode& err); virtual const char* getTestDataPath(UErrorCode& err) override; static const char* getSourceTestData(UErrorCode& err); + // Gets the path for the top-level testdata/ directory + static const char* getSharedTestData(UErrorCode& err); static char *getUnidataPath(char path[]); char16_t *ReadAndConvertFile(const char *fileName, int &ulen, const char *encoding, UErrorCode &status); diff --git a/icu4c/source/test/intltest/messageformat2test_read_json.cpp b/icu4c/source/test/intltest/messageformat2test_read_json.cpp index b52b3c48d4be..33e65a92ce85 100644 --- a/icu4c/source/test/intltest/messageformat2test_read_json.cpp +++ b/icu4c/source/test/intltest/messageformat2test_read_json.cpp @@ -39,7 +39,13 @@ static UErrorCode getExpectedErrorFromString(const std::string& errorName) { } static UErrorCode getExpectedRuntimeErrorFromString(const std::string& errorName) { - if (errorName == "unresolved-var") { + if (errorName == "parse-error" || errorName == "empty-token" || errorName == "extra-content") { + return U_MF_SYNTAX_ERROR; + } + if (errorName == "key-mismatch") { + return U_MF_VARIANT_KEY_MISMATCH_ERROR; + } + if (errorName == "missing-var" || errorName == "unresolved-var") { return U_MF_UNRESOLVED_VARIABLE_ERROR; } if (errorName == "unsupported-annotation") { @@ -137,7 +143,8 @@ static void runValidTest(TestMessageFormat2& icuTest, // Certain ICU4J tests don't work yet in ICU4C. // See ICU-22754 - if (!j_object["ignoreTest"].is_null()) { + // ignoreCpp => only works in Java + if (!j_object["ignoreCpp"].is_null()) { return; } @@ -210,7 +217,7 @@ static void runSyntaxErrorTest(TestMessageFormat2& icuTest, static void runICU4JSyntaxTestsFromJsonFile(TestMessageFormat2& t, const std::string& fileName, IcuTestErrorCode& errorCode) { - const char* testDataDirectory = IntlTest::getSourceTestData(errorCode); + const char* testDataDirectory = IntlTest::getSharedTestData(errorCode); CHECK_ERROR(errorCode); std::string testFileName(testDataDirectory); @@ -252,7 +259,7 @@ static void runICU4JSyntaxTestsFromJsonFile(TestMessageFormat2& t, static void runICU4JSelectionTestsFromJsonFile(TestMessageFormat2& t, const std::string& fileName, IcuTestErrorCode& errorCode) { - const char* testDataDirectory = IntlTest::getSourceTestData(errorCode); + const char* testDataDirectory = IntlTest::getSharedTestData(errorCode); CHECK_ERROR(errorCode); std::string testFileName(testDataDirectory); @@ -270,7 +277,7 @@ static void runICU4JSelectionTestsFromJsonFile(TestMessageFormat2& t, auto variations = j_object["variations"]; // Skip ignored tests - if (!j_object["ignoreTest"].is_null()) { + if (!j_object["ignoreCpp"].is_null()) { return; } @@ -281,7 +288,7 @@ static void runICU4JSelectionTestsFromJsonFile(TestMessageFormat2& t, messageText += piece; } - t.logln("ICU4J selectors tests:"); + t.logln(u_str("ICU4J selectors tests: " + fileName)); t.logln(u_str(iter->dump())); TestCase::Builder test; @@ -315,7 +322,7 @@ static void runICU4JSelectionTestsFromJsonFile(TestMessageFormat2& t, static void runValidTestsFromJsonFile(TestMessageFormat2& t, const std::string& fileName, IcuTestErrorCode& errorCode) { - const char* testDataDirectory = IntlTest::getSourceTestData(errorCode); + const char* testDataDirectory = IntlTest::getSharedTestData(errorCode); CHECK_ERROR(errorCode); std::string testFileName(testDataDirectory); @@ -341,7 +348,7 @@ static void runValidTestsFromJsonFile(TestMessageFormat2& t, static void runDataModelErrorTestsFromJsonFile(TestMessageFormat2& t, const std::string& fileName, IcuTestErrorCode& errorCode) { - const char* testDataDirectory = IntlTest::getSourceTestData(errorCode); + const char* testDataDirectory = IntlTest::getSharedTestData(errorCode); CHECK_ERROR(errorCode); std::string dataModelErrorsFileName(testDataDirectory); @@ -368,7 +375,7 @@ static void runDataModelErrorTestsFromJsonFile(TestMessageFormat2& t, for (auto messagesIter = messages.begin(); messagesIter != messages.end(); ++messagesIter) { makeTestName(testName, sizeof(testName), errorName, testNum); testBuilder.setName(testName); - t.logln(testName); + t.logln(u_str(fileName + ": " + testName)); testNum++; UnicodeString messageText = u_str(*messagesIter); t.logln(messageText); @@ -386,7 +393,7 @@ static void runDataModelErrorTestsFromJsonFile(TestMessageFormat2& t, static void runSyntaxErrorTestsFromJsonFile(TestMessageFormat2& t, const std::string& fileName, IcuTestErrorCode& errorCode) { - const char* testDataDirectory = IntlTest::getSourceTestData(errorCode); + const char* testDataDirectory = IntlTest::getSharedTestData(errorCode); CHECK_ERROR(errorCode); std::string syntaxErrorsFileName(testDataDirectory); @@ -420,7 +427,7 @@ static void runSyntaxErrorTestsFromJsonFile(TestMessageFormat2& t, static void runSyntaxTestsWithDiagnosticsFromJsonFile(TestMessageFormat2& t, const std::string& fileName, IcuTestErrorCode& errorCode) { - const char* testDataDirectory = IntlTest::getSourceTestData(errorCode); + const char* testDataDirectory = IntlTest::getSharedTestData(errorCode); CHECK_ERROR(errorCode); std::string testFileName(testDataDirectory); @@ -446,7 +453,7 @@ static void runFunctionTestsFromJsonFile(TestMessageFormat2& t, const std::string& fileName, IcuTestErrorCode& errorCode) { // Get the test data directory - const char* testDataDirectory = IntlTest::getSourceTestData(errorCode); + const char* testDataDirectory = IntlTest::getSharedTestData(errorCode); CHECK_ERROR(errorCode); std::string functionTestsFileName(testDataDirectory); @@ -460,7 +467,7 @@ static void runFunctionTestsFromJsonFile(TestMessageFormat2& t, for (auto iter = tests.begin(); iter != tests.end(); ++iter) { int32_t testNum = 0; auto functionName = iter->first; - t.logln("Function tests:"); + t.logln(u_str("Function tests: " + fileName)); t.logln(u_str(iter->second.dump())); // Array of tests @@ -487,6 +494,7 @@ void TestMessageFormat2::jsonTestsFromFiles(IcuTestErrorCode& errorCode) { // Do spec tests for syntax errors runSyntaxErrorTestsFromJsonFile(*this, "spec/syntax-errors.json", errorCode); + runSyntaxErrorTestsFromJsonFile(*this, "more-syntax-errors.json", errorCode); // Do tests for data model errors runDataModelErrorTestsFromJsonFile(*this, "spec/data-model-errors.json", errorCode); @@ -540,13 +548,10 @@ void TestMessageFormat2::jsonTestsFromFiles(IcuTestErrorCode& errorCode) { runSyntaxTestsWithDiagnosticsFromJsonFile(*this, "syntax-errors-diagnostics-multiline.json", errorCode); // ICU4J tests - runFunctionTestsFromJsonFile(*this, "icu4j/icu-test-functions.json", errorCode); - // Changes made to get these tests to work: - // * removed double backslashes from "Hello \\t \\n \\r \\{ world!" - // * added empty {{}} pattern at the end of the tests under "Simple messages, with declarations" - // and the first one under "Multiple declarations in one message" - runICU4JSyntaxTestsFromJsonFile(*this, "icu4j/icu-parser-tests.json", errorCode); - runICU4JSelectionTestsFromJsonFile(*this, "icu4j/icu-test-selectors.json", errorCode); + runFunctionTestsFromJsonFile(*this, "icu-test-functions.json", errorCode); + runICU4JSyntaxTestsFromJsonFile(*this, "icu-parser-tests.json", errorCode); + runICU4JSelectionTestsFromJsonFile(*this, "icu-test-selectors.json", errorCode); + runValidTestsFromJsonFile(*this, "icu-test-previous-release.json", errorCode); } #endif /* #if !UCONFIG_NO_MF2 */ diff --git a/icu4c/source/test/testdata/message2/README.txt b/icu4c/source/test/testdata/message2/README.txt deleted file mode 100644 index 65996cab7bde..000000000000 --- a/icu4c/source/test/testdata/message2/README.txt +++ /dev/null @@ -1,40 +0,0 @@ -© 2024 and later: Unicode, Inc. and others. -License & terms of use: http://www.unicode.org/copyright.html - -The format of the JSON files in this directory follows the same format as `test-core.json` -in the spec, described in: - -https://github.com/unicode-org/message-format-wg/blob/main/test/README.md - -The `parts` and `cleanSrc` fields are not used. - -Some extensions: - -Additional "char" and "line" fields may be present with integer values, -used for tests expected to trigger a syntax error. -If present, "char" reflects the expected character offset and "line" -reflects the expected line number in the parse error. -The files with "diagnostics" in the name have these fields filled in. - -An additional `comment` field may be present, which is only for human readers. - -A "srcs" field, whose value is an array of strings, may be present instead -of "src". The strings are concatenated to get the message. - -In the "params" field, a date parameter can be expressed as: -{ "date": n } -where n is a number representing a Unix timestamp. - -An optional field, "ignoreTest", can be used for tests that are -currently expected to fail. The field may have any value; if it's -present, the test is ignored. (The value can be a comment explaining -why it's expected to fail.) - -Tests in the `icu4j/` subdirectory are taken from: - icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2 - and need to be manually synced with those files. The format is a bit - different in some cases. - -Tests in the `spec/` subdirectory are taken from https://github.com/unicode-org/message-format-wg/blob/main/test -and need to be manually updated if the contents change upstream. - diff --git a/icu4c/source/test/testdata/message2/invalid-options.json b/icu4c/source/test/testdata/message2/invalid-options.json deleted file mode 100644 index 0ae458375be7..000000000000 --- a/icu4c/source/test/testdata/message2/invalid-options.json +++ /dev/null @@ -1,10 +0,0 @@ -[ - { "src": ".local $foo = {1 :number minimumIntegerDigits=-1} {{bar {$foo}}}", "errors": [{"type": "bad-option"}]}, - { "src": ".local $foo = {1 :number minimumIntegerDigits=foo} {{bar {$foo}}}", "errors": [{"type": "bad-option"}]}, - { "src": ".local $foo = {1 :number minimumFractionDigits=foo} {{bar {$foo}}}", "errors": [{"type": "bad-option"}]}, - { "src": ".local $foo = {1 :number maximumFractionDigits=foo} {{bar {$foo}}}", "errors": [{"type": "bad-option"}]}, - { "src": ".local $foo = {1 :number minimumSignificantDigits=foo} {{bar {$foo}}}", "errors": [{"type": "bad-option"}]}, - { "src": ".local $foo = {1 :number maximumSignificantDigits=foo} {{bar {$foo}}}", "errors": [{"type": "bad-option"}]}, - { "src": ".local $foo = {1 :integer minimumIntegerDigits=foo} {{bar {$foo}}}", "errors": [{"type": "bad-option"}]}, - { "src": ".local $foo = {1 :integer maximumSignificantDigits=foo} {{bar {$foo}}}", "errors": [{"type": "bad-option"}]} -] diff --git a/icu4c/source/test/testdata/message2/reserved-syntax.json b/icu4c/source/test/testdata/message2/reserved-syntax.json deleted file mode 100644 index 29397f54cc27..000000000000 --- a/icu4c/source/test/testdata/message2/reserved-syntax.json +++ /dev/null @@ -1,39 +0,0 @@ -[ - { "src": "hello {|4.2| %number}", "errors": [{ "type": "unsupported-annotation" }] }, - { "src": "hello {|4.2| %n|um|ber}", "errors": [{ "type": "unsupported-annotation" }] }, - { "src": "{+42}", "errors": [{ "type": "unsupported-annotation" }] }, - { "src": "hello {|4.2| &num|be|r}", "errors": [{ "type": "unsupported-annotation" }] }, - { "src": "hello {|4.2| ^num|be|r}", "errors": [{ "type": "unsupported-annotation" }] }, - { "src": "hello {|4.2| +num|be|r}", "errors": [{ "type": "unsupported-annotation" }] }, - { "src": "hello {|4.2| ?num|be||r|s}", "errors": [{ "type": "unsupported-annotation" }] }, - { "src": "hello {|foo| !number}", "errors": [{ "type": "unsupported-annotation" }] }, - { "src": "hello {|foo| *number}", "errors": [{ "type": "unsupported-annotation" }] }, - { "src": "hello {?number}", "errors": [{ "type": "unsupported-annotation" }] }, - { "src": "{xyzz }", "errors": [{ "type": "unsupported-annotation" }] }, - { "src": "hello {$foo ~xyzz }", "errors": [{ "type": "unsupported-annotation" }] }, - { "src": "hello {$x xyzz }", "errors": [{ "type": "unsupported-annotation" }] }, - { "src": "{ !xyzz }", "errors": [{ "type": "unsupported-annotation" }] }, - { "src": "{~xyzz }", "errors": [{ "type": "unsupported-annotation" }] }, - { "src": "{ num x \\\\ abcde |aaa||3.14||42| r }", "errors": [{ "type": "unsupported-annotation" }] }, - { "src": "hello {$foo >num x \\\\ abcde |aaa||3.14| |42| r }", "errors": [{ "type": "unsupported-annotation" }] } -] diff --git a/icu4c/source/test/testdata/message2/resolution-errors.json b/icu4c/source/test/testdata/message2/resolution-errors.json deleted file mode 100644 index 82595b4a27ce..000000000000 --- a/icu4c/source/test/testdata/message2/resolution-errors.json +++ /dev/null @@ -1,8 +0,0 @@ -[ - { "src": "{$oops}", "exp": "{$oops}", "errors": [{ "type": "unresolved-var" }]}, - { "src": ".input {$x :number} {{{$x}}}", "exp": "{$x}", "errors": [{ "type": "unresolved-var" }]}, - { "src": ".local $foo = {$bar} .match {$foo :number} one {{one}} * {{other}}", "exp": "other", "errors": [{ "type": "unresolved-var" }]}, - { "src": ".local $foo = {$bar} .match {$foo :number} one {{one}} * {{other}}", "exp": "other", "errors": [{ "type": "unresolved-var" }]}, - { "src": ".local $bar = {$none :number} .match {$foo :string} one {{one}} * {{{$bar}}}", "exp": "{$none}", "errors": [{ "type": "unresolved-var" }]}, - { "src": "The value is {horse :func}.", "exp": "The value is {|horse|}.", "errors": [{ "type": "missing-func" }]} -] diff --git a/icu4c/source/test/testdata/message2/spec/test-functions.json b/icu4c/source/test/testdata/message2/spec/test-functions.json deleted file mode 100644 index 3258d2a5b39f..000000000000 --- a/icu4c/source/test/testdata/message2/spec/test-functions.json +++ /dev/null @@ -1,306 +0,0 @@ -{ - "date": [ - { "src": "{:date}", "exp": "{:date}", "errors": [{ "type": "bad-input" }] }, - { - "src": "{horse :date}", - "exp": "{|horse|}", - "errors": [{ "type": "bad-input" }] - }, - { "src": "{|2006-01-02| :date}", "exp": "1/2/06" }, - { "src": "{|2006-01-02T15:04:06| :date}", "exp": "1/2/06" }, - { "src": "{|2006-01-02| :date style=long}", "exp": "January 2, 2006" }, - { - "src": ".local $d = {|2006-01-02| :date style=long} {{{$d :date}}}", - "exp": "January 2, 2006" - }, - { - "src": ".local $t = {|2006-01-02T15:04:06| :time} {{{$t :date}}}", - "exp": "1/2/06" - } - ], - "time": [ - { "src": "{:time}", "exp": "{:time}", "errors": [{ "type": "bad-input" }] }, - { - "src": "{horse :time}", - "exp": "{|horse|}", - "errors": [{ "type": "bad-input" }] - }, - { "src": "{|2006-01-02T15:04:06| :time}", "exp": "3:04\u202FPM" }, - { - "src": "{|2006-01-02T15:04:06| :time style=medium}", - "exp": "3:04:06\u202FPM" - }, - { - "src": ".local $t = {|2006-01-02T15:04:06| :time style=medium} {{{$t :time}}}", - "exp": "3:04:06\u202FPM" - }, - { - "src": ".local $d = {|2006-01-02T15:04:06| :date} {{{$d :time}}}", - "exp": "3:04\u202FPM" - } - ], - "datetime": [ - { - "src": "{:datetime}", - "exp": "{:datetime}", - "errors": [{ "type": "bad-input" }] - }, - { - "src": "{$x :datetime}", - "exp": "{$x}", - "params": { "x": true }, - "errors": [{ "type": "bad-input" }] - }, - { - "src": "{horse :datetime}", - "exp": "{|horse|}", - "errors": [{ "name": "RangeError" }] - }, - { "src": "{|2006-01-02T15:04:06| :datetime}", "exp": "1/2/06, 3:04\u202FPM" }, - { - "src": "{|2006-01-02T15:04:06| :datetime year=numeric month=|2-digit|}", - "exp": "01/2006" - }, - { - "src": "{|2006-01-02T15:04:06| :datetime dateStyle=long}", - "exp": "January 2, 2006" - }, - { - "src": "{|2006-01-02T15:04:06| :datetime timeStyle=medium}", - "exp": "3:04:06\u202FPM" - }, - { - "src": "{$dt :datetime}", - "params": { "dt": "2006-01-02T15:04:06" }, - "exp": "1/2/06, 3:04\u202FPM" - } - ], - "integer": [ - { "src": "hello {4.2 :integer}", "exp": "hello 4" }, - { "src": "hello {-4.20 :integer}", "exp": "hello -4" }, - { "src": "hello {0.42e+1 :integer}", "exp": "hello 4" }, - { - "src": ".match {$foo :integer} one {{one}} * {{other}}", - "params": { "foo": 1.2 }, - "exp": "one" - } - ], - "number": [ - { "src": "hello {4.2 :number}", "exp": "hello 4.2" }, - { "src": "hello {-4.20 :number}", "exp": "hello -4.2" }, - { "src": "hello {0.42e+1 :number}", "exp": "hello 4.2" }, - { - "src": "hello {foo :number}", - "exp": "hello {|foo|}", - "errors": [{ "type": "bad-input" }] - }, - { - "src": "hello {:number}", - "exp": "hello {:number}", - "errors": [{ "type": "bad-input" }] - }, - { - "src": "hello {4.2 :number minimumFractionDigits=2}", - "exp": "hello 4.20" - }, - { - "src": "hello {|4.2| :number minimumFractionDigits=|2|}", - "exp": "hello 4.20" - }, - { - "src": "hello {4.2 :number minimumFractionDigits=$foo}", - "params": { "foo": 2 }, - "exp": "hello 4.20" - }, - { - "src": "hello {|4.2| :number minimumFractionDigits=$foo}", - "params": { "foo": "2" }, - "exp": "hello 4.20" - }, - { - "src": ".local $foo = {$bar :number} {{bar {$foo}}}", - "params": { "bar": 4.2 }, - "exp": "bar 4.2" - }, - { - "src": ".local $foo = {$bar :number minimumFractionDigits=2} {{bar {$foo}}}", - "params": { "bar": 4.2 }, - "exp": "bar 4.20" - }, - { - "src": ".local $foo = {$bar :number minimumFractionDigits=foo} {{bar {$foo}}}", - "params": { "bar": 4.2 }, - "exp": "bar {$bar}", - "errors": [{ "type": "bad-option" }] - }, - { - "src": ".local $foo = {$bar :number} {{bar {$foo}}}", - "params": { "bar": "foo" }, - "exp": "bar {$bar}", - "errors": [{ "type": "bad-input" }] - }, - { - "src": ".input {$foo :number} {{bar {$foo}}}", - "params": { "foo": 4.2 }, - "exp": "bar 4.2" - }, - { - "src": ".input {$foo :number minimumFractionDigits=2} {{bar {$foo}}}", - "params": { "foo": 4.2 }, - "exp": "bar 4.20" - }, - { - "src": ".input {$foo :number minimumFractionDigits=foo} {{bar {$foo}}}", - "params": { "foo": 4.2 }, - "exp": "bar {$foo}", - "errors": [{ "type": "bad-option" }] - }, - { - "src": ".input {$foo :number} {{bar {$foo}}}", - "params": { "foo": "foo" }, - "exp": "bar {$foo}", - "errors": [{ "type": "bad-input" }] - }, - { - "src": ".match {$foo :number} one {{one}} * {{other}}", - "params": { "foo": 1 }, - "exp": "one" - }, - { - "src": ".match {$foo :number} 1 {{=1}} one {{one}} * {{other}}", - "params": { "foo": 1 }, - "exp": "=1" - }, - { - "src": ".match {$foo :number} one {{one}} 1 {{=1}} * {{other}}", - "params": { "foo": 1 }, - "exp": "=1" - }, - { - "src": ".match {$foo :number} {$bar :number} one one {{one one}} one * {{one other}} * * {{other}}", - "params": { "foo": 1, "bar": 1 }, - "exp": "one one" - }, - { - "src": ".match {$foo :number} {$bar :number} one one {{one one}} one * {{one other}} * * {{other}}", - "params": { "foo": 1, "bar": 2 }, - "exp": "one other" - }, - { - "src": ".match {$foo :number} {$bar :number} one one {{one one}} one * {{one other}} * * {{other}}", - "params": { "foo": 2, "bar": 2 }, - "exp": "other" - }, - { - "src": ".input {$foo :number} .match {$foo} one {{one}} * {{other}}", - "params": { "foo": 1 }, - "exp": "one" - }, - { - "src": ".local $foo = {$bar :number} .match {$foo} one {{one}} * {{other}}", - "params": { "bar": 1 }, - "exp": "one" - }, - { - "src": ".input {$foo :number} .local $bar = {$foo} .match {$bar} one {{one}} * {{other}}", - "params": { "foo": 1 }, - "exp": "one" - }, - { - "src": ".input {$bar :number} .match {$bar} one {{one}} * {{other}}", - "params": { "bar": 2 }, - "exp": "other" - }, - { - "src": ".input {$bar} .match {$bar :number} one {{one}} * {{other}}", - "params": { "bar": 1 }, - "exp": "one" - }, - { - "src": ".input {$bar} .match {$bar :number} one {{one}} * {{other}}", - "params": { "bar": 2 }, - "exp": "other" - }, - { - "src": ".input {$bar} .match {$bar :number} one {{one}} * {{other}}", - "params": { "bar": 1 }, - "exp": "one" - }, - { - "src": ".input {$bar} .match {$bar :number} one {{one}} * {{other}}", - "params": { "bar": 2 }, - "exp": "other" - }, - { - "src": ".input {$none} .match {$foo :number} one {{one}} * {{{$none}}}", - "params": { "foo": 1 }, - "exp": "one" - }, - { - "src": ".local $bar = {$none} .match {$foo :number} one {{one}} * {{{$bar}}}", - "params": { "foo": 1 }, - "exp": "one" - }, - { - "src": ".local $bar = {$none} .match {$foo :number} one {{one}} * {{{$bar}}}", - "params": { "foo": 2 }, - "exp": "{$none}", - "errors": [{ "type": "unresolved-var" }] - }, - { - "src": "{42 :number @foo @bar=13}", - "exp": "42", - "parts": [ - { "type": "number", "parts": [{ "type": "integer", "value": "42" }] } - ] - } - ], - "ordinal": [ - { - "src": ".match {$foo :ordinal} one {{st}} two {{nd}} few {{rd}} * {{th}}", - "params": { "foo": 1 }, - "exp": "th", - "errors": [{ "type": "missing-func" }, { "type": "not-selectable" }] - }, - { - "src": "hello {42 :ordinal}", - "exp": "hello {|42|}", - "errors": [{ "type": "missing-func" }] - } - ], - "plural": [ - { - "src": ".match {$foo :plural} one {{one}} * {{other}}", - "params": { "foo": 1 }, - "exp": "other", - "errors": [{ "type": "missing-func" }, { "type": "not-selectable" }] - }, - { - "src": "hello {42 :plural}", - "exp": "hello {|42|}", - "errors": [{ "type": "missing-func" }] - } - ], - "string": [ - { - "src": ".match {$foo :string} |1| {{one}} * {{other}}", - "params": { "foo": "1" }, - "exp": "one" - }, - { - "src": ".match {$foo :string} 1 {{one}} * {{other}}", - "params": { "foo": 1 }, - "exp": "one" - }, - { - "src": ".match {$foo :string} 1 {{one}} * {{other}}", - "params": { "foo": null }, - "exp": "other" - }, - { - "src": ".match {$foo :string} 1 {{one}} * {{other}}", - "exp": "other", - "errors": [{ "type": "unresolved-var" }] - } - ] -} diff --git a/icu4c/source/test/testdata/message2/syntax-errors-diagnostics.json b/icu4c/source/test/testdata/message2/syntax-errors-diagnostics.json deleted file mode 100644 index b4b86c68e8e7..000000000000 --- a/icu4c/source/test/testdata/message2/syntax-errors-diagnostics.json +++ /dev/null @@ -1,352 +0,0 @@ -[ - { "src": "}{|xyz|", "char": 0 }, - { "src": "}", "char": 0 }, - { - "src": "{{{%\\y{}}", - "char": 5, - "comment": "Backslash followed by non-backslash followed by a '{' -- this should be an error immediately after the first backslash" - }, - { - "src": "{%abc|\\z}}", - "char": 7, - "comment": "Reserved chars followed by a '|' that doesn't begin a valid literal -- this should be an error at the first invalid char in the literal" - }, - { - "src": "{%\\y{p}}", - "char": 3, - "comment": "Same pattern, but with a valid reserved-char following the erroneous reserved-escape -- the offset should be the same as with the previous one" - }, - { - "src": "{{{%ab|\\z|cd}}", - "char": 8, - "comment": "Erroneous literal inside a reserved string -- the error should be at the first erroneous literal char" - }, - { - "src": "hello {|4.2| %num\\ber}}", - "char": 18, - "comment": "Single backslash not allowed" - }, - { - "src": "hello {|4.2| %num{be\\|r}}", - "char": 17, - "comment": "Unescaped '{' not allowed" - }, - { - "src": "hello {|4.2| %num}be\\|r}}", - "char": 21, - "comment": "Unescaped '}' -- will be interpreted as the end of the reserved string, and the error will be reported at the index of '|', which is when the parser determines that the escaped '|' isn't a valid text-escape" - }, - { - "src": "hello {|4.2| %num\\{be|r}}", - "char": 25, - "comment": "Unescaped '|' -- will be interpreted as the beginning of a literal. Error at end of input" - }, - { - "src": "a\\qbc", - "char": 2, - "comment": "Invalid escape sequence in a `text` -- the error should be at the character following the backslash" - }, - { - "src": ".match{|y|}|y|{{{|||}}}", - "char": 19, - "comment": "No spaces are required here. The error should be in the pattern, not before" - }, - { - "src": ".match {|y|}|foo|bar {{{a}}}", - "char": 17, - "comment": "Missing spaces between keys" - }, - { - "src": ".match {|y|} |quux| |foo|bar {{{a}}}", - "char": 25, - "comment": "Missing spaces between keys" - }, - { - "src": ".match {|y|} |quux| |foo||bar| {{{a}}}", - "char": 26, - "comment": "Missing spaces between keys" - }, - { - "src": ".match {|y|} |\\q| * %{! {z}", - "char": 16, - "comment": "Error parsing the first key -- the error should be there, not in the also-erroneous third key" - }, - { - "src": ".match {|y|} * %{! {z} |\\q|", - "char": 16, - "comment": "Error parsing the second key -- the error should be there, not in the also-erroneous third key" - }, - { - "src": ".match {|y|} * |\\q| {\\z}", - "char": 18, - "comment": "Error parsing the last key -- the error should be there, not in the erroneous pattern" - }, - { - "src": ".match {|y|} {\\|} {@} * * * {{a}}", - "char": 14, - "comment": "Non-expression as scrutinee in pattern -- error should be at the first non-expression, not the later non-expression" - }, - { - "src": ".match {|y|} $foo * {{a}} when * :bar {{b}}", - "char": 14, - "comment": "Non-key in variant -- error should be there, not in the next erroneous variant" - }, - { - "src": "{{ foo {|bar|} \\q baz ", - "char": 16, - "comment": "Error should be within the first erroneous `text` or expression" - }, - { - "src": "{{{: }}}", - "char": 4, - "comment": "':' has to be followed by a function name -- the error should be at the first whitespace character" - }, - { - "src": ".local $x = }|foo|}", - "char": 12, - "comment": "Expression not starting with a '{'" - }, - { - "src": ".local $x = {|foo|} .l $y = {|bar|} .local $z {|quux|}", - "char": 22, - "comment": "Error should be at the first declaration not starting with a `.local`" - }, - { - "src": ".local $bar {|foo|} {{$bar}}", - "char": 12, - "comment": "Missing '=' in `.local` declaration" - }, - { - "src": ".local bar = {|foo|} {{$bar}}", - "char": 7, - "comment": "LHS of declaration doesn't start with a '$'" - }, - { - "src": ".local $bar = |foo| {{$bar}}", - "char": 14, - "comment": "`.local` RHS isn't an expression" - }, - { - "src": "{{extra}}content", - "char": 9, - "comment": "Trailing characters that are not whitespace" - }, - { - "src": ".match {|x|} * {{foo}}extra", - "char": 28, - "comment": "Trailing characters that are not whitespace" - }, - { - "src": ".match {$foo :string} {$bar :string} one * {{one}} * * {{other}} ", - "char": 66, - "comment": "Trailing whitespace at end of message should not be accepted either" - }, - { - "src": "{{hi}} ", - "char": 6, - "comment": "Trailing whitespace at end of message should not be accepted either" - }, - { - "src": "empty { }", - "char": 8, - "comment": "Empty expression" - }, - { - "src": ".match {} * {{foo}}", - "char": 8, - "comment": "Empty expression" - }, - { - "src": "bad {:}", - "char": 6, - "comment": "':' not preceding a function name" - }, - { - "src": "{{no-equal {|42| :number m }}}", - "char": 27, - "comment": "Missing '=' after option name" - }, - { - "src": "{{no-equal {|42| :number minimumFractionDigits 2}}}", - "char": 47, - "comment": "Missing '=' after option name" - }, - { - "src": "bad {:placeholder option value}", - "char": 25, - "comment": "Missing '=' after option name" - }, - { - "src": "hello {|4.2| :number min=2=3}", - "char": 26, - "comment": "Extra '=' after option name" - }, - { - "src": "hello {|4.2| :number min=2max=3}", - "char": 26, - "comment": "Extra '=' after option name" - }, - { - "src": "hello {|4.2| :number min=|a|max=3}", - "char": 28, - "comment": "Missing whitespace between valid options" - }, - { - "src": "hello {|4.2| :number min=|\\a|}", - "char": 27, - "comment": "Ill-formed RHS of option -- the error should be within the RHS, not after parsing options" - }, - { - "src": "no-equal {|42| :number {}", - "char": 25, - "comment": "Junk after annotation" - }, - { - "src": "bad {:placeholder option=}", - "char": 25, - "comment": "Missing RHS of option" - }, - { - "src": "bad {:placeholder option}", - "char": 24, - "comment": "Missing RHS of option" - }, - { - "src": "bad {$placeholder option}", - "char": 18, - "comment": "Annotation is not a function or reserved text" - }, - { - "src": "no {$placeholder end", - "char": 17, - "comment": "Annotation is not a function or reserved text" - }, - { - "src": ".match * {{foo}}", - "char": 8, - "comment": "Missing expression in selectors" - }, - { - "src": ".match |x| * {{foo}}", - "char": 7, - "comment": "Non-expression in selectors" - }, - { - "src": ".match {|x|} * foo", - "char": 19, - "comment": "Missing RHS in variant" - }, - { - "src": "{$:abc}", - "char": 2, - "comment": "Variable names can't start with a : or -" - }, - { - "src": "{$-abc}", - "char": 2, - "comment": "Variable names can't start with a : or -" - }, - { - "src": "{$bar+foo}", - "char": 5, - "comment": "Missing space before annotation. Note that {{$bar:foo}} and {{$bar-foo}} are valid, because variable names can contain a ':' or a '-'" - }, - { - "src": "{|3.14|:foo}", - "char": 7, - "comment": "Missing space before annotation." - }, - { - "src": "{|3.14|-foo}", - "char": 7, - "comment": "Missing space before annotation." - }, - { - "src": "{|3.14|+foo}", - "char": 7, - "comment": "Missing space before annotation." - }, - { - "src": ".local $foo = {$bar} .match {$foo} :one {one} * {other}", - "char": 36, - "comment": "Unquoted literals can't begin with a ':'" - }, - { - "src": ".local $foo = {$bar :fun option=:a} {{bar {$foo}}}", - "char": 32, - "comment": "Unquoted literals can't begin with a ':'" - }, - { "src": "{|foo| {#markup}}", "char": 7, "comment": "Markup in wrong place" }, - { "src": "{|foo| #markup}", "char": 7, "comment": "Markup in wrong place" }, - { "src": "{|foo| {#markup/}}", "char": 7, "comment": "Markup in wrong place" }, - { "src": "{|foo| {/markup}}", "char": 7, "comment": "Markup in wrong place" }, - { "src": ".input $x = {|1|} {{{$x}}}", "char": 7, "comment": ".input with non-variable-expression" }, - { "src": ".input $x = {:number} {{{$x}}}", "char": 7, "comment": ".input with non-variable-expression" }, - { "src": ".input {|1| :number} {{{$x}}}", "char": 7, "comment": ".input with non-variable-expression" }, - { "src": ".input {:number} {{{$x}}}", "char": 7, "comment": ".input with non-variable-expression" }, - { "src": ".input {|1|} {{{$x}}}", "char": 7, "comment": ".input with non-variable-expression" }, - { "src": ".", "char": 1}, - { "src": "{", "char": 1}, - { "src": "}", "char": 0}, - { "src": "{}", "char": 1}, - { "src": "{{", "char": 2}, - { "src": "{{}", "char": 3}, - { "src": "{{}}}", "char": 4}, - { "src": "{|foo| #markup}", "char": 7}, - { "src": "{{missing end brace}", "char": 20}, - { "src": "{{missing end braces", "char": 20}, - { "src": "{{missing end {$braces", "char": 22}, - { "src": "{{extra}} content", "char": 9}, - { "src": "empty { } placeholder", "char": 8}, - { "src": "missing space {42:func}", "char": 17}, - { "src": "missing space {|foo|:func}", "char": 20}, - { "src": "missing space {|foo|@bar}", "char": 20}, - { "src": "missing space {:func@bar}", "char": 20}, - { "src": "{:func @bar@baz}", "char": 11}, - { "src": "{:func @bar=42@baz}", "char": 14}, - { "src": "{+reserved@bar}", "char": 10}, - { "src": "{&private@bar}", "char": 9}, - { "src": "bad {:} placeholder", "char": 6}, - { "src": "bad {\\u0000placeholder}", "char": 5}, - { "src": "no-equal {|42| :number minimumFractionDigits 2}", "char": 45}, - { "src": "bad {:placeholder option=}", "char": 25}, - { "src": "bad {:placeholder option value}", "char": 25}, - { "src": "bad {:placeholder option:value}", "char": 30}, - { "src": "bad {:placeholder option}", "char": 24}, - { "src": "bad {:placeholder:}", "char": 18}, - { "src": "bad {::placeholder}", "char": 6}, - { "src": "bad {:placeholder::foo}", "char": 18}, - { "src": "bad {:placeholder option:=x}", "char": 25}, - { "src": "bad {:placeholder :option=x}", "char": 18}, - { "src": "bad {:placeholder option::x=y}", "char": 25}, - { "src": "bad {$placeholder option}", "char": 18}, - { "src": "bad {:placeholder @attribute=}", "char": 29}, - { "src": "bad {:placeholder @attribute=@foo}", "char": 29}, - { "src": "no {placeholder end", "char": 16}, - { "src": "no {$placeholder end", "char": 17}, - { "src": "no {:placeholder end", "char": 20}, - { "src": "no {|placeholder| end", "char": 18}, - { "src": "no {|literal} end", "char": 17}, - { "src": "no {|literal or placeholder end", "char": 31}, - { "src": ".local bar = {|foo|} {{_}}", "char": 7}, - { "src": ".local #bar = {|foo|} {{_}}", "char": 7}, - { "src": ".local $bar {|foo|} {{_}}", "char": 12}, - { "src": ".local $bar = |foo| {{_}}", "char": 14}, - { "src": ".match {#foo} * {{foo}}", "char": 8}, - { "src": ".match {} * {{foo}}", "char": 8}, - { "src": ".match {|foo| :x} {|bar| :x} ** {{foo}}", "char": 30}, - { "src": ".match * {{foo}}", "char": 7}, - { "src": ".match {|x| :x} * foo", "char": 21}, - { "src": ".match {|x| :x} * {{foo}} extra", "char": 31}, - { "src": ".match |x| * {{foo}}", "char": 7}, - { "src": ".match {|foo| :string} o:ne {{one}} * {{other}}", "char": 24, "comment" : "tests for ':' in unquoted literals (not allowed)" }, - { "src": ".match {|foo| :string} one: {{one}} * {{other}}", "char": 26, "comment" : "tests for ':' in unquoted literals (not allowed)" }, - { "src": ".local $foo = {|42| :number option=a:b} {{bar {$foo}}}", "char": 36, "comment" : "tests for ':' in unquoted literals (not allowed)" }, - { "src": ".local $foo = {|42| :number option=a:b:c} {{bar {$foo}}}", "char": 36, "comment" : "tests for ':' in unquoted literals (not allowed)" }, - { "src": "{$bar:foo}", "char": 5, "comment" : "tests for ':' in unquoted literals (not allowed)" }, - { - "src": ".match {1} {{_}}", - "char": 12, - "comment": "Disambiguating a wrong .match from an unsupported statement" - } -] diff --git a/icu4j/main/common_tests/src/test/java/com/ibm/icu/dev/test/message2/Mf2FeaturesTest.java b/icu4j/main/common_tests/src/test/java/com/ibm/icu/dev/test/message2/Mf2FeaturesTest.java index 4bd787416b27..77c0fa3604d7 100644 --- a/icu4j/main/common_tests/src/test/java/com/ibm/icu/dev/test/message2/Mf2FeaturesTest.java +++ b/icu4j/main/common_tests/src/test/java/com/ibm/icu/dev/test/message2/Mf2FeaturesTest.java @@ -330,7 +330,7 @@ public void testSpecialPluralWithDecimals() { message = ".local $amount = {$count :number}\n" + ".match {$amount :number}\n" + " 1 {{I have {$amount} dollar.}}\n" - + " * {{I have {$amount} dollars.}}\n"; + + " * {{I have {$amount} dollars.}}"; TestUtils.runTestCase(new TestCase.Builder() .pattern(message) .locale("en-US") @@ -340,7 +340,7 @@ public void testSpecialPluralWithDecimals() { message = ".local $amount = {$count :number minimumFractionDigits=2}\n" + ".match {$amount :number minimumFractionDigits=2}\n" + " one {{I have {$amount} dollar.}}\n" - + " * {{I have {$amount} dollars.}}\n"; + + " * {{I have {$amount} dollars.}}"; TestUtils.runTestCase(new TestCase.Builder() .pattern(message) .locale("en-US") @@ -369,7 +369,7 @@ public void testDefaultFunctionAndOptions() { public void testSimpleSelection() { String message = ".match {$count :number}\n" + " 1 {{You have one notification.}}\n" - + " * {{You have {$count} notifications.}}\n"; + + " * {{You have {$count} notifications.}}"; TestUtils.runTestCase(new TestCase.Builder() .pattern(message) @@ -440,7 +440,7 @@ public void testLocaleVariableWithSelect() { + ".local $exp = {$expDate :date year=numeric month=short day=numeric weekday=short}\n" + ".match {$count :number}\n" + " 1 {{Your ticket expires on {$exp}.}}\n" - + " * {{Your {$count} tickets expire on {$exp}.}}\n"; + + " * {{Your {$count} tickets expire on {$exp}.}}"; TestUtils.runTestCase(new TestCase.Builder() .pattern(message) diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFParser.java b/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFParser.java index 4a88bd04e1d6..073aafd8ef32 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFParser.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFParser.java @@ -65,7 +65,6 @@ private MFDataModel.Message parseImpl() throws MFParseException { MFDataModel.Pattern pattern = getPattern(); result = new MFDataModel.PatternMessage(new ArrayList<>(), pattern); } - skipOptionalWhitespaces(); checkCondition(input.atEnd(), "Content detected after the end of the message."); new MFDataModelValidator(result).validate(); return result; @@ -174,14 +173,30 @@ private MFDataModel.Expression getPlaceholder() throws MFParseException { return result; } - private MFDataModel.Annotation getAnnotation() throws MFParseException { + private MFDataModel.Annotation getAnnotation(boolean whitespaceRequired) throws MFParseException { int position = input.getPosition(); - skipOptionalWhitespaces(); + // Handle absent annotation first (before parsing mandatory whitespace) int cp = input.peekChar(); + if (cp == '}') { + return null; + } + + int whitespaceCount = 0; + if (whitespaceRequired) { + whitespaceCount = skipMandatoryWhitespaces(); + } else { + whitespaceCount = skipOptionalWhitespaces(); + } + + cp = input.peekChar(); switch (cp) { - case '}': + case '}': { + // No annotation -- push the whitespace back, + // in case it's the required whitespace before an attribute + input.backup(whitespaceCount); return null; + } case ':': // annotation, function // abnf: function = ":" identifier *(s option) input.readCodePoint(); // Consume the sigil @@ -232,13 +247,13 @@ private MFDataModel.Expression getLiteralExpression() throws MFParseException { checkCondition(literal != null, "Literal expression expected."); MFDataModel.Annotation annotation = null; - int wsCount = skipWhitespaces(); - if (wsCount > 0) { // we might have an annotation - annotation = getAnnotation(); + boolean hasWhitespace = StringUtils.isWhitespace(input.peekChar()); + if (hasWhitespace) { // we might have an annotation + annotation = getAnnotation(true); if (annotation == null) { // We had some spaces, but no annotation. // So we put (some) back for the possible attributes. - input.backup(1); + // input.backup(1); } } @@ -251,7 +266,7 @@ private MFDataModel.Expression getLiteralExpression() throws MFParseException { // abnf: variable-expression = "{" [s] variable [s annotation] *(s attribute) [s] "}" private MFDataModel.VariableExpression getVariableExpression() throws MFParseException { MFDataModel.VariableRef variableRef = getVariableRef(); - MFDataModel.Annotation annotation = getAnnotation(); + MFDataModel.Annotation annotation = getAnnotation(true); List attributes = getAttributes(); // Variable without a function, for example {$foo} return new MFDataModel.VariableExpression(variableRef, annotation, attributes); @@ -259,7 +274,7 @@ private MFDataModel.VariableExpression getVariableExpression() throws MFParseExc // abnf: annotation-expression = "{" [s] annotation *(s attribute) [s] "}" private MFDataModel.Expression getAnnotationExpression() throws MFParseException { - MFDataModel.Annotation annotation = getAnnotation(); + MFDataModel.Annotation annotation = getAnnotation(false); List attributes = getAttributes(); if (annotation instanceof MFDataModel.FunctionAnnotation) { @@ -286,6 +301,9 @@ private MFDataModel.Markup getMarkup() throws MFParseException { MFDataModel.Annotation annotation = getMarkupAnnotation(); List attributes = getAttributes(); + // Parse optional whitespace after attribute list + skipOptionalWhitespaces(); + cp = input.peekChar(); if (cp == '/') { kind = MFDataModel.Markup.Kind.STANDALONE; @@ -397,16 +415,28 @@ private String getIdentifier() throws MFParseException { // abnf helper: *(s option) private Map getOptions() throws MFParseException { Map options = new LinkedHashMap<>(); + boolean first = true; + int skipCount = 0; while (true) { MFDataModel.Option option = getOption(); if (option == null) { break; } + // function = ":" identifier *(s option) + checkCondition(first || skipCount != 0, + "Expected whitespace before option " + option.name); + first = false; if (options.containsKey(option.name)) { error("Duplicated option '" + option.name + "'"); } options.put(option.name, option); + // Can't just call skipMandatoryWhitespaces() here, because it + // might be the last option. So check for whitespace when + // parsing the next option instead. + skipCount = skipOptionalWhitespaces(); } + // Restore the last chunk of whitespace in case there's an attribute following + input.backup(skipCount); return options; } @@ -422,7 +452,7 @@ private MFDataModel.Option getOption() throws MFParseException { skipOptionalWhitespaces(); int cp = input.readCodePoint(); checkCondition(cp == '=', "Expected '='"); - // skipOptionalWhitespaces(); + skipOptionalWhitespaces(); MFDataModel.LiteralOrVariableRef litOrVar = getLiteralOrVariableRef(); if (litOrVar == null) { error("Options must have a value. An empty string should be quoted."); @@ -478,8 +508,9 @@ private MFDataModel.Literal getQuotedLiteral() throws MFParseException { result.appendCodePoint(cp); } else if (cp == '\\') { cp = input.readCodePoint(); - checkCondition(cp == '|', "Invalid escape sequence, only \"\\|\" is valid here"); - result.appendCodePoint('|'); + boolean isValidEscape = cp == '|' || cp == '\\' || cp == '{' || cp == '}'; + checkCondition(isValidEscape, "Invalid escape sequence inside quoted literal"); + result.appendCodePoint(cp); } else { break; } @@ -511,13 +542,14 @@ private MFDataModel.Literal getNumberLiteral() { return null; } - private void skipMandatoryWhitespaces() throws MFParseException { + private int skipMandatoryWhitespaces() throws MFParseException { int count = skipWhitespaces(); checkCondition(count > 0, "Space expected"); + return count; } - private void skipOptionalWhitespaces() { - skipWhitespaces(); + private int skipOptionalWhitespaces() { + return skipWhitespaces(); } private int skipWhitespaces() { @@ -554,13 +586,11 @@ private MFDataModel.Message getComplexMessage() throws MFParseException { } else { // Expect {{...}} or end of message skipOptionalWhitespaces(); int cp = input.peekChar(); - if (cp == EOF) { - // Only declarations, no pattern - return new MFDataModel.PatternMessage(declarations, null); - } else { - MFDataModel.Pattern pattern = getQuotedPattern(); - return new MFDataModel.PatternMessage(declarations, pattern); - } + // complex-message = *(declaration [s]) complex-body + checkCondition(cp != EOF, "Expected a quoted pattern or .match; got end-of-input"); + MFDataModel.Pattern pattern = getQuotedPattern(); + checkCondition(input.atEnd(), "Content detected after the end of the message."); + return new MFDataModel.PatternMessage(declarations, pattern); } } @@ -576,7 +606,11 @@ private MFDataModel.SelectMessage getMatch(List declara // Look for selectors List expressions = new ArrayList<>(); while (true) { - skipMandatoryWhitespaces(); + // Whitespace not required between selectors: + // match 1*([s] selector) + // Whitespace not required before first variant: + // matcher = match-statement 1*([s] variant) + skipOptionalWhitespaces(); MFDataModel.Expression expression = getPlaceholder(); if (expression == null) { break; @@ -597,6 +631,7 @@ private MFDataModel.SelectMessage getMatch(List declara } variants.add(variant); } + checkCondition(input.atEnd(), "Content detected after the end of the message."); return new MFDataModel.SelectMessage(declarations, expressions, variants); } @@ -613,7 +648,12 @@ private MFDataModel.Variant getVariant() throws MFParseException { } keys.add(key); } - skipOptionalWhitespaces(); + // Only want to skip whitespace if we parsed at least one key -- + // otherwise, we might fail to catch trailing whitespace at the end of + // the message, which is a parse error + if (!keys.isEmpty()) { + skipOptionalWhitespaces(); + } if (input.atEnd()) { checkCondition( keys.isEmpty(), "After selector keys it is mandatory to have a pattern."); @@ -624,17 +664,26 @@ private MFDataModel.Variant getVariant() throws MFParseException { } private MFDataModel.LiteralOrCatchallKey getKey(boolean requireSpaces) throws MFParseException { + int cp = input.peekChar(); + // Whitespace not required between last key and pattern: + // variant = key *(s key) [s] quoted-pattern + if (cp == '{') { + return null; + } + int skipCount = 0; if (requireSpaces) { - skipMandatoryWhitespaces(); + skipCount = skipMandatoryWhitespaces(); } else { - skipOptionalWhitespaces(); + skipCount = skipOptionalWhitespaces(); } - int cp = input.peekChar(); + cp = input.peekChar(); if (cp == '*') { input.readCodePoint(); // consume the '*' return new MFDataModel.CatchallKey(); } if (cp == EOF) { + // Restore whitespace, in order to detect the error case of whitespace at the end of a message + input.backup(skipCount); return null; } return getLiteral(); @@ -666,14 +715,11 @@ private MFDataModel.Declaration getDeclaration() throws MFParseException { skipMandatoryWhitespaces(); expression = getPlaceholder(); String inputVarName = null; - if (expression instanceof MFDataModel.VariableExpression) { - inputVarName = ((MFDataModel.VariableExpression) expression).arg.name; - } - if (expression instanceof MFDataModel.VariableExpression) { - return new MFDataModel.InputDeclaration( - inputVarName, (MFDataModel.VariableExpression) expression); - } - break; + checkCondition(expression instanceof MFDataModel.VariableExpression, + "Variable expression required in .input declaration"); + inputVarName = ((MFDataModel.VariableExpression) expression).arg.name; + return new MFDataModel.InputDeclaration(inputVarName, + (MFDataModel.VariableExpression) expression); case "local": // abnf: local-declaration = local s variable [s] "=" [s] expression skipMandatoryWhitespaces(); @@ -696,6 +742,17 @@ private MFDataModel.Declaration getDeclaration() throws MFParseException { List expressions = new ArrayList<>(); while (true) { skipOptionalWhitespaces(); + // Look ahead to detect the end of the unsupported statement + // (next token either begins a placeholder or begins a complex body) + cp = input.readCodePoint(); + int cp1 = input.readCodePoint(); + if (cp == '{' && cp1 == '{') { + // End of unsupported statement + input.backup(2); + break; + } else { + input.backup(2); + } expression = getPlaceholder(); // This also covers != null if (expression instanceof MFDataModel.VariableExpression) { diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/CoreTest.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/CoreTest.java index 66c40ccfbdc9..3305f2435a50 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/CoreTest.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/CoreTest.java @@ -14,14 +14,27 @@ @SuppressWarnings({"static-method", "javadoc"}) @RunWith(JUnit4.class) public class CoreTest extends CoreTestFmwk { - private static final String JSON_FILE = "test-core.json"; + private static final String[] JSON_FILES = {"alias-selector-annotations.json", + "duplicate-declarations.json", + "invalid-options.json", + "markup.json", + "matches-whitespace.json", + "reserved-syntax.json", + "resolution-errors.json", + "runtime-errors.json", + "spec/test-core.json", + "syntax-errors-diagnostics.json", + "tricky-declarations.json", + "valid-tests.json"}; @Test public void test() throws Exception { - try (Reader reader = TestUtils.jsonReader(JSON_FILE)) { - Unit[] unitList = TestUtils.GSON.fromJson(reader, Unit[].class); - for (Unit unit : unitList) { - TestUtils.runTestCase(unit); + for (String jsonFile : JSON_FILES) { + try (Reader reader = TestUtils.jsonReader(jsonFile)) { + Unit[] unitList = TestUtils.GSON.fromJson(reader, Unit[].class); + for (Unit unit : unitList) { + TestUtils.runTestCase(unit); + } } } } diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/CustomFormatterPersonTest.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/CustomFormatterPersonTest.java index 0fec7a46b4e5..97dcfb3f287d 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/CustomFormatterPersonTest.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/CustomFormatterPersonTest.java @@ -165,7 +165,7 @@ public void testCustomFunctionsComplexMessage() { + " * 0 {{{$hostName} does not give a party.}}\n" + " * 1 {{{$hostName} invites {$guestName} to their party.}}\n" + " * 2 {{{$hostName} invites {$guestName} and one other person to their party.}}\n" - + " * * {{{$hostName} invites {$guestName} and {$guestsOther} other people to their party.}}\n"; + + " * * {{{$hostName} invites {$guestName} and {$guestsOther} other people to their party.}}"; TestUtils.runTestCase(CUSTOM_FUNCTION_REGISTRY, new TestCase.Builder() .pattern(message) diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/DataModelErrorsTest.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/DataModelErrorsTest.java index 162fade64c92..d8ba5699c260 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/DataModelErrorsTest.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/DataModelErrorsTest.java @@ -19,20 +19,23 @@ @SuppressWarnings({"static-method", "javadoc"}) @RunWith(JUnit4.class) public class DataModelErrorsTest extends CoreTestFmwk { - private static final String JSON_FILE = "data-model-errors.json"; + private static final String[] JSON_FILES = {"spec/data-model-errors.json", + "more-data-model-errors.json"}; @Test public void test() throws Exception { - try (Reader reader = TestUtils.jsonReader(JSON_FILE)) { - Type mapType = new TypeToken>(){/* not code */}.getType(); - Map unitList = TestUtils.GSON.fromJson(reader, mapType); - for (Entry tests : unitList.entrySet()) { - for (String pattern : tests.getValue()) { - try { - MessageFormatter.builder().setPattern(pattern).build().formatToString(null); - fail("Undetected errors in '" + tests.getKey() + "': '" + pattern + "'"); - } catch (Exception e) { - // We expected an error, so it's all good + for (String jsonFile : JSON_FILES) { + try (Reader reader = TestUtils.jsonReader(jsonFile)) { + Type mapType = new TypeToken>(){/* not code */}.getType(); + Map unitList = TestUtils.GSON.fromJson(reader, mapType); + for (Entry tests : unitList.entrySet()) { + for (String pattern : tests.getValue()) { + try { + MessageFormatter.builder().setPattern(pattern).build().formatToString(null); + fail("Undetected errors in '" + tests.getKey() + "': '" + pattern + "'"); + } catch (Exception e) { + // We expected an error, so it's all good + } } } } diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/FunctionsTest.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/FunctionsTest.java index 6a3e15df0a09..c76c60a50327 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/FunctionsTest.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/FunctionsTest.java @@ -18,16 +18,19 @@ @SuppressWarnings({"static-method", "javadoc"}) @RunWith(JUnit4.class) public class FunctionsTest extends CoreTestFmwk { - private static final String JSON_FILE = "test-functions.json"; + private static final String[] JSON_FILES = {"spec/test-functions.json", + "more-functions.json"}; @Test public void test() throws Exception { - try (Reader reader = TestUtils.jsonReader(JSON_FILE)) { - Type mapType = new TypeToken>(){/* not code */}.getType(); - Map unitList = TestUtils.GSON.fromJson(reader, mapType); - for (Entry testGroup : unitList.entrySet()) { - for (Unit unit : testGroup.getValue()) { - TestUtils.runTestCase(unit); + for (String jsonFile : JSON_FILES) { + try (Reader reader = TestUtils.jsonReader(jsonFile)) { + Type mapType = new TypeToken>(){/* not code */}.getType(); + Map unitList = TestUtils.GSON.fromJson(reader, mapType); + for (Entry testGroup : unitList.entrySet()) { + for (Unit unit : testGroup.getValue()) { + TestUtils.runTestCase(unit); + } } } } diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/IcuFunctionsTest.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/IcuFunctionsTest.java index 7930e6ccb18f..bd0af39d085c 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/IcuFunctionsTest.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/IcuFunctionsTest.java @@ -21,13 +21,6 @@ public class IcuFunctionsTest extends CoreTestFmwk { private static final String JSON_FILE = "icu-test-functions.json"; - // Some default parameters for all messages, to use if the message does not have its own - private static final Map ARGS = - Args.of( - "user", "John", - "exp", new Date(2024 - 1900, 7, 3, 21, 43, 57), // Aug 3, 2024, at 9:43:57 pm - "tsOver", "full"); - @Test public void test() throws Exception { try (Reader reader = TestUtils.jsonReader(JSON_FILE)) { @@ -38,7 +31,7 @@ public void test() throws Exception { Map unitList = TestUtils.GSON.fromJson(reader, mapType); for (Entry testGroup : unitList.entrySet()) { for (Unit unit : testGroup.getValue()) { - TestUtils.runTestCase(unit, ARGS); + TestUtils.runTestCase(unit); } } } diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/MessageFormat2Test.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/MessageFormat2Test.java index 33005b33c893..6ac37d6085fd 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/MessageFormat2Test.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/MessageFormat2Test.java @@ -131,7 +131,7 @@ public void testPlural() { String message = "" + ".match {$count :number}\n" + " 1 {{You have one notification.}}\n" - + " * {{You have {$count} notifications.}}\n"; + + " * {{You have {$count} notifications.}}"; MessageFormatter mf2 = MessageFormatter.builder() .setPattern(message) @@ -154,7 +154,7 @@ public void testPluralOrdinal() { + " one {{You got in the {$place}st place}}\n" + " two {{You got in the {$place}nd place}}\n" + " few {{You got in the {$place}rd place}}\n" - + " * {{You got in the {$place}th place}}\n" + + " * {{You got in the {$place}th place}}" ; MessageFormatter mf2 = MessageFormatter.builder() @@ -336,7 +336,7 @@ public void testPluralWithOffset() { + " 1 {{Anna}}\n" + " 2 {{Anna and Bob}}\n" + " one {{Anna, Bob, and {$count :number icu:offset=2} other guest}}\n" - + " * {{Anna, Bob, and {$count :number icu:offset=2} other guests}}\n"; + + " * {{Anna, Bob, and {$count :number icu:offset=2} other guests}}"; MessageFormatter mf2 = MessageFormatter.builder() .setPattern(message) .build(); @@ -365,7 +365,7 @@ public void testPluralWithOffsetAndLocalVar() { + " 1 {{Anna}}\n" + " 2 {{Anna and Bob}}\n" + " one {{Anna, Bob, and {$foo} other guest}}\n" - + " * {{Anna, Bob, and {$foo} other guests}}\n"; + + " * {{Anna, Bob, and {$foo} other guests}}"; MessageFormatter mf2 = MessageFormatter.builder() .setPattern(message) .build(); @@ -393,7 +393,7 @@ public void testPluralWithOffsetAndLocalVar2() { + ".match {$foo :number}\n" // should "inherit" the offset + " 1 {{Last dollar}}\n" + " one {{{$foo} dollar}}\n" - + " * {{{$foo} dollars}}\n"; + + " * {{{$foo} dollars}}"; MessageFormatter mf2 = MessageFormatter.builder() .setPattern(message) .build(); @@ -415,7 +415,7 @@ public void testPluralWithOffsetAndLocalVar2Options() { + ".match {$foo :number}\n" // should "inherit" the offset + " 1 {{Last dollar}}\n" + " one {{{$foo} dollar}}\n" - + " * {{{$foo} dollars}}\n"; + + " * {{{$foo} dollars}}"; MessageFormatter mf2 = MessageFormatter.builder() .setPattern(message) .build(); @@ -436,7 +436,7 @@ public void testLoopOnLocalVars() { + ".local $foo = {$baz :number}\n" + ".local $bar = {$foo}\n" + ".local $baz = {$bar}\n" - + "{{The message uses {$baz} and works}}\n"; + + "{{The message uses {$baz} and works}}"; // Circular references on variables is now detected. // So we check that this throws (see expected in the @Test above) MessageFormatter.builder() @@ -451,7 +451,7 @@ public void testVariableOptionsInSelector() { + " 1 {{A}}\n" + " 2 {{A and B}}\n" + " one {{A, B, and {$count :number icu:offset=$delta} more character}}\n" - + " * {{A, B, and {$count :number icu:offset=$delta} more characters}}\n"; + + " * {{A, B, and {$count :number icu:offset=$delta} more characters}}"; MessageFormatter mfVar = MessageFormatter.builder() .setPattern(messageVar) .build(); @@ -468,7 +468,7 @@ public void testVariableOptionsInSelector() { + ".match {$count :number icu:offset=$delta}\n" + " 1 {{Exactly 1}}\n" + " 2 {{Exactly 2}}\n" - + " * {{Count = {$count :number icu:offset=$delta} and delta={$delta}.}}\n"; + + " * {{Count = {$count :number icu:offset=$delta} and delta={$delta}.}}"; MessageFormatter mfVar2 = MessageFormatter.builder() .setPattern(messageVar2) .build(); @@ -509,7 +509,7 @@ public void testVariableOptionsInSelectorWithLocalVar() { + " 1 {{A}}\n" + " 2 {{A and B}}\n" + " one {{A, B, and {$offCount} more character}}\n" - + " * {{A, B, and {$offCount} more characters}}\n"; + + " * {{A, B, and {$offCount} more characters}}"; MessageFormatter mfFix = MessageFormatter.builder() .setPattern(messageFix) .build(); @@ -524,7 +524,7 @@ public void testVariableOptionsInSelectorWithLocalVar() { + " 1 {{A}}\n" + " 2 {{A and B}}\n" + " one {{A, B, and {$offCount} more character}}\n" - + " * {{A, B, and {$offCount} more characters}}\n"; + + " * {{A, B, and {$offCount} more characters}}"; MessageFormatter mfVar = MessageFormatter.builder() .setPattern(messageVar) .build(); @@ -542,7 +542,7 @@ public void testVariableOptionsInSelectorWithLocalVar() { + ".match {$offCount :number}\n" + " 1 {{Exactly 1}}\n" + " 2 {{Exactly 2}}\n" - + " * {{Count = {$count}, OffCount = {$offCount}, and delta={$delta}.}}\n"; + + " * {{Count = {$count}, OffCount = {$offCount}, and delta={$delta}.}}"; MessageFormatter mfVar2 = MessageFormatter.builder() .setPattern(messageVar2) .build(); diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/SerializationTest.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/SerializationTest.java index 01e18c586df4..2655d42569fe 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/SerializationTest.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/SerializationTest.java @@ -43,12 +43,12 @@ public void test() throws Exception { + "1 {{You got the gold medal}}\n" + "2 {{You got the silver medal}}\n" + "3 {{You got the bronze medal}}\n" - + "few {{You fininshed in the {$place}rd place}}\n", + + "few {{You fininshed in the {$place}rd place}}", ".match {$fileCount :number} {$folderCount :number}\n" + "* * {{You deleted {$fileCount} files in {$folderCount} folders}}\n" + "one one {{You deleted {$fileCount} file in {$folderCount} folder}}\n" + "one * {{You deleted {$fileCount} file in {$folderCount} folders}}\n" - + "* one {{You deleted {$fileCount} files in {$folderCount} folder}}\n", + + "* one {{You deleted {$fileCount} files in {$folderCount} folder}}", "{$count :number minimumFractionDigits=2} dollars", "{$count :number minimumFractionDigits=3} dollars", "{|3.1415| :number minimumFractionDigits=5} dollars", @@ -57,7 +57,7 @@ public void test() throws Exception { + ".match {$c}\n" + "one {{{$c} dollar}}\n" + "* {{{$c} dollars}}", - ".local $c = {$count} .foobar |asd asd asd asd| {$bar1} {$bar2} {$bar3} .local $b = {$bar} {{Foo bar}}\n", + ".local $c = {$count} .foobar |asd asd asd asd| {$bar1} {$bar2} {$bar3} .local $b = {$bar} {{Foo bar}}", ".local $c = {1 :number minimumFractionDigits=2}\n" + ".match {$c}\n" + "one {{{$c} dollar}}\n" diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/SyntaxErrorsTest.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/SyntaxErrorsTest.java index 30ad47f189a2..204f79555161 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/SyntaxErrorsTest.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/SyntaxErrorsTest.java @@ -15,18 +15,21 @@ @SuppressWarnings({"static-method", "javadoc"}) @RunWith(JUnit4.class) public class SyntaxErrorsTest extends CoreTestFmwk { - private static final String JSON_FILE = "syntax-errors.json"; + private static final String[] JSON_FILES = {"more-syntax-errors.json", + "spec/syntax-errors.json"}; @Test public void test() throws Exception { - try (Reader reader = TestUtils.jsonReader(JSON_FILE)) { - String[] srcList = TestUtils.GSON.fromJson(reader, String[].class); - for (String source : srcList) { - try { - MessageFormatter.builder().setPattern(source).build(); - fail("Pattern expected to fail, but didn't: '" + source + "'"); - } catch (Exception e) { - // If we get here it is fine + for (String jsonFile : JSON_FILES) { + try (Reader reader = TestUtils.jsonReader(jsonFile)) { + String[] srcList = TestUtils.GSON.fromJson(reader, String[].class); + for (String source : srcList) { + try { + MessageFormatter.builder().setPattern(source).build(); + fail("Pattern expected to fail, but didn't: '" + source + "'"); + } catch (Exception e) { + // If we get here it is fine + } } } } diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/TestUtils.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/TestUtils.java index ae8868d03f20..572da833e67a 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/TestUtils.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/TestUtils.java @@ -14,6 +14,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Date; import java.util.Locale; import java.util.Map; @@ -71,6 +72,42 @@ private static String reportCase(TestCase testCase) { // ======= Same functionality with Unit, usable with JSON ======== + static void rewriteDates(Map params) { + // For each value in `params` that's a map with the single key + // `date` and a double value d, + // return a map with that value changed to Date(d) + // In JSON this looks like: + // "params": {"exp": { "date": 1722746637000 } } + for (Map.Entry pair : params.entrySet()) { + if (pair.getValue() instanceof Map) { + Map innerMap = (Map) pair.getValue(); + if (innerMap.size() == 1 && innerMap.containsKey("date") && innerMap.get("date") instanceof Double) { + Long dateValue = Double.valueOf((Double) innerMap.get("date")).longValue(); + params.put(pair.getKey(), new Date(dateValue)); + } + } + } + } + + static void rewriteDecimals(Map params) { + // For each value in `params` that's a map with the single key + // `decimal` and a string value s + // return a map with that value changed to Decimal(s) + // In JSON this looks like: + // "params": {"val": {"decimal": "1234567890123456789.987654321"}}, + for (Map.Entry pair : params.entrySet()) { + if (pair.getValue() instanceof Map) { + Map innerMap = (Map) pair.getValue(); + if (innerMap.size() == 1 && innerMap.containsKey("decimal") + && innerMap.get("decimal") instanceof String) { + String decimalValue = (String) innerMap.get("decimal"); + params.put(pair.getKey(), new com.ibm.icu.math.BigDecimal(decimalValue)); + } + } + } + } + + static boolean expectsErrors(Unit unit) { return unit.errors != null && !unit.errors.isEmpty(); } @@ -80,7 +117,7 @@ static void runTestCase(Unit unit) { } static void runTestCase(Unit unit, Map params) { - if (unit.ignore != null) { + if (unit.ignoreJava != null) { return; } @@ -107,6 +144,8 @@ static void runTestCase(Unit unit, Map params) { MessageFormatter mf = mfBuilder.build(); if (unit.params != null) { params = unit.params; + rewriteDates(params); + rewriteDecimals(params); } String result = mf.formatToString(params); if (expectsErrors(unit)) { @@ -134,10 +173,25 @@ static Reader jsonReader(String jsonFileName) throws URISyntaxException, IOExcep return Files.newBufferedReader(json, StandardCharsets.UTF_8); } - private static Path getTestFile(Class cls, String fileName) throws URISyntaxException { + private static Path getTestFile(Class cls, String fileName) throws URISyntaxException, IOException { String packageName = cls.getPackage().getName().replace('.', '/'); URI getPath = cls.getClassLoader().getResource(packageName).toURI(); Path filePath = Paths.get(getPath); Path json = Paths.get(fileName); - return filePath.resolve(json); + // First, check the top level of the source directory, + // in case we're in a source tarball + Path icuTestdataInSourceDir = filePath.resolve("../../../../../../../../../../../testdata/message2/").normalize(); + Path icuTestdataDir = icuTestdataInSourceDir; + if (!Files.isDirectory(icuTestdataInSourceDir)) { + // If that doesn't exist, check one directory higher, in case we're + // in a checked-out repo + Path icuTestdataInRepo = Paths.get("../").resolve(icuTestdataInSourceDir).normalize(); + if (!Files.isDirectory(icuTestdataInRepo)) { + throw new java.io.FileNotFoundException("Test data directory does not exist: tried " + + icuTestdataInSourceDir + " and " + + icuTestdataInRepo); + } + icuTestdataDir = icuTestdataInSourceDir; + } + return icuTestdataDir.resolve(json); }} diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/Unit.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/Unit.java index facacca75234..3ff18ecd7cc2 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/Unit.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/Unit.java @@ -15,7 +15,7 @@ class Unit { final String locale; final Map params; final String exp; - final String ignore; + final String ignoreJava; final List errors; Unit( @@ -24,14 +24,14 @@ class Unit { String locale, Map params, String exp, - String ignore, + String ignoreJava, List errors) { this.src = src; this.srcs = srcs; this.locale = locale; this.params = params; this.exp = exp; - this.ignore = ignore; + this.ignoreJava = ignoreJava; this.errors = errors; } @@ -80,7 +80,7 @@ public Unit merge(Unit other) { String newLocale = other.locale != null ? other.locale : this.locale; Map newParams = other.params != null ? other.params : this.params; String newExp = other.exp != null ? other.exp : this.exp; - String newIgnore = other.ignore != null ? other.ignore : this.ignore; + String newIgnore = other.ignoreJava != null ? other.ignoreJava : this.ignoreJava; List newErrors = other.errors != null ? other.errors : this.errors; return new Unit(newSrc, newSrcs, newLocale, newParams, newExp, newIgnore, newErrors); } diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/data-model-errors.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/data-model-errors.json deleted file mode 100644 index 0a6bd67641b6..000000000000 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/data-model-errors.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "Variant Key Mismatch": [ - ".match {$foo :x} * * {{foo}}", - ".match {$foo :x} {$bar :x} * {{foo}}" - ], - "Missing Fallback Variant": [ - ".match {:foo} 1 {{_}}", - ".match {:foo} other {{_}}", - ".match {:foo} {:bar} * 1 {{_}} 1 * {{_}}" - ], - "Missing Selector Annotation": [ - ".match {$foo} one {{one}} * {{other}}", - ".input {$foo} .match {$foo} one {{one}} * {{other}}", - ".local $foo = {$bar} .match {$foo} one {{one}} * {{other}}" - ], - "Duplicate Declaration": [ - ".input {$foo} .input {$foo} {{_}}", - ".input {$foo} .local $foo = {42} {{_}}", - ".local $foo = {42} .input {$foo} {{_}}", - ".local $foo = {:unknown} .local $foo = {42} {{_}}", - ".local $foo = {$bar} .local $bar = {42} {{_}}", - ".local $foo = {$foo} {{_}}", - ".local $foo = {$bar} .local $bar = {$baz} {{_}}", - ".local $foo = {$bar :func} .local $bar = {$baz} {{_}}", - ".local $foo = {42 :func opt=$foo} {{_}}", - ".local $foo = {42 :func opt=$bar} .local $bar = {42} {{_}}" - ], - "Duplicate Option Name": [ - "bad {:placeholder option=x option=x}", - "bad {:placeholder ns:option=x ns:option=y}" - ] -} diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-parser-tests.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-parser-tests.json deleted file mode 100644 index 56e3094e58d7..000000000000 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-parser-tests.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "Simple messages": [ - "", - "Hello", - "Hello world!", - "Hello \\t \\n \\r \\{ world!", - "Hello world {:datetime}", - "Hello world {foo}", - "Hello {0} world", - "Hello {123} world", - "Hello {-123} world", - "Hello {3.1416} world", - "Hello {-3.1416} world", - "Hello {123E+2} world", - "Hello {123E-2} world", - "Hello {123.456E+2} world", - "Hello {123.456E-2} world", - "Hello {-123.456E+2} world", - "Hello {-123.456E-2} world", - "Hello {-123E+2} world", - "Hello {-123E-2} world", - "Hello world {$exp}", - "Hello world {$exp :datetime}", - "Hello world {|2024-02-27| :datetime}", - "Hello world {$exp :datetime style=long} and more", - "Hello world {$exp :function number=1234} and more", - "Hello world {$exp :function unquoted=left } and more", - "Hello world {$exp :function quoted=|Something| } and more", - "Hello world {$exp :function quoted=|Something with spaces| } and more", - "Hello world {$exp :function quoted=|Something with \\| spaces and \\| escapes| } and more", - "Hello world {$exp :function number=1234 unquoted=left quoted=|Something|}", - "Hello world {$exp :function number=1234 unquoted=left quoted=|Something longer|}", - "Hello world {$exp :function number=1234 unquoted=left quoted=|Something \\| longer|}" - ], - "Attributes": [ - "Hello world {$exp}", - "Hello world {$exp @attr}", - "Hello world {$exp @valid @attr=a @attrb=123 @atrn=|foo bar|}", - "Hello world {$exp :date @valid @attr=aaaa @attrb=123 @atrn=|foo bar|}", - "Hello world {$exp :date year=numeric month=long day=numeric int=12 @valid @attr=a @attrb=123 @atrn=|foo bar|}" - ], - "Reserved and private": [ - "Reserved {$exp &foo |something more protected|} and more", - "Reserved {$exp %foo |something quoted \\| inside|} and more", - "{{.starting with dot is OK here}}", - "{{Some string pattern, with {$foo} and {$exp :date style=long}!}}" - ], - "Simple messages, with declarations": [ - ".input {$pi :number}", - ".input {$exp :date}", - ".local $foo = {$exp}", - ".local $foo = {$exp :date}", - ".local $foo = {$exp :date year=numeric month=long day=numeric}", - ".local $bar = {$foo :date month=medium}", - ".something |reserved=| {$foo :date}" - ], - "Multiple declarations in one message": [ - ".input {$a :date} .local $b = {$a :date year=numeric month=long day=numeric} .local $c = {$b :date month=medium} .someting |reserved = \\| and more| {$x :date} {$y :date} {$z :number}", - ".input {$a :date} .local $exp = {$a :date style=full} {{Your card expires on {$exp}!}}" - ] -} diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/syntax-errors.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/syntax-errors.json deleted file mode 100644 index fc4537131c83..000000000000 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/syntax-errors.json +++ /dev/null @@ -1,56 +0,0 @@ -[ - ".", - "{", - "}", - "{}", - "{{", - "{{}", - "{{}}}", - "{|foo| #markup}", - "{{missing end brace}", - "{{missing end braces", - "{{missing end {$braces", - "{{extra}} content", - "empty { } placeholder", - "missing space {42:func}", - "missing space {|foo|:func}", - "missing space {|foo|@bar}", - "missing space {:func@bar}", - "missing space {:func @bar@baz}", - "missing space {:func @bar=42@baz}", - "missing space {+reserved@bar}", - "missing space {&private@bar}", - "bad {:} placeholder", - "bad {\\u0000placeholder}", - "no-equal {|42| :number minimumFractionDigits 2}", - "bad {:placeholder option=}", - "bad {:placeholder option value}", - "bad {:placeholder option:value}", - "bad {:placeholder option}", - "bad {:placeholder:}", - "bad {::placeholder}", - "bad {:placeholder::foo}", - "bad {:placeholder option:=x}", - "bad {:placeholder :option=x}", - "bad {:placeholder option::x=y}", - "bad {$placeholder option}", - "bad {:placeholder @attribute=}", - "bad {:placeholder @attribute=@foo}", - "no {placeholder end", - "no {$placeholder end", - "no {:placeholder end", - "no {|placeholder| end", - "no {|literal} end", - "no {|literal or placeholder end", - ".local bar = {|foo|} {{_}}", - ".local #bar = {|foo|} {{_}}", - ".local $bar {|foo|} {{_}}", - ".local $bar = |foo| {{_}}", - ".match {#foo} * {{foo}}", - ".match {} * {{foo}}", - ".match {|foo| :x} {|bar| :x} ** {{foo}}", - ".match * {{foo}}", - ".match {|x| :x} * foo", - ".match {|x| :x} * {{foo}} extra", - ".match |x| * {{foo}}" -] diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/test-core.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/test-core.json deleted file mode 100644 index 4400b69774ee..000000000000 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/test-core.json +++ /dev/null @@ -1,212 +0,0 @@ -[ - { "src": "hello", "exp": "hello" }, - { "src": "hello {world}", "exp": "hello world" }, - { - "src": "hello { world\t\n}", - "exp": "hello world", - "cleanSrc": "hello {world}" - }, - { - "src": "hello {\u3000world\r}", - "exp": "hello world", - "cleanSrc": "hello {world}" - }, - { "src": "hello {|world|}", "exp": "hello world" }, - { "src": "hello {||}", "exp": "hello " }, - { - "src": "hello {$place}", - "params": { "place": "world" }, - "exp": "hello world" - }, - { - "src": "hello {$place-.}", - "params": { "place-.": "world" }, - "exp": "hello world" - }, - { - "src": "hello {$place}", - "errorsJs": [{ "type": "unresolved-var" }], - "exp": "hello {$place}" - }, - { - "src": "{$one} and {$two}", - "params": { "one": 1.3, "two": 4.2 }, - "exp": "1.3 and 4.2" - }, - { - "src": "{$one} et {$two}", - "locale": "fr", - "params": { "one": 1.3, "two": 4.2 }, - "exp": "1,3 et 4,2" - }, - { "src": ".local $foo = {bar} {{bar {$foo}}}", "exp": "bar bar" }, - { "src": ".local $foo = {|bar|} {{bar {$foo}}}", "exp": "bar bar" }, - { - "src": ".local $foo = {|bar|} {{bar {$foo}}}", - "params": { "foo": "foo" }, - "exp": "bar bar" - }, - { - "src": ".local $foo = {$bar} {{bar {$foo}}}", - "params": { "bar": "foo" }, - "exp": "bar foo" - }, - { - "src": ".local $foo = {$baz} .local $bar = {$foo} {{bar {$bar}}}", - "params": { "baz": "foo" }, - "exp": "bar foo" - }, - { - "src": ".input {$foo} {{bar {$foo}}}", - "params": { "foo": "foo" }, - "exp": "bar foo" - }, - { - "src": ".input {$foo} .local $bar = {$foo} {{bar {$bar}}}", - "params": { "foo": "foo" }, - "exp": "bar foo" - }, - { - "src": ".local $foo = {$baz} .local $bar = {$foo} {{bar {$bar}}}", - "params": { "baz": "foo" }, - "exp": "bar foo" - }, - { "src": ".local $x = {42} .local $y = {$x} {{{$x} {$y}}}", "exp": "42 42" }, - { - "src": "{#tag}", - "exp": "", - "parts": [{ "type": "markup", "kind": "open", "name": "tag" }] - }, - { - "src": "{#tag}content", - "exp": "content", - "parts": [ - { "type": "markup", "kind": "open", "name": "tag" }, - { "type": "literal", "value": "content" } - ] - }, - { - "src": "{#ns:tag}content{/ns:tag}", - "exp": "content", - "parts": [ - { "type": "markup", "kind": "open", "name": "ns:tag" }, - { "type": "literal", "value": "content" }, - { "type": "markup", "kind": "close", "name": "ns:tag" } - ] - }, - { - "src": "{/tag}content", - "exp": "content", - "parts": [ - { "type": "markup", "kind": "close", "name": "tag" }, - { "type": "literal", "value": "content" } - ] - }, - { - "src": "{#tag foo=bar}", - "exp": "", - "parts": [ - { - "type": "markup", - "kind": "open", - "name": "tag", - "options": { "foo": "bar" } - } - ] - }, - { - "src": "{#tag foo=bar/}", - "cleanSrc": "{#tag foo=bar /}", - "exp": "", - "parts": [ - { - "type": "markup", - "kind": "standalone", - "name": "tag", - "options": { "foo": "bar" } - } - ] - }, - { - "src": "{#tag a:foo=|foo| b:bar=$bar}", - "params": { "bar": "b a r" }, - "exp": "", - "parts": [ - { - "type": "markup", - "kind": "open", - "name": "tag", - "options": { "a:foo": "foo", "b:bar": "b a r" } - } - ] - }, - { - "src": "{/tag foo=bar}", - "exp": "", - "parts": [ - { - "type": "markup", - "kind": "close", - "name": "tag", - "options": { "foo": "bar" } - } - ] - }, - { - "src": "{42 @foo @bar=13}", - "exp": "42", - "parts": [{ "type": "string", "value": "42" }] - }, - { - "src": "{42 @foo=$bar}", - "exp": "42", - "parts": [{ "type": "string", "value": "42" }] - }, - { - "src": "foo {+reserved}", - "exp": "foo {+}", - "parts": [ - { "type": "literal", "value": "foo " }, - { "type": "fallback", "source": "+" } - ], - "errorsJs": [{ "type": "unsupported-annotation" }] - }, - { - "src": "foo {&private}", - "exp": "foo {&}", - "parts": [ - { "type": "literal", "value": "foo " }, - { "type": "fallback", "source": "&" } - ], - "errorsJs": [{ "type": "unsupported-annotation" }] - }, - { - "src": "foo {?reserved @a @b=$c}", - "exp": "foo {?}", - "parts": [ - { "type": "literal", "value": "foo " }, - { "type": "fallback", "source": "?" } - ], - "errorsJs": [{ "type": "unsupported-annotation" }] - }, - { - "src": ".foo {42} {{bar}}", - "exp": "bar", - "parts": [{ "type": "literal", "value": "bar" }], - "errorsJs": [{ "type": "unsupported-statement" }] - }, - { - "src": ".foo{42}{{bar}}", - "cleanSrc": ".foo {42} {{bar}}", - "exp": "bar", - "parts": [{ "type": "literal", "value": "bar" }], - "errorsJs": [{ "type": "unsupported-statement" }] - }, - { - "src": ".foo |}lit{| {42}{{bar}}", - "cleanSrc": ".foo |}lit{| {42} {{bar}}", - "exp": "bar", - "parts": [{ "type": "literal", "value": "bar" }], - "errorsJs": [{ "type": "unsupported-statement" }] - } -] diff --git a/icu4j/pom.xml b/icu4j/pom.xml index a17662603fd0..eeee1cb8c3fc 100644 --- a/icu4j/pom.xml +++ b/icu4j/pom.xml @@ -279,6 +279,7 @@ ${rootlocation}/.. LICENSE + testdata/ diff --git a/testdata/message2/README.txt b/testdata/message2/README.txt new file mode 100644 index 000000000000..17c7c656c320 --- /dev/null +++ b/testdata/message2/README.txt @@ -0,0 +1,88 @@ +© 2024 and later: Unicode, Inc. and others. +License & terms of use: http://www.unicode.org/copyright.html + +The format of the JSON files in this directory follows the same format as `test-core.json` +in the spec, described in: + +https://github.com/unicode-org/message-format-wg/blob/main/test/README.md + +The `parts` field is not used. + +# JSON extensions + +An additional `comment` field may be present, which is only for human readers. + +A "srcs" field, whose value is an array of strings, may be present instead +of "src". The strings are concatenated to get the message. + +In the "params" field, a date parameter can be expressed as: +{ "date": n } +where n is a number representing a Unix timestamp. + +In the "params" field, a decimal string parameter can be expressed as: +{ "decimal": s } +where s is a string. + +Optional fields, "ignoreJava" and "ignoreCpp" can be used +for tests currently expected to fail in the respective language. +The field may have any value; if it's +present, the test is ignored. (The value can be a comment explaining +why it's expected to fail.) + +Tests in the `spec/` subdirectory are taken from https://github.com/unicode-org/message-format-wg/blob/main/test . +If the contents change upstream, then the corresponding tests in CLDR +need to be updated (also see https://unicode-org.atlassian.net/browse/ICU-22812 ). + +## ICU4J only + +The `cleanSrc` fields is used to represent normalized input (ICU4C has its +own function for normalizing input). + +## ICU4C only + +Additional "char" and "line" fields may be present with integer values, +used for tests expected to trigger a syntax error. +If present, "char" reflects the expected character offset and "line" +reflects the expected line number in the parse error. +The files with "diagnostics" in the name have these fields filled in. + +# ICU4C vs. ICU4J tests + +The following tests are run in both ICU4C and ICU4J: + +* alias-selector-annotations.json +* duplicate-declarations.json +* icu-parser-tests.json + - Two tests removed while single-sourcing tests, because a `{{}}` message body + had to be added to get it to parse in ICU4C, and this broke the test in ICU4J. + These tests are in icu-parser-tests-old.json +* icu-test-functions.json + - Some tests marked as ignored +* icu-test-previous-release.json + - Some tests marked as ignored +* icu-test-selectors.json +* markup.json +* matches-whitespace.json + - Some tests marked as ignored +* more-data-model-errors.json +* more-syntax-errors.json +* reserved-syntax.json + - All tests marked as ignored in Java (resolution errors are suppressed) +* resolution-errors.json + - All tests marked as ignored in Java (resolution errors are suppressed) +* runtime-errors.json + - All tests marked as ignored in Java (message function errors are suppressed) +* syntax-errors-diagnostics.json +* tricky-declarations.json +* valid-tests.json + - Some tests marked as ignored +* spec/* + - Some tests in test-core.json and test-functions.json marked as ignored + +The following tests are only run in ICU4C, either because ICU4J doesn't check +for invalid options, or because ICU4J doesn't report line/column numbers for +parse errors: +* invalid-number-literals-diagnostics.json +* invalid-options.json +* syntax-errors-diagnostics-multiline.json +* syntax-errors-end-of-input.json diff --git a/icu4c/source/test/testdata/message2/alias-selector-annotations.json b/testdata/message2/alias-selector-annotations.json similarity index 100% rename from icu4c/source/test/testdata/message2/alias-selector-annotations.json rename to testdata/message2/alias-selector-annotations.json diff --git a/icu4c/source/test/testdata/message2/duplicate-declarations.json b/testdata/message2/duplicate-declarations.json similarity index 100% rename from icu4c/source/test/testdata/message2/duplicate-declarations.json rename to testdata/message2/duplicate-declarations.json diff --git a/icu4c/source/test/testdata/message2/icu4j/icu-parser-tests.json b/testdata/message2/icu-parser-tests.json similarity index 98% rename from icu4c/source/test/testdata/message2/icu4j/icu-parser-tests.json rename to testdata/message2/icu-parser-tests.json index b9eb24d2330f..0cdfe0a467aa 100644 --- a/icu4c/source/test/testdata/message2/icu4j/icu-parser-tests.json +++ b/testdata/message2/icu-parser-tests.json @@ -55,7 +55,7 @@ ".something |reserved=| {$foo :date} {{}}" ], "Multiple declarations in one message": [ - ".input {$a :date} .local $b = {$a :date year=numeric month=long day=numeric} .local $c = {$b :date month=medium} .someting |reserved = \\| and more| {$x :date} {$y :date} {$z :number} {{}}", - ".input {$a :date} .local $exp = {$a :date style=full} {{Your card expires on {$exp}!}}" + ".input {$a :date} .local $exp = {$a :date style=full} {{Your card expires on {$exp}!}}", + ".input {$a :date} .local $b = {$a :date year=numeric month=long day=numeric} .local $c = {$b :date month=medium} .someting |reserved = \\| and more| {$x :date} {$y :date} {$z :number} {{}}" ] } diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-test-functions.json b/testdata/message2/icu-test-functions.json similarity index 66% rename from icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-test-functions.json rename to testdata/message2/icu-test-functions.json index 9dc118bd07df..2dfd91cb8030 100644 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-test-functions.json +++ b/testdata/message2/icu-test-functions.json @@ -2,71 +2,91 @@ "Date and time formats": [ { "src": "Expires on {$exp}", - "exp": "Expires on 8/3/24, 9:43 PM" + "exp": "Expires on 8/3/24, 9:43 PM", + "comment": "Modified from ICU4J copy to add params (likewise with the other date/time tests); 1722746637000 is 2024-08-03 21:43:57 PDT", + "params": {"exp": { "date": 1722746637000 } } }, { "src": "Expires on {$exp :datetime}", - "exp": "Expires on 8/3/24, 9:43 PM" + "exp": "Expires on 8/3/24, 9:43 PM", + "params": {"exp": { "date": 1722746637000 } } }, { "src": "Expires on {$exp :datetime icu:skeleton=yMMMMdjmsSSEE}", - "exp": "Expires on Sat, August 3, 2024 at 9:43:57.00 PM" + "exp": "Expires on Sat, August 3, 2024 at 9:43:57.00 PM", + "params": {"exp": { "date": 1722746637000 } }, + "ignoreCpp": "ICU-22754 Skeleton option not implemented yet" }, { "src": "Expires on {$exp :datetime dateStyle=full}", - "exp": "Expires on Saturday, August 3, 2024" + "exp": "Expires on Saturday, August 3, 2024", + "params": {"exp": { "date": 1722746637000 } } }, { "src": "Expires on {$exp :datetime dateStyle=long}", - "exp": "Expires on August 3, 2024" + "exp": "Expires on August 3, 2024", + "params": {"exp": { "date": 1722746637000 } } }, { "src": "Expires on {$exp :datetime dateStyle=medium}", - "exp": "Expires on Aug 3, 2024" + "exp": "Expires on Aug 3, 2024", + "params": {"exp": { "date": 1722746637000 } } }, { "src": "Expires on {$exp :datetime timeStyle=long}", - "exp": "Expires on 9:43:57 PM PDT" + "exp": "Expires on 9:43:57 PM PDT", + "params": {"exp": { "date": 1722746637000 } } }, { "src": "Expires on {$exp :datetime timeStyle=medium}", - "exp": "Expires on 9:43:57 PM" + "exp": "Expires on 9:43:57 PM", + "params": {"exp": { "date": 1722746637000 } } }, { "src": "Expires on {$exp :datetime timeStyle=short}", - "exp": "Expires on 9:43 PM" + "exp": "Expires on 9:43 PM", + "params": {"exp": { "date": 1722746637000 } } }, { "src": "Expires on {$exp :datetime dateStyle=full timeStyle=medium}", - "exp": "Expires on Saturday, August 3, 2024 at 9:43:57 PM" + "exp": "Expires on Saturday, August 3, 2024 at 9:43:57 PM", + "params": {"exp": { "date": 1722746637000 } } }, { "src": "Expires on {$exp :datetime year=numeric month=long}", - "exp": "Expires on August 2024" + "exp": "Expires on August 2024", + "params": {"exp": { "date": 1722746637000 } } }, { "src": "Expires on {$exp :datetime year=numeric month=medium day=numeric weekday=long hour=numeric minute=numeric}", - "exp": "Expires on 3 Saturday 2024, 9:43 PM" + "exp": "Expires on 3 Saturday 2024, 9:43 PM", + "params": {"exp": { "date": 1722746637000 } } }, { "comment": "Make sure we ignore date / time fields if needed", "src": "Expires on {$exp :date year=numeric month=medium day=numeric weekday=long hour=numeric minute=numeric}", - "exp": "Expires on 3 Saturday 2024" + "exp": "Expires on 3 Saturday 2024", + "params": {"exp": { "date": 1722746637000 } }, + "ignoreCpp": "ICU-22754 ICU4C doesn't accept field options for `:date` or `:time` -- see spec" }, { "comment": "Make sure we ignore date / time fields if needed", "src": "Expires at {$exp :time year=numeric month=medium day=numeric weekday=long hour=numeric minute=numeric}", - "exp": "Expires at 9:43 PM" + "exp": "Expires at 9:43 PM", + "params": {"exp": { "date": 1722746637000 } }, + "ignoreCpp": "ICU-22754 ICU4C doesn't accept field options for `:date` or `:time` -- see spec" }, { "comment": "Make sure we ignore date / time fields if needed", "src": "Expires at {$exp :time style=long dateStyle=full timeStyle=medium}", - "exp": "Expires at 9:43:57 PM PDT" + "exp": "Expires at 9:43:57 PM PDT", + "params": {"exp": { "date": 1722746637000 } } }, { "comment": "Make sure we ignore date / time fields if needed", "src": "Expires on {$exp :date style=long dateStyle=full timeStyle=medium}", - "exp": "Expires on August 3, 2024" + "exp": "Expires on August 3, 2024", + "params": {"exp": { "date": 1722746637000 } } } ], "Literals" : [ @@ -88,11 +108,13 @@ }, { "src": "Expires at {|2024-07-02T19:23:45Z| :datetime timeStyle=long}", - "exp": "Expires at 7:23:45 PM GMT" + "exp": "Expires at 7:23:45 PM GMT", + "ignoreCpp": "ICU-22754 Time zones not working yet (bug)" }, { "src": "Expires at {|2024-07-02T19:23:45+03:30| :datetime timeStyle=full}", - "exp": "Expires at 7:23:45 PM GMT+03:30" + "exp": "Expires at 7:23:45 PM GMT+03:30", + "ignoreCpp": "ICU-22754 Time zones not working yet (bug)" } ], "Chaining" : [ @@ -105,7 +127,9 @@ ".local $zooExp = {$exp :datetime dateStyle=short timeStyle=$tsOver}\n", "{{Hello John, you want '{$exp}', '{$longExp}', or '{$zooExp}' or even '{$exp :datetime dateStyle=full}'?}}" ], - "exp": "Hello John, you want '9:43 PM', 'August 3, 2024 at 9:43 PM', or '8/3/24, 9:43:57 PM Pacific Daylight Time' or even 'Saturday, August 3, 2024 at 9:43 PM'?" + "exp": "Hello John, you want '9:43 PM', 'August 3, 2024 at 9:43 PM', or '8/3/24, 9:43:57 PM Pacific Daylight Time' or even 'Saturday, August 3, 2024 at 9:43 PM'?", + "params": {"exp": { "date": 1722746637000 }, "user": "John", "tsOver" : "full" }, + "ignoreCpp": "ICU-22754 ICU4C doesn't implement this kind of function composition yet. See https://github.com/unicode-org/message-format-wg/issues/515" }, { "srcs": [ @@ -113,7 +137,8 @@ ".local $longExp = {$exp :datetime month=long weekday=long}\n", "{{Expires on '{$exp}' ('{$longExp}').}}" ], - "exp": "Expires on '8/03/2024' ('Saturday, August 03, 2024')." + "exp": "Expires on '8/03/2024' ('Saturday, August 03, 2024').", + "params": {"exp": { "date": 1722746637000 } } } ], "Number formatter" : [ @@ -125,7 +150,8 @@ { "src": "Format {123456789.9876} number", "locale": "en-IN", - "exp": "Format 12,34,56,789.9876 number" + "exp": "Format 12,34,56,789.9876 number", + "ignoreCpp": "ICU-22754 No default formatting for numbers, so it's formatted as a literal string. Is this in the spec?" }, { "src": "Format {|3.1416|} number", @@ -135,7 +161,8 @@ { "src": "Format {|3.1416|} number", "locale": "ar-AR-u-nu-arab", - "exp": "Format ٣٫١٤١٦ number" + "exp": "Format ٣٫١٤١٦ number", + "ignoreCpp": "ICU-22754 No default formatting for numbers, so it's formatted as a literal string. Is this in the spec?" }, { "src": "Format {3.1415926 :number}", diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-test-previous-release.json b/testdata/message2/icu-test-previous-release.json similarity index 70% rename from icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-test-previous-release.json rename to testdata/message2/icu-test-previous-release.json index 6c235c2a3add..cc863411db95 100644 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-test-previous-release.json +++ b/testdata/message2/icu-test-previous-release.json @@ -19,7 +19,8 @@ { "src": "hello {$place}", "exp": "hello {$place}", - "errorsJs": [{ "type": "missing-var" }] + "errors": [{ "type": "unresolved-var" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, { "src": "{$one} and {$two}", @@ -43,13 +44,15 @@ }, { "src": "hello {|foo| :number}", - "exp": "hello {|foo|}" + "exp": "hello {|foo|}", + "errors": [{ "type": "bad-input" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, { "src": "hello {:number}", - "exp": "hello {|foo|}", + "exp": "hello {:number}", "comment": "This is different from JS, should be an error.", - "errors": [{ "type": "ICU4J: exception" }] + "errors": [{ "type": "bad-input" }] }, { "src": "hello {|4.2| :number minimumFractionDigits=2}", @@ -61,7 +64,7 @@ }, { "src": "hello {|4.2| :number minimumFractionDigits=$foo}", - "params": { "foo": 2f }, + "params": { "foo": 2.0 }, "exp": "hello 4.20" }, { @@ -97,16 +100,18 @@ "exp": "bar 4.20" }, { - "ignore": "Maybe. Because `minimumFractionDigits=foo`", + "ignoreJava": "Maybe. Because `minimumFractionDigits=foo`", "src": ".local $foo = {$bar :number minimumFractionDigits=foo} {{bar {$foo}}}", "params": { "bar": 4.2 }, "exp": "bar 4.2", - "errors": [{ "type": "invalid-type" }] + "errors": [{ "type": "bad-option" }] }, { "src": ".local $foo = {$bar :number} {{bar {$foo}}}", "params": { "bar": "foo" }, - "exp": "bar {|foo|}" + "exp": "bar {$bar}", + "errors": [{ "type": "bad-input" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, { "src": ".local $bar = {$baz} .local $foo = {$bar} {{bar {$foo}}}", @@ -122,9 +127,8 @@ { "src": ".match {$foo :number} 1 {{one}} * {{other}}", "params": { "foo": "1" }, - "comment": "Should this be error? Plural on string?", - "expectedJs": "one", - "exp": "other" + "exp": "one", + "ignoreJava": "See ICU-22809" }, { "src": ".match {$foo :string} 1 {{one}} * {{other}}", @@ -143,7 +147,7 @@ "exp": "one" }, { - "ignore": "Can't pass null in a map", + "ignoreJava": "Can't pass null in a map", "src": ".match {$foo} 1 {{one}} * {{other}}", "params": { "foo": null }, "exp": "other" @@ -210,25 +214,11 @@ }, { "srcJs": ".local $bar = {$none} .match {$foo} one {{one}} * {{{$bar}}}", - "src": ".local $bar = {$none :number} .match {$foo} one {{one}} * {{{$bar}}}", + "src": ".local $bar = {$none :number} .match {$foo :string} one {{one}} * {{{$bar}}}", "params": { "foo": 2 }, - "exp": "{$bar}", - "errors": [{ "type": "missing-var" }] - }, - { - "src": ".local bar = {|foo|} {{$bar}}", - "exp": "{$bar}", - "errors": [{ "type": "missing-char" }, { "type": "missing-var" }] - }, - { - "src": ".local $bar {|foo|} {{$bar}}", - "exp": "foo", - "errors": [{ "type": "missing-char" }] - }, - { - "src": ".local $bar = |foo| {{$bar}}", - "exp": "{$bar}", - "errors": [{ "type": "missing-char" }, { "type": "junk-element" }] + "exp": "{$none}", + "errors": [{ "type": "unresolved-var" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, { "src": "{{#tag}}", @@ -281,25 +271,11 @@ "exp": "no braces 2", "errorsJs": [{ "type": "parse-error" }, { "type": "junk-element" }] }, - { - "src": "{missing end brace", - "exp": "missing end brace", - "errors": [{ "type": "missing-char" }] - }, - { - "src": "{missing end {$brace", - "exp": "missing end {$brace}", - "errors": [{ "type": "missing-char" }, { "type": "missing-char" }, { "type": "missing-var" }] - }, - { - "src": "{{extra}} content", - "exp": "extra", - "errors": [{ "type": "extra-content" }] - }, { "src": "empty { }", "exp": "empty ", - "errors": [{ "type": "parse-error" }, { "type": "junk-element" }] + "errors": [{ "type": "parse-error" }, { "type": "junk-element" }], + "ignoreCpp": "Fallback is unclear. See https://github.com/unicode-org/message-format-wg/issues/703" }, { "src": "bad {:}", @@ -311,80 +287,27 @@ "exp": "bad placeholder", "errorsJs": [{ "type": "parse-error" }, { "type": "extra-content" }, { "type": "junk-element" }] }, - { - "src": "no-equal {|42| :number minimumFractionDigits 2}", - "exp": "no-equal 42.00", - "errors": [{ "type": "missing-char" }] - }, - { - "src": "bad {:placeholder option=}", - "exp": "bad {:placeholder}", - "errors": [{ "type": "empty-token" }, { "type": "missing-func" }] - }, - { - "src": "bad {:placeholder option value}", - "exp": "bad {:placeholder}", - "errors": [{ "type": "missing-char" }, { "type": "missing-func" }] - }, - { - "src": "{bad {:placeholder option}}", - "exp": "bad {:placeholder}", - "errors": [{ "type": "missing-char" }, { "type": "empty-token" }, { "type": "missing-func" }] - }, { "src": "{bad {$placeholder option}}", "exp": "bad {$placeholder}", - "errors": [{ "type": "extra-content" }, { "type": "extra-content" }, { "type": "missing-var" }] - }, - { - "src": "{no {$placeholder end}", - "exp": "no {$placeholder}", - "errors": [{ "type": "extra-content" }, { "type": "missing-var" }] - }, - { - "src": ".match {} * {{foo}}", - "exp": "foo", - "errors": [{ "type": "parse-error" }, { "type": "bad-selector" }, { "type": "junk-element" }] - }, - { - "src": ".match {#foo} * {{foo}}", - "exp": "foo", - "errors": [{ "type": "bad-selector" }] + "errors": [{ "type": "extra-content" }, { "type": "extra-content" }, { "type": "missing-var" }], + "ignoreCpp": "Fallback is unclear. See https://github.com/unicode-org/message-format-wg/issues/703" }, { - "src": ".match {|foo|} *{{foo}}", - "exp": "foo", - "errors": [{ "type": "missing-char" }] - }, - { - "src": ".match * {{foo}}", - "exp": "foo", - "errors": [{ "type": "empty-token" }] - }, - { - "src": ".match {|x|} * foo", - "exp": "", - "errors": [{ "type": "key-mismatch" }, { "type": "missing-char" }] - }, - { - "src": ".match {|x|} * {{foo}} extra", - "exp": "foo", - "errors": [{ "type": "extra-content" }] - }, - { - "src": ".match |x| * {{foo}}", - "exp": "", - "errors": [{ "type": "empty-token" }, { "type": "extra-content" }] + "src": ".match {|foo| :string} *{{foo}}", + "exp": "foo" }, { - "src": ".match {$foo} * * {{foo}}", + "src": ".match {$foo :string} * * {{foo}}", "exp": "foo", - "errors": [{ "type": "key-mismatch" }, { "type": "missing-var" }] + "errors": [{ "type": "key-mismatch" }, { "type": "missing-var" }], + "ignoreCpp": "Fallback is unclear. See https://github.com/unicode-org/message-format-wg/issues/735" }, { - "src": ".match {$foo} {$bar} * {{foo}}", + "src": ".match {$foo :string} {$bar :string} * {{foo}}", "exp": "foo", - "errors": [{ "type": "key-mismatch" }, { "type": "missing-var" }, { "type": "missing-var" }] + "errors": [{ "type": "key-mismatch" }, { "type": "missing-var" }, { "type": "missing-var" }], + "ignoreCpp": "Fallback is unclear. See https://github.com/unicode-org/message-format-wg/issues/735" } ] diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-test-selectors.json b/testdata/message2/icu-test-selectors.json similarity index 98% rename from icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-test-selectors.json rename to testdata/message2/icu-test-selectors.json index 1f237254f198..efe67e7c4c9e 100644 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-test-selectors.json +++ b/testdata/message2/icu-test-selectors.json @@ -48,7 +48,7 @@ "1 {{You got the gold medal}}\n", "2 {{You got the silver medal}}\n", "3 {{You got the bronze medal}}\n", - "few {{You finished in the {$place}rd place}}\n" + "few {{You finished in the {$place}rd place}}" ] }, "variations" : [ @@ -70,7 +70,7 @@ " * * {{You found {$fileCount} files in {$folderCount} folders}}\n", " one one {{You found {$fileCount} file in {$folderCount} folder}}\n", " one * {{You found {$fileCount} file in {$folderCount} folders}}\n", - " * one {{You found {$fileCount} files in {$folderCount} folder}}\n" + " * one {{You found {$fileCount} files in {$folderCount} folder}}" ] }, "variations" : [ diff --git a/icu4c/source/test/testdata/message2/invalid-number-literals-diagnostics.json b/testdata/message2/invalid-number-literals-diagnostics.json similarity index 100% rename from icu4c/source/test/testdata/message2/invalid-number-literals-diagnostics.json rename to testdata/message2/invalid-number-literals-diagnostics.json diff --git a/testdata/message2/invalid-options.json b/testdata/message2/invalid-options.json new file mode 100644 index 000000000000..8b4740ee9bcb --- /dev/null +++ b/testdata/message2/invalid-options.json @@ -0,0 +1,34 @@ +[ + { "src": ".local $foo = {1 :number minimumIntegerDigits=-1} {{bar {$foo}}}", + "errors": [{"type": "bad-option"}], + "ignoreJava": "ICU4J doesn't validate options" + }, + { "src": ".local $foo = {1 :number minimumIntegerDigits=foo} {{bar {$foo}}}", + "errors": [{"type": "bad-option"}], + "ignoreJava": "ICU4J doesn't validate options" + }, + { "src": ".local $foo = {1 :number minimumFractionDigits=foo} {{bar {$foo}}}", + "errors": [{"type": "bad-option"}], + "ignoreJava": "ICU4J doesn't validate options" + }, + { "src": ".local $foo = {1 :number maximumFractionDigits=foo} {{bar {$foo}}}", + "errors": [{"type": "bad-option"}], + "ignoreJava": "ICU4J doesn't validate options" + }, + { "src": ".local $foo = {1 :number minimumSignificantDigits=foo} {{bar {$foo}}}", + "errors": [{"type": "bad-option"}], + "ignoreJava": "ICU4J doesn't validate options" + }, + { "src": ".local $foo = {1 :number maximumSignificantDigits=foo} {{bar {$foo}}}", + "errors": [{"type": "bad-option"}], + "ignoreJava": "ICU4J doesn't validate options" + }, + { "src": ".local $foo = {1 :integer minimumIntegerDigits=foo} {{bar {$foo}}}", + "errors": [{"type": "bad-option"}], + "ignoreJava": "ICU4J doesn't validate options" + }, + { "src": ".local $foo = {1 :integer maximumSignificantDigits=foo} {{bar {$foo}}}", + "errors": [{"type": "bad-option"}], + "ignoreJava": "ICU4J doesn't validate options" + } +] diff --git a/icu4c/source/test/testdata/message2/markup.json b/testdata/message2/markup.json similarity index 100% rename from icu4c/source/test/testdata/message2/markup.json rename to testdata/message2/markup.json diff --git a/icu4c/source/test/testdata/message2/matches-whitespace.json b/testdata/message2/matches-whitespace.json similarity index 82% rename from icu4c/source/test/testdata/message2/matches-whitespace.json rename to testdata/message2/matches-whitespace.json index c70bef04b650..d0b2c4ecdfe7 100644 --- a/icu4c/source/test/testdata/message2/matches-whitespace.json +++ b/testdata/message2/matches-whitespace.json @@ -2,19 +2,25 @@ { "src": ".match {one :string} {bar :string} one * {{one}} * * {{other}}", "exp": "one" }, { "src": ".match {foo :string} {bar :string}one * {{one}} * * {{other}}", - "exp": "other" }, + "exp": "other" + }, { "src": ".match {foo :string}{bar :string} one * {{one}} * * {{other}}", - "exp": "other" }, + "exp": "other" + }, { "src": ".match {one :string}{bar :string}one * {{one}} * * {{other}}", - "exp": "one" }, + "exp": "one" + }, { "src": ".match{foo :string} {bar :string} one * {{one}} * * {{other}}", - "exp": "other" }, + "exp": "other" + }, { "src": ".match {foo :string} {bar :string} one * {{one}}* * {{other}}", "exp": "other" }, { "src": ".match {foo :string} {bar :string}one * {{one}}* * {{other}}", - "exp": "other" }, + "exp": "other" + }, { "src": ".match {foo :string} {bar :string} one *{{one}} * * {{foo}}", - "exp": "foo" }, + "exp": "foo" + }, { "src": ".match {foo :string} {bar :string} one * {{one}} * * {{foo}}", "exp": "foo" } ] diff --git a/icu4c/source/test/testdata/message2/more-data-model-errors.json b/testdata/message2/more-data-model-errors.json similarity index 94% rename from icu4c/source/test/testdata/message2/more-data-model-errors.json rename to testdata/message2/more-data-model-errors.json index ee26a1ffdf2f..0c86c58a5a12 100644 --- a/icu4c/source/test/testdata/message2/more-data-model-errors.json +++ b/testdata/message2/more-data-model-errors.json @@ -30,9 +30,5 @@ "{:foo a=1 a=1}", "{:foo a=1 a=2}", "{|x| :foo a=1 a=2}" - ], - "Unsupported Statement": [ - ".matc {-1} {{hello}}", - ".m {-1} {{hello}}" ] } diff --git a/icu4c/source/test/testdata/message2/more-functions.json b/testdata/message2/more-functions.json similarity index 87% rename from icu4c/source/test/testdata/message2/more-functions.json rename to testdata/message2/more-functions.json index 161379a63c55..99640a4daed3 100644 --- a/icu4c/source/test/testdata/message2/more-functions.json +++ b/testdata/message2/more-functions.json @@ -67,7 +67,8 @@ "locale": "en-IN", "params": {"val": 1234567890.97531}, "errors": [{"type": "bad-input"}], - "comment": "Should fail because number literals are not treated as localized numbers" + "comment": "Should fail because number literals are not treated as localized numbers", + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, { "src": "From literal: {|123456789.531| :number}!", @@ -99,7 +100,16 @@ "exp": "Default int64: 1.234.567.890.123.456.800!", "locale": "ro", "params": {"val": 1234567890123456789}, - "comment": "Rounded due to JSON not supporting full 64-bit ints" + "comment": "Rounded due to JSON not supporting full 64-bit ints", + "ignoreJava": "See https://unicode-org.atlassian.net/browse/ICU-22754?focusedCommentId=175932" + }, + { + "src": "Default int64: {$val}!", + "exp": "Default int64: 1.234.567.890.123.456.770!", + "locale": "ro", + "params": {"val": 1234567890123456789}, + "comment": "Rounded due to JSON not supporting full 64-bit ints", + "ignoreCpp": "See https://unicode-org.atlassian.net/browse/ICU-22754?focusedCommentId=175932" }, { "src": "Default number: {$val}!", diff --git a/testdata/message2/more-syntax-errors.json b/testdata/message2/more-syntax-errors.json new file mode 100644 index 000000000000..48f145b452c0 --- /dev/null +++ b/testdata/message2/more-syntax-errors.json @@ -0,0 +1,5 @@ +[ + ".input {$x :number}", + ".local $foo = {|1|}", + ".unsupported |statement| {$x :number}" +] diff --git a/testdata/message2/reserved-syntax.json b/testdata/message2/reserved-syntax.json new file mode 100644 index 000000000000..9efd6fe6b9c9 --- /dev/null +++ b/testdata/message2/reserved-syntax.json @@ -0,0 +1,40 @@ +[ + { "src": "hello {|4.2| %number}", "errors": [{ "type": "unsupported-annotation" }], "ignoreJava": "ICU4J doesn't error out on reserved annotations" }, + { "src": "hello {|4.2| %n|um|ber}", "errors": [{ "type": "unsupported-annotation" }], "ignoreJava": "ICU4J doesn't error out on reserved annotations" }, + { "src": "{+42}", "errors": [{ "type": "unsupported-annotation" }], "ignoreJava": "ICU4J doesn't error out on reserved annotations" }, + { "src": "hello {|4.2| &num|be|r}", "errors": [{ "type": "unsupported-annotation" }], "ignoreJava": "ICU4J doesn't error out on reserved annotations" }, + { "src": "hello {|4.2| ^num|be|r}", "errors": [{ "type": "unsupported-annotation" }], "ignoreJava": "ICU4J doesn't error out on reserved annotations" }, + { "src": "hello {|4.2| +num|be|r}", "errors": [{ "type": "unsupported-annotation" }], "ignoreJava": "ICU4J doesn't error out on reserved annotations" }, + { "src": "hello {|4.2| ?num|be||r|s}", "errors": [{ "type": "unsupported-annotation" }], "ignoreJava": "ICU4J doesn't error out on reserved annotations" }, + { "src": "hello {|foo| !number}", "errors": [{ "type": "unsupported-annotation" }], "ignoreJava": "ICU4J doesn't error out on reserved annotations" }, + { "src": "hello {|foo| *number}", "errors": [{ "type": "unsupported-annotation" }], "ignoreJava": "ICU4J doesn't error out on reserved annotations" }, + { "src": "hello {?number}", "errors": [{ "type": "unsupported-annotation" }], "ignoreJava": "ICU4J doesn't error out on reserved annotations" }, + { "src": "{xyzz }", "errors": [{ "type": "unsupported-annotation" }], "ignoreJava": "ICU4J doesn't error out on reserved annotations" }, + { "src": "hello {$foo ~xyzz }", "errors": [{ "type": "unsupported-annotation" }], "ignoreJava": "ICU4J doesn't error out on reserved annotations" }, + { "src": "hello {$x xyzz }", "errors": [{ "type": "unsupported-annotation" }], "ignoreJava": "ICU4J doesn't error out on reserved annotations" }, + { "src": "{ !xyzz }", "errors": [{ "type": "unsupported-annotation" }], "ignoreJava": "ICU4J doesn't error out on reserved annotations" }, + { "src": "{~xyzz }", "errors": [{ "type": "unsupported-annotation" }], "ignoreJava": "ICU4J doesn't error out on reserved annotations" }, + { "src": "{ num x \\\\ abcde |aaa||3.14||42| r }", "errors": [{ "type": "unsupported-annotation" }], "ignoreJava": "ICU4J doesn't error out on reserved annotations" }, + { "src": "hello {$foo >num x \\\\ abcde |aaa||3.14| |42| r }", "errors": [{ "type": "unsupported-annotation" }], "ignoreJava": "ICU4J doesn't error out on reserved annotations" } +] + diff --git a/testdata/message2/resolution-errors.json b/testdata/message2/resolution-errors.json new file mode 100644 index 000000000000..077e8b0e9c3b --- /dev/null +++ b/testdata/message2/resolution-errors.json @@ -0,0 +1,14 @@ +[ + { "src": "{$oops}", "exp": "{$oops}", "errors": [{ "type": "unresolved-var" }], "ignoreJava": "ICU4J doesn't signal unresolved variable errors?"}, + { "src": ".input {$x :number} {{{$x}}}", "exp": "{$x}", "errors": [{ "type": "unresolved-var" }], "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782"}, + { "src": ".local $foo = {$bar} .match {$foo :number} one {{one}} * {{other}}", "exp": "other", "errors": [{ "type": "unresolved-var" }], "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782"}, + { "src": ".local $bar = {$none :number} .match {$foo :string} one {{one}} * {{{$bar}}}", "exp": "{$none}", "errors": [{ "type": "unresolved-var" }], "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782"}, + { "src": "The value is {horse :func}.", "exp": "The value is {|horse|}.", "errors": [{ "type": "missing-func" }], "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782"}, + { "src": ".matc {-1} {{hello}}", + "errors": [{ "type": "unsupported-statement" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + }, + { "src": ".m {-1} {{hello}}", + "errors": [{ "type": "unsupported-statement" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" } +] diff --git a/icu4c/source/test/testdata/message2/runtime-errors.json b/testdata/message2/runtime-errors.json similarity index 61% rename from icu4c/source/test/testdata/message2/runtime-errors.json rename to testdata/message2/runtime-errors.json index 5b93f54d7566..9c7bfbc58443 100644 --- a/icu4c/source/test/testdata/message2/runtime-errors.json +++ b/testdata/message2/runtime-errors.json @@ -2,21 +2,25 @@ { "src": ".match {|horse| :date}\n 1 {{The value is one.}}\n * {{Formatter used as selector.}}", "exp": "Formatter used as selector.", - "errors": [{"type": "selector-error"}] + "errors": [{"type": "selector-error"}], + "ignoreJava": "ICU4J doesn't signal runtime errors?" }, { "src": ".match {|horse| :string}\n 1 {{The value is one.}}\n * {{Selector used as formatter: {|horse| :string}}}", "exp": "Selector used as formatter: {|horse|}", - "errors": [{"type": "formatting-error"}] + "errors": [{"type": "formatting-error"}], + "ignoreJava": "ICU4J doesn't signal runtime errors?" }, { "src": ".match {|horse| :number}\n 1 {{The value is one.}}\n * {{horse is not a number.}}", "exp": "horse is not a number.", - "errors": [{"type": "selector-error"}] + "errors": [{"type": "selector-error"}], + "ignoreJava": "ICU4J doesn't signal runtime errors?" }, { "src": ".local $sel = {|horse| :number}\n .match {$sel}\n 1 {{The value is one.}}\n * {{horse is not a number.}}", "exp": "horse is not a number.", - "errors": [{"type": "selector-error"}] + "errors": [{"type": "selector-error"}], + "ignoreJava": "ICU4J doesn't signal runtime errors?" } ] diff --git a/icu4c/source/test/testdata/message2/spec/data-model-errors.json b/testdata/message2/spec/data-model-errors.json similarity index 100% rename from icu4c/source/test/testdata/message2/spec/data-model-errors.json rename to testdata/message2/spec/data-model-errors.json diff --git a/icu4c/source/test/testdata/message2/spec/syntax-errors.json b/testdata/message2/spec/syntax-errors.json similarity index 100% rename from icu4c/source/test/testdata/message2/spec/syntax-errors.json rename to testdata/message2/spec/syntax-errors.json diff --git a/icu4c/source/test/testdata/message2/spec/test-core.json b/testdata/message2/spec/test-core.json similarity index 83% rename from icu4c/source/test/testdata/message2/spec/test-core.json rename to testdata/message2/spec/test-core.json index 0e7049fdb1a3..2ee0786e62de 100644 --- a/icu4c/source/test/testdata/message2/spec/test-core.json +++ b/testdata/message2/spec/test-core.json @@ -26,7 +26,8 @@ { "src": "hello {$place}", "errors": [{ "type": "unresolved-var" }], - "exp": "hello {$place}" + "exp": "hello {$place}", + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, { "src": "{$one} and {$two}", @@ -169,7 +170,8 @@ { "type": "literal", "value": "foo " }, { "type": "fallback", "source": "+" } ], - "errors": [{ "type": "unsupported-annotation" }] + "errors": [{ "type": "unsupported-annotation" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, { "src": "foo {&private}", @@ -178,7 +180,8 @@ { "type": "literal", "value": "foo " }, { "type": "fallback", "source": "&" } ], - "errors": [{ "type": "unsupported-annotation" }] + "errors": [{ "type": "unsupported-annotation" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, { "src": "foo {?reserved @a @b=$c}", @@ -187,26 +190,30 @@ { "type": "literal", "value": "foo " }, { "type": "fallback", "source": "?" } ], - "errors": [{ "type": "unsupported-annotation" }] + "errors": [{ "type": "unsupported-annotation" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, { "src": ".foo {42} {{bar}}", "exp": "bar", "parts": [{ "type": "literal", "value": "bar" }], - "errors": [{ "type": "unsupported-statement" }] + "errors": [{ "type": "unsupported-statement" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, { "src": ".foo{42}{{bar}}", "cleanSrc": ".foo {42} {{bar}}", "exp": "bar", "parts": [{ "type": "literal", "value": "bar" }], - "errors": [{ "type": "unsupported-statement" }] + "errors": [{ "type": "unsupported-statement" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, { "src": ".foo |}lit{| {42}{{bar}}", "cleanSrc": ".foo |}lit{| {42} {{bar}}", "exp": "bar", "parts": [{ "type": "literal", "value": "bar" }], - "errors": [{ "type": "unsupported-statement" }] + "errors": [{ "type": "unsupported-statement" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" } ] diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/test-functions.json b/testdata/message2/spec/test-functions.json similarity index 77% rename from icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/test-functions.json rename to testdata/message2/spec/test-functions.json index 8cf1b7a25325..a8979106947e 100644 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/test-functions.json +++ b/testdata/message2/spec/test-functions.json @@ -1,10 +1,16 @@ { "date": [ - { "src": "{:date}", "exp": "{:date}", "errorsJs": [{ "type": "bad-input" }] }, + { "src": "{:date}", + "exp": "{:date}", + "errors": [{ "type": "bad-input" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + }, { "src": "{horse :date}", "exp": "{|horse|}", - "errorsJs": [{ "name": "RangeError" }] + "errors": [{ "type": "bad-input" }], + "errorsJs": [{ "name": "RangeError" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, { "src": "{|2006-01-02| :date}", "exp": "1/2/06" }, { "src": "{|2006-01-02T15:04:06| :date}", "exp": "1/2/06" }, @@ -14,17 +20,20 @@ "exp": "January 2, 2006" }, { - "ignore": "Can't chain :time and :date, they are different types", + "ignoreJava": "Can't chain :time and :date, they are different types", "src": ".local $t = {|2006-01-02T15:04:06| :time} {{{$t :date}}}", "exp": "1/2/06" } ], "time": [ - { "src": "{:time}", "exp": "{:time}", "errorsJs": [{ "type": "bad-input" }] }, + { "src": "{:time}", "exp": "{:time}", "errors": [{ "type": "bad-input" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + }, { "src": "{horse :time}", "exp": "{|horse|}", - "errorsJs": [{ "name": "RangeError" }] + "errors": [{ "type": "bad-input" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, { "src": "{|2006-01-02T15:04:06| :time}", "expJs": "3:04 PM", "exp": "3:04 PM" }, { @@ -38,16 +47,17 @@ "exp": "3:04:06 PM" }, { - "ignore": "Can't chain :time and :date, they are different types", + "ignoreJava": "Can't chain :time and :date, they are different types", "src": ".local $d = {|2006-01-02T15:04:06| :date} {{{$d :time}}}", - "exp": "3:04 PM" + "exp": "3:04 PM" } ], "datetime": [ { "src": "{:datetime}", "exp": "{:datetime}", - "errorsJs": [{ "type": "bad-input" }] + "errors": [{ "type": "bad-input" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, { "src": "{$x :datetime}", @@ -58,7 +68,8 @@ { "src": "{horse :datetime}", "exp": "{|horse|}", - "errorsJs": [{ "name": "RangeError" }] + "errors": [{ "type": "bad-input" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, { "src": "{|2006-01-02T15:04:06| :datetime}", "expJs": "1/2/06, 3:04 PM", "exp": "1/2/06, 3:04 PM" }, { @@ -81,10 +92,11 @@ "exp": "1/2/06, 3:04 PM" }, { - "ignore": "Can't chain :time and :date, they are different types", + "ignoreJava": "Can't chain :time and :date, they are different types", + "ignoreCpp": "Same reason as Java", "src": ".input {$dt :time style=medium} {{{$dt :datetime dateStyle=long}}}", "params": { "dt": "2006-01-02T15:04:06" }, - "exp": "January 2, 2006 at 3:04:06 PM" + "exp": "January 2, 2006 at 3:04:06 PM" } ], "integer": [ @@ -104,7 +116,8 @@ { "src": "hello {foo :number}", "exp": "hello {|foo|}", - "errorsJs": [{ "type": "bad-input" }] + "errors": [{ "type": "bad-input" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, { "src": "hello {:number}", @@ -150,9 +163,9 @@ { "src": ".local $foo = {$bar :number} {{bar {$foo}}}", "params": { "bar": "foo" }, - "expJs": "bar {$bar}", - "exp": "bar {|foo|}", - "errorsJs": [{ "type": "bad-input" }] + "exp": "bar {$bar}", + "errors": [{ "type": "bad-input" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, { "src": ".input {$foo :number} {{bar {$foo}}}", @@ -175,9 +188,9 @@ { "src": ".input {$foo :number} {{bar {$foo}}}", "params": { "foo": "foo" }, - "expJs": "bar {$foo}", - "exp": "bar {|foo|}", - "errorsJs": [{ "type": "bad-input" }] + "exp": "bar {$foo}", + "errors": [{ "type": "bad-input" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, { "src": ".match {$foo :number} one {{one}} * {{other}}", @@ -278,12 +291,14 @@ "src": ".match {$foo :ordinal} one {{st}} two {{nd}} few {{rd}} * {{th}}", "params": { "foo": 1 }, "exp": "th", - "errors": [{ "type": "missing-func" }, { "type": "not-selectable" }] + "errors": [{ "type": "missing-func" }, { "type": "not-selectable" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, { "src": "hello {42 :ordinal}", "exp": "hello {|42|}", - "errorsJs": [{ "type": "missing-func" }] + "errors": [{ "type": "missing-func" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" } ], "plural": [ @@ -291,12 +306,14 @@ "src": ".match {$foo :plural} one {{one}} * {{other}}", "params": { "foo": 1 }, "exp": "other", - "errors": [{ "type": "missing-func" }, { "type": "not-selectable" }] + "errors": [{ "type": "missing-func" }, { "type": "not-selectable" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, { "src": "hello {42 :plural}", "exp": "hello {|42|}", - "errorsJs": [{ "type": "missing-func" }] + "errors": [{ "type": "missing-func" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" } ], "string": [ @@ -308,8 +325,8 @@ { "src": ".match {$foo :string} 1 {{one}} * {{other}}", "params": { "foo": 1 }, - "expJs": "one", - "exp": "other" + "exp": "one", + "ignoreJava": "See https://unicode-org.atlassian.net/browse/ICU-22754?focusedCommentId=175933" }, { "src": ".match {$foo :string} 1 {{one}} * {{other}}", @@ -319,7 +336,8 @@ { "src": ".match {$foo :string} 1 {{one}} * {{other}}", "exp": "other", - "errorsJs": [{ "type": "unresolved-var" }] + "errors": [{ "type": "unresolved-var" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" } ] } diff --git a/icu4c/source/test/testdata/message2/syntax-errors-diagnostics-multiline.json b/testdata/message2/syntax-errors-diagnostics-multiline.json similarity index 100% rename from icu4c/source/test/testdata/message2/syntax-errors-diagnostics-multiline.json rename to testdata/message2/syntax-errors-diagnostics-multiline.json diff --git a/testdata/message2/syntax-errors-diagnostics.json b/testdata/message2/syntax-errors-diagnostics.json new file mode 100644 index 000000000000..2b0188f6b557 --- /dev/null +++ b/testdata/message2/syntax-errors-diagnostics.json @@ -0,0 +1,402 @@ +[ + { "src": "}{|xyz|", "char": 0, "errors": [{"type": "parse-error"}] }, + { "src": "}", "char": 0, "errors": [{"type": "parse-error"}] }, + { + "src": "{{{%\\y{}}", + "char": 5, + "comment": "Backslash followed by non-backslash followed by a '{' -- this should be an error immediately after the first backslash", + "errors": [{"type": "parse-error"}] + }, + { + "src": "{%abc|\\z}}", + "char": 7, + "comment": "Reserved chars followed by a '|' that doesn't begin a valid literal -- this should be an error at the first invalid char in the literal", + "errors": [{"type": "parse-error"}] + }, + { + "src": "{%\\y{p}}", + "char": 3, + "comment": "Same pattern, but with a valid reserved-char following the erroneous reserved-escape -- the offset should be the same as with the previous one", + "errors": [{"type": "parse-error"}] + }, + { + "src": "{{{%ab|\\z|cd}}", + "char": 8, + "comment": "Erroneous literal inside a reserved string -- the error should be at the first erroneous literal char", + "errors": [{"type": "parse-error"}] + }, + { + "src": "hello {|4.2| %num\\ber}}", + "char": 18, + "comment": "Single backslash not allowed", + "errors": [{"type": "parse-error"}] + }, + { + "src": "hello {|4.2| %num{be\\|r}}", + "char": 17, + "comment": "Unescaped '{' not allowed", + "errors": [{"type": "parse-error"}] + }, + { + "src": "hello {|4.2| %num}be\\|r}}", + "char": 21, + "comment": "Unescaped '}' -- will be interpreted as the end of the reserved string, and the error will be reported at the index of '|', which is when the parser determines that the escaped '|' isn't a valid text-escape", + "errors": [{"type": "parse-error"}] + }, + { + "src": "hello {|4.2| %num\\{be|r}}", + "char": 25, + "comment": "Unescaped '|' -- will be interpreted as the beginning of a literal. Error at end of input", + "errors": [{"type": "parse-error"}] + }, + { + "src": ".match{|y|}|y|{{{|||}}}", + "char": 19, + "comment": "No spaces are required here. The error should be in the pattern, not before", + "errors": [{"type": "parse-error"}] + }, + { + "src": ".match {|y|}|foo|bar {{{a}}}", + "char": 17, + "comment": "Missing spaces between keys", + "errors": [{"type": "parse-error"}] + }, + { + "src": ".match {|y|} |quux| |foo|bar {{{a}}}", + "char": 25, + "comment": "Missing spaces between keys", + "errors": [{"type": "parse-error"}] + }, + { + "src": ".match {|y|} |quux| |foo||bar| {{{a}}}", + "char": 26, + "comment": "Missing spaces between keys", + "errors": [{"type": "parse-error"}] + }, + { + "src": ".match {|y|} |\\q| * %{! {z}", + "char": 16, + "comment": "Error parsing the first key -- the error should be there, not in the also-erroneous third key", + "errors": [{"type": "parse-error"}] + }, + { + "src": ".match {|y|} * %{! {z} |\\q|", + "char": 16, + "comment": "Error parsing the second key -- the error should be there, not in the also-erroneous third key", + "errors": [{"type": "parse-error"}] + }, + { + "src": ".match {|y|} * |\\q| {\\z}", + "char": 18, + "comment": "Error parsing the last key -- the error should be there, not in the erroneous pattern", + "errors": [{"type": "parse-error"}] + }, + { + "src": ".match {|y|} {\\|} {@} * * * {{a}}", + "char": 14, + "comment": "Non-expression as scrutinee in pattern -- error should be at the first non-expression, not the later non-expression", + "errors": [{"type": "parse-error"}] + }, + { + "src": ".match {|y|} $foo * {{a}} when * :bar {{b}}", + "char": 14, + "comment": "Non-key in variant -- error should be there, not in the next erroneous variant", + "errors": [{"type": "parse-error"}] + }, + { + "src": "{{ foo {|bar|} \\q baz ", + "char": 16, + "comment": "Error should be within the first erroneous `text` or expression", + "errors": [{"type": "parse-error"}] + }, + { + "src": "{{{: }}}", + "char": 4, + "comment": "':' has to be followed by a function name -- the error should be at the first whitespace character", + "errors": [{"type": "parse-error"}] + }, + { + "src": ".local $x = }|foo|}", + "char": 12, + "comment": "Expression not starting with a '{'", + "errors": [{"type": "parse-error"}] + }, + { + "src": ".local $x = {|foo|} .l $y = {|bar|} .local $z {|quux|}", + "char": 22, + "comment": "Error should be at the first declaration not starting with a `.local`", + "errors": [{"type": "parse-error"}] + }, + { + "src": ".local $bar {|foo|} {{$bar}}", + "char": 12, + "comment": "Missing '=' in `.local` declaration", + "errors": [{"type": "parse-error"}] + }, + { + "src": ".local bar = {|foo|} {{$bar}}", + "char": 7, + "comment": "LHS of declaration doesn't start with a '$'", + "errors": [{"type": "parse-error"}] + }, + { + "src": ".local $bar = |foo| {{$bar}}", + "char": 14, + "comment": "`.local` RHS isn't an expression", + "errors": [{"type": "parse-error"}] + }, + { + "src": "{{extra}}content", + "char": 9, + "comment": "Trailing characters that are not whitespace", + "errors": [{"type": "parse-error"}] + }, + { + "src": ".match {|x|} * {{foo}}extra", + "char": 28, + "comment": "Trailing characters that are not whitespace", + "errors": [{"type": "parse-error"}] + }, + { + "src": ".match {$foo :string} {$bar :string} one * {{one}} * * {{other}} ", + "char": 66, + "comment": "Trailing whitespace at end of message should not be accepted either", + "errors": [{"type": "parse-error"}] + }, + { + "src": "{{hi}} ", + "char": 6, + "comment": "Trailing whitespace at end of message should not be accepted either", + "errors": [{"type": "parse-error"}] + }, + { + "src": "empty { }", + "char": 8, + "comment": "Empty expression", + "errors": [{"type": "parse-error"}] + }, + { + "src": ".match {} * {{foo}}", + "char": 8, + "comment": "Empty expression", + "errors": [{"type": "parse-error"}] + }, + { + "src": "bad {:}", + "char": 6, + "comment": "':' not preceding a function name", + "errors": [{"type": "parse-error"}] + }, + { + "src": "{{no-equal {|42| :number m }}}", + "char": 27, + "comment": "Missing '=' after option name", + "errors": [{"type": "parse-error"}] + }, + { + "src": "{{no-equal {|42| :number minimumFractionDigits 2}}}", + "char": 47, + "comment": "Missing '=' after option name", + "errors": [{"type": "parse-error"}] + }, + { + "src": "bad {:placeholder option value}", + "char": 25, + "comment": "Missing '=' after option name", + "errors": [{"type": "parse-error"}] + }, + { + "src": "hello {|4.2| :number min=2=3}", + "char": 26, + "comment": "Extra '=' after option name", + "errors": [{"type": "parse-error"}] + }, + { + "src": "hello {|4.2| :number min=2max=3}", + "char": 26, + "comment": "Missing space between options", + "errors": [{"type": "parse-error"}] + }, + { + "src": "hello {|4.2| :number min=|a|max=3}", + "char": 28, + "comment": "Missing whitespace between valid options", + "errors": [{"type": "parse-error"}] + }, + { + "src": "hello {|4.2| :number min=|\\a|}", + "char": 27, + "comment": "Ill-formed RHS of option -- the error should be within the RHS, not after parsing options", + "errors": [{"type": "parse-error"}] + }, + { + "src": "no-equal {|42| :number {}", + "char": 25, + "comment": "Junk after annotation", + "errors": [{"type": "parse-error"}] + }, + { + "src": "bad {:placeholder option=}", + "char": 25, + "comment": "Missing RHS of option", + "errors": [{"type": "parse-error"}] + }, + { + "src": "bad {:placeholder option}", + "char": 24, + "comment": "Missing RHS of option", + "errors": [{"type": "parse-error"}] + }, + { + "src": "bad {$placeholder option}", + "char": 18, + "comment": "Annotation is not a function or reserved text", + "errors": [{"type": "parse-error"}] + }, + { + "src": "no {$placeholder end", + "char": 17, + "comment": "Annotation is not a function or reserved text", + "errors": [{"type": "parse-error"}] + }, + { + "src": ".match * {{foo}}", + "char": 8, + "comment": "Missing expression in selectors", + "errors": [{"type": "parse-error"}] + }, + { + "src": ".match |x| * {{foo}}", + "char": 7, + "comment": "Non-expression in selectors", + "errors": [{"type": "parse-error"}] + }, + { + "src": ".match {|x|} * foo", + "char": 19, + "comment": "Missing RHS in variant", + "errors": [{"type": "parse-error"}] + }, + { + "src": "{$:abc}", + "char": 2, + "comment": "Variable names can't start with a : or -", + "errors": [{"type": "parse-error"}] + }, + { + "src": "{$-abc}", + "char": 2, + "comment": "Variable names can't start with a : or -", + "errors": [{"type": "parse-error"}] + }, + { + "src": "{$bar+foo}", + "char": 5, + "comment": "Missing space before annotation. Note that {{$bar:foo}} and {{$bar-foo}} are valid, because variable names can contain a ':' or a '-'", + "errors": [{"type": "parse-error"}] + }, + { + "src": "{|3.14|:foo}", + "char": 7, + "comment": "Missing space before annotation.", + "errors": [{"type": "parse-error"}] + }, + { + "src": "{|3.14|-foo}", + "char": 7, + "comment": "Missing space before annotation.", + "errors": [{"type": "parse-error"}] + }, + { + "src": "{|3.14|+foo}", + "char": 7, + "comment": "Missing space before annotation.", + "errors": [{"type": "parse-error"}] + }, + { + "src": ".local $foo = {$bar} .match {$foo} :one {one} * {other}", + "char": 36, + "comment": "Unquoted literals can't begin with a ':'", + "errors": [{"type": "parse-error"}] + }, + { + "src": ".local $foo = {$bar :fun option=:a} {{bar {$foo}}}", + "char": 32, + "comment": "Unquoted literals can't begin with a ':'", + "errors": [{"type": "parse-error"}] + }, + { "src": "{|foo| {#markup}}", "char": 7, "comment": "Markup in wrong place", "errors": [{"type": "parse-error"}] }, + { "src": "{|foo| #markup}", "char": 7, "comment": "Markup in wrong place", "errors": [{"type": "parse-error"}] }, + { "src": "{|foo| {#markup/}}", "char": 7, "comment": "Markup in wrong place", "errors": [{"type": "parse-error"}] }, + { "src": "{|foo| {/markup}}", "char": 7, "comment": "Markup in wrong place", "errors": [{"type": "parse-error"}] }, + { "src": ".input $x = {|1|} {{{$x}}}", "char": 7, "comment": ".input with non-variable-expression", "errors": [{"type": "parse-error"}] }, + { "src": ".input $x = {:number} {{{$x}}}", "char": 7, "comment": ".input with non-variable-expression", "errors": [{"type": "parse-error"}] }, + { "src": ".input {|1| :number} {{{$x}}}", "char": 7, "comment": ".input with non-variable-expression", "errors": [{"type": "parse-error"}] }, + { "src": ".input {:number} {{{$x}}}", "char": 7, "comment": ".input with non-variable-expression", "errors": [{"type": "parse-error"}] }, + { "src": ".input {|1|} {{{$x}}}", "char": 7, "comment": ".input with non-variable-expression", "errors": [{"type": "parse-error"}] }, + { "src": ".", "char": 1, "errors": [{"type": "parse-error"}]}, + { "src": "{", "char": 1, "errors": [{"type": "parse-error"}]}, + { "src": "}", "char": 0, "errors": [{"type": "parse-error"}]}, + { "src": "{}", "char": 1, "errors": [{"type": "parse-error"}]}, + { "src": "{{", "char": 2, "errors": [{"type": "parse-error"}]}, + { "src": "{{}", "char": 3, "errors": [{"type": "parse-error"}]}, + { "src": "{{}}}", "char": 4, "errors": [{"type": "parse-error"}]}, + { "src": "{|foo| #markup}", "char": 7, "errors": [{"type": "parse-error"}]}, + { "src": "{{missing end brace}", "char": 20, "errors": [{"type": "parse-error"}]}, + { "src": "{{missing end braces", "char": 20, "errors": [{"type": "parse-error"}]}, + { "src": "{{missing end {$braces", "char": 22, "errors": [{"type": "parse-error"}]}, + { "src": "{{extra}} content", "char": 9, "errors": [{"type": "parse-error"}]}, + { "src": "empty { } placeholder", "char": 8, "errors": [{"type": "parse-error"}]}, + { "src": "missing space {42:func}", "char": 17, "errors": [{"type": "parse-error"}]}, + { "src": "missing space {|foo|:func}", "char": 20, "errors": [{"type": "parse-error"}]}, + { "src": "missing space {|foo|@bar}", "char": 20, "errors": [{"type": "parse-error"}]}, + { "src": "missing space {:func@bar}", "char": 20, "errors": [{"type": "parse-error"}]}, + { "src": "{:func @bar@baz}", "char": 11, "errors": [{"type": "parse-error"}]}, + { "src": "{:func @bar=42@baz}", "char": 14, "errors": [{"type": "parse-error"}]}, + { "src": "{+reserved@bar}", "char": 10, "errors": [{"type": "parse-error"}]}, + { "src": "{&private@bar}", "char": 9, "errors": [{"type": "parse-error"}]}, + { "src": "bad {:} placeholder", "char": 6, "errors": [{"type": "parse-error"}]}, + { "src": "bad {\\u0000placeholder}", "char": 5, "errors": [{"type": "parse-error"}]}, + { "src": "no-equal {|42| :number minimumFractionDigits 2}", "char": 45, "errors": [{"type": "parse-error"}]}, + { "src": "bad {:placeholder option=}", "char": 25, "errors": [{"type": "parse-error"}]}, + { "src": "bad {:placeholder option value}", "char": 25, "errors": [{"type": "parse-error"}]}, + { "src": "bad {:placeholder option:value}", "char": 30, "errors": [{"type": "parse-error"}]}, + { "src": "bad {:placeholder option}", "char": 24, "errors": [{"type": "parse-error"}]}, + { "src": "bad {:placeholder:}", "char": 18, "errors": [{"type": "parse-error"}]}, + { "src": "bad {::placeholder}", "char": 6, "errors": [{"type": "parse-error"}]}, + { "src": "bad {:placeholder::foo}", "char": 18, "errors": [{"type": "parse-error"}]}, + { "src": "bad {:placeholder option:=x}", "char": 25, "errors": [{"type": "parse-error"}]}, + { "src": "bad {:placeholder :option=x}", "char": 18, "errors": [{"type": "parse-error"}]}, + { "src": "bad {:placeholder option::x=y}", "char": 25, "errors": [{"type": "parse-error"}]}, + { "src": "bad {$placeholder option}", "char": 18, "errors": [{"type": "parse-error"}]}, + { "src": "bad {:placeholder @attribute=}", "char": 29, "errors": [{"type": "parse-error"}]}, + { "src": "bad {:placeholder @attribute=@foo}", "char": 29, "errors": [{"type": "parse-error"}]}, + { "src": "no {placeholder end", "char": 16, "errors": [{"type": "parse-error"}]}, + { "src": "no {$placeholder end", "char": 17, "errors": [{"type": "parse-error"}]}, + { "src": "no {:placeholder end", "char": 20, "errors": [{"type": "parse-error"}]}, + { "src": "no {|placeholder| end", "char": 18, "errors": [{"type": "parse-error"}]}, + { "src": "no {|literal} end", "char": 17, "errors": [{"type": "parse-error"}]}, + { "src": "no {|literal or placeholder end", "char": 31, "errors": [{"type": "parse-error"}]}, + { "src": ".local bar = {|foo|} {{_}}", "char": 7, "errors": [{"type": "parse-error"}]}, + { "src": ".local #bar = {|foo|} {{_}}", "char": 7, "errors": [{"type": "parse-error"}]}, + { "src": ".local $bar {|foo|} {{_}}", "char": 12, "errors": [{"type": "parse-error"}]}, + { "src": ".local $bar = |foo| {{_}}", "char": 14, "errors": [{"type": "parse-error"}]}, + { "src": ".match {#foo} * {{foo}}", "char": 8, "errors": [{"type": "parse-error"}]}, + { "src": ".match {} * {{foo}}", "char": 8, "errors": [{"type": "parse-error"}]}, + { "src": ".match {|foo| :x} {|bar| :x} ** {{foo}}", "char": 30, "errors": [{"type": "parse-error"}]}, + { "src": ".match * {{foo}}", "char": 7, "errors": [{"type": "parse-error"}]}, + { "src": ".match {|x| :x} * foo", "char": 21, "errors": [{"type": "parse-error"}]}, + { "src": ".match {|x| :x} * {{foo}} extra", "char": 31, "errors": [{"type": "parse-error"}]}, + { "src": ".match |x| * {{foo}}", "char": 7, "errors": [{"type": "parse-error"}]}, + { "src": ".match {|foo| :string} o:ne {{one}} * {{other}}", "char": 24, "comment" : "tests for ':' in unquoted literals (not allowed)" , "errors": [{"type": "parse-error"}]}, + { "src": ".match {|foo| :string} one: {{one}} * {{other}}", "char": 26, "comment" : "tests for ':' in unquoted literals (not allowed)" , "errors": [{"type": "parse-error"}]}, + { "src": ".local $foo = {|42| :number option=a:b} {{bar {$foo}}}", "char": 36, "comment" : "tests for ':' in unquoted literals (not allowed)" , "errors": [{"type": "parse-error"}]}, + { "src": ".local $foo = {|42| :number option=a:b:c} {{bar {$foo}}}", "char": 36, "comment" : "tests for ':' in unquoted literals (not allowed)" , "errors": [{"type": "parse-error"}]}, + { "src": "{$bar:foo}", "char": 5, "comment" : "tests for ':' in unquoted literals (not allowed)", "errors": [{"type": "parse-error"}]}, + { + "src": ".match {1} {{_}}", + "char": 12, + "comment": "Disambiguating a wrong .match from an unsupported statement", + "errors": [{"type": "parse-error"}] + } +] diff --git a/icu4c/source/test/testdata/message2/syntax-errors-end-of-input.json b/testdata/message2/syntax-errors-end-of-input.json similarity index 100% rename from icu4c/source/test/testdata/message2/syntax-errors-end-of-input.json rename to testdata/message2/syntax-errors-end-of-input.json diff --git a/icu4c/source/test/testdata/message2/tricky-declarations.json b/testdata/message2/tricky-declarations.json similarity index 100% rename from icu4c/source/test/testdata/message2/tricky-declarations.json rename to testdata/message2/tricky-declarations.json diff --git a/icu4c/source/test/testdata/message2/valid-tests.json b/testdata/message2/valid-tests.json similarity index 68% rename from icu4c/source/test/testdata/message2/valid-tests.json rename to testdata/message2/valid-tests.json index c0047a2c3c2e..f037c49d4b48 100644 --- a/icu4c/source/test/testdata/message2/valid-tests.json +++ b/testdata/message2/valid-tests.json @@ -1,8 +1,8 @@ [ { "src": "hello {|4.2| :number}", "exp": "hello 4.2"}, { "src": "hello {|4.2| :number minimumFractionDigits=2}", "exp": "hello 4.20"}, - { "src": "hello {|4.2| :number minimumFractionDigits = 2}", "exp": "hello 4.20"}, - { "src": "hello {|4.2| :number minimumFractionDigits= 2}", "exp": "hello 4.20"}, + { "src": "hello {|4.2| :number minimumFractionDigits = 2}", "exp": "hello 4.20" }, + { "src": "hello {|4.2| :number minimumFractionDigits= 2}", "exp": "hello 4.20" }, { "src": "hello {|4.2| :number minimumFractionDigits =2}", "exp": "hello 4.20"}, { "src": "hello {|4.2| :number minimumFractionDigits=2 }", "exp": "hello 4.20"}, { "src": "hello {|4.2| :number minimumFractionDigits=2 bar=3}", "exp": "hello 4.20"}, @@ -46,34 +46,60 @@ { "src": "{-1}", "exp": "-1"}, { "src": "{0}", "exp": "0"}, { "src": "{0.0123}", "exp": "0.0123"}, - { "src": "{1.234e5}", "exp": "1.234e5"}, - { "src": "{1.234E5}", "exp": "1.234E5"}, - { "src": "{1.234E+5}", "exp": "1.234E+5"}, - { "src": "{1.234e-5}", "exp": "1.234e-5"}, - { "src": "{42e5}", "exp": "42e5"}, - { "src": "{42e0}", "exp": "42e0"}, - { "src": "{42e000}", "exp": "42e000"}, - { "src": "{42e369}", "exp": "42e369"}, + { "src": "{1.234e5}", "exp": "1.234e5", + "ignoreJava": "See ICU-22810"}, + { "src": "{1.234E5}", "exp": "1.234E5", + "ignoreJava": "See ICU-22810"}, + { "src": "{1.234E+5}", "exp": "1.234E+5", + "ignoreJava": "See ICU-22810"}, + { "src": "{1.234e-5}", "exp": "1.234e-5", + "ignoreJava": "See ICU-22810"}, + { "src": "{42e5}", "exp": "42e5", + "ignoreJava": "See ICU-22810"}, + { "src": "{42e0}", "exp": "42e0", + "ignoreJava": "See ICU-22810"}, + { "src": "{42e000}", "exp": "42e000", + "ignoreJava": "See ICU-22810"}, + { "src": "{42e369}", "exp": "42e369", + "ignoreJava": "See ICU-22810"}, { "src": "hello {|3| :number }", "exp": "hello 3" }, - { "src": "{:foo}", "errors": [{ "type": "missing-func" }] }, - { "src": "{:foo }", "errors": [{ "type": "missing-func" }] }, - { "src": "{:foo }", "errors": [{ "type": "missing-func" }] }, - { "src": "{:foo k=v}", "errors": [{ "type": "missing-func" }] }, - { "src": "{:foo k=v }", "errors": [{ "type": "missing-func" }] }, - { "src": "{:foo k1=v1 k2=v2}", "errors": [{ "type": "missing-func" }] }, - { "src": "{:foo k1=v1 k2=v2 }", "errors": [{ "type": "missing-func" }] }, + { "src": "{:foo}", "errors": [{ "type": "missing-func" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, + { "src": "{:foo }", "errors": [{ "type": "missing-func" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, + { "src": "{:foo }", "errors": [{ "type": "missing-func" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, + { "src": "{:foo k=v}", "errors": [{ "type": "missing-func" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, + { "src": "{:foo k=v }", "errors": [{ "type": "missing-func" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, + { "src": "{:foo k1=v1 k2=v2}", "errors": [{ "type": "missing-func" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, + { "src": "{:foo k1=v1 k2=v2 }", "errors": [{ "type": "missing-func" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, { "src": "{|3.14| }", "exp": "3.14" }, { "src": "{|3.14| }", "exp": "3.14" }, { "src": "{|3.14| :number}", "exp": "3.14" }, { "src": "{|3.14| :number }", "exp": "3.14" }, - { "src": "{$bar }", "errors": [{ "type": "unresolved-var" }] }, - { "src": "{$bar }", "errors": [{ "type": "unresolved-var" }] }, - { "src": "{$bar :foo}", "errors": [{ "type": "unresolved-var" }] }, - { "src": "{$bar :foo }", "errors": [{ "type": "unresolved-var" }] }, - { "src": "{$bar-foo}", "errors": [{ "type": "unresolved-var" }] }, + { "src": "{$bar }", "errors": [{ "type": "unresolved-var" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, + { "src": "{$bar }", "errors": [{ "type": "unresolved-var" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, + { "src": "{$bar :foo}", "errors": [{ "type": "unresolved-var" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, + { "src": "{$bar :foo }", "errors": [{ "type": "unresolved-var" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, + { "src": "{$bar-foo}", "errors": [{ "type": "unresolved-var" }], + "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" }, { "src": ".local $foo = {|hello|} .local $foo = {$foo} {{{$foo}}}", "errors": [{ "type": "duplicate-declaration" }] }, { "src": "good {placeholder}", "exp": "good placeholder" }, + { + "src": "a\\\\qbc", + "exp": "a\\qbc", + "comment": "pattern -> escaped-char -> backslash backslash" + }, + { "src": "{$one} and {$two}", "params": { "one": "1.3", "two": "4.2" },