Skip to content

Commit

Permalink
Merge branch 'main' of github.com:unicode-org/conformance
Browse files Browse the repository at this point in the history
  • Loading branch information
sven-oly committed Jan 11, 2024
2 parents 350165f + 4b641e1 commit 3c1068a
Show file tree
Hide file tree
Showing 15 changed files with 285 additions and 96 deletions.
20 changes: 18 additions & 2 deletions executors/cpp/number_fmt.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ UNumberSignDisplay set_sign_display(json_object* options_obj) {
const string test_numfmt(json_object *json_in) {
UErrorCode status = U_ZERO_ERROR;

// Locale information
// label information
json_object *label_obj = json_object_object_get(json_in, "label");
string label_string = json_object_get_string(label_obj);

Expand All @@ -250,6 +250,15 @@ const string test_numfmt(json_object *json_in) {

const Locale displayLocale(locale_string.c_str());

// Try using the skeleton to create the formatter
json_object *skelecton_obj = json_object_object_get(json_in, "skeleton");
UnicodeString unicode_skeleton_string;
string skeleton_string;
if (skeleton_obj) {
skeleton_string = json_object_get_string(skeleton_obj);
unicode_skeleton_string = skeleton_string.c_str();
}

// Get options
json_object *notation_obj;
json_object *unit_obj;
Expand Down Expand Up @@ -278,7 +287,7 @@ const string test_numfmt(json_object *json_in) {
IntegerWidth integerWidth_setting = IntegerWidth::zeroFillTo(1);
MeasureUnit unit_setting = NoUnit::base();
Notation notation_setting = Notation::simple();
Precision precision_setting = Precision::unlimited();
Precision precision_setting = Precision::integer(); // TODO? = Precision::unlimited();
Scale scale_setting = Scale::none();
UNumberSignDisplay signDisplay_setting = UNUM_SIGN_AUTO;
UNumberFormatRoundingMode rounding_setting = UNUM_ROUND_HALFEVEN;
Expand Down Expand Up @@ -471,6 +480,12 @@ const string test_numfmt(json_object *json_in) {
}
}

if (skeleton_obj) {
cout << "# SKELETON " << skeleton_string << endl;
cout << "# LOCALE " << locale_string << endl;
nf = NumberFormatter::forSkeleton(unicode_skeleton_string, status).locale(displayLocale);
}
else {
// Use settings to initialize the formatter
nf = NumberFormatter::withLocale(displayLocale)
.notation(notation_setting)
Expand All @@ -484,6 +499,7 @@ const string test_numfmt(json_object *json_in) {
.sign(signDisplay_setting)
.unit(unit_setting)
.unitWidth(unit_width_setting);
}

if (U_FAILURE(status)) {
test_result = error_message.c_str();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import java.util.Optional;
import org.unicode.conformance.testtype.ITestType;
import org.unicode.conformance.testtype.collator.CollatorTester;
import org.unicode.conformance.testtype.langnames.LangNamesTester;

/**
* Hello world!
Expand Down Expand Up @@ -111,7 +112,9 @@ public static String getTestCaseResponse(String inputLine) throws Exception {
String testTypeStr = testTypeOpt.get();
ITestType testType;
if (testTypeStr.equals("collation_short")) {
testType = new CollatorTester();
testType = CollatorTester.INSTANCE;
} else if (testTypeStr.equals("lang_names")) {
testType = LangNamesTester.INSTANCE;
} else {
io.lacuna.bifurcan.IMap<String,String> response =
parsedInputPersistentMap
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package org.unicode.conformance.testtype.langnames;

import org.unicode.conformance.testtype.ITestTypeInputJson;

public class LangNamesInputJson implements ITestTypeInputJson {

public String test_type;

public String label;

public String language_label;

public String locale_label;

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package org.unicode.conformance.testtype.langnames;

import org.unicode.conformance.testtype.ITestTypeOutputJson;

public class LangNamesOutputJson implements ITestTypeOutputJson {

public String test_type;

public String label;

public String language_label;

public String locale_label;

public String result;

public String error;

public String error_message;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package org.unicode.conformance.testtype.langnames;

import com.ibm.icu.util.ULocale;
import io.lacuna.bifurcan.Map;
import org.unicode.conformance.ExecutorUtils;
import org.unicode.conformance.testtype.ITestType;
import org.unicode.conformance.testtype.ITestTypeInputJson;
import org.unicode.conformance.testtype.ITestTypeOutputJson;
import org.unicode.conformance.testtype.collator.CollatorInputJson;
import org.unicode.conformance.testtype.collator.CollatorOutputJson;

public class LangNamesTester implements ITestType {

public static LangNamesTester INSTANCE = new LangNamesTester();

@Override
public ITestTypeInputJson inputMapToJson(Map<String, String> inputMapData) {
LangNamesInputJson result = new LangNamesInputJson();

result.test_type = inputMapData.get("test_type", null);
result.label = inputMapData.get("label", null);

result.language_label = inputMapData.get("language_label", null);
result.locale_label = inputMapData.get("locale_label", null);

return result;
}

@Override
public ITestTypeOutputJson execute(ITestTypeInputJson inputJson) {
LangNamesInputJson input = (LangNamesInputJson) inputJson;

// partially construct output
LangNamesOutputJson output = new LangNamesOutputJson();
output.label = input.label;

try {
String displayNameResult = getDisplayLanguageString(input);
output.result = displayNameResult;
} catch (Exception e) {
output.error = "error running test";
output.error = e.getMessage();
return output;
}

// If we get here, it's a pass/fail result (supported options and no runtime errors/exceptions)
return output;
}

@Override
public String formatOutputJson(ITestTypeOutputJson outputJson) {
return ExecutorUtils.GSON.toJson((LangNamesOutputJson) outputJson);
}

public String getDisplayLanguageString(LangNamesInputJson input) {
String localeID = input.language_label;
String displayLocaleID = input.locale_label;
return ULocale.getDisplayNameWithDialect(localeID, displayLocaleID);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package org.unicode.conformance.langnames;

import static org.junit.Assert.assertEquals;

import org.junit.Test;
import org.unicode.conformance.testtype.langnames.LangNamesOutputJson;
import org.unicode.conformance.testtype.langnames.LangNamesTester;

public class LangNamesTest {

@Test
public void testLocaleAndDisplayLocale() {
String testInput =
"{\"test_type\": \"lang_names\", \"label\": \"01\", \"language_label\": \"fr\", \"locale_label\": \"de\"}";

LangNamesOutputJson output =
(LangNamesOutputJson) LangNamesTester.INSTANCE.getStructuredOutputFromInputStr(testInput);

assertEquals("Französisch", output.result);
}

}
45 changes: 28 additions & 17 deletions executors/test_strings.txt
Original file line number Diff line number Diff line change
Expand Up @@ -44,23 +44,23 @@
{"test_type": "collation_short", "label":"00008","s1":"ä","s2":"ä","compare_type":"=","test_description":" simple CEs & expansions","rules":"&\\x01\n<<<\\u0300\n&9<\\x00\n&\\uA00A\\uA00B=\\uA002\n&\\uA00A\\uA00B\\u00050005=\\uA003"}

=======
{"test_type": "display_names", "label": "01", "language_label": "en", "locale_label": "af"}
{"test_type": "display_names", "label": "01", "language_label": "", "locale_label": "fr"}
{"test_type": "display_names", "label": "01", "language_label": "de", "locale_label": "fr"}
{"test_type": "display_names", "label": "02", "language_label": "fr", "locale_label": "de"}
{"test_type": "display_names", "label": "LANG_ABC", "language_label": "en", "locale_label": "ja"}
{"test_type": "display_names", "label": "LANG_ABC", "language_label": "en", "locale_label": "pt-PT"}
{"test_type": "display_names", "label": "LANG_ABC", "language_label": "en", "locale_label": "zh-CN"}
{"test_type": "display_names", "label": "LANG_ABC", "language_label": "en", "locale_label": "es"}
{"test_type": "display_names", "label": "LANG_af_NA", "language_label": "en", "locale_label": "af_NA"}
{"label":"188691","language_label":"zh_MO","locale_label":"en_150","test_type":"language_display_name"}
{"label":"188691","language_label":"zh-TW","locale_label":"en","test_type":"language_display_name"}
{"label":"188691","language_label":"zh","locale_label":"en","test_type":"language_display_name"}
{"label":"zh zh","language_label":"zh","locale_label":"zh","test_type":"language_display_name"}
{"label":"en zh","language_label":"en","locale_label":"zh-CN","test_type":"language_display_name"}
{"label":"188691","language_label":"zh-CN","locale_label":"en","test_type":"language_display_name"}

{"label":"zh en-150","language_label":"zh","locale_label":"en_150","test_type":"language_display_name"}
{"test_type": "lang_names", "label": "01", "language_label": "en", "locale_label": "af"}
{"test_type": "lang_names", "label": "01", "language_label": "", "locale_label": "fr"}
{"test_type": "lang_names", "label": "01", "language_label": "de", "locale_label": "fr"}
{"test_type": "lang_names", "label": "02", "language_label": "fr", "locale_label": "de"}
{"test_type": "lang_names", "label": "LANG_ABC", "language_label": "en", "locale_label": "ja"}
{"test_type": "lang_names", "label": "LANG_ABC", "language_label": "en", "locale_label": "pt-PT"}
{"test_type": "lang_names", "label": "LANG_ABC", "language_label": "en", "locale_label": "zh-CN"}
{"test_type": "lang_names", "label": "LANG_ABC", "language_label": "en", "locale_label": "es"}
{"test_type": "lang_names", "label": "LANG_af_NA", "language_label": "en", "locale_label": "af_NA"}
{"label":"188691","language_label":"zh_MO","locale_label":"en_150","test_type":"lang_names"}
{"label":"188691","language_label":"zh-TW","locale_label":"en","test_type":"lang_names"}
{"label":"188691","language_label":"zh","locale_label":"en","test_type":"lang_names"}
{"label":"zh zh","language_label":"zh","locale_label":"zh","test_type":"lang_names"}
{"label":"en zh","language_label":"en","locale_label":"zh-CN","test_type":"lang_names"}
{"label":"188691","language_label":"zh-CN","locale_label":"en","test_type":"lang_names"}

{"label":"zh en-150","language_label":"zh","locale_label":"en_150","test_type":"lang_names"}

# COLLATION
{"test_type": "coll_shift_short", "label": "COLL_ABC1", "string1": "de", "string2" : "da"}
Expand Down Expand Up @@ -147,6 +147,17 @@
# FOR TESTING REJECTION OF UNKNOWN TYPE
{"test_type": "bogus_fmt", "label":"NUM_2345", "input": "23.45","locale":"it","options":{"maximumFractionDigits":3}}

{"test_type": "number_fmt", "label":"0019","locale":"es-MX","skeleton":"compact-short currency/EUR unit-width-narrow","input":"91827.3645","options":{"notation":"compact","compactDisplay":"short","style":"currency","currencyDisplay":"narrowSymbol","currency":"EUR","unitDisplay":"narrow","maximumFractionDigits":2}}
{"test_type": "number_fmt", "label":"0019-2","locale":"es-MX","skeleton":"compact-short currency/EUR unit-width-narrow","input":"91827.3645","options":{"notation":"compact","compactDisplay":"short","style":"currency","currencyDisplay":"narrowSymbol","currency":"EUR","unitDisplay":"narrow"}}

{"test_type": "number_fmt", "label":"0648","locale":"es-MX","skeleton":"compact-short percent decimal-always","input":"0","options":{"notation":"compact","compactDisplay":"short","style":"unit","unit":"percent","conformanceDecimalAlways":true,"maximumFractionDigits":2}}
{"test_type": "number_fmt", "label":"0648-no-max-fraction","locale":"es-MX","skeleton":"compact-short percent decimal-always","input":"0","options":{"notation":"compact","compactDisplay":"short","style":"unit","unit":"percent","conformanceDecimalAlways":true}}
{"test_type": "number_fmt", "label":"0649","locale":"es-MX","skeleton":"compact-short percent decimal-always","input":"91827.3645","options":{"notation":"compact","compactDisplay":"short","style":"unit","unit":"percent","conformanceDecimalAlways":true,"maximumFractionDigits":2}}
{"test_type": "number_fmt", "label":"0649","locale":"es-MX","skeleton":"compact-short percent decimal-always","input":"91827.3645","options":{"notation":"compact","compactDisplay":"short","style":"unit","unit":"percent","conformanceDecimalAlways":true}}
{"test_type": "number_fmt", "label":"5913","op":"format", "locale": "en", "skeleton":".00","input":"123456789","options":{"roundingMode":"halfEven","minimumIntegerDigits":1,"maximumFractionDigits":0,"useGrouping":false}}
{"test_type": "number_fmt", "label":"5913","op":"format", "locale":"es-MX", "skeleton":"percent .##/@@@+","input":"123456789.9876543210","options":{"roundingMode":"halfEven","minimumIntegerDigits":1,"maximumFractionDigits":0,"useGrouping":false}}


# LOCALE_INFO
{"test_type":"likely_subtags", "option":"maximize", "locale":"en", "label":"en_max"}
{"test_type":"likely_subtags", "option":"minimize", "locale":"en-US", "label":"en_min"}
Expand Down
15 changes: 15 additions & 0 deletions run_config.json
Original file line number Diff line number Diff line change
Expand Up @@ -240,5 +240,20 @@
],
"per_execution": 10000
}
},
{
"prereq": {
"name": "mvn-icu4j-73-shaded",
"version": "73",
"command": "mvn -f ../executors/icu4j/73/executor-icu4j/pom.xml package"
},
"run": {
"icu_version": "icu73",
"exec": "icu4j",
"test_type": [
"lang_names"
],
"per_execution": 10000
}
}
]
2 changes: 1 addition & 1 deletion schema/collation_short/test_schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
"description":" Obsolete tag to be removed and replaced with test Type"
},
"tests": {
"description": "list of N test for collation each of type collation_short",
"description": "list of N tests for collation each of type collation_short",
"type": ["array", "null"],
"items": {
"type": "object",
Expand Down
2 changes: 1 addition & 1 deletion schema/language_names/test_schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
}
},
"tests": {
"description": "list of N tests for likely subtags",
"description": "list of N tests for language names",
"type": "array",
"items": {
"type": "object",
Expand Down
6 changes: 5 additions & 1 deletion schema/number_format/test_schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,12 @@
"description": "language tag for formatting the output",
"type": "string"
},
"pattern": {
"description": "Pattern-style description of parameters. May be used to create a skeleton.",
"type": "string"
},
"skeleton": {
"description": "Skeleton-style description of parameters. Informational only. ",
"description": "Skeleton-style description of parameters. ",
"type": "string"
},
"input": {
Expand Down
16 changes: 3 additions & 13 deletions schema/schema_validator.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,6 @@ def validate_json_file(self, schema_file_path, data_file_path):
# Now check this!
try:
validate(data_to_check, schema)
logging.info('This test output file validates: %s, %s:',
data_file_path, schema_file_path)
return True, None
except ValidationError as err:
logging.error('ValidationError for test output %s and schema %s',
Expand All @@ -92,9 +90,6 @@ def validate_test_data_with_schema(self):
all_results = []
for test_type in self.test_types:
for icu_version in self.icu_versions:
if self.debug > 0:
logging.info('Checking test data %s, %s', test_type, icu_version)
logging.info('Checking %s, %s', test_type, icu_version)
result_data = self.check_test_data_schema(icu_version, test_type)
logging.debug('test result data = %s', result_data)
msg = result_data['err_info']
Expand All @@ -111,7 +106,7 @@ def validate_test_data_with_schema(self):

def check_test_data_schema(self, icu_version, test_type):
# Check the generated test data for structure agains the schema
logging.info('Validating %s with icu version %s', test_type, icu_version)
logging.info('Validating %s with %s', test_type, icu_version)

# Check test output vs. the test data schema
schema_verify_file = os.path.join( self.schema_base, test_type, 'test_schema.json')
Expand Down Expand Up @@ -150,7 +145,7 @@ def check_test_data_schema(self, icu_version, test_type):

def check_test_output_schema(self, icu_version, test_type, executor):
# Check the output of the tests for structure against the schema
logging.info('Validating %s with icu version %s', test_type, icu_version)
logging.info('Validating test output: %s %s %s', executor , test_type, icu_version)

# Check test output vs. the schema
schema_file_name = SCHEMA_FILE_MAP[test_type]['result_data']['schema_file']
Expand All @@ -177,9 +172,7 @@ def check_test_output_schema(self, icu_version, test_type, executor):
result, err_msg = self.validate_json_file(schema_verify_file, test_result_file)
results['result'] = result
results['err_info'] = err_msg
if result:
logging.info('Result data %s validated with %s, ICU %s', test_type, executor, icu_version)
else:
if not result:
logging.error('Result data %s FAILED with %s ICU %s: %s', test_type, executor, icu_version, err_msg)

return results
Expand Down Expand Up @@ -235,16 +228,13 @@ def validate_test_output_with_schema(self):
for executor in self.executors:
for icu_version in self.icu_versions:
for test_type in self.test_types:
logging.info('Checking %s, %s, %s', test_type, icu_version, executor)
results = self.check_test_output_schema(icu_version, test_type, executor)
if not results['data_file_name']:
# This is not an error but simple a test that wasn't run.
continue
if not results['result']:
logging.warning('VALIDATION FAILS: %s %s %s. MSG=%s',
test_type, icu_version, executor, results['err_info'])
else:
logging.info('VALIDATION WORKS: %s %s %s', test_type, icu_version, executor)
all_results.append(results)
return all_results

Expand Down
Loading

0 comments on commit 3c1068a

Please sign in to comment.