Skip to content

Commit

Permalink
Nodejs collation tests: Change "fail" to "unsupported" when a rule or…
Browse files Browse the repository at this point in the history 
… complex comparison is used. (unicode-org#321)

* NodeJS with collator - moving rules items to unsupported

* Node Collation: tests with rules or non-standard compare --> 'unsupported'

* Remove run_config.json from this PR

* Updating collation in node

* Fix collation rules, improving results

* Fixing some collation problems

* amend

* Fixing attribute list handling in ICU4J collation

* Restructured collation data generation

* Node Collation: reclassify problems from error to fail

* Remove extra cout statements
  • Loading branch information
@sven-oly
sven-oly authored Jan 8, 2025
1 parent 46539fc commit 5b47f49
Show file tree
Hide file tree
Showing 9 changed files with 136 additions and 56 deletions.
31 changes: 21 additions & 10 deletions executors/cpp/coll.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ auto TestCollator(json_object *json_in) -> string {
json_object *str1 = json_object_object_get(json_in, "s1");
json_object *str2 = json_object_object_get(json_in, "s2");

// Unescape the input strings?
string string1 = json_object_get_string(str1);
string string2 = json_object_get_string(str2);

Expand All @@ -78,6 +79,9 @@ auto TestCollator(json_object *json_in) -> string {
string compare_type_string;
if (compare_type_obj != nullptr) {
compare_type_string = json_object_get_string(compare_type_obj);
if (compare_type_string.substr(0,4) == "<") {
compare_type_string = "<" + compare_type_string.substr(4,1);
}
}

// Strength of comparison
Expand All @@ -95,7 +99,7 @@ auto TestCollator(json_object *json_in) -> string {
strength_type = Collator::TERTIARY;
} else if (strength_string == "quaternary") {
strength_type = Collator::QUATERNARY;
} else if (strength_string == "IDENTICAL") {
} else if (strength_string == "identical") {
strength_type = Collator::IDENTICAL;
}
}
Expand All @@ -108,13 +112,6 @@ auto TestCollator(json_object *json_in) -> string {
}
UnicodeString uni_rules = UnicodeString::fromUTF8(rules_string);

// Allow for different levels or types of comparison.
json_object *compare_type = json_object_object_get(json_in, "compare_type");
if (compare_type != nullptr) {
// TODO: Apply this in tests.
const char *comparison_type = json_object_get_string(compare_type);
}

// Handle some options
json_object *ignore_obj =
json_object_object_get(json_in, "ignorePunctuation");
Expand Down Expand Up @@ -153,9 +150,16 @@ auto TestCollator(json_object *json_in) -> string {
} else {
// Not a rule-based collator.
if (strlen(locale_string) <= 0) {
// Uses the default Locale.
uni_coll = Collator::createInstance(status);
} else {
uni_coll = Collator::createInstance(Locale(locale_string), status);
Locale this_locale;
if (locale_string == "root") {
this_locale = Locale::getRoot();
} else {
this_locale = Locale(locale_string);
}
uni_coll = Collator::createInstance(this_locale, status);
}

if (check_icu_error(
Expand Down Expand Up @@ -209,7 +213,14 @@ auto TestCollator(json_object *json_in) -> string {
}
}

coll_result = (uni_result != UCOL_GREATER);
// Use the compare_type to see if "<" or "=" should be applied.
if (compare_type_string == "" || compare_type_string.substr(0, 1) == "<") {
// Default checking for <= 0.
coll_result = (uni_result != UCOL_GREATER);
} else {
coll_result = (uni_result == UCOL_EQUAL);
}

if (!coll_result) {
// Test did not succeed!
// Include data compared in the failing test
Expand Down
6 changes: 3 additions & 3 deletions executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/Icu4jExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,13 +151,13 @@ public static String getTestCaseResponse(String inputLine) throws Exception {

try {
return testType.getFinalOutputFromInput(parsedInputPersistentMap);
} catch (Exception e) {
} catch (Throwable t) {
ITestTypeOutputJson defaultOutput = testType.getDefaultOutputJson();
return ExecutorUtils.formatAsJson(
testType.convertOutputToMap(defaultOutput)
.put("label", parsedInputPersistentMap.get("label", null))
.put("error", "Error in input" + e.getMessage())
.put("error_message", "Error in handling test case: " + e.getMessage())
.put("error", "Error in input" + t.getMessage())
.put("error_message", "Error in handling test case: " + t.getMessage())
);
}
}
Expand Down
3 changes: 2 additions & 1 deletion ...utor-icu4j/src/main/java/org/unicode/conformance/testtype/collator/CollatorInputJson.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package org.unicode.conformance.testtype.collator;

import java.util.ArrayList;
import org.unicode.conformance.testtype.ITestTypeInputJson;

public class CollatorInputJson implements ITestTypeInputJson {
Expand All @@ -21,7 +22,7 @@ public class CollatorInputJson implements ITestTypeInputJson {

public String test_description;

public String[] attributes;
public ArrayList<String> attributes;

public String rules;

Expand Down
22 changes: 16 additions & 6 deletions ...xecutor-icu4j/src/main/java/org/unicode/conformance/testtype/collator/CollatorTester.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import com.ibm.icu.util.ULocale;
import io.lacuna.bifurcan.IMap;
import io.lacuna.bifurcan.Map;
import java.util.ArrayList;
import java.util.Optional;
import org.unicode.conformance.ExecutorUtils;
import org.unicode.conformance.testtype.ITestType;
Expand Down Expand Up @@ -42,16 +43,24 @@ public ITestTypeInputJson inputMapToJson(Map<String, Object> inputMapData) {
result.ignorePunctuation = (boolean) inputMapData.get("ignorePunctuation", false);
result.line = (int) ((double) inputMapData.get("line", 0.0));

// Resolve "&lt;"
result.compare_type = (String) inputMapData.get("compare_type", null);
if (result.compare_type != null && ! result.compare_type.equals("") && result.compare_type.length() > 4) {
String first_part = result.compare_type.substring(0,4);
if (first_part.equals("&lt;")) {
String next_part = result.compare_type.substring(4,5);
result.compare_type = "<" + next_part;
}
}
result.test_description = (String) inputMapData.get("test_description", null);

// TODO: implement this correctly recursively (either using APIs or else DIY)
String[] attrs;
Optional<Object> attrsString = inputMapData.get("attributes");
if (attrsString.isPresent()) {
attrs = new String[]{ (String) attrsString.get() };
ArrayList<String> attrs;
Optional<Object> attrsListOpt = inputMapData.get("attributes");
if (attrsListOpt.isPresent()) {
attrs = (ArrayList<String>) attrsListOpt.get();
} else {
attrs = new String[]{};
attrs = new ArrayList<>();
}
result.attributes = attrs;

Expand Down Expand Up @@ -90,6 +99,7 @@ public ITestTypeOutputJson execute(ITestTypeInputJson inputJson) {

try {
int collResult = coll.compare(input.s1, input.s2);
// TODO! Use compare_type to check for <= or ==.
if (collResult > 0) {
// failure
output.result = false;
Expand Down Expand Up @@ -137,7 +147,7 @@ public String formatOutputJson(ITestTypeOutputJson outputJson) {
public Collator getCollatorForInput(CollatorInputJson input) {
RuleBasedCollator result = null;

if (input.locale == null) {
if (input.locale == null || input.locale == "root") {
if (input.rules == null) {
result = (RuleBasedCollator) Collator.getInstance(ULocale.ROOT);
} else {
Expand Down
20 changes: 20 additions & 0 deletions .../74/executor-icu4j/src/test/java/org/unicode/conformance/collator/icu74/CollatorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,24 @@ public void testNonEscaped() {
assertTrue(output.result);
}

@Test
public void testAttributesAsArrayList() {
String testInput =
"{\"test_type\": \"collation_short\", \"label\":\"00099\",\"s1\":\"\",\"s2\":\"a\",\"line\":293,\"compare_type\":\"&lt;1\",\"test_description\":\" discontiguous contractions\",\"attributes\":[[\"strength\",\"primary\"],[\"strength\",\"secondary\"],[\"strength\",\"tertiary\"],[\"strength\",\"quaternary\"],[\"strength\",\"identical\"]],\"strength\":\"primary\",\"hexhash\":\"4c4c61ac18e9c222aaa457daa5d72a72ce0490d2\"}";

CollatorOutputJson output =
(CollatorOutputJson) CollatorTester.INSTANCE.getStructuredOutputFromInputStr(testInput);

assertTrue(output.result);
}

/* @Test
public void testCompareLT2() {
String testInput =
"{\"test_type\": \"collation_short\", \"label\":\"00115\",\"s1\":\"cote\",\"s2\":\"coté\",\"line\":329,\"source_file\":\"collationtest.txt\",\"compare_type\":\"&lt;2\",\"test_description\":\" discontiguous contractions\",\"hexhash\":\"b56b2f345f58f7044c14e392ea94304c075cbaf5\"}";
CollatorOutputJson output =
(CollatorOutputJson) CollatorTester.INSTANCE.getStructuredOutputFromInputStr(testInput);
assertTrue(output.result);
}*/
}
85 changes: 57 additions & 28 deletions executors/node/collator.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
// The Collator used for the actual testing.

// Collation: determine the sensitivity that corresponds to the strength.
module.exports = {

testCollationShort: function(json) {
Expand All @@ -13,19 +12,47 @@ module.exports = {
}
let testCollOptions = {};
if ('ignorePunctuation' in json) {
testCollOptions = {
ignorePunctuation:json['ignorePunctuation']}
testCollOptions['ignorePunctuation'] = json['ignorePunctuation'];
}

if ('numeric' in json) {
testCollOptions['numeric'] = true;
}
if ('caseFirst' in json) {
testCollOptions['caseFirst'] = json['case_first'];
}
const strength = json['strength'];
if (strength) {
if (strength == 'primary') {
testCollOptions['sensitivity'] = 'base';
} else
if (strength == 'secondary') {
testCollOptions['sensitivity'] = 'accent';
} else
if (strength == 'tertiary') {
testCollOptions['sensitivity'] = 'case';
}
}

let outputLine = {'label':json['label']};
// Get other fields if provided
let rules = undefined;
if ('rules' in json) {
rules = json['rules'];
outputLine['unsupported'] = 'Collator rules not available';
outputLine['error_detail'] = 'Rules not supported';
return outputLine;
}

let compare_type = undefined;
let compare_type;
if ('compare_type' in json) {
compare_type = json['compare_type'];
compare_type = json['compare_type'].trim();
compare_type = compare_type.replace('&lt;', '<');
}

let reoder;
if ('reorder' in json) {
reorder = json['reorder'];
}

// Set up collator object with optional locale and testOptions.
Expand All @@ -38,46 +65,48 @@ module.exports = {

// Should we check with < or <=?
const compared = coll.compare(d1, d2);
let result = compared<= 0 ? true : false;
let result_bool = true;
if (compared > 0) {
result_bool = false;

let result = false;
// Check for strict equality comparison
if (compare_type) {
if (compare_type == '=' && compared == 0) {
result = true;
} else
// Check results with different compare types
if (compare_type[0] == '<' && compared < 0) {
result = true;
}
} else {
// Default comparison method.
result = (compared <= 0);
}
outputLine = {'label':json['label'],
}

outputLine['result'] = result;
if (result == true) {
// Only output result field if result is true.
outputLine['result'] = result_bool;
outputLine['compare_result'] = compared;
} else {
// Additional info for the comparison
outputLine['compare'] = compared;
if (rules) {
outputLine['unsupported'] = 'Collator rules not available';
outputLine['error_detail'] = 'No rules';
outputLine['error'] = 'rules';
}
else {
outputLine['actual_options'] = JSON.stringify(coll.resolvedOptions()); //.toString();
outputLine['compare_result'] = compared;
outputLine['result'] = result_bool;
}
outputLine['actual_options'] = JSON.stringify(coll.resolvedOptions());
outputLine['compare_result'] = compared;
outputLine['result'] = result;
}

} catch (error) {
const error_message = error.message;

if (testLocale == "root" || error_message == "Incorrect locale information provided") {
if (testLocale == "root" ||
error_message == "Incorrect locale information provided") {
outputLine = {'label': json['label'],
'error_message': error.message,
'unsupported': 'root locale',
'error_detail': error_message + ': ' + testLocale,
'error': 'Unsupported locale'
};
} else {
// Another kind of error.
outputLine = {'label': json['label'],
'error_message': error_message,
'error_message': error.message,
'error_detail': testLocale,
'error': 'Something wrong'
'error': error.name
};
}
}
Expand Down
3 changes: 2 additions & 1 deletion testgen/generators/collation_short.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ def set_patterns(self):
self.comparison_line = re.compile("^([<=]\S*)(\s*)(\S*)(\s*)#?(.*)")

self.input_pattern_with_comment = re.compile("^([^#]+)#?(.*)")

self.attribute_test = re.compile("^% (\S+)\s*=\s*(.+)")
self.reorder_test = re.compile("^% (reorder)\s+(.+)")

Expand Down Expand Up @@ -166,7 +167,7 @@ def check_parse_compare(self, line_index, lines, filename):
tests.append(new_test)
line_index += 1

# Check for string conversion errors. ???
# Check for string conversion errors. ???
if string2_errors:
pass

Expand Down
10 changes: 7 additions & 3 deletions verifier/testreport.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -585,13 +585,17 @@ def flatten_and_combine(self, input_dict, input_simple):
def characterize_results_by_options(self, test_list, category):
# User self.failing_tests, looking at options
results = defaultdict(lambda : defaultdict(list))
if not test_list:
# no test --> no characterizations
return results

results['locale'] = {} # Dictionary of labels for each locale

# Look at particular test types
if self.test_type == 'plural_rules' and test_list:
if self.test_type == 'plural_rules':
self.characterize_plural_rules_tests(test_list, results)

if self.test_type == 'datetime_fmt' and test_list:
if self.test_type == 'datetime_fmt':
self.characterize_datetime_tests(test_list, results)

for test in test_list:
Expand Down Expand Up @@ -780,7 +784,7 @@ def add_to_results_by_key(self, label, results, input_data, test, key_list):
pass
except:
pass

def check_simple_text_diffs(self, test_list, category):
results = defaultdict(list)
all_checks = ['insert', 'delete', 'insert_digit', 'insert_space', 'delete_digit',
Expand Down
12 changes: 8 additions & 4 deletions verifier/verify_plan.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,11 +186,15 @@ def compare_test_to_expected(self):

# Remember details about the test
test['input_data'] = test_data
test['expected'] = expected_result
if actual_result == expected_result:
self.report.record_pass(test)
if 'unsupported' in test:
self.report.record_unsupported(test)
else:
self.report.record_fail(test)
# This should be a supported test type.
test['expected'] = expected_result
if actual_result == expected_result:
self.report.record_pass(test)
else:
self.report.record_fail(test)

except (AttributeError, KeyError):
# Add input information to the test results
Expand Down

0 comments on commit 5b47f49

Please sign in to comment.