fix: Update schema to detect duplicates

This change updates the schema as suggested in silnrsi#17 to detect duplicates
in tags. It also adds a unit tests to detect duplicates that appear in
multiple entries, as suggested in silnrsi#16.

This change does not update the data and thus will cause failing tests.

source/langtags_schema.json

@@ -27,12 +27,14 @@
         "tags": {
           "type": "array",
           "items": { "$ref": "#/definitions/bcp47" },
-          "additionalItems": false
+          "additionalItems": false,
+          "uniqueItems": true
         },
         "variants": {
           "type": "array",
           "items": { "$ref": "#/definitions/bcp47_variant" },
-          "additionalItems": false
+          "additionalItems": false,
+          "uniqueItems": true
         },
         "iso639_3": {
           "$ref": "#/definitions/iso639_3"
@@ -43,21 +45,24 @@
         "regions": {
           "type": "array",
           "items": { "$ref": "#/definitions/iso3166_1" },
-          "additionalItems": false
+          "additionalItems": false,
+          "uniqueItems": true
         },
         "regionname": {
           "type": "string"
         },
         "iana": {
           "type": "array",
-          "items": { "type": "string" }
+          "items": { "type": "string" },
+          "uniqueItems": true
         },
         "name": {
           "type": "string"
         },
         "names": {
           "type": "array",
-          "items": { "type": "string" }
+          "items": { "type": "string" },
+          "uniqueItems": true
         },
         "localname": {
           "type": "string"
@@ -76,11 +81,13 @@
         },
         "localnames": {
           "type": "array",
-          "items": { "type": "string" }
+          "items": { "type": "string" },
+          "uniqueItems": true
         },
         "latnnames": {
           "type": "array",
-          "items": { "type": "string" }
+          "items": { "type": "string" },
+          "uniqueItems": true
         },
         "suppress": {
           "type": "boolean"

tests/test_json.py

@@ -5,6 +5,7 @@
 schemapath = os.path.join(os.path.dirname(__file__), '..', 'source', 'langtags_schema.json')
 testfile = os.path.join(os.path.dirname(__file__), '..', 'pub', 'langtags.json')
 
+
 class JsonSchemaTest(unittest.TestCase):
     ''' Tests that generated JSON conforms to the schema '''
     def test_jsonschema(self):
@@ -25,7 +26,25 @@ def test_jsonschema(self):
             testdata = json.load(inf)
         for error in validator.iter_errors(testdata):
             errors.append(error.message)
-        if len(errors):
+        if len(errors) > 0:
+            self.fail("\n".join(errors))
+
+    def test_duplicate_tags(self):
+        'test for duplicate tags not found by schema'
+        unique_values = set()
+        locations = {}
+        errors = []
+        with open(testfile) as data:
+            testdata = json.load(data)
+        for item in testdata:
+            if 'tags' in item:
+                for value in item['tags']:
+                    if value in unique_values:
+                        errors.append(f'tag {value} is repeated at {item["full"]} and {locations[value]}')
+                    else:
+                        unique_values.add(value)
+                        locations[value] = item['full']
+        if len(errors) > 0:
             self.fail("\n".join(errors))