From 0b9f4fdf86eaacf70e104e85ffa9f5c785175c42 Mon Sep 17 00:00:00 2001 From: Duncan Dewhurst Date: Thu, 29 Jun 2023 08:06:55 +1200 Subject: [PATCH 01/15] Schema reference docs initial draft --- .gitmodules | 1 - docs/_static/docson | 1 + docs/conf.py | 2 +- docs/data_model/index.md | 1 + docs/data_model/schema.md | 87 ++++++++++++++++++++++++++ manage.py | 125 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 215 insertions(+), 2 deletions(-) create mode 160000 docs/_static/docson create mode 100644 docs/data_model/schema.md diff --git a/.gitmodules b/.gitmodules index 831c3f27..91a432db 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,3 @@ [submodule "docs/_static/docson"] path = docs/_static/docson url = https://github.com/OpenDataServices/docson - branch = master diff --git a/docs/_static/docson b/docs/_static/docson new file mode 160000 index 00000000..8ea21cd9 --- /dev/null +++ b/docs/_static/docson @@ -0,0 +1 @@ +Subproject commit 8ea21cd9c34e7379a2ac0aba640933901345d7d3 diff --git a/docs/conf.py b/docs/conf.py index 79b49083..2698e5b3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -174,7 +174,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ['_static', '../schema'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied diff --git a/docs/data_model/index.md b/docs/data_model/index.md index cfc0c2e5..6f4045ab 100644 --- a/docs/data_model/index.md +++ b/docs/data_model/index.md @@ -63,5 +63,6 @@ ______________________________________________________________________ vulnerability loss codelists + schema ``` diff --git a/docs/data_model/schema.md b/docs/data_model/schema.md new file mode 100644 index 00000000..63d94805 --- /dev/null +++ b/docs/data_model/schema.md @@ -0,0 +1,87 @@ +# Schema + +The schema provides the authoritative definition of the structure of Risk Data Library Standard (RDLS) data, the meaning of each field, and the rules that must be followed to publish RDLS data. It is used to validate the structure and format of RDLS data. + +For this version of RDLS, the canonical URL of the schema is []](). Use the canonical URL to make sure that your software, documentation or other resources refer to the specific version of the schema with which they were tested. + +This page presents the schema in an [interactive browser](#browser) and in [reference tables](#reference-tables) with additional information in paragraphs. You can also download the canonical version of the schema as [JSON Schema](../../schema/rdl_schema_0.1.json) or download it as a CSV spreadsheet [TODO]. + +```{note} + If any conflicts are found between the text on this page and the text within the schema, the text within the schema takes precedence. +``` + +## Browser + +Click on schema elements to expand the tree, or use the '+' icon to expand all elements. Use { } to view the underlying schema for any section. Required fields are indicated in **bold**. + + + +## Reference tables + +### Dataset + +```{jsonschema} ../../schema/rdl_schema_0.1.json +:addtargets: +``` + +### Hazard + +```{jsonschema} ../../schema/rdl_schema_0.1.json +:pointer: /anyOf/0/properties/hazard +:collapse: +:addtargets: +``` + +### Exposure + +```{jsonschema} ../../schema/rdl_schema_0.1.json +:pointer: /anyOf/1/properties/exposure +:collapse: +:addtargets: +``` + +### Vulnerability + +```{jsonschema} ../../schema/rdl_schema_0.1.json +:pointer: /anyOf/2/properties/vulnerability +:collapse: +:addtargets: +``` + +### Loss + +```{jsonschema} ../../schema/rdl_schema_0.1.json +:pointer: /anyOf/3/properties/loss +:collapse: +:addtargets: +``` + +### Sub-schemas + +#### common_aggregation_type + +```{jsonschema} ../../schema/rdl_schema_0.1.json +:pointer: /$defs/common_aggregation_type +:collapse: +:addtargets: +``` + + "common_analysis_type": + "common_calc_method": + "common_exp_category": + "common_exp_occupancy": + "common_exp_val_type": + "common_frequency_type": + "common_hazard_type": + "common_impact_type": + "common_iso": + "common_license": + "common_occupancy_time": + "common_process_type": + "im_code": + "loss_loss_type": + "loss_metric": + "vulnerability_function_type": + "vulnerability_f_subtype": + "vulnerability_f_relationship": + "vulnerability_f_math": diff --git a/manage.py b/manage.py index 91a0d401..62bb8f76 100644 --- a/manage.py +++ b/manage.py @@ -229,6 +229,128 @@ def update_codelist_docs(schema): write_lines(referencedir / 'codelists.md', codelist_reference) +def get_definition_references(schema, defn, parents=None, full_schema=None): + """ + Recursively generate a list of JSON pointers that reference a definition in JSON schema. + + :param schema: The JSON schema + :defn: The name of the definition + :parents: A list of the parents of schema + :full_schema: The full schema + """ + + references = [] + + if parents is None: + parents = [] + + if full_schema is None: + full_schema = schema + + if 'properties' in schema: + for key, value in schema['properties'].items(): + if value.get('type') == 'array' and '$ref' in value['items']: + if value['items']['$ref'] == f"#/$defs/{defn}": + references.append(parents + [key, '0']) + else: + references.extend(get_definition_references(full_schema['$defs'][value['items']['$ref'].split('/')[-1]], defn, parents + [key, '0'], full_schema)) + elif '$ref' in value: + if value['$ref'] == f"#/$defs/{defn}": + references.append(parents + [key]) + else: + references.extend(get_definition_references(full_schema['$defs'][value['$ref'].split('/')[-1]], defn, parents + [key], full_schema)) + elif 'properties' in value: + references.extend(get_definition_references(value, defn, parents + [key], full_schema)) + + if '$defs' in schema: + for key, value in schema['$defs'].items(): + references.extend(get_definition_references(value, defn, [key], full_schema)) + + return references + + +def update_schema_docs(schema): + """Update schema.md""" + + # Load schema reference + schema_reference = read_lines(referencedir / 'schema.md') + + # Preserve content that appears before the generated reference content for each component + components_index = schema_reference.index("### Sub-schemas\n") + 3 + + for i in range(components_index, len(schema_reference)): + if schema_reference[i][:5] == "#### ": + defn = schema_reference[i][5:-1] + + # Drop definitions that don't appear in the schema + if defn in schema["$defs"]: + schema["$defs"][defn]["content"] = [] + j = i+1 + + # while j < len(schema_reference) and not schema_reference[j].startswith("```{admonition}") and schema_reference[j] != 'This component is referenced by the following properties:\n': + while j < len(schema_reference) and not schema_reference[j].startswith("```{admonition}") and schema_reference[j] != f"`{defn}` is defined as:\n": + schema["$defs"][defn]["content"].append(schema_reference[j]) + j = j+1 + + # Preserve introductory content up to and including the sentence below the ### Sub-schemas heading + schema_reference = schema_reference[:components_index] + schema_reference.append("\n") + + # Generate standard reference content for each definition + for defn, definition in schema["$defs"].items(): + definition["content"] = definition.get("content", []) + + # Add heading + definition["content"].insert(0, f"#### {defn}\n") + + # Add description + definition["content"].extend([ + f"`{defn}` is defined as:\n\n", + "```{jsoninclude-quote} ../../schema/rdl_schema_0.1.json\n", + f":jsonpointer: /$defs/{defn}/description\n", + "```\n\n" + ]) + + # Add a list of properties that reference this definition + definition["references"] = get_definition_references(schema, defn) + definition["content"].append("This component is referenced by the following properties:\n\n") + + for ref in definition["references"]: + # Remove array indices because they do not appear in the HTML anchors generated by the json schema directive + ref = [part for part in ref if part != '0'] + + # Ideally, these would be relative links - see https://github.com/OpenDataServices/sphinxcontrib-opendataservices/issues/43 + url = 'rdl_schema_0.1.json,' + + # Omit nested references + if ref[0] in schema['$defs'] and len(ref) == 2: + url += '/$defs/' + elif len(ref) == 1: + url += ',' + else: + continue + + url += ','.join(ref) + definition["content"].append(f"- [`{'/'.join(ref)}`]({url})\n") + + if definition.get('additionalProperties') == False: + definition["content"].append(f"\nAdditional properties are not permitted within `{defn}` objects.\n") + + # Add schema table + definition["content"].extend([ + f"\nEach `{defn}` has the following fields:\n\n", + "```{jsonschema} ../../schema/rdl_schema_0.1.json\n", + f":pointer: /$defs/{defn}\n", + f":collapse: {','.join(definition['properties'].keys())}\n", + ":addtargets:\n", + "```\n\n", + ]) + + schema_reference.extend(definition["content"]) + + write_lines(referencedir / 'schema.md', schema_reference) + + @click.group() def cli(): pass @@ -242,6 +364,9 @@ def pre_commit(): # Load schema schema = json_load('rdl_schema_0.1.json') + # Update schema.md and subschemas/* + update_schema_docs(schema) + # Update codelists.md update_codelist_docs(schema) From e58076a4f1a6f96626f05d217c54506e4f7814d7 Mon Sep 17 00:00:00 2001 From: Duncan Dewhurst Date: Tue, 4 Jul 2023 11:48:43 +1200 Subject: [PATCH 02/15] docs/data_model: Integrate content from general.md to schema.md --- docs/data_model/general.md | 28 ---------------------------- docs/data_model/index.md | 1 - docs/data_model/schema.md | 12 ++++++++++++ 3 files changed, 12 insertions(+), 29 deletions(-) delete mode 100644 docs/data_model/general.md diff --git a/docs/data_model/general.md b/docs/data_model/general.md deleted file mode 100644 index a6f1a957..00000000 --- a/docs/data_model/general.md +++ /dev/null @@ -1,28 +0,0 @@ -# General attributes - -In addition to schema-specific attributes, each dataset is identified by a list of attributes based on the [Dublin Core Metadata Initiative Metadata Terms](https://www.dublincore.org/specifications/dublin-core/dcmi-terms). - -| **Required** | **Attribute** | **Description** | **Type** | -| :----------: | ------------- | ----------------------------------------------------------------------------------------------- | --------------------------------------------------- | -| \* | Component | Schema to be used |