From 39faa60f2a460cbd9aef4008b0c5f9a9a150752c Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Mon, 20 May 2024 19:33:56 +0530 Subject: [PATCH 01/32] initial commit --- ndcube/asdf/__init__.py | 0 ndcube/asdf/converters/ndcube_converter.py | 30 ++++++++++++++ ndcube/asdf/entry_points.py | 40 +++++++++++++++++++ .../resources/manifests/ndcube-1.0.0.yaml | 10 +++++ .../asdf/resources/schemas/NDCube-1.0.0.yaml | 21 ++++++++++ pyproject.toml | 6 +++ 6 files changed, 107 insertions(+) create mode 100644 ndcube/asdf/__init__.py create mode 100644 ndcube/asdf/converters/ndcube_converter.py create mode 100644 ndcube/asdf/entry_points.py create mode 100644 ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml create mode 100644 ndcube/asdf/resources/schemas/NDCube-1.0.0.yaml diff --git a/ndcube/asdf/__init__.py b/ndcube/asdf/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py new file mode 100644 index 000000000..a6ff4be70 --- /dev/null +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -0,0 +1,30 @@ +from asdf.extension import Converter +import numpy as np + + +class NDCubeConverter(Converter): + tags = ["tag:sunpy.org:ndcube/ndcube/NDCube-*"] + + @property + def types(self): + from ndcube.ndcube import NDCube + return [NDCube] + + def select_tag(self, obj, tags, ctx): + # Sort the tags in reverse alphabetical order and pick the first (i.e. + # the one with the highest version). This assumes that all the tags for + # this converter are named the same other than the version number. + tags = list(sorted(tags, reverse=True)) + return tags[0] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube import ndcube + data = np.asanyarray(node["data"]) + wcs = node["wcs"] + return(ndcube.NDCube(data, wcs)) + + def to_yaml_tree(self, ndcube, tag, ctx): + node = {} + node["data"] = np.asarray(ndcube.data) + node["wcs"] = ndcube.wcs + return node diff --git a/ndcube/asdf/entry_points.py b/ndcube/asdf/entry_points.py new file mode 100644 index 000000000..2dc63a334 --- /dev/null +++ b/ndcube/asdf/entry_points.py @@ -0,0 +1,40 @@ +""" +This file contains the entry points for asdf. +""" +import importlib.resources as importlib_resources +from asdf.extension import ManifestExtension +from asdf.resource import DirectoryResourceMapping + + +def get_resource_mappings(): + """ + Get the resource mapping instances for myschemas + and manifests. This method is registered with the + asdf.resource_mappings entry point. + + Returns + ------- + list of collections.abc.Mapping + """ + from ndcube.asdf import resources + resources_root = importlib_resources.files(resources) + return [ + DirectoryResourceMapping( + resources_root / "schemas", "asdf://sunpy.org/ndcube/schemas/"), + DirectoryResourceMapping( + resources_root / "manifests", "asdf://sunpy.org/ndcube/manifests/"), + ] + + +def get_extensions(): + """ + Get the list of extensions. + """ + from ndcube.asdf.converters.ndcube_converter import NDCubeConverter + + ndcube_converters = [NDCubeConverter()] + _manifest_uri = "asdf://sunpy.org/ndcube/manifests/ndcube-1.0.0" + + return [ + ManifestExtension.from_uri(_manifest_uri, converters=ndcube_converters) + ] diff --git a/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml b/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml new file mode 100644 index 000000000..aab04090e --- /dev/null +++ b/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml @@ -0,0 +1,10 @@ +%YAML 1.1 +--- +id: asdf://sunpy.org/ndcube/manifests/ndcube-1.0.0 +extension_uri: asdf://sunpy.org/extensions/ndcube-1.0.0 +title: NDCube ASDF Manifest +description: ASDF schemas and tags for NDCube classes. + +tags: + - tag_uri: "tag:sunpy.org:ndcube/ndcube/NDCube-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/NDCube-1.0.0" \ No newline at end of file diff --git a/ndcube/asdf/resources/schemas/NDCube-1.0.0.yaml b/ndcube/asdf/resources/schemas/NDCube-1.0.0.yaml new file mode 100644 index 000000000..2d4862745 --- /dev/null +++ b/ndcube/asdf/resources/schemas/NDCube-1.0.0.yaml @@ -0,0 +1,21 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/NDCube-1.0.0" + +title: | + Represents the ndcube.NDCube class object + +description: + Test ndcube class + +type: object +properties: + data: + tag: "tag:stsci.edu:asdf/core/ndarray-1.0.0" + wcs: + $ref: "http://stsci.edu/schemas/gwcs/wcs-1.1.0" + +required: [data , wcs] +allowAdditionalProperties: False +... \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index d82798b40..7cce3ed73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,12 @@ exclude = ["ndcube._dev*"] [tool.setuptools_scm] write_to = "ndcube/_version.py" +[project.entry-points."asdf.resource_mappings"] +ndcube = "ndcube.asdf.entry_points:get_resource_mappings" + +[project.entry-points."asdf.extensions"] +ndcube = "ndcube.asdf.entry_points:get_extensions" + [tool.towncrier] package = "ndcube" filename = "CHANGELOG.rst" From 9644bbb640663b5118fde80f1f98805b477f1c15 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Mon, 20 May 2024 21:04:24 +0530 Subject: [PATCH 02/32] version alignment --- ndcube/asdf/entry_points.py | 2 +- ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml | 10 ++++++++++ ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml | 10 ---------- .../schemas/{NDCube-1.0.0.yaml => NDCube-0.1.0.yaml} | 6 +++--- 4 files changed, 14 insertions(+), 14 deletions(-) create mode 100644 ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml delete mode 100644 ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml rename ndcube/asdf/resources/schemas/{NDCube-1.0.0.yaml => NDCube-0.1.0.yaml} (70%) diff --git a/ndcube/asdf/entry_points.py b/ndcube/asdf/entry_points.py index 2dc63a334..36126d25c 100644 --- a/ndcube/asdf/entry_points.py +++ b/ndcube/asdf/entry_points.py @@ -33,7 +33,7 @@ def get_extensions(): from ndcube.asdf.converters.ndcube_converter import NDCubeConverter ndcube_converters = [NDCubeConverter()] - _manifest_uri = "asdf://sunpy.org/ndcube/manifests/ndcube-1.0.0" + _manifest_uri = "asdf://sunpy.org/ndcube/manifests/ndcube-0.1.0" return [ ManifestExtension.from_uri(_manifest_uri, converters=ndcube_converters) diff --git a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml new file mode 100644 index 000000000..1d45b4af7 --- /dev/null +++ b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml @@ -0,0 +1,10 @@ +%YAML 1.1 +--- +id: asdf://sunpy.org/ndcube/manifests/ndcube-0.1.0 +extension_uri: asdf://sunpy.org/extensions/ndcube-0.1.0 +title: NDCube ASDF Manifest +description: ASDF schemas and tags for NDCube classes. + +tags: + - tag_uri: "tag:sunpy.org:ndcube/ndcube/NDCube-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/NDCube-0.1.0" \ No newline at end of file diff --git a/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml b/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml deleted file mode 100644 index aab04090e..000000000 --- a/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml +++ /dev/null @@ -1,10 +0,0 @@ -%YAML 1.1 ---- -id: asdf://sunpy.org/ndcube/manifests/ndcube-1.0.0 -extension_uri: asdf://sunpy.org/extensions/ndcube-1.0.0 -title: NDCube ASDF Manifest -description: ASDF schemas and tags for NDCube classes. - -tags: - - tag_uri: "tag:sunpy.org:ndcube/ndcube/NDCube-1.0.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/NDCube-1.0.0" \ No newline at end of file diff --git a/ndcube/asdf/resources/schemas/NDCube-1.0.0.yaml b/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml similarity index 70% rename from ndcube/asdf/resources/schemas/NDCube-1.0.0.yaml rename to ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml index 2d4862745..5154938e1 100644 --- a/ndcube/asdf/resources/schemas/NDCube-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml @@ -1,13 +1,13 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/NDCube-1.0.0" +id: "asdf://sunpy.org/ndcube/schemas/NDCube-0.1.0" title: | - Represents the ndcube.NDCube class object + Represents the ndcube NDCube object description: - Test ndcube class + Represents the ndcube NDCube object type: object properties: From af7dccba720b45a56acf2e519df576dc0e6fa918 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Thu, 23 May 2024 04:16:00 +0530 Subject: [PATCH 03/32] Remove method to remove any delay and --- ndcube/asdf/converters/ndcube_converter.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index a6ff4be70..f84f17603 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -4,18 +4,7 @@ class NDCubeConverter(Converter): tags = ["tag:sunpy.org:ndcube/ndcube/NDCube-*"] - - @property - def types(self): - from ndcube.ndcube import NDCube - return [NDCube] - - def select_tag(self, obj, tags, ctx): - # Sort the tags in reverse alphabetical order and pick the first (i.e. - # the one with the highest version). This assumes that all the tags for - # this converter are named the same other than the version number. - tags = list(sorted(tags, reverse=True)) - return tags[0] + types = ["ndcube.ndcube.NDCube"] def from_yaml_tree(self, node, tag, ctx): from ndcube import ndcube @@ -25,6 +14,6 @@ def from_yaml_tree(self, node, tag, ctx): def to_yaml_tree(self, ndcube, tag, ctx): node = {} - node["data"] = np.asarray(ndcube.data) + node["data"] =np.array(ndcube.data) node["wcs"] = ndcube.wcs return node From 67b8805a32724bc405f155d09278a102c77c6dde Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Thu, 23 May 2024 04:17:05 +0530 Subject: [PATCH 04/32] Update tag and --- ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml b/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml index 5154938e1..207890b0f 100644 --- a/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml @@ -3,7 +3,7 @@ $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" id: "asdf://sunpy.org/ndcube/schemas/NDCube-0.1.0" -title: | +title: Represents the ndcube NDCube object description: @@ -12,9 +12,11 @@ description: type: object properties: data: - tag: "tag:stsci.edu:asdf/core/ndarray-1.0.0" + tag: "tag:stsci.edu:asdf/core/ndarray-1.*" wcs: - $ref: "http://stsci.edu/schemas/gwcs/wcs-1.1.0" + anyOf: + - $ref: "http://stsci.edu/schemas/gwcs/wcs-1.1.0" + - $ref: "http://stsci.edu/schemas/gwcs/wcs-1.0.0" required: [data , wcs] allowAdditionalProperties: False From b62ac94f19714314a893cb6abea024945c19a6ad Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Thu, 23 May 2024 04:18:24 +0530 Subject: [PATCH 05/32] enable schema test --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 7cce3ed73..4e7bc35fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,10 @@ exclude = ["ndcube._dev*"] [tool.setuptools_scm] write_to = "ndcube/_version.py" +[tool.pytest.ini_options] +asdf_schema_root = 'ndcube/asdf/resources/schemas' +asdf_schema_tests_enabled = true + [project.entry-points."asdf.resource_mappings"] ndcube = "ndcube.asdf.entry_points:get_resource_mappings" From e2928ead17feb2988fbe524ca32ea561370a8967 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Thu, 23 May 2024 05:05:45 +0530 Subject: [PATCH 06/32] revert small change --- ndcube/asdf/converters/ndcube_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index f84f17603..86739dbae 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -14,6 +14,6 @@ def from_yaml_tree(self, node, tag, ctx): def to_yaml_tree(self, ndcube, tag, ctx): node = {} - node["data"] =np.array(ndcube.data) + node["data"] =np.asarray(ndcube.data) node["wcs"] = ndcube.wcs return node From c2b240feebefaccae277686a93fec69e2bc5d134 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Thu, 30 May 2024 21:54:55 +0530 Subject: [PATCH 07/32] Add the converters for ExtraCoords and TimeTableCoordinate class --- .../asdf/converters/extracoords_converter.py | 29 +++++++++++++++++++ ndcube/asdf/converters/ndcube_converter.py | 12 ++++++-- .../asdf/converters/tablecoord_converter.py | 29 +++++++++++++++++++ 3 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 ndcube/asdf/converters/extracoords_converter.py create mode 100644 ndcube/asdf/converters/tablecoord_converter.py diff --git a/ndcube/asdf/converters/extracoords_converter.py b/ndcube/asdf/converters/extracoords_converter.py new file mode 100644 index 000000000..1532efe2b --- /dev/null +++ b/ndcube/asdf/converters/extracoords_converter.py @@ -0,0 +1,29 @@ +from asdf.extension import Converter + + +class ExtraCoordsConverter(Converter): + tags = ["tag:sunpy.org:ndcube/extra_coords/extra_coords/ExtraCoords-*"] + types = ["ndcube.extra_coords.extra_coords.ExtraCoords"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.extra_coords.extra_coords import ExtraCoords + extra_coords = ExtraCoords() + extra_coords._wcs = node.get("wcs") + extra_coords._mapping = node.get("mapping") + if "lookup_tables" in node: + extra_coords._lookup_tables = node.get("lookup_tables") + extra_coords._dropped_tables = node.get("dropped_tables") + extra_coords._ndcube = node.get("ndcube") + return extra_coords + + def to_yaml_tree(self, extracoords, tag, ctx): + node = {} + if extracoords._wcs is not None: + node["wcs"] = extracoords._wcs + if extracoords._mapping is not None: + node["mapping"] = extracoords._mapping + if extracoords._lookup_tables: + node["lookup_tables"] = extracoords._lookup_tables + node["dropped_tables"] = extracoords._dropped_tables + node["ndcube"] = extracoords._ndcube + return node diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index 86739dbae..1b2d7e8e4 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -7,13 +7,19 @@ class NDCubeConverter(Converter): types = ["ndcube.ndcube.NDCube"] def from_yaml_tree(self, node, tag, ctx): - from ndcube import ndcube + from ndcube.ndcube import NDCube + data = np.asanyarray(node["data"]) wcs = node["wcs"] - return(ndcube.NDCube(data, wcs)) + ndcube = NDCube(data, wcs) + ndcube._extra_coords = node["extra_coords"] + + return ndcube def to_yaml_tree(self, ndcube, tag, ctx): node = {} - node["data"] =np.asarray(ndcube.data) + node["data"] = np.asarray(ndcube.data) node["wcs"] = ndcube.wcs + node["extra_coords"] = ndcube.extra_coords + return node diff --git a/ndcube/asdf/converters/tablecoord_converter.py b/ndcube/asdf/converters/tablecoord_converter.py new file mode 100644 index 000000000..728703dd6 --- /dev/null +++ b/ndcube/asdf/converters/tablecoord_converter.py @@ -0,0 +1,29 @@ +from asdf.extension import Converter + + +class TableCoordConverter(Converter): + tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/TimeTableCoordinate-*"] + types = ["ndcube.extra_coords.table_coord.TimeTableCoordinate"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.extra_coords.table_coord import TimeTableCoordinate + + table = node.get("table") + names = node.get("names") + physical_types = node.get("physical_types") + reference_time = node.get("reference_time") + timetablecoordinate = TimeTableCoordinate( + table, names=names, physical_types=physical_types, reference_time=reference_time) + + return timetablecoordinate + + def to_yaml_tree(self, timetablecoordinate, tag, ctx): + node = {} + node["table"] = timetablecoordinate.table + node["names"] = timetablecoordinate.names + node["mesh"] = timetablecoordinate.mesh + if timetablecoordinate.physical_types is not None: + node["physical_types"] = timetablecoordinate.physical_types + node["reference_time"] = timetablecoordinate.reference_time + + return node From 94f2065571119aea7854211261d80a4bf0811688 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Thu, 30 May 2024 21:59:22 +0530 Subject: [PATCH 08/32] Add ExtraCoords and TimeTableCoord Schema --- .../resources/schemas/ExtraCoords-0.1.0.yaml | 34 +++++++++++++++++++ .../asdf/resources/schemas/NDCube-0.1.0.yaml | 6 ++-- .../schemas/TimeTableCoordinate-0.1.0.yaml | 27 +++++++++++++++ 3 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml create mode 100644 ndcube/asdf/resources/schemas/TimeTableCoordinate-0.1.0.yaml diff --git a/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml b/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml new file mode 100644 index 000000000..d71be90e8 --- /dev/null +++ b/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml @@ -0,0 +1,34 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/ExtraCoords-0.1.0" + +title: + Represents the ndcube ExtraCoords object + +description: + Represents the ndcube ExtraCoords object + +type: object +properties: + wcs: + tag: "tag:stsci.edu:gwcs/wcs-1.*" + mapping: + type: array + lookup_tables: + type: array + items: + type: array + properties: + index: + type: array + table: + $ref: "asdf://sunpy.org/ndcube/schemas/TimeTableCoordinate-0.1.0" + dropped_tables: + type: array + ndcube: + $ref: "asdf://sunpy.org/ndcube/schemas/NDCube-0.1.0" + +required: [ndcube] +allowAdditionalProperties: False +... \ No newline at end of file diff --git a/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml b/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml index 207890b0f..c4edd7210 100644 --- a/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml @@ -14,9 +14,9 @@ properties: data: tag: "tag:stsci.edu:asdf/core/ndarray-1.*" wcs: - anyOf: - - $ref: "http://stsci.edu/schemas/gwcs/wcs-1.1.0" - - $ref: "http://stsci.edu/schemas/gwcs/wcs-1.0.0" + tag: "tag:stsci.edu:gwcs/wcs-1.*" + extra_coords: + $ref: "asdf://sunpy.org/ndcube/schemas/ExtraCoords-0.1.0" required: [data , wcs] allowAdditionalProperties: False diff --git a/ndcube/asdf/resources/schemas/TimeTableCoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/TimeTableCoordinate-0.1.0.yaml new file mode 100644 index 000000000..b52b01783 --- /dev/null +++ b/ndcube/asdf/resources/schemas/TimeTableCoordinate-0.1.0.yaml @@ -0,0 +1,27 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/TimeTableCoordinate-0.1.0" + +title: + Represents the TimeTableCoords class + +description: + Represents the TimeTableCoords class + +type: object +properties: + table: + tag: "tag:stsci.edu:asdf/time/time-1*" + names: + type: array + mesh: + type: boolean + physical_types: + type: array + reference_time: + tag: "tag:stsci.edu:asdf/time/time-1*" + +required: ["table"] +additionalProperties: False +... From cc65cc3e20b7ac7f60a7cc05a444cb70278192fe Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Thu, 30 May 2024 22:01:58 +0530 Subject: [PATCH 09/32] Update manifest and entry_point.py --- ndcube/asdf/entry_points.py | 4 +++- ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ndcube/asdf/entry_points.py b/ndcube/asdf/entry_points.py index 36126d25c..0398ba91e 100644 --- a/ndcube/asdf/entry_points.py +++ b/ndcube/asdf/entry_points.py @@ -31,8 +31,10 @@ def get_extensions(): Get the list of extensions. """ from ndcube.asdf.converters.ndcube_converter import NDCubeConverter + from ndcube.asdf.converters.extracoords_converter import ExtraCoordsConverter + from ndcube.asdf.converters.tablecoord_converter import TableCoordConverter - ndcube_converters = [NDCubeConverter()] + ndcube_converters = [NDCubeConverter(),ExtraCoordsConverter(),TableCoordConverter()] _manifest_uri = "asdf://sunpy.org/ndcube/manifests/ndcube-0.1.0" return [ diff --git a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml index 1d45b4af7..1fa8638dd 100644 --- a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml @@ -7,4 +7,10 @@ description: ASDF schemas and tags for NDCube classes. tags: - tag_uri: "tag:sunpy.org:ndcube/ndcube/NDCube-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/NDCube-0.1.0" \ No newline at end of file + schema_uri: "asdf://sunpy.org/ndcube/schemas/NDCube-0.1.0" + + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/extra_coords/ExtraCoords-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/ExtraCoords-0.1.0" + + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/TimeTableCoordinate-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/TimeTableCoordinate-0.1.0" \ No newline at end of file From 5ffb493127dcc16a1d6bfd67d4ac8fd421db268c Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Sat, 1 Jun 2024 01:46:42 +0530 Subject: [PATCH 10/32] Add the QuantityTableCoordinate and SkyCoordTableCoordinate converters --- .../asdf/converters/tablecoord_converter.py | 58 ++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/ndcube/asdf/converters/tablecoord_converter.py b/ndcube/asdf/converters/tablecoord_converter.py index 728703dd6..2d292ab7f 100644 --- a/ndcube/asdf/converters/tablecoord_converter.py +++ b/ndcube/asdf/converters/tablecoord_converter.py @@ -1,7 +1,7 @@ from asdf.extension import Converter -class TableCoordConverter(Converter): +class TimeTableCoordConverter(Converter): tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/TimeTableCoordinate-*"] types = ["ndcube.extra_coords.table_coord.TimeTableCoordinate"] @@ -27,3 +27,59 @@ def to_yaml_tree(self, timetablecoordinate, tag, ctx): node["reference_time"] = timetablecoordinate.reference_time return node + + +class QuantityTableCoordinateConverter(Converter): + tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/QuantityTableCoordinate-*"] + types = ["ndcube.extra_coords.table_coord.QuantityTableCoordinate"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.extra_coords.table_coord import QuantityTableCoordinate + + unit = node.get("unit") + table = node.get("table") + names = node.get("names") + mesh = node.get("mesh") + physical_types = node.get("physical_types") + quantitytablecoordinate = QuantityTableCoordinate(*table, + names=names, physical_types=physical_types,) + quantitytablecoordinate.unit = unit + quantitytablecoordinate.mesh = mesh + return quantitytablecoordinate + + def to_yaml_tree(self, quantitytablecoordinate, tag, ctx): + node = {} + node["unit"] = quantitytablecoordinate.unit + node["table"] = quantitytablecoordinate.table + node["names"] = quantitytablecoordinate.names + node["mesh"] = quantitytablecoordinate.mesh + if quantitytablecoordinate.physical_types is not None: + node["physical_types"] = quantitytablecoordinate.physical_types + + return node + + +class SkyCoordTableCoordinateConverter(Converter): + tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/SkyCoordTableCoordinate-*"] + types = ["ndcube.extra_coords.table_coord.SkyCoordTableCoordinate"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.extra_coords.table_coord import SkyCoordTableCoordinate + + table = node.get("table") + names = node.get("names") + mesh = node.get("mesh") + physical_types = node.get("physical_types") + skycoordinatetablecoordinate = SkyCoordTableCoordinate(table, mesh=mesh, + names=names, physical_types=physical_types) + return skycoordinatetablecoordinate + + def to_yaml_tree(self, skycoordinatetablecoordinate, tag, ctx): + node = {} + node["table"] = skycoordinatetablecoordinate.table + node["names"] = skycoordinatetablecoordinate.names + node["mesh"] = skycoordinatetablecoordinate.mesh + if skycoordinatetablecoordinate.physical_types is not None: + node["physical_types"] = skycoordinatetablecoordinate.physical_types + + return node From 695f8f69c14c91a8e660180d7689889e18d12455 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Sat, 1 Jun 2024 01:49:02 +0530 Subject: [PATCH 11/32] Add the schema for QuantityTableCoordinate and SkyCoordTableCoordinate --- .../QuantityTableCoordinate-0.1.0.yaml | 29 +++++++++++++++++++ .../SkyCoordTableCoordinate-0.1.0.yaml | 25 ++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml create mode 100644 ndcube/asdf/resources/schemas/SkyCoordTableCoordinate-0.1.0.yaml diff --git a/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml new file mode 100644 index 000000000..28072f043 --- /dev/null +++ b/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml @@ -0,0 +1,29 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/QuantityTableCoordinate-0.1.0" + +title: + Represents the QuantityTableCoords class + +description: + Represents the QuantityTableCoords class + +type: object +properties: + unit: + tag: "tag:stsci.edu:asdf/unit/unit-*" + table: + type: array + properties: + tag: "tag:stsci.edu:asdf/unit/quantity-*" + names: + type: array + mesh: + type: boolean + physical_types: + type: array + +required: ["table"] +additionalProperties: False +... diff --git a/ndcube/asdf/resources/schemas/SkyCoordTableCoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/SkyCoordTableCoordinate-0.1.0.yaml new file mode 100644 index 000000000..5d9051918 --- /dev/null +++ b/ndcube/asdf/resources/schemas/SkyCoordTableCoordinate-0.1.0.yaml @@ -0,0 +1,25 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/SkyCoordTableCoordinate-0.1.0" + +title: + Represents the SkyCoordTableCoordinate class + +description: + Represents the SkyCoordTableCoordinate class + +type: object +properties: + table: + tag: "tag:astropy.org:astropy/coordinates/skycoord-*" + names: + type: array + mesh: + type: boolean + physical_types: + type: array + +required: ["table"] +additionalProperties: False +... From 9e38352b7dc932f42ca14b8cf548b44bed82a827 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Sat, 1 Jun 2024 01:51:42 +0530 Subject: [PATCH 12/32] Update manifest and entry_point.py --- ndcube/asdf/entry_points.py | 6 ++++-- ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml | 8 +++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ndcube/asdf/entry_points.py b/ndcube/asdf/entry_points.py index 0398ba91e..8b001db7b 100644 --- a/ndcube/asdf/entry_points.py +++ b/ndcube/asdf/entry_points.py @@ -32,9 +32,11 @@ def get_extensions(): """ from ndcube.asdf.converters.ndcube_converter import NDCubeConverter from ndcube.asdf.converters.extracoords_converter import ExtraCoordsConverter - from ndcube.asdf.converters.tablecoord_converter import TableCoordConverter + from ndcube.asdf.converters.tablecoord_converter import TimeTableCoordConverter + from ndcube.asdf.converters.tablecoord_converter import QuantityTableCoordinateConverter + from ndcube.asdf.converters.tablecoord_converter import SkyCoordTableCoordinateConverter - ndcube_converters = [NDCubeConverter(),ExtraCoordsConverter(),TableCoordConverter()] + ndcube_converters = [NDCubeConverter(),ExtraCoordsConverter(),TimeTableCoordConverter(),QuantityTableCoordinateConverter(),SkyCoordTableCoordinateConverter()] _manifest_uri = "asdf://sunpy.org/ndcube/manifests/ndcube-0.1.0" return [ diff --git a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml index 1fa8638dd..66d893d74 100644 --- a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml @@ -13,4 +13,10 @@ tags: schema_uri: "asdf://sunpy.org/ndcube/schemas/ExtraCoords-0.1.0" - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/TimeTableCoordinate-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/TimeTableCoordinate-0.1.0" \ No newline at end of file + schema_uri: "asdf://sunpy.org/ndcube/schemas/TimeTableCoordinate-0.1.0" + + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/QuantityTableCoordinate-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/QuantityTableCoordinate-0.1.0" + + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/SkyCoordTableCoordinate-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/SkyCoordTableCoordinate-0.1.0" \ No newline at end of file From a9c58688bd664d6af0b8a41e66fda871a6e6cbc7 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Wed, 5 Jun 2024 03:28:52 +0530 Subject: [PATCH 13/32] Add validation for schema and manifests --- pytest.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pytest.ini b/pytest.ini index 4fdcf72ec..7c1f1a272 100644 --- a/pytest.ini +++ b/pytest.ini @@ -25,6 +25,8 @@ addopts = -p no:threadexception -m "not mpl_image_compare" --doctest-ignore-import-errors +asdf_schema_tests_enabled = true +asdf_schema_root = ndcube/asdf/resources remote_data_strict = True filterwarnings = # Turn all warnings into errors so they do not pass silently. @@ -49,3 +51,4 @@ filterwarnings = ignore:Animating a NDCube does not support transposing the array. The world axes may not display as expected because the array will not be transposed:UserWarning # This is raised by the Windows and mac os build for visualization.rst ignore:FigureCanvasAgg is non-interactive, and thus cannot be shown:UserWarning + From 3bb70e4f0a95f2a9e4e42e9e91c23f07c874cc6e Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 5 Jun 2024 12:02:41 +0100 Subject: [PATCH 14/32] pre-commit --- ndcube/asdf/converters/ndcube_converter.py | 3 ++- ndcube/asdf/entry_points.py | 11 +++++++---- ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml | 2 +- ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml | 2 +- ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml | 2 +- .../schemas/QuantityTableCoordinate-0.1.0.yaml | 2 +- .../schemas/SkyCoordTableCoordinate-0.1.0.yaml | 2 +- pyproject.toml | 2 +- 8 files changed, 15 insertions(+), 11 deletions(-) diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index 1b2d7e8e4..cc96557b9 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -1,6 +1,7 @@ -from asdf.extension import Converter import numpy as np +from asdf.extension import Converter + class NDCubeConverter(Converter): tags = ["tag:sunpy.org:ndcube/ndcube/NDCube-*"] diff --git a/ndcube/asdf/entry_points.py b/ndcube/asdf/entry_points.py index 8b001db7b..0b51aae81 100644 --- a/ndcube/asdf/entry_points.py +++ b/ndcube/asdf/entry_points.py @@ -2,6 +2,7 @@ This file contains the entry points for asdf. """ import importlib.resources as importlib_resources + from asdf.extension import ManifestExtension from asdf.resource import DirectoryResourceMapping @@ -30,11 +31,13 @@ def get_extensions(): """ Get the list of extensions. """ - from ndcube.asdf.converters.ndcube_converter import NDCubeConverter from ndcube.asdf.converters.extracoords_converter import ExtraCoordsConverter - from ndcube.asdf.converters.tablecoord_converter import TimeTableCoordConverter - from ndcube.asdf.converters.tablecoord_converter import QuantityTableCoordinateConverter - from ndcube.asdf.converters.tablecoord_converter import SkyCoordTableCoordinateConverter + from ndcube.asdf.converters.ndcube_converter import NDCubeConverter + from ndcube.asdf.converters.tablecoord_converter import ( + QuantityTableCoordinateConverter, + SkyCoordTableCoordinateConverter, + TimeTableCoordConverter, + ) ndcube_converters = [NDCubeConverter(),ExtraCoordsConverter(),TimeTableCoordConverter(),QuantityTableCoordinateConverter(),SkyCoordTableCoordinateConverter()] _manifest_uri = "asdf://sunpy.org/ndcube/manifests/ndcube-0.1.0" diff --git a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml index 66d893d74..7bd5e9e53 100644 --- a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml @@ -19,4 +19,4 @@ tags: schema_uri: "asdf://sunpy.org/ndcube/schemas/QuantityTableCoordinate-0.1.0" - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/SkyCoordTableCoordinate-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/SkyCoordTableCoordinate-0.1.0" \ No newline at end of file + schema_uri: "asdf://sunpy.org/ndcube/schemas/SkyCoordTableCoordinate-0.1.0" diff --git a/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml b/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml index d71be90e8..e8f318504 100644 --- a/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml @@ -31,4 +31,4 @@ properties: required: [ndcube] allowAdditionalProperties: False -... \ No newline at end of file +... diff --git a/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml b/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml index c4edd7210..dcf4dac44 100644 --- a/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml @@ -20,4 +20,4 @@ properties: required: [data , wcs] allowAdditionalProperties: False -... \ No newline at end of file +... diff --git a/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml index 28072f043..b04a19246 100644 --- a/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml @@ -11,7 +11,7 @@ description: type: object properties: - unit: + unit: tag: "tag:stsci.edu:asdf/unit/unit-*" table: type: array diff --git a/ndcube/asdf/resources/schemas/SkyCoordTableCoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/SkyCoordTableCoordinate-0.1.0.yaml index 5d9051918..178eedb37 100644 --- a/ndcube/asdf/resources/schemas/SkyCoordTableCoordinate-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/SkyCoordTableCoordinate-0.1.0.yaml @@ -17,7 +17,7 @@ properties: type: array mesh: type: boolean - physical_types: + physical_types: type: array required: ["table"] diff --git a/pyproject.toml b/pyproject.toml index 617306e24..babadcbb9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,7 +75,7 @@ write_to = "ndcube/_version.py" [tool.pytest.ini_options] asdf_schema_root = 'ndcube/asdf/resources/schemas' asdf_schema_tests_enabled = true - + [project.entry-points."asdf.resource_mappings"] ndcube = "ndcube.asdf.entry_points:get_resource_mappings" From 04409b59246e6353b0a54b276060810d49395adc Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Wed, 5 Jun 2024 21:37:06 +0530 Subject: [PATCH 15/32] Update the tox.ini and CI workflow --- .github/workflows/ci.yml | 1 + pyproject.toml | 4 ---- tox.ini | 11 +++++++++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8cb06f27..8bb7e6985 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,6 +55,7 @@ jobs: - windows: py311 - macos: py310 - linux: py310-oldestdeps + - linux: schema secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/pyproject.toml b/pyproject.toml index babadcbb9..3063406d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,10 +72,6 @@ exclude = ["ndcube._dev*"] [tool.setuptools_scm] write_to = "ndcube/_version.py" -[tool.pytest.ini_options] -asdf_schema_root = 'ndcube/asdf/resources/schemas' -asdf_schema_tests_enabled = true - [project.entry-points."asdf.resource_mappings"] ndcube = "ndcube.asdf.entry_points:get_resource_mappings" diff --git a/tox.ini b/tox.ini index 32f4932b8..62b223d5e 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ envlist = py310-oldestdeps codestyle build_docs + schema [testenv] # We use bash in some of our environments so we have to whitelist it. @@ -89,6 +90,16 @@ commands = {toxinidir}/docs \ {posargs} +[testenv:schema] +description = Run schema tests +deps = + pytest + asdf +set_env = + asdf_schema_root = {toxinidir}/ndcube/asdf/resources +commands= + pytest {env:asdf_schema_root} + [testenv:build_docs] change_dir = docs description = Invoke sphinx-build to build the HTML docs From 76cc27eceefa8af1662c444563915716c7b4d035 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Wed, 5 Jun 2024 21:51:36 +0530 Subject: [PATCH 16/32] Update schemas --- ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml | 2 +- .../asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml b/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml index e8f318504..d3ff106eb 100644 --- a/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml @@ -27,7 +27,7 @@ properties: dropped_tables: type: array ndcube: - $ref: "asdf://sunpy.org/ndcube/schemas/NDCube-0.1.0" + tag: tag:sunpy.org:ndcube/ndcube/NDCube-0.1.0 required: [ndcube] allowAdditionalProperties: False diff --git a/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml index b04a19246..9a3e88a81 100644 --- a/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml @@ -15,7 +15,7 @@ properties: tag: "tag:stsci.edu:asdf/unit/unit-*" table: type: array - properties: + items: tag: "tag:stsci.edu:asdf/unit/quantity-*" names: type: array From 3ea5e65ad44e91b00f9490d4de4aaa368c92362d Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Mon, 17 Jun 2024 17:34:14 +0530 Subject: [PATCH 17/32] Add the converter and schema for GlobalCoords --- .../asdf/converters/globalcoords_converter.py | 28 +++++++++++++++++ .../resources/schemas/GlobalCoords-0.1.0.yaml | 30 +++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 ndcube/asdf/converters/globalcoords_converter.py create mode 100644 ndcube/asdf/resources/schemas/GlobalCoords-0.1.0.yaml diff --git a/ndcube/asdf/converters/globalcoords_converter.py b/ndcube/asdf/converters/globalcoords_converter.py new file mode 100644 index 000000000..221543c76 --- /dev/null +++ b/ndcube/asdf/converters/globalcoords_converter.py @@ -0,0 +1,28 @@ +from typing import OrderedDict + +from asdf.extension import Converter + + +class GlobalCoordsConverter(Converter): + tags = ["tag:sunpy.org:ndcube/global_coords/GlobalCoords-*"] + types = ["ndcube.global_coords.GlobalCoords"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.global_coords import GlobalCoords + + globalcoords = GlobalCoords() + if node.get("internal_coords") is not None: + globalcoords._internal_coords = OrderedDict(node.get("internal_coords")) + globalcoords._ndcube = node["ndcube"] + + return globalcoords + + def to_yaml_tree(self, globalcoords, tag, ctx): + node = {} + node["ndcube"] = globalcoords._ndcube + if globalcoords._internal_coords: + node["internal_coords"] = dict(globalcoords._internal_coords) + # Todo: Include `_all_coords` as a node key to preserve the dropped dimensions + # after ndcube support serialization of sliced NDCube object to asdf. + + return node diff --git a/ndcube/asdf/resources/schemas/GlobalCoords-0.1.0.yaml b/ndcube/asdf/resources/schemas/GlobalCoords-0.1.0.yaml new file mode 100644 index 000000000..370be3bda --- /dev/null +++ b/ndcube/asdf/resources/schemas/GlobalCoords-0.1.0.yaml @@ -0,0 +1,30 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/GlobalCoords-0.1.0" + +title: + Represents the ndcube GlobalCoords object + +description: + Represents the ndcube GlobalCoords object + +type: object +properties: + internal_coords: + type: object + patternProperties: + .*: + type: array + items: + - type: string + - type: object + oneOf: + - tag: "tag:stsci.edu:asdf/unit/quantity-*" + - tag: "tag:astropy.org:astropy/coordinates/skycoord-*" + ndcube: + tag: tag:sunpy.org:ndcube/ndcube/NDCube-0.1.0 + +required: [ndcube] +allowAdditionalProperties: False +... From a31f5fcc2636ce060887564a52ed482e6a65f86d Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Mon, 17 Jun 2024 17:37:27 +0530 Subject: [PATCH 18/32] Update converters --- ndcube/asdf/converters/extracoords_converter.py | 6 +++--- ndcube/asdf/converters/ndcube_converter.py | 2 ++ ndcube/asdf/converters/tablecoord_converter.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ndcube/asdf/converters/extracoords_converter.py b/ndcube/asdf/converters/extracoords_converter.py index 1532efe2b..79c99bba2 100644 --- a/ndcube/asdf/converters/extracoords_converter.py +++ b/ndcube/asdf/converters/extracoords_converter.py @@ -10,8 +10,7 @@ def from_yaml_tree(self, node, tag, ctx): extra_coords = ExtraCoords() extra_coords._wcs = node.get("wcs") extra_coords._mapping = node.get("mapping") - if "lookup_tables" in node: - extra_coords._lookup_tables = node.get("lookup_tables") + extra_coords._lookup_tables = node.get("lookup_tables") extra_coords._dropped_tables = node.get("dropped_tables") extra_coords._ndcube = node.get("ndcube") return extra_coords @@ -24,6 +23,7 @@ def to_yaml_tree(self, extracoords, tag, ctx): node["mapping"] = extracoords._mapping if extracoords._lookup_tables: node["lookup_tables"] = extracoords._lookup_tables - node["dropped_tables"] = extracoords._dropped_tables + if extracoords._dropped_tables: + node["dropped_tables"] = extracoords._dropped_tables node["ndcube"] = extracoords._ndcube return node diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index cc96557b9..2c116b0e5 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -14,6 +14,7 @@ def from_yaml_tree(self, node, tag, ctx): wcs = node["wcs"] ndcube = NDCube(data, wcs) ndcube._extra_coords = node["extra_coords"] + ndcube._global_coords = node["global_coords"] return ndcube @@ -22,5 +23,6 @@ def to_yaml_tree(self, ndcube, tag, ctx): node["data"] = np.asarray(ndcube.data) node["wcs"] = ndcube.wcs node["extra_coords"] = ndcube.extra_coords + node["global_coords"] = ndcube.global_coords return node diff --git a/ndcube/asdf/converters/tablecoord_converter.py b/ndcube/asdf/converters/tablecoord_converter.py index 2d292ab7f..3b3b4886a 100644 --- a/ndcube/asdf/converters/tablecoord_converter.py +++ b/ndcube/asdf/converters/tablecoord_converter.py @@ -42,7 +42,7 @@ def from_yaml_tree(self, node, tag, ctx): mesh = node.get("mesh") physical_types = node.get("physical_types") quantitytablecoordinate = QuantityTableCoordinate(*table, - names=names, physical_types=physical_types,) + names=names, physical_types=physical_types) quantitytablecoordinate.unit = unit quantitytablecoordinate.mesh = mesh return quantitytablecoordinate From f313395b818df42604c287422c40af7b16b31c71 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Mon, 17 Jun 2024 17:45:13 +0530 Subject: [PATCH 19/32] Update entry_points,schemas and manifest --- ndcube/asdf/entry_points.py | 3 ++- ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml | 3 +++ .../asdf/resources/schemas/ExtraCoords-0.1.0.yaml | 15 +++++++++------ ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml | 2 ++ 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/ndcube/asdf/entry_points.py b/ndcube/asdf/entry_points.py index 0b51aae81..72b0ecf17 100644 --- a/ndcube/asdf/entry_points.py +++ b/ndcube/asdf/entry_points.py @@ -32,6 +32,7 @@ def get_extensions(): Get the list of extensions. """ from ndcube.asdf.converters.extracoords_converter import ExtraCoordsConverter + from ndcube.asdf.converters.globalcoords_converter import GlobalCoordsConverter from ndcube.asdf.converters.ndcube_converter import NDCubeConverter from ndcube.asdf.converters.tablecoord_converter import ( QuantityTableCoordinateConverter, @@ -39,7 +40,7 @@ def get_extensions(): TimeTableCoordConverter, ) - ndcube_converters = [NDCubeConverter(),ExtraCoordsConverter(),TimeTableCoordConverter(),QuantityTableCoordinateConverter(),SkyCoordTableCoordinateConverter()] + ndcube_converters = [NDCubeConverter(),ExtraCoordsConverter(),TimeTableCoordConverter(),QuantityTableCoordinateConverter(),SkyCoordTableCoordinateConverter(),GlobalCoordsConverter()] _manifest_uri = "asdf://sunpy.org/ndcube/manifests/ndcube-0.1.0" return [ diff --git a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml index 7bd5e9e53..3d5ef1d19 100644 --- a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml @@ -20,3 +20,6 @@ tags: - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/SkyCoordTableCoordinate-0.1.0" schema_uri: "asdf://sunpy.org/ndcube/schemas/SkyCoordTableCoordinate-0.1.0" + + - tag_uri: "tag:sunpy.org:ndcube/global_coords/GlobalCoords-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/GlobalCoords-0.1.0" diff --git a/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml b/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml index d3ff106eb..6acfa48aa 100644 --- a/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml @@ -18,12 +18,15 @@ properties: lookup_tables: type: array items: - type: array - properties: - index: - type: array - table: - $ref: "asdf://sunpy.org/ndcube/schemas/TimeTableCoordinate-0.1.0" + type: array + items: + - oneOf: + - type: number + - type: array + - oneOf: + - $ref: "asdf://sunpy.org/ndcube/schemas/QuantityTableCoordinate-0.1.0" + - $ref: "asdf://sunpy.org/ndcube/schemas/SkyCoordTableCoordinate-0.1.0" + - $ref: "asdf://sunpy.org/ndcube/schemas/TimeTableCoordinate-0.1.0" dropped_tables: type: array ndcube: diff --git a/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml b/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml index dcf4dac44..7a08d2395 100644 --- a/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml @@ -17,6 +17,8 @@ properties: tag: "tag:stsci.edu:gwcs/wcs-1.*" extra_coords: $ref: "asdf://sunpy.org/ndcube/schemas/ExtraCoords-0.1.0" + globa_coords: + $ref: "asdf://sunpy.org/ndcube/schemas/GlobalCoords-0.1.0" required: [data , wcs] allowAdditionalProperties: False From 3d9d9e8dc74335ce0fa15827ee510272ab801e72 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Wed, 19 Jun 2024 18:51:00 +0530 Subject: [PATCH 20/32] minor change --- ndcube/asdf/converters/globalcoords_converter.py | 2 -- ndcube/asdf/entry_points.py | 9 ++++++++- .../resources/schemas/QuantityTableCoordinate-0.1.0.yaml | 2 +- tox.ini | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ndcube/asdf/converters/globalcoords_converter.py b/ndcube/asdf/converters/globalcoords_converter.py index 221543c76..4f6a76d64 100644 --- a/ndcube/asdf/converters/globalcoords_converter.py +++ b/ndcube/asdf/converters/globalcoords_converter.py @@ -22,7 +22,5 @@ def to_yaml_tree(self, globalcoords, tag, ctx): node["ndcube"] = globalcoords._ndcube if globalcoords._internal_coords: node["internal_coords"] = dict(globalcoords._internal_coords) - # Todo: Include `_all_coords` as a node key to preserve the dropped dimensions - # after ndcube support serialization of sliced NDCube object to asdf. return node diff --git a/ndcube/asdf/entry_points.py b/ndcube/asdf/entry_points.py index 72b0ecf17..55462842e 100644 --- a/ndcube/asdf/entry_points.py +++ b/ndcube/asdf/entry_points.py @@ -40,7 +40,14 @@ def get_extensions(): TimeTableCoordConverter, ) - ndcube_converters = [NDCubeConverter(),ExtraCoordsConverter(),TimeTableCoordConverter(),QuantityTableCoordinateConverter(),SkyCoordTableCoordinateConverter(),GlobalCoordsConverter()] + ndcube_converters = [ + NDCubeConverter(), + ExtraCoordsConverter(), + TimeTableCoordConverter(), + QuantityTableCoordinateConverter(), + SkyCoordTableCoordinateConverter(), + GlobalCoordsConverter(), + ] _manifest_uri = "asdf://sunpy.org/ndcube/manifests/ndcube-0.1.0" return [ diff --git a/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml index 9a3e88a81..e24c7e08f 100644 --- a/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml @@ -24,6 +24,6 @@ properties: physical_types: type: array -required: ["table"] +required: ["table","unit"] additionalProperties: False ... diff --git a/tox.ini b/tox.ini index 62b223d5e..115504e36 100644 --- a/tox.ini +++ b/tox.ini @@ -97,7 +97,7 @@ deps = asdf set_env = asdf_schema_root = {toxinidir}/ndcube/asdf/resources -commands= +commands = pytest {env:asdf_schema_root} [testenv:build_docs] From d031c35d2b28a9f6bc96beee25ac54f8e698cd0c Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Sat, 22 Jun 2024 00:27:29 +0530 Subject: [PATCH 21/32] lowercase schema URIs and use wildcards --- .../asdf/converters/extracoords_converter.py | 2 +- .../asdf/converters/globalcoords_converter.py | 2 +- ndcube/asdf/converters/ndcube_converter.py | 8 +++---- .../asdf/converters/tablecoord_converter.py | 6 ++--- .../resources/manifests/ndcube-0.1.0.yaml | 24 +++++++++---------- ...rds-0.1.0.yaml => extra_coords-0.1.0.yaml} | 12 +++++----- ...ds-0.1.0.yaml => global_coords-0.1.0.yaml} | 6 ++--- .../{NDCube-0.1.0.yaml => ndcube-0.1.0.yaml} | 10 ++++---- ...aml => quantitytablecoordinate-0.1.0.yaml} | 4 ++-- ...aml => skycoordtablecoordinate-0.1.0.yaml} | 2 +- ....0.yaml => timetablecoordinate-0.1.0.yaml} | 2 +- 11 files changed, 38 insertions(+), 40 deletions(-) rename ndcube/asdf/resources/schemas/{ExtraCoords-0.1.0.yaml => extra_coords-0.1.0.yaml} (55%) rename ndcube/asdf/resources/schemas/{GlobalCoords-0.1.0.yaml => global_coords-0.1.0.yaml} (79%) rename ndcube/asdf/resources/schemas/{NDCube-0.1.0.yaml => ndcube-0.1.0.yaml} (54%) rename ndcube/asdf/resources/schemas/{QuantityTableCoordinate-0.1.0.yaml => quantitytablecoordinate-0.1.0.yaml} (83%) rename ndcube/asdf/resources/schemas/{SkyCoordTableCoordinate-0.1.0.yaml => skycoordtablecoordinate-0.1.0.yaml} (86%) rename ndcube/asdf/resources/schemas/{TimeTableCoordinate-0.1.0.yaml => timetablecoordinate-0.1.0.yaml} (87%) diff --git a/ndcube/asdf/converters/extracoords_converter.py b/ndcube/asdf/converters/extracoords_converter.py index 79c99bba2..95d4d1cda 100644 --- a/ndcube/asdf/converters/extracoords_converter.py +++ b/ndcube/asdf/converters/extracoords_converter.py @@ -2,7 +2,7 @@ class ExtraCoordsConverter(Converter): - tags = ["tag:sunpy.org:ndcube/extra_coords/extra_coords/ExtraCoords-*"] + tags = ["tag:sunpy.org:ndcube/extra_coords/extra_coords/extracoords-*"] types = ["ndcube.extra_coords.extra_coords.ExtraCoords"] def from_yaml_tree(self, node, tag, ctx): diff --git a/ndcube/asdf/converters/globalcoords_converter.py b/ndcube/asdf/converters/globalcoords_converter.py index 4f6a76d64..85f7bf340 100644 --- a/ndcube/asdf/converters/globalcoords_converter.py +++ b/ndcube/asdf/converters/globalcoords_converter.py @@ -4,7 +4,7 @@ class GlobalCoordsConverter(Converter): - tags = ["tag:sunpy.org:ndcube/global_coords/GlobalCoords-*"] + tags = ["tag:sunpy.org:ndcube/global_coords/globalcoords-*"] types = ["ndcube.global_coords.GlobalCoords"] def from_yaml_tree(self, node, tag, ctx): diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index 2c116b0e5..c1bcc43b5 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -1,16 +1,14 @@ -import numpy as np - from asdf.extension import Converter class NDCubeConverter(Converter): - tags = ["tag:sunpy.org:ndcube/ndcube/NDCube-*"] + tags = ["tag:sunpy.org:ndcube/ndcube/ndcube-*"] types = ["ndcube.ndcube.NDCube"] def from_yaml_tree(self, node, tag, ctx): from ndcube.ndcube import NDCube - data = np.asanyarray(node["data"]) + data = node["data"] wcs = node["wcs"] ndcube = NDCube(data, wcs) ndcube._extra_coords = node["extra_coords"] @@ -20,7 +18,7 @@ def from_yaml_tree(self, node, tag, ctx): def to_yaml_tree(self, ndcube, tag, ctx): node = {} - node["data"] = np.asarray(ndcube.data) + node["data"] = ndcube.data node["wcs"] = ndcube.wcs node["extra_coords"] = ndcube.extra_coords node["global_coords"] = ndcube.global_coords diff --git a/ndcube/asdf/converters/tablecoord_converter.py b/ndcube/asdf/converters/tablecoord_converter.py index 3b3b4886a..70daccd4b 100644 --- a/ndcube/asdf/converters/tablecoord_converter.py +++ b/ndcube/asdf/converters/tablecoord_converter.py @@ -2,7 +2,7 @@ class TimeTableCoordConverter(Converter): - tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/TimeTableCoordinate-*"] + tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/timetablecoordinate-*"] types = ["ndcube.extra_coords.table_coord.TimeTableCoordinate"] def from_yaml_tree(self, node, tag, ctx): @@ -30,7 +30,7 @@ def to_yaml_tree(self, timetablecoordinate, tag, ctx): class QuantityTableCoordinateConverter(Converter): - tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/QuantityTableCoordinate-*"] + tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/quantitytablecoordinate-*"] types = ["ndcube.extra_coords.table_coord.QuantityTableCoordinate"] def from_yaml_tree(self, node, tag, ctx): @@ -60,7 +60,7 @@ def to_yaml_tree(self, quantitytablecoordinate, tag, ctx): class SkyCoordTableCoordinateConverter(Converter): - tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/SkyCoordTableCoordinate-*"] + tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/skycoordtablecoordinate-*"] types = ["ndcube.extra_coords.table_coord.SkyCoordTableCoordinate"] def from_yaml_tree(self, node, tag, ctx): diff --git a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml index 3d5ef1d19..4559a5499 100644 --- a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml @@ -6,20 +6,20 @@ title: NDCube ASDF Manifest description: ASDF schemas and tags for NDCube classes. tags: - - tag_uri: "tag:sunpy.org:ndcube/ndcube/NDCube-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/NDCube-0.1.0" + - tag_uri: "tag:sunpy.org:ndcube/ndcube/ndcube-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/ndcube-0.1.0" - - tag_uri: "tag:sunpy.org:ndcube/extra_coords/extra_coords/ExtraCoords-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/ExtraCoords-0.1.0" + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/extra_coords/extracoords-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/extra_coords-0.1.0" - - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/TimeTableCoordinate-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/TimeTableCoordinate-0.1.0" + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/timetablecoordinate-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/timetablecoordinate-0.1.0" - - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/QuantityTableCoordinate-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/QuantityTableCoordinate-0.1.0" + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/quantitytablecoordinate-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/quantitytablecoordinate-0.1.0" - - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/SkyCoordTableCoordinate-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/SkyCoordTableCoordinate-0.1.0" + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/skycoordtablecoordinate-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/skycoordtablecoordinate-0.1.0" - - tag_uri: "tag:sunpy.org:ndcube/global_coords/GlobalCoords-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/GlobalCoords-0.1.0" + - tag_uri: "tag:sunpy.org:ndcube/global_coords/globalcoords-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/global_coords-0.1.0" diff --git a/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml b/ndcube/asdf/resources/schemas/extra_coords-0.1.0.yaml similarity index 55% rename from ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml rename to ndcube/asdf/resources/schemas/extra_coords-0.1.0.yaml index 6acfa48aa..6b16e3c9c 100644 --- a/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/extra_coords-0.1.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/ExtraCoords-0.1.0" +id: "asdf://sunpy.org/ndcube/schemas/extra_coords-0.1.0" title: Represents the ndcube ExtraCoords object @@ -24,14 +24,14 @@ properties: - type: number - type: array - oneOf: - - $ref: "asdf://sunpy.org/ndcube/schemas/QuantityTableCoordinate-0.1.0" - - $ref: "asdf://sunpy.org/ndcube/schemas/SkyCoordTableCoordinate-0.1.0" - - $ref: "asdf://sunpy.org/ndcube/schemas/TimeTableCoordinate-0.1.0" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/quantitytablecoordinate-0.*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/skycoordtablecoordinate-0.*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/timetablecoordinate-0.*" dropped_tables: type: array ndcube: - tag: tag:sunpy.org:ndcube/ndcube/NDCube-0.1.0 + tag: "tag:sunpy.org:ndcube/ndcube/ndcube-0.*" required: [ndcube] -allowAdditionalProperties: False +additionalProperties: false ... diff --git a/ndcube/asdf/resources/schemas/GlobalCoords-0.1.0.yaml b/ndcube/asdf/resources/schemas/global_coords-0.1.0.yaml similarity index 79% rename from ndcube/asdf/resources/schemas/GlobalCoords-0.1.0.yaml rename to ndcube/asdf/resources/schemas/global_coords-0.1.0.yaml index 370be3bda..3f1068096 100644 --- a/ndcube/asdf/resources/schemas/GlobalCoords-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/global_coords-0.1.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/GlobalCoords-0.1.0" +id: "asdf://sunpy.org/ndcube/schemas/global_coords-0.1.0" title: Represents the ndcube GlobalCoords object @@ -23,8 +23,8 @@ properties: - tag: "tag:stsci.edu:asdf/unit/quantity-*" - tag: "tag:astropy.org:astropy/coordinates/skycoord-*" ndcube: - tag: tag:sunpy.org:ndcube/ndcube/NDCube-0.1.0 + tag: "tag:sunpy.org:ndcube/ndcube/ndcube-0.*" required: [ndcube] -allowAdditionalProperties: False +additionalProperties: false ... diff --git a/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml similarity index 54% rename from ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml rename to ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml index 7a08d2395..14bb43ddf 100644 --- a/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml @@ -12,14 +12,14 @@ description: type: object properties: data: - tag: "tag:stsci.edu:asdf/core/ndarray-1.*" + description: "Must be compatible with ASDF serialization/deserialization and supported by NDCube." wcs: tag: "tag:stsci.edu:gwcs/wcs-1.*" extra_coords: - $ref: "asdf://sunpy.org/ndcube/schemas/ExtraCoords-0.1.0" - globa_coords: - $ref: "asdf://sunpy.org/ndcube/schemas/GlobalCoords-0.1.0" + tag: "tag:sunpy.org:ndcube/extra_coords/extra_coords/extracoords-0.*" + global_coords: + tag: "tag:sunpy.org:ndcube/global_coords/globalcoords-0.*" required: [data , wcs] -allowAdditionalProperties: False +additionalProperties: true ... diff --git a/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/quantitytablecoordinate-0.1.0.yaml similarity index 83% rename from ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml rename to ndcube/asdf/resources/schemas/quantitytablecoordinate-0.1.0.yaml index e24c7e08f..1d31476fb 100644 --- a/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/quantitytablecoordinate-0.1.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/QuantityTableCoordinate-0.1.0" +id: "asdf://sunpy.org/ndcube/schemas/quantitytablecoordinate-0.1.0" title: Represents the QuantityTableCoords class @@ -24,6 +24,6 @@ properties: physical_types: type: array -required: ["table","unit"] +required: ["table", "unit"] additionalProperties: False ... diff --git a/ndcube/asdf/resources/schemas/SkyCoordTableCoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/skycoordtablecoordinate-0.1.0.yaml similarity index 86% rename from ndcube/asdf/resources/schemas/SkyCoordTableCoordinate-0.1.0.yaml rename to ndcube/asdf/resources/schemas/skycoordtablecoordinate-0.1.0.yaml index 178eedb37..97f22843a 100644 --- a/ndcube/asdf/resources/schemas/SkyCoordTableCoordinate-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/skycoordtablecoordinate-0.1.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/SkyCoordTableCoordinate-0.1.0" +id: "asdf://sunpy.org/ndcube/schemas/skycoordtablecoordinate-0.1.0" title: Represents the SkyCoordTableCoordinate class diff --git a/ndcube/asdf/resources/schemas/TimeTableCoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/timetablecoordinate-0.1.0.yaml similarity index 87% rename from ndcube/asdf/resources/schemas/TimeTableCoordinate-0.1.0.yaml rename to ndcube/asdf/resources/schemas/timetablecoordinate-0.1.0.yaml index b52b01783..d57cd6a00 100644 --- a/ndcube/asdf/resources/schemas/TimeTableCoordinate-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/timetablecoordinate-0.1.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/TimeTableCoordinate-0.1.0" +id: "asdf://sunpy.org/ndcube/schemas/timetablecoordinate-0.1.0" title: Represents the TimeTableCoords class From 171dcd6f261543d8c00e081f850b9560ae7d8ce7 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Tue, 2 Jul 2024 02:22:04 +0530 Subject: [PATCH 22/32] Add tests and GWCS objects --- .../converters/tests/test_ndcube_converter.py | 43 +++++ ndcube/conftest.py | 175 ++++++++++++++++++ 2 files changed, 218 insertions(+) create mode 100644 ndcube/asdf/converters/tests/test_ndcube_converter.py diff --git a/ndcube/asdf/converters/tests/test_ndcube_converter.py b/ndcube/asdf/converters/tests/test_ndcube_converter.py new file mode 100644 index 000000000..a3a5df2ed --- /dev/null +++ b/ndcube/asdf/converters/tests/test_ndcube_converter.py @@ -0,0 +1,43 @@ +import numpy as np +import pytest + +import asdf + +from ndcube.tests.helpers import assert_cubes_equal + + +@pytest.mark.parametrize("ndc",[("ndcube_gwcs_2d_ln_lt"), + ("ndcube_gwcs_3d_ln_lt_l"), + ("ndcube_gwcs_3d_ln_lt_l_ec_gc"), + ("ndcube_gwcs_3d_rotated"), + ("ndcube_gwcs_4d_ln_lt_l_t"), + ], indirect=("ndc",)) +def test_serialization(ndc, tmp_path): + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["ndcube_gwcs"] = ndc + af.write_to(file_path) + + with asdf.open(file_path) as af: + assert_cubes_equal(af["ndcube_gwcs"], ndc) + +@pytest.mark.xfail(reason="Serialization of sliced ndcube not supported") +def test_serialization_sliced_ndcube(ndcube_gwcs_3d_ln_lt_l, tmp_path): + sndc = ndcube_gwcs_3d_ln_lt_l[np.s_[0, :, :]] + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["ndcube_gwcs"] = sndc + af.write_to(file_path) + + with asdf.open(file_path) as af: + assert_cubes_equal(af["ndcube_gwcs"], sndc) + +@pytest.mark.xfail(reason="Serialization of ndcube with .wcs attribute as astropy.wcs.wcs.WCS not supported") +def test_serialization_ndcube_wcs(ndcube_3d_ln_lt_l, tmp_path): + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["ndcube"] = ndcube_3d_ln_lt_l + af.write_to(file_path) + + with asdf.open(file_path) as af: + assert_cubes_equal(af["ndcube"], ndcube_3d_ln_lt_l) diff --git a/ndcube/conftest.py b/ndcube/conftest.py index 130941e6a..38c98a227 100644 --- a/ndcube/conftest.py +++ b/ndcube/conftest.py @@ -7,10 +7,14 @@ import dask.array import numpy as np import pytest +from gwcs import coordinate_frames as cf +from gwcs import wcs import astropy.nddata import astropy.units as u +from astropy import coordinates as coord from astropy.coordinates import SkyCoord +from astropy.modeling import models from astropy.nddata import StdDevUncertainty from astropy.time import Time, TimeDelta from astropy.wcs import WCS @@ -81,6 +85,136 @@ def gen_ndcube_3d_l_ln_lt_ectime(wcs_3d_lt_ln_l, time_axis, time_base, global_co # WCS Fixtures ################################################################################ +@pytest.fixture +def gwcs_4d_t_l_lt_ln(): + """ + Creates a 4D GWCS object with time, wavelength, and celestial coordinates. + + - Time: Axis 0 + - Wavelength: Axis 1 + - Sky: Axes 2 and 3 + + Returns: + wcs.WCS: 4D GWCS object. + """ + + time_model = models.Identity(1) + time_frame = cf.TemporalFrame(axes_order=(0, ), unit=u.s, + reference_frame=Time("2000-01-01T00:00:00")) + + wave_frame = cf.SpectralFrame(axes_order=(1, ), unit=u.m, axes_names=('wavelength',)) + wave_model = models.Scale(0.2) + + shift = models.Shift(-5) & models.Shift(0) + scale = models.Scale(5) & models.Scale(20) + tan = models.Pix2Sky_TAN() + celestial_rotation = models.RotateNative2Celestial(0, 0, 180) + cel_model = shift | scale | tan | celestial_rotation + sky_frame = cf.CelestialFrame(axes_order=(2, 3), name='icrs', + reference_frame=coord.ICRS(), + axes_names=("longitude", "latitude")) + + transform = time_model & wave_model & cel_model + + frame = cf.CompositeFrame([time_frame, wave_frame, sky_frame]) + detector_frame = cf.CoordinateFrame(name="detector", naxes=4, + axes_order=(0, 1, 2, 3), + axes_type=("pixel", "pixel", "pixel", "pixel"), + unit=(u.pix, u.pix, u.pix, u.pix)) + + return (wcs.WCS(forward_transform=transform, output_frame=frame, input_frame=detector_frame)) + +@pytest.fixture +def gwcs_3d_lt_ln_l(): + """ + Creates a 3D GWCS object with celestial coordinates and wavelength. + + - Sky: Axes 0 and 1 + - Wavelength: Axis 2 + + Returns: + wcs.WCS: 3D GWCS object. + """ + + shift = models.Shift(-5) & models.Identity(1) + scale = models.Scale(5) & models.Scale(10) + tan = models.Pix2Sky_TAN() + celestial_rotation = models.RotateNative2Celestial(0, 0, 180) + cel_model = shift | scale | tan | celestial_rotation + sky_frame = cf.CelestialFrame(axes_order=(0, 1), name='icrs', + reference_frame=coord.ICRS(), + axes_names=("longitude", "latitude")) + + wave_model = models.Identity(1) | models.Scale(0.2) | models.Shift(10) + wave_frame = cf.SpectralFrame(axes_order=(2, ), unit=u.nm, axes_names=("wavelength",)) + + transform = cel_model & wave_model + + frame = cf.CompositeFrame([sky_frame, wave_frame]) + detector_frame = cf.CoordinateFrame(name="detector", naxes=3, + axes_order=(0, 1, 2), + axes_type=("pixel", "pixel", "pixel"), + axes_names=("x", "y", "z"), unit=(u.pix, u.pix, u.pix)) + + return (wcs.WCS(forward_transform=transform, output_frame=frame, input_frame=detector_frame)) + +@pytest.fixture +def gwcs_3d_ln_lt_t_rotated(): + """ + Creates a 3D GWCS object with celestial coordinates and wavelength, including rotation. + + - Sky: Axes 0 and 1 + - Wavelength: Axis 2 + + Returns: + wcs.WCS: 3D GWCS object with rotation. + """ + shift = models.Shift(-5) & models.Identity(1) + scale = models.Scale(5) & models.Scale(10) + matrix = np.array([[1.290551569736E-05, 5.9525007864732E-06], + [5.0226382102765E-06 , -1.2644844123757E-05]]) + rotation = models.AffineTransformation2D(matrix) + tan = models.Pix2Sky_TAN() + celestial_rotation = models.RotateNative2Celestial(0, 0, 180) + cel_model = shift | scale| rotation | tan | celestial_rotation + sky_frame = cf.CelestialFrame(axes_order=(0, 1), name='icrs', + reference_frame=coord.ICRS(), + axes_names=("longitude", "latitude")) + + wave_model = models.Identity(1) | models.Scale(0.2) | models.Shift(10) + wave_frame = cf.SpectralFrame(axes_order=(2, ), unit=u.nm, axes_names=("wavelength",)) + + transform = cel_model & wave_model + + frame = cf.CompositeFrame([sky_frame, wave_frame]) + detector_frame = cf.CoordinateFrame(name="detector", naxes=3, + axes_order=(0, 1, 2), + axes_type=("pixel", "pixel", "pixel"), + axes_names=("x", "y", "z"), unit=(u.pix, u.pix, u.pix)) + + return (wcs.WCS(forward_transform=transform, output_frame=frame, input_frame=detector_frame)) + +@pytest.fixture +def gwcs_2d_lt_ln(): + """ + Creates a 2D GWCS object with celestial coordinates. + + - Sky: Axes 0 and 1 + + Returns: + wcs.WCS: 2D GWCS object. + """ + shift = models.Shift(-5) & models.Shift(-5) + scale = models.Scale(2) & models.Scale(4) + tan = models.Pix2Sky_TAN() + celestial_rotation = models.RotateNative2Celestial(0, 0, 180) + cel_model = shift | scale | tan | celestial_rotation + input_frame = cf.Frame2D(name="detector", axes_names=("x", "y")) + sky_frame = cf.CelestialFrame(axes_order=(0, 1), name='icrs', + reference_frame=coord.ICRS(), + axes_names=("longitude", "latitude")) + + return (wcs.WCS(forward_transform=cel_model, output_frame=sky_frame, input_frame=input_frame)) @pytest.fixture def wcs_4d_t_l_lt_ln(): @@ -318,6 +452,47 @@ def extra_coords_sharing_axis(): # NDCube Fixtures ################################################################################ +@pytest.fixture +def ndcube_gwcs_4d_ln_lt_l_t(gwcs_4d_t_l_lt_ln): + shape = (5, 8, 10, 12) + gwcs_4d_t_l_lt_ln.array_shape = shape + data_cube = data_nd(shape) + return NDCube(data_cube, wcs=gwcs_4d_t_l_lt_ln) + +@pytest.fixture +def ndcube_gwcs_3d_ln_lt_l(gwcs_3d_lt_ln_l): + shape = (2, 3, 4) + gwcs_3d_lt_ln_l.array_shape = shape + data_cube = data_nd(shape) + return NDCube(data_cube, wcs=gwcs_3d_lt_ln_l) + +@pytest.fixture +def ndcube_gwcs_3d_rotated(gwcs_3d_lt_ln_l, simple_extra_coords_3d): + data_rotated = np.array([[[1, 2, 3, 4, 6], [2, 4, 5, 3, 1], [0, -1, 2, 4, 2], [3, 5, 1, 2, 0]], + [[2, 4, 5, 1, 3], [1, 5, 2, 2, 4], [2, 3, 4, 0, 5], [0, 1, 2, 3, 4]]]) + cube = NDCube( + data_rotated, + wcs=gwcs_3d_lt_ln_l) + cube._extra_coords = simple_extra_coords_3d + return cube + +@pytest.fixture +def ndcube_gwcs_3d_ln_lt_l_ec_gc(gwcs_3d_lt_ln_l, simple_extra_coords_3d): + shape = (2, 3, 4) + gwcs_3d_lt_ln_l.array_shape = shape + data_cube = data_nd(shape) + cube = NDCube(data_cube, wcs=gwcs_3d_lt_ln_l) + coord1 = 1 * u.m + cube.global_coords.add('name1', 'custom:physical_type1', coord1) + cube._extra_coords = simple_extra_coords_3d + return cube + +@pytest.fixture +def ndcube_gwcs_2d_ln_lt(gwcs_2d_lt_ln): + shape = (10, 12) + data_cube = data_nd(shape) + return NDCube(data_cube, wcs=gwcs_2d_lt_ln) + @pytest.fixture def ndcube_4d_ln_l_t_lt(wcs_4d_lt_t_l_ln): shape = (5, 10, 12, 8) From 0a84f8b0f3099a891a2f71ec2af914a0fb7fd72d Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Thu, 4 Jul 2024 19:13:09 +0530 Subject: [PATCH 23/32] revert small change --- pytest.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 9c8e2eefa..83a4a3980 100644 --- a/pytest.ini +++ b/pytest.ini @@ -27,7 +27,6 @@ addopts = --doctest-ignore-import-errors asdf_schema_tests_enabled = true asdf_schema_root = ndcube/asdf/resources -remote_data_strict = True filterwarnings = # Turn all warnings into errors so they do not pass silently. error From cd31d95a1f59ddba2c0b00f27f047155ae5b604c Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Sat, 6 Jul 2024 23:53:34 +0530 Subject: [PATCH 24/32] apply suggestions from code review --- .../asdf/converters/extracoords_converter.py | 5 ++--- .../asdf/converters/globalcoords_converter.py | 8 +++----- ndcube/asdf/converters/ndcube_converter.py | 4 +--- ndcube/asdf/converters/tablecoord_converter.py | 12 ++++-------- .../converters/tests/test_ndcube_converter.py | 5 +++-- .../resources/schemas/global_coords-0.1.0.yaml | 17 ++++++++--------- .../asdf/resources/schemas/ndcube-0.1.0.yaml | 2 +- ndcube/conftest.py | 18 ++++++++++++++++-- 8 files changed, 38 insertions(+), 33 deletions(-) diff --git a/ndcube/asdf/converters/extracoords_converter.py b/ndcube/asdf/converters/extracoords_converter.py index 95d4d1cda..c94795943 100644 --- a/ndcube/asdf/converters/extracoords_converter.py +++ b/ndcube/asdf/converters/extracoords_converter.py @@ -10,7 +10,7 @@ def from_yaml_tree(self, node, tag, ctx): extra_coords = ExtraCoords() extra_coords._wcs = node.get("wcs") extra_coords._mapping = node.get("mapping") - extra_coords._lookup_tables = node.get("lookup_tables") + extra_coords._lookup_tables = node.get("lookup_tables", []) extra_coords._dropped_tables = node.get("dropped_tables") extra_coords._ndcube = node.get("ndcube") return extra_coords @@ -23,7 +23,6 @@ def to_yaml_tree(self, extracoords, tag, ctx): node["mapping"] = extracoords._mapping if extracoords._lookup_tables: node["lookup_tables"] = extracoords._lookup_tables - if extracoords._dropped_tables: - node["dropped_tables"] = extracoords._dropped_tables + node["dropped_tables"] = extracoords._dropped_tables node["ndcube"] = extracoords._ndcube return node diff --git a/ndcube/asdf/converters/globalcoords_converter.py b/ndcube/asdf/converters/globalcoords_converter.py index 85f7bf340..9700a5890 100644 --- a/ndcube/asdf/converters/globalcoords_converter.py +++ b/ndcube/asdf/converters/globalcoords_converter.py @@ -1,5 +1,3 @@ -from typing import OrderedDict - from asdf.extension import Converter @@ -11,8 +9,8 @@ def from_yaml_tree(self, node, tag, ctx): from ndcube.global_coords import GlobalCoords globalcoords = GlobalCoords() - if node.get("internal_coords") is not None: - globalcoords._internal_coords = OrderedDict(node.get("internal_coords")) + if "internal_coords" in node: + globalcoords._internal_coords = node["internal_coords"] globalcoords._ndcube = node["ndcube"] return globalcoords @@ -21,6 +19,6 @@ def to_yaml_tree(self, globalcoords, tag, ctx): node = {} node["ndcube"] = globalcoords._ndcube if globalcoords._internal_coords: - node["internal_coords"] = dict(globalcoords._internal_coords) + node["internal_coords"] = globalcoords._internal_coords return node diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index c1bcc43b5..18003d928 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -8,9 +8,7 @@ class NDCubeConverter(Converter): def from_yaml_tree(self, node, tag, ctx): from ndcube.ndcube import NDCube - data = node["data"] - wcs = node["wcs"] - ndcube = NDCube(data, wcs) + ndcube = NDCube(node["data"], node["wcs"]) ndcube._extra_coords = node["extra_coords"] ndcube._global_coords = node["global_coords"] diff --git a/ndcube/asdf/converters/tablecoord_converter.py b/ndcube/asdf/converters/tablecoord_converter.py index 70daccd4b..2c4305001 100644 --- a/ndcube/asdf/converters/tablecoord_converter.py +++ b/ndcube/asdf/converters/tablecoord_converter.py @@ -8,12 +8,11 @@ class TimeTableCoordConverter(Converter): def from_yaml_tree(self, node, tag, ctx): from ndcube.extra_coords.table_coord import TimeTableCoordinate - table = node.get("table") names = node.get("names") physical_types = node.get("physical_types") reference_time = node.get("reference_time") timetablecoordinate = TimeTableCoordinate( - table, names=names, physical_types=physical_types, reference_time=reference_time) + node["table"], names=names, physical_types=physical_types, reference_time=reference_time) return timetablecoordinate @@ -36,14 +35,12 @@ class QuantityTableCoordinateConverter(Converter): def from_yaml_tree(self, node, tag, ctx): from ndcube.extra_coords.table_coord import QuantityTableCoordinate - unit = node.get("unit") - table = node.get("table") names = node.get("names") mesh = node.get("mesh") physical_types = node.get("physical_types") - quantitytablecoordinate = QuantityTableCoordinate(*table, + quantitytablecoordinate = QuantityTableCoordinate(*node["table"], names=names, physical_types=physical_types) - quantitytablecoordinate.unit = unit + quantitytablecoordinate.unit = node["unit"] quantitytablecoordinate.mesh = mesh return quantitytablecoordinate @@ -66,11 +63,10 @@ class SkyCoordTableCoordinateConverter(Converter): def from_yaml_tree(self, node, tag, ctx): from ndcube.extra_coords.table_coord import SkyCoordTableCoordinate - table = node.get("table") names = node.get("names") mesh = node.get("mesh") physical_types = node.get("physical_types") - skycoordinatetablecoordinate = SkyCoordTableCoordinate(table, mesh=mesh, + skycoordinatetablecoordinate = SkyCoordTableCoordinate(node["table"], mesh=mesh, names=names, physical_types=physical_types) return skycoordinatetablecoordinate diff --git a/ndcube/asdf/converters/tests/test_ndcube_converter.py b/ndcube/asdf/converters/tests/test_ndcube_converter.py index a3a5df2ed..b327561b7 100644 --- a/ndcube/asdf/converters/tests/test_ndcube_converter.py +++ b/ndcube/asdf/converters/tests/test_ndcube_converter.py @@ -8,9 +8,10 @@ @pytest.mark.parametrize("ndc",[("ndcube_gwcs_2d_ln_lt"), ("ndcube_gwcs_3d_ln_lt_l"), - ("ndcube_gwcs_3d_ln_lt_l_ec_gc"), + ("ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim"), + ("ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc"), ("ndcube_gwcs_3d_rotated"), - ("ndcube_gwcs_4d_ln_lt_l_t"), + ("ndcube_gwcs_4d_ln_lt_l_t") ], indirect=("ndc",)) def test_serialization(ndc, tmp_path): file_path = tmp_path / "test.asdf" diff --git a/ndcube/asdf/resources/schemas/global_coords-0.1.0.yaml b/ndcube/asdf/resources/schemas/global_coords-0.1.0.yaml index 3f1068096..09657fede 100644 --- a/ndcube/asdf/resources/schemas/global_coords-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/global_coords-0.1.0.yaml @@ -13,15 +13,14 @@ type: object properties: internal_coords: type: object - patternProperties: - .*: - type: array - items: - - type: string - - type: object - oneOf: - - tag: "tag:stsci.edu:asdf/unit/quantity-*" - - tag: "tag:astropy.org:astropy/coordinates/skycoord-*" + additionalProperties: + type: array + items: + - type: string + - type: object + oneOf: + - tag: "tag:stsci.edu:asdf/unit/quantity-*" + - tag: "tag:astropy.org:astropy/coordinates/skycoord-*" ndcube: tag: "tag:sunpy.org:ndcube/ndcube/ndcube-0.*" diff --git a/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml index 14bb43ddf..8b4270be6 100644 --- a/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml @@ -20,6 +20,6 @@ properties: global_coords: tag: "tag:sunpy.org:ndcube/global_coords/globalcoords-0.*" -required: [data , wcs] +required: [data, wcs] additionalProperties: true ... diff --git a/ndcube/conftest.py b/ndcube/conftest.py index 38c98a227..56b8a0165 100644 --- a/ndcube/conftest.py +++ b/ndcube/conftest.py @@ -39,6 +39,10 @@ # Helper Functions ################################################################################ +def time_lut(shape): + base_time = Time('2000-01-01', format='fits', scale='utc') + timestamps = Time([base_time + TimeDelta(60 * i, format='sec') for i in range(shape[0])]) + return timestamps def skycoord_2d_lut(shape): total_len = np.prod(shape) @@ -477,14 +481,24 @@ def ndcube_gwcs_3d_rotated(gwcs_3d_lt_ln_l, simple_extra_coords_3d): return cube @pytest.fixture -def ndcube_gwcs_3d_ln_lt_l_ec_gc(gwcs_3d_lt_ln_l, simple_extra_coords_3d): +def ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim(gwcs_3d_lt_ln_l, time_and_simple_extra_coords_2d): shape = (2, 3, 4) gwcs_3d_lt_ln_l.array_shape = shape data_cube = data_nd(shape) cube = NDCube(data_cube, wcs=gwcs_3d_lt_ln_l) + cube._extra_coords = time_and_simple_extra_coords_2d[0] + return cube + +@pytest.fixture +def ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc(gwcs_3d_lt_ln_l): + shape = (3, 3, 4) + gwcs_3d_lt_ln_l.array_shape = shape + data_cube = data_nd(shape) + cube = NDCube(data_cube, wcs=gwcs_3d_lt_ln_l) coord1 = 1 * u.m cube.global_coords.add('name1', 'custom:physical_type1', coord1) - cube._extra_coords = simple_extra_coords_3d + cube.extra_coords.add("time", 0, time_lut(shape)) + cube.extra_coords.add("exposure_lut", 1, range(shape[1]) * u.s) return cube @pytest.fixture From 7c04d357b3eb136d530a65e333abcfee5cdf4354 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Thu, 11 Jul 2024 13:08:57 +0530 Subject: [PATCH 25/32] Remove mesh as a property validator --- ndcube/asdf/converters/tablecoord_converter.py | 1 - ndcube/asdf/resources/schemas/timetablecoordinate-0.1.0.yaml | 2 -- 2 files changed, 3 deletions(-) diff --git a/ndcube/asdf/converters/tablecoord_converter.py b/ndcube/asdf/converters/tablecoord_converter.py index 2c4305001..b07e630e6 100644 --- a/ndcube/asdf/converters/tablecoord_converter.py +++ b/ndcube/asdf/converters/tablecoord_converter.py @@ -20,7 +20,6 @@ def to_yaml_tree(self, timetablecoordinate, tag, ctx): node = {} node["table"] = timetablecoordinate.table node["names"] = timetablecoordinate.names - node["mesh"] = timetablecoordinate.mesh if timetablecoordinate.physical_types is not None: node["physical_types"] = timetablecoordinate.physical_types node["reference_time"] = timetablecoordinate.reference_time diff --git a/ndcube/asdf/resources/schemas/timetablecoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/timetablecoordinate-0.1.0.yaml index d57cd6a00..3c5a828ce 100644 --- a/ndcube/asdf/resources/schemas/timetablecoordinate-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/timetablecoordinate-0.1.0.yaml @@ -15,8 +15,6 @@ properties: tag: "tag:stsci.edu:asdf/time/time-1*" names: type: array - mesh: - type: boolean physical_types: type: array reference_time: From 266f88e31372bd79a597e31f7280ebb7d0404db6 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Fri, 12 Jul 2024 00:18:59 +0530 Subject: [PATCH 26/32] Update the dependencies version --- pyproject.toml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3063406d1..5439fe9e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,9 +16,10 @@ authors = [ { name = "The SunPy Community", email = "sunpy@googlegroups.com" }, ] dependencies = [ - "astropy>=5.0.6,!=5.1.0", - "gwcs>=0.18", - "numpy>=1.23.0" + "astropy>=5.3", + "gwcs>=0.20", + "numpy>=1.23.0", + "asdf>=2.14.4" ] dynamic = ["version"] From 67358b46bbb95dc0db1cdaac9a170544ca01e5ed Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Sat, 20 Jul 2024 00:51:06 +0530 Subject: [PATCH 27/32] Style changes and add warnings to NDCube converter --- ndcube/asdf/converters/ndcube_converter.py | 26 +++++++++++++++++++ .../converters/tests/test_ndcube_converter.py | 2 ++ .../quantitytablecoordinate-0.1.0.yaml | 4 ++- tox.ini | 4 +-- 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index 18003d928..355e54ce5 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -1,3 +1,5 @@ +import warnings + from asdf.extension import Converter @@ -15,10 +17,34 @@ def from_yaml_tree(self, node, tag, ctx): return ndcube def to_yaml_tree(self, ndcube, tag, ctx): + """ + Notes + ----- + This methods serializes the primary components of the NDCube object, + including the `data`, `wcs`, `extra_coords`, and `global_coords` attributes. + Issues a warning if unsupported attributes (uncertainty, mask, meta, unit) are present, + which are not currently serialized to ASDF. + + Warnings + -------- + UserWarning + Warns if the NDCube object has attributes 'uncertainty', 'mask', 'meta', + or 'unit' that are present but not being saved in the ASDF serialization. + This ensures that users are aware of potentially important information + that is not included in the serialized output. + """ node = {} node["data"] = ndcube.data node["wcs"] = ndcube.wcs node["extra_coords"] = ndcube.extra_coords node["global_coords"] = ndcube.global_coords + attributes = ['uncertainty', 'mask', 'unit'] + for attr in attributes: + if getattr(ndcube, attr) is not None: + warnings.warn(f"Attribute '{attr}' is present but not being saved in ASDF serialization.", UserWarning) + + if len(ndcube.meta) > 0: + warnings.warn("Attribute 'meta' is present but not being saved in ASDF serialization.", UserWarning) + return node diff --git a/ndcube/asdf/converters/tests/test_ndcube_converter.py b/ndcube/asdf/converters/tests/test_ndcube_converter.py index b327561b7..5104193d3 100644 --- a/ndcube/asdf/converters/tests/test_ndcube_converter.py +++ b/ndcube/asdf/converters/tests/test_ndcube_converter.py @@ -22,6 +22,7 @@ def test_serialization(ndc, tmp_path): with asdf.open(file_path) as af: assert_cubes_equal(af["ndcube_gwcs"], ndc) + @pytest.mark.xfail(reason="Serialization of sliced ndcube not supported") def test_serialization_sliced_ndcube(ndcube_gwcs_3d_ln_lt_l, tmp_path): sndc = ndcube_gwcs_3d_ln_lt_l[np.s_[0, :, :]] @@ -33,6 +34,7 @@ def test_serialization_sliced_ndcube(ndcube_gwcs_3d_ln_lt_l, tmp_path): with asdf.open(file_path) as af: assert_cubes_equal(af["ndcube_gwcs"], sndc) + @pytest.mark.xfail(reason="Serialization of ndcube with .wcs attribute as astropy.wcs.wcs.WCS not supported") def test_serialization_ndcube_wcs(ndcube_3d_ln_lt_l, tmp_path): file_path = tmp_path / "test.asdf" diff --git a/ndcube/asdf/resources/schemas/quantitytablecoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/quantitytablecoordinate-0.1.0.yaml index 1d31476fb..728d46353 100644 --- a/ndcube/asdf/resources/schemas/quantitytablecoordinate-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/quantitytablecoordinate-0.1.0.yaml @@ -12,7 +12,9 @@ description: type: object properties: unit: - tag: "tag:stsci.edu:asdf/unit/unit-*" + anyOf: + - tag: "tag:stsci.edu:asdf/unit/unit-*" + - tag: "tag:astropy.org:astropy/units/unit-1.*" table: type: array items: diff --git a/tox.ini b/tox.ini index a4d6b35ea..c834dab47 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,7 @@ envlist = py310-oldestdeps codestyle build_docs - schema + asdf_schemas [testenv] # We use bash in some of our environments so we have to whitelist it. @@ -91,7 +91,7 @@ commands = {toxinidir}/docs \ {posargs} -[testenv:schema] +[testenv:asdf_schemas] description = Run schema tests deps = pytest From 6e7a33853aaf7db2001c6a63477cb53feb310275 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Sat, 20 Jul 2024 02:17:39 +0530 Subject: [PATCH 28/32] env name update --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8bb7e6985..238aa6ba3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: - windows: py311 - macos: py310 - linux: py310-oldestdeps - - linux: schema + - linux: asdf_schemas secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} From c1a6b49cacb31bab9483ccfbe61c6917d77bb530 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Mon, 5 Aug 2024 01:55:41 +0530 Subject: [PATCH 29/32] Add asdf as an optional dep --- pyproject.toml | 16 ++++++++++------ tox.ini | 1 + 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5439fe9e1..82a92042c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,10 +16,9 @@ authors = [ { name = "The SunPy Community", email = "sunpy@googlegroups.com" }, ] dependencies = [ - "astropy>=5.3", - "gwcs>=0.20", - "numpy>=1.23.0", - "asdf>=2.14.4" + "astropy>=5.0.6,!=5.1.0", + "gwcs>=0.18", + "numpy>=1.23.0" ] dynamic = ["version"] @@ -54,11 +53,16 @@ plotting = [ reproject = [ "reproject>=0.7.1", ] +asdf = [ + "asdf>=2.14.4", + "astropy>=5.3.0", + "gwcs>=0.20" +] all = [ - "ndcube[plotting,reproject]", + "ndcube[plotting,reproject,asdf]", ] dev = [ - "ndcube[tests,docs,plotting,reproject]", + "ndcube[tests,docs,plotting,reproject,asdf]", ] [project.urls] repository = "https://docs.sunpy.org/projects/ndcube" diff --git a/tox.ini b/tox.ini index c834dab47..ca6b085af 100644 --- a/tox.ini +++ b/tox.ini @@ -63,6 +63,7 @@ deps = figure-!devdeps: dask # The following indicates which extras_require will be installed extras = + asdf plotting reproject tests From 0546cadc55266dfabcc581887edb8ff94aa2fb00 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Mon, 5 Aug 2024 12:19:08 +0530 Subject: [PATCH 30/32] Remove asdf as an optional dependency --- ndcube/asdf/converters/tests/test_ndcube_converter.py | 3 +++ pyproject.toml | 9 ++------- tox.ini | 1 - 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/ndcube/asdf/converters/tests/test_ndcube_converter.py b/ndcube/asdf/converters/tests/test_ndcube_converter.py index 5104193d3..19feefa1b 100644 --- a/ndcube/asdf/converters/tests/test_ndcube_converter.py +++ b/ndcube/asdf/converters/tests/test_ndcube_converter.py @@ -1,5 +1,7 @@ import numpy as np import pytest +from gwcs import __version__ as gwcs_version +from packaging.version import Version import asdf @@ -13,6 +15,7 @@ ("ndcube_gwcs_3d_rotated"), ("ndcube_gwcs_4d_ln_lt_l_t") ], indirect=("ndc",)) +@pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") def test_serialization(ndc, tmp_path): file_path = tmp_path / "test.asdf" with asdf.AsdfFile() as af: diff --git a/pyproject.toml b/pyproject.toml index 82a92042c..3063406d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,16 +53,11 @@ plotting = [ reproject = [ "reproject>=0.7.1", ] -asdf = [ - "asdf>=2.14.4", - "astropy>=5.3.0", - "gwcs>=0.20" -] all = [ - "ndcube[plotting,reproject,asdf]", + "ndcube[plotting,reproject]", ] dev = [ - "ndcube[tests,docs,plotting,reproject,asdf]", + "ndcube[tests,docs,plotting,reproject]", ] [project.urls] repository = "https://docs.sunpy.org/projects/ndcube" diff --git a/tox.ini b/tox.ini index ca6b085af..c834dab47 100644 --- a/tox.ini +++ b/tox.ini @@ -63,7 +63,6 @@ deps = figure-!devdeps: dask # The following indicates which extras_require will be installed extras = - asdf plotting reproject tests From e072a9a9026eb81160390dcd550662ac3bba40ff Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Wed, 7 Aug 2024 02:21:09 +0530 Subject: [PATCH 31/32] Add support for meta in converter and schema --- ndcube/asdf/converters/ndcube_converter.py | 14 +++++++------- ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml | 2 ++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index 355e54ce5..34d372019 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -10,9 +10,11 @@ class NDCubeConverter(Converter): def from_yaml_tree(self, node, tag, ctx): from ndcube.ndcube import NDCube - ndcube = NDCube(node["data"], node["wcs"]) - ndcube._extra_coords = node["extra_coords"] - ndcube._global_coords = node["global_coords"] + ndcube = NDCube(node["data"], node["wcs"], meta=node.get("meta")) + if "extra_coords" in node: + ndcube._extra_coords = node["extra_coords"] + if "global_coords" in node: + ndcube._global_coords = node["global_coords"] return ndcube @@ -28,7 +30,7 @@ def to_yaml_tree(self, ndcube, tag, ctx): Warnings -------- UserWarning - Warns if the NDCube object has attributes 'uncertainty', 'mask', 'meta', + Warns if the NDCube object has attributes 'uncertainty', 'mask', or 'unit' that are present but not being saved in the ASDF serialization. This ensures that users are aware of potentially important information that is not included in the serialized output. @@ -38,13 +40,11 @@ def to_yaml_tree(self, ndcube, tag, ctx): node["wcs"] = ndcube.wcs node["extra_coords"] = ndcube.extra_coords node["global_coords"] = ndcube.global_coords + node["meta"] = ndcube.meta attributes = ['uncertainty', 'mask', 'unit'] for attr in attributes: if getattr(ndcube, attr) is not None: warnings.warn(f"Attribute '{attr}' is present but not being saved in ASDF serialization.", UserWarning) - if len(ndcube.meta) > 0: - warnings.warn("Attribute 'meta' is present but not being saved in ASDF serialization.", UserWarning) - return node diff --git a/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml index 8b4270be6..db10a488e 100644 --- a/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml @@ -19,6 +19,8 @@ properties: tag: "tag:sunpy.org:ndcube/extra_coords/extra_coords/extracoords-0.*" global_coords: tag: "tag:sunpy.org:ndcube/global_coords/globalcoords-0.*" + meta: + type: object required: [data, wcs] additionalProperties: true From 5cfa4f9ee22bc0df08dd0e3d6ad172082cc3530d Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Wed, 21 Aug 2024 20:41:55 +0530 Subject: [PATCH 32/32] Add mask as a validator property --- ndcube/asdf/converters/ndcube_converter.py | 9 +++++++-- ndcube/asdf/converters/tests/test_ndcube_converter.py | 2 +- ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml | 2 ++ ndcube/conftest.py | 9 +++++++-- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index 34d372019..c88b8bdff 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -10,7 +10,10 @@ class NDCubeConverter(Converter): def from_yaml_tree(self, node, tag, ctx): from ndcube.ndcube import NDCube - ndcube = NDCube(node["data"], node["wcs"], meta=node.get("meta")) + ndcube = NDCube(node["data"], + node["wcs"], + meta = node.get("meta"), + mask = node.get("mask")) if "extra_coords" in node: ndcube._extra_coords = node["extra_coords"] if "global_coords" in node: @@ -41,8 +44,10 @@ def to_yaml_tree(self, ndcube, tag, ctx): node["extra_coords"] = ndcube.extra_coords node["global_coords"] = ndcube.global_coords node["meta"] = ndcube.meta + if ndcube.mask is not None: + node["mask"] = ndcube.mask - attributes = ['uncertainty', 'mask', 'unit'] + attributes = ['uncertainty', 'unit'] for attr in attributes: if getattr(ndcube, attr) is not None: warnings.warn(f"Attribute '{attr}' is present but not being saved in ASDF serialization.", UserWarning) diff --git a/ndcube/asdf/converters/tests/test_ndcube_converter.py b/ndcube/asdf/converters/tests/test_ndcube_converter.py index 19feefa1b..ad5df5ee7 100644 --- a/ndcube/asdf/converters/tests/test_ndcube_converter.py +++ b/ndcube/asdf/converters/tests/test_ndcube_converter.py @@ -8,7 +8,7 @@ from ndcube.tests.helpers import assert_cubes_equal -@pytest.mark.parametrize("ndc",[("ndcube_gwcs_2d_ln_lt"), +@pytest.mark.parametrize("ndc",[("ndcube_gwcs_2d_ln_lt_mask"), ("ndcube_gwcs_3d_ln_lt_l"), ("ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim"), ("ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc"), diff --git a/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml index db10a488e..5529ccbfa 100644 --- a/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml @@ -21,6 +21,8 @@ properties: tag: "tag:sunpy.org:ndcube/global_coords/globalcoords-0.*" meta: type: object + mask: + type: object required: [data, wcs] additionalProperties: true diff --git a/ndcube/conftest.py b/ndcube/conftest.py index 56b8a0165..6d0dcf9c5 100644 --- a/ndcube/conftest.py +++ b/ndcube/conftest.py @@ -502,10 +502,15 @@ def ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc(gwcs_3d_lt_ln_l): return cube @pytest.fixture -def ndcube_gwcs_2d_ln_lt(gwcs_2d_lt_ln): +def ndcube_gwcs_2d_ln_lt_mask(gwcs_2d_lt_ln): shape = (10, 12) data_cube = data_nd(shape) - return NDCube(data_cube, wcs=gwcs_2d_lt_ln) + mask = np.zeros(shape, dtype=bool) + mask[1, 1] = True + mask[2, 0] = True + mask[3, 3] = True + mask[4:6, :4] = True + return NDCube(data_cube, wcs=gwcs_2d_lt_ln, mask=mask) @pytest.fixture def ndcube_4d_ln_l_t_lt(wcs_4d_lt_t_l_ln):