From a67d27028c04ca243c6afd497b7ab5dcb2fb4ddb Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Fri, 18 Oct 2024 10:37:20 -0400 Subject: [PATCH 1/5] Remove units from the ref files --- .../schemas/reference_files/dark-1.0.0.yaml | 45 +++++++------------ .../reference_files/distortion-1.0.0.yaml | 13 ------ .../schemas/reference_files/gain-1.0.0.yaml | 15 +++---- .../inverselinearity-1.0.0.yaml | 13 ------ .../reference_files/linearity-1.0.0.yaml | 13 ------ .../reference_files/pixelarea-1.0.0.yaml | 18 ++------ .../reference_files/readnoise-1.0.0.yaml | 15 +++---- .../schemas/reference_files/refpix-1.0.0.yaml | 9 ---- .../reference_files/saturation-1.0.0.yaml | 15 +++---- .../reference_files/wfi_img_photom-1.0.0.yaml | 24 ++-------- 10 files changed, 37 insertions(+), 143 deletions(-) diff --git a/src/rad/resources/schemas/reference_files/dark-1.0.0.yaml b/src/rad/resources/schemas/reference_files/dark-1.0.0.yaml index d5cabf6e..b5b44639 100644 --- a/src/rad/resources/schemas/reference_files/dark-1.0.0.yaml +++ b/src/rad/resources/schemas/reference_files/dark-1.0.0.yaml @@ -57,16 +57,11 @@ properties: description: | The dark current array represents the integrated number of counts due to the accumulation of dark current electrons in the pixels. - tag: tag:stsci.edu:asdf/unit/quantity-1.* - properties: - value: - tag: tag:stsci.edu:asdf/core/ndarray-1.* - datatype: float32 - exact_datatype: true - ndim: 3 - unit: - tag: tag:astropy.org:astropy/units/unit-1.* - enum: ["DN"] + tag: tag:stsci.edu:asdf/core/ndarray-1.* + datatype: float32 + exact_datatype: true + ndim: 3 + unit: DN dq: title: 2-D Data Quality Array description: | @@ -81,31 +76,21 @@ properties: The dark current rate array represents the slope of the integrated number of counts due to the accumulation of dark current electrons in the pixels calculated from slope fitting the Dark Current Array. - tag: tag:stsci.edu:asdf/unit/quantity-1.* - properties: - value: - tag: tag:stsci.edu:asdf/core/ndarray-1.* - datatype: float32 - exact_datatype: true - ndim: 2 - unit: - tag: tag:astropy.org:astropy/units/unit-1.* - enum: ["DN / s"] + tag: tag:stsci.edu:asdf/core/ndarray-1.* + datatype: float32 + exact_datatype: true + ndim: 2 + unit: DN / s dark_slope_error: title: Dark Current Rate Uncertainty Array description: | The uncertainty calculated from the slope fitting of the Dark Current Array. - tag: tag:stsci.edu:asdf/unit/quantity-1.* - properties: - value: - tag: tag:stsci.edu:asdf/core/ndarray-1.* - datatype: float32 - exact_datatype: true - ndim: 2 - unit: - tag: tag:astropy.org:astropy/units/unit-1.* - enum: ["DN / s"] + tag: tag:stsci.edu:asdf/core/ndarray-1.* + datatype: float32 + exact_datatype: true + ndim: 2 + unit: DN / s required: [meta, data, dq, dark_slope, dark_slope_error] flowStyle: block propertyOrder: [meta, data, dq, dark_slope, dark_slope_error] diff --git a/src/rad/resources/schemas/reference_files/distortion-1.0.0.yaml b/src/rad/resources/schemas/reference_files/distortion-1.0.0.yaml index 65e4a1da..24890cf2 100644 --- a/src/rad/resources/schemas/reference_files/distortion-1.0.0.yaml +++ b/src/rad/resources/schemas/reference_files/distortion-1.0.0.yaml @@ -17,19 +17,6 @@ properties: reftype: type: string enum: [DISTORTION] - input_units: - title: Input Model Units - description: | - The pixel input units of the detector coordinate model. - tag: tag:stsci.edu:asdf/unit/unit-1.* - enum: ["pixel"] - output_units: - title: Output Model Units - description: | - The V2/V3 coordinates output units after the model is applied. - tag: tag:stsci.edu:asdf/unit/unit-1.* - enum: ["arcsec"] - required: [output_units, input_units] - $ref: asdf://stsci.edu/datamodels/roman/schemas/reference_files/ref_optical_element-1.0.0 coordinate_distortion_transform: title: Distortion Transform Model diff --git a/src/rad/resources/schemas/reference_files/gain-1.0.0.yaml b/src/rad/resources/schemas/reference_files/gain-1.0.0.yaml index 7bc25869..eeb421d2 100644 --- a/src/rad/resources/schemas/reference_files/gain-1.0.0.yaml +++ b/src/rad/resources/schemas/reference_files/gain-1.0.0.yaml @@ -22,16 +22,11 @@ properties: description: | The Gain Data Array represents the pixel to pixel conversion from digital numbers (DN) to electrons (e). The units are e/DN. - tag: tag:stsci.edu:asdf/unit/quantity-1.* - properties: - value: - tag: tag:stsci.edu:asdf/core/ndarray-1.* - datatype: float32 - exact_datatype: true - ndim: 2 - unit: - tag: tag:astropy.org:astropy/units/unit-1.* - enum: ["electron / DN"] + tag: tag:stsci.edu:asdf/core/ndarray-1.* + datatype: float32 + exact_datatype: true + ndim: 2 + unit: electron / DN required: [meta, data] flowStyle: block propertyOrder: [meta, data] diff --git a/src/rad/resources/schemas/reference_files/inverselinearity-1.0.0.yaml b/src/rad/resources/schemas/reference_files/inverselinearity-1.0.0.yaml index 29b3a3b6..a364deee 100644 --- a/src/rad/resources/schemas/reference_files/inverselinearity-1.0.0.yaml +++ b/src/rad/resources/schemas/reference_files/inverselinearity-1.0.0.yaml @@ -17,19 +17,6 @@ properties: reftype: type: string enum: [INVERSELINEARITY] - input_units: - title: Inverse Linearity Input Units - description: | - Units of the input to the inverse linearity polynomial. - tag: tag:astropy.org:astropy/units/unit-1.* - enum: ["DN"] - output_units: - title: Inverse Linearity Output Units - description: | - Units of the output of the inverse linearity polynomial. - tag: tag:astropy.org:astropy/units/unit-1.* - enum: ["DN"] - required: [output_units, input_units] coeffs: title: Inverse Linearity Coefficients description: | diff --git a/src/rad/resources/schemas/reference_files/linearity-1.0.0.yaml b/src/rad/resources/schemas/reference_files/linearity-1.0.0.yaml index d9c9ec7d..c11ee5df 100644 --- a/src/rad/resources/schemas/reference_files/linearity-1.0.0.yaml +++ b/src/rad/resources/schemas/reference_files/linearity-1.0.0.yaml @@ -17,19 +17,6 @@ properties: reftype: type: string enum: [LINEARITY] - input_units: - title: Units of the Input to the Linearity Polynomial - description: | - Units of the input to the linearity polynomial. - tag: tag:astropy.org:astropy/units/unit-1.* - enum: ["DN"] - output_units: - title: Units of the Output of the Linearity Polynomial - description: | - Units of the output to the linearity polynomial. - tag: tag:astropy.org:astropy/units/unit-1.* - enum: ["DN"] - required: [output_units, input_units] coeffs: title: Linearity Coefficients description: | diff --git a/src/rad/resources/schemas/reference_files/pixelarea-1.0.0.yaml b/src/rad/resources/schemas/reference_files/pixelarea-1.0.0.yaml index 89bb8f86..977321af 100644 --- a/src/rad/resources/schemas/reference_files/pixelarea-1.0.0.yaml +++ b/src/rad/resources/schemas/reference_files/pixelarea-1.0.0.yaml @@ -25,26 +25,16 @@ properties: description: | The nominal pixel area in steradians. anyOf: - - tag: tag:stsci.edu:asdf/unit/quantity-1.* - properties: - datatype: - enum: ["float64"] - unit: - tag: tag:stsci.edu:asdf/unit/unit-1.* - enum: ["sr"] + - type: number + unit: sr - type: "null" pixelarea_arcsecsq: title: Pixel Area (arcsec^2) description: The nominal pixel area in arcec^2. anyOf: - - tag: tag:stsci.edu:asdf/unit/quantity-1.* - properties: - datatype: - enum: ["float64"] - unit: - tag: tag:stsci.edu:asdf/unit/unit-1.* - enum: ["arcsec**2"] + - type: number + unit: arcsec**2 - type: "null" required: [pixelarea_steradians, pixelarea_arcsecsq] required: [photometry] diff --git a/src/rad/resources/schemas/reference_files/readnoise-1.0.0.yaml b/src/rad/resources/schemas/reference_files/readnoise-1.0.0.yaml index 5ff64cf5..36d2f87f 100644 --- a/src/rad/resources/schemas/reference_files/readnoise-1.0.0.yaml +++ b/src/rad/resources/schemas/reference_files/readnoise-1.0.0.yaml @@ -22,16 +22,11 @@ properties: description: | The pixel-by-pixel map read noise data array is used in estimating the expected noise in each pixel. - tag: tag:stsci.edu:asdf/unit/quantity-1.* - properties: - value: - tag: tag:stsci.edu:asdf/core/ndarray-1.* - datatype: float32 - exact_datatype: true - ndim: 2 - unit: - tag: tag:astropy.org:astropy/units/unit-1.* - enum: ["DN"] + tag: tag:stsci.edu:asdf/core/ndarray-1.* + datatype: float32 + exact_datatype: true + ndim: 2 + unit: DN required: [meta, data] flowStyle: block propertyOrder: [meta, data] diff --git a/src/rad/resources/schemas/reference_files/refpix-1.0.0.yaml b/src/rad/resources/schemas/reference_files/refpix-1.0.0.yaml index c1292a95..1a5dc672 100644 --- a/src/rad/resources/schemas/reference_files/refpix-1.0.0.yaml +++ b/src/rad/resources/schemas/reference_files/refpix-1.0.0.yaml @@ -19,15 +19,6 @@ properties: reftype: type: string enum: [REFPIX] - input_units: - title: Units of the input to the linearity polynomial. - tag: tag:astropy.org:astropy/units/unit-1.* - enum: ["DN"] - output_units: - title: Units of the output of the linearity polynomial. - tag: tag:astropy.org:astropy/units/unit-1.* - enum: ["DN"] - required: [output_units, input_units] gamma: title: Left column correction coefficients diff --git a/src/rad/resources/schemas/reference_files/saturation-1.0.0.yaml b/src/rad/resources/schemas/reference_files/saturation-1.0.0.yaml index 73378525..17396f99 100644 --- a/src/rad/resources/schemas/reference_files/saturation-1.0.0.yaml +++ b/src/rad/resources/schemas/reference_files/saturation-1.0.0.yaml @@ -22,16 +22,11 @@ properties: description: | The pixel level threshold for determining saturation before non-linearity corrections are applied. - tag: tag:stsci.edu:asdf/unit/quantity-1.* - properties: - value: - tag: tag:stsci.edu:asdf/core/ndarray-1.* - datatype: float32 - exact_datatype: true - ndim: 2 - unit: - tag: tag:astropy.org:astropy/units/unit-1.* - enum: ["DN"] + tag: tag:stsci.edu:asdf/core/ndarray-1.* + datatype: float32 + exact_datatype: true + ndim: 2 + unit: DN dq: title: 2-D Data Quality Array description: | diff --git a/src/rad/resources/schemas/reference_files/wfi_img_photom-1.0.0.yaml b/src/rad/resources/schemas/reference_files/wfi_img_photom-1.0.0.yaml index caab6cfe..a4570c93 100644 --- a/src/rad/resources/schemas/reference_files/wfi_img_photom-1.0.0.yaml +++ b/src/rad/resources/schemas/reference_files/wfi_img_photom-1.0.0.yaml @@ -32,39 +32,21 @@ properties: description: | Surface brightness, in MJy / steradian. anyOf: - - tag: tag:stsci.edu:asdf/unit/quantity-1.* - properties: - value: - type: number - unit: - tag: tag:stsci.edu:asdf/unit/unit-1.* - enum: [MJy.sr**-1] + - type: number - type: "null" uncertainty: title: Surface Brightness Uncertainty description: | Uncertainty of surface brightness, in MJy / steradian. anyOf: - - tag: tag:stsci.edu:asdf/unit/quantity-1.* - properties: - value: - type: number - unit: - tag: tag:stsci.edu:asdf/unit/unit-1.* - enum: [MJy.sr**-1] + - type: number - type: "null" pixelareasr: title: Pixel Area description: | The nominal pixel area, in steradian. anyOf: - - tag: tag:stsci.edu:asdf/unit/quantity-1.* - properties: - value: - type: number - unit: - tag: tag:stsci.edu:asdf/unit/unit-1.* - enum: [sr] + - type: number - type: "null" required: [photmjsr, uncertainty, pixelareasr] additionalProperties: false From e7457eb5fcc4838ce4f182e3dcc4cc54e92e7b98 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Mon, 21 Oct 2024 11:22:28 -0400 Subject: [PATCH 2/5] Fix #489 --- .../schemas/reference_files/dark-1.0.0.yaml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/src/rad/resources/schemas/reference_files/dark-1.0.0.yaml b/src/rad/resources/schemas/reference_files/dark-1.0.0.yaml index b5b44639..7a4cde6f 100644 --- a/src/rad/resources/schemas/reference_files/dark-1.0.0.yaml +++ b/src/rad/resources/schemas/reference_files/dark-1.0.0.yaml @@ -20,21 +20,6 @@ properties: exposure: type: object properties: - ngroups: - title: Number of Resultants - description: | - The number of resultants averaged according to - Multi-Accumulation (MA) Table read pattern. - type: integer - nframes: - title: Number of frames per resultant # should be removed - type: integer - groupgap: - title: Number of Skips Between Resultants - description: | - The number of frames skipped between resultants according to - MA Table read pattern. - type: integer ma_table_name: title: Multi-Accumulation Table Name description: | @@ -48,7 +33,7 @@ properties: Table that keeps the same name will have a new ma_table_number. type: integer - required: [ngroups, nframes, groupgap, ma_table_name, ma_table_number] + required: [ma_table_name, ma_table_number] required: [exposure] - $ref: asdf://stsci.edu/datamodels/roman/schemas/reference_files/ref_exposure_type-1.0.0 - $ref: asdf://stsci.edu/datamodels/roman/schemas/reference_files/ref_optical_element-1.0.0 From a57713b66a72d3561d81e425ba28956e6dfa6664 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Mon, 13 Jan 2025 11:56:24 -0500 Subject: [PATCH 3/5] Move ref_common check to rad from roman_datamodels --- tests/test_schemas.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_schemas.py b/tests/test_schemas.py index c8c0660e..ff7e0f6c 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -21,6 +21,7 @@ asdf.schema.load_schema("asdf://stsci.edu/datamodels/roman/schemas/wfi_optical_element-1.0.0")["enum"] ) EXPOSURE_TYPE_ELEMENTS = list(asdf.schema.load_schema("asdf://stsci.edu/datamodels/roman/schemas/exposure_type-1.0.0")["enum"]) +EXPECTED_COMMON_REFERENCE = {"$ref": "asdf://stsci.edu/datamodels/roman/schemas/reference_files/ref_common-1.0.0"} @pytest.fixture(scope="session", params=SCHEMA_URIS) @@ -263,6 +264,19 @@ def test_reftype_tag(ref_file_uris): assert asdf.util.uri_match(f"asdf://stsci.edu/datamodels/roman/schemas/reference_files/*{reftype}-*", schema_uri) +def test_ref_file_meta_common(ref_file_schema): + """ + Test that the meta for all reference files contains a reference to `ref_common` + """ + all_of = ref_file_schema["properties"]["meta"]["allOf"] + + for item in all_of: + if item == EXPECTED_COMMON_REFERENCE: + break + else: + raise ValueError("ref_common not found in meta") + + # don't test tvac or fps schemas as they are static @pytest.mark.parametrize( "uri", From e3cf85d5036c67b7b950a55dfb13a83f43a7a520 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Mon, 13 Jan 2025 11:57:16 -0500 Subject: [PATCH 4/5] Add change log --- changes/490.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/490.feature.rst diff --git a/changes/490.feature.rst b/changes/490.feature.rst new file mode 100644 index 00000000..ba8046de --- /dev/null +++ b/changes/490.feature.rst @@ -0,0 +1 @@ +Remove units from the reference file schemas. From 2b7a04a05530ce0d4dd7dfd0c760ff9797e19500 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 16 Jan 2025 10:55:40 -0500 Subject: [PATCH 5/5] Document input/output units in schema --- .../resources/schemas/reference_files/distortion-1.0.0.yaml | 6 +++++- .../schemas/reference_files/inverselinearity-1.0.0.yaml | 4 ++++ .../resources/schemas/reference_files/linearity-1.0.0.yaml | 4 ++++ src/rad/resources/schemas/reference_files/refpix-1.0.0.yaml | 4 ++++ 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/rad/resources/schemas/reference_files/distortion-1.0.0.yaml b/src/rad/resources/schemas/reference_files/distortion-1.0.0.yaml index 24890cf2..22f51582 100644 --- a/src/rad/resources/schemas/reference_files/distortion-1.0.0.yaml +++ b/src/rad/resources/schemas/reference_files/distortion-1.0.0.yaml @@ -13,13 +13,17 @@ properties: allOf: - $ref: asdf://stsci.edu/datamodels/roman/schemas/reference_files/ref_common-1.0.0 - type: object + + input_units: "pixel" + output_units: "arcsec" + properties: reftype: type: string enum: [DISTORTION] - $ref: asdf://stsci.edu/datamodels/roman/schemas/reference_files/ref_optical_element-1.0.0 coordinate_distortion_transform: - title: Distortion Transform Model + title: Distortion Transform Model with inputs in "pixel" and outputs in "arcsec" description: | The astropy.modeling.Model instance of of the distortion transform model. type: object diff --git a/src/rad/resources/schemas/reference_files/inverselinearity-1.0.0.yaml b/src/rad/resources/schemas/reference_files/inverselinearity-1.0.0.yaml index a364deee..d6b7d908 100644 --- a/src/rad/resources/schemas/reference_files/inverselinearity-1.0.0.yaml +++ b/src/rad/resources/schemas/reference_files/inverselinearity-1.0.0.yaml @@ -13,6 +13,10 @@ properties: allOf: - $ref: asdf://stsci.edu/datamodels/roman/schemas/reference_files/ref_common-1.0.0 - type: object + + input_units: "DN" + output_units: "DN" + properties: reftype: type: string diff --git a/src/rad/resources/schemas/reference_files/linearity-1.0.0.yaml b/src/rad/resources/schemas/reference_files/linearity-1.0.0.yaml index c11ee5df..8b56ceba 100644 --- a/src/rad/resources/schemas/reference_files/linearity-1.0.0.yaml +++ b/src/rad/resources/schemas/reference_files/linearity-1.0.0.yaml @@ -13,6 +13,10 @@ properties: allOf: - $ref: asdf://stsci.edu/datamodels/roman/schemas/reference_files/ref_common-1.0.0 - type: object + + input_units: "DN" + output_units: "DN" + properties: reftype: type: string diff --git a/src/rad/resources/schemas/reference_files/refpix-1.0.0.yaml b/src/rad/resources/schemas/reference_files/refpix-1.0.0.yaml index 1a5dc672..ea2a14fb 100644 --- a/src/rad/resources/schemas/reference_files/refpix-1.0.0.yaml +++ b/src/rad/resources/schemas/reference_files/refpix-1.0.0.yaml @@ -15,6 +15,10 @@ properties: allOf: - $ref: asdf://stsci.edu/datamodels/roman/schemas/reference_files/ref_common-1.0.0 - type: object + + input_units: "DN" + output_units: "DN" + properties: reftype: type: string