Merge remote-tracking branch 'upstream/main' into schema_cleanup

.cruft.json

@@ -1,6 +1,6 @@
 {
   "template": "https://github.com/sunpy/package-template",
-  "commit": "29a55419c27cd7a72bfd60fee7cd92fb59769425",
+  "commit": "1f45be08b3825bde2152b5361cbc2355cf0857e3",
   "checkout": null,
   "context": {
     "cookiecutter": {

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 repos:
     # This should be before any formatting hooks like isort
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.2.2"
+    rev: "v0.3.4"
     hooks:
       - id: ruff
         args: ["--fix"]

.ruff.toml

@@ -5,10 +5,28 @@ exclude = [
   "__pycache__",
   "build",
   "dkist/version.py",
+  "dkist/_dev/",
 ]
 
 [lint]
-select = ["E", "F"]
+select = [
+  "F",
+  "E",
+  "W",
+  "UP",
+  "C4",
+  "ICN",
+  "G",
+  "INP",
+  "PT",
+  "Q",
+  "RSE",
+  "RET",
+  "TID",
+  "PTH",
+  "NPY",
+  "RUF",
+]
 extend-ignore = [
   # pycodestyle (E, W)
   "E501", # LineTooLong                             # TODO! fix
@@ -25,6 +43,17 @@ extend-ignore = [
   "PT007", # Parametrize should be lists of tuples  # TODO! fix
   "PT011", # Too broad exception assert             # TODO! fix
   "PT023", # Always use () on pytest decorators
+  # pyupgrade
+  "UP038", # Use | in isinstance - not compatible with models and is slower
+  # Returns (RET)
+  "RET502",  # Do not implicitly return None in function able to return non-None value
+  "RET503",  # Missing explicit return at the end of function able to return non-None value
+  # Pathlib (PTH)
+  "PTH123",  # open() should be replaced by Path.open()
+  # Ruff
+  "RUF012",  # Mutable class attributes should be annotated with `typing.ClassVar`
+  "RUF013",  # PEP 484 prohibits implicit `Optional`
+  "RUF015",  # Prefer `next(iter(...))` over single element slice
 ]
 
 [lint.per-file-ignores]
@@ -39,6 +68,11 @@ extend-ignore = [
 ]
 "__init__.py" = ["E402", "F401", "F403"]
 "test_*.py" = ["B011", "D", "E402", "PGH001", "S101"]
+"dkist/logger.py" = ["PTH"]
+
+[lint.flake8-import-conventions.extend-aliases]
+"astropy.units" = "u"
+"sunpy.net.attrs" = "a"
 
 [lint.pydocstyle]
 convention = "numpy"

changelog/309.bugfix.rst

@@ -0,0 +1 @@
+Fix broadcasting issues during pixel -> world conversion for models with a Ravel component.

changelog/344.trivial.rst

@@ -0,0 +1 @@
+Refactor various subclasses of VaryingCelestialTransform to centralise the calculations in preparation for improving performance.

conftest.py

@@ -1,9 +1,10 @@
-import matplotlib
+import matplotlib as mpl
 
-matplotlib.use("Agg")
+mpl.use("Agg")
 
 
 def pytest_configure(config):
     # pre-cache the IERS file for astropy to prevent downloads
     # which will cause errors with remote_data off
-    from astropy.utils.iers import IERS_Auto; IERS_Auto.open()
+    from astropy.utils.iers import IERS_Auto
+    IERS_Auto.open()

dkist/__init__.py

@@ -14,7 +14,7 @@
     __version__ = "unknown"
 
 
-__all__ = ['TiledDataset', 'Dataset', 'load_dataset', 'system_info']
+__all__ = ["TiledDataset", "Dataset", "load_dataset", "system_info"]
 
 
 def write_default_config(overwrite=False):
@@ -30,5 +30,5 @@ def write_default_config(overwrite=False):
 
 
 # Do internal imports last (so logger etc is initialised)
-from dkist.dataset import Dataset, TiledDataset, load_dataset  # noqa
-from dkist.utils.sysinfo import system_info  # noqa
+from dkist.dataset import Dataset, TiledDataset, load_dataset
+from dkist.utils.sysinfo import system_info

dkist/_dev/scm_version.py

@@ -5,8 +5,8 @@
 try:
     from setuptools_scm import get_version
 
-    version = get_version(root=os.path.join('..', '..'), relative_to=__file__)
+    version = get_version(root=os.path.join("..", ".."), relative_to=__file__)
 except ImportError:
     raise
 except Exception as e:
-    raise ValueError('setuptools_scm can not determine version.') from e
+    raise ValueError("setuptools_scm can not determine version.") from e

dkist/config/__init__.py

@@ -1,12 +1,12 @@
 from astropy.config import ConfigItem as _AstropyConfigItem
 from astropy.config import ConfigNamespace as _AstropyConfigNamespace
 
-__all__ = ['ConfigItem', 'ConfigNamespace']
+__all__ = ["ConfigItem", "ConfigNamespace"]
 
 
 class ConfigNamespace(_AstropyConfigNamespace):
-    rootname = 'dkist'
+    rootname = "dkist"
 
 
 class ConfigItem(_AstropyConfigItem):
-    rootname = 'dkist'
+    rootname = "dkist"

dkist/conftest.py

@@ -36,7 +36,7 @@ def caplog_dkist(caplog):
 
 @pytest.fixture
 def array():
-    shape = 2**np.random.randint(2, 7, size=2)
+    shape = 2**np.random.randint(2, 7, size=2)  # noqa: NPY002
     x = np.ones(np.prod(shape)) + 10
     x = x.reshape(shape)
     return da.from_array(x, tuple(shape))
@@ -65,7 +65,7 @@ def identity_gwcs():
     """
     identity = m.Multiply(1*u.arcsec/u.pixel) & m.Multiply(1*u.arcsec/u.pixel)
     sky_frame = cf.CelestialFrame(axes_order=(0, 1),
-                                  name='helioprojective',
+                                  name="helioprojective",
                                   reference_frame=Helioprojective(obstime="2018-01-01"),
                                   unit=(u.arcsec, u.arcsec),
                                   axis_physical_types=("custom:pos.helioprojective.lat",
@@ -89,7 +89,7 @@ def identity_gwcs_3d():
     identity = (TwoDScale(1 * u.arcsec / u.pixel) &
                 m.Multiply(1 * u.nm / u.pixel))
 
-    sky_frame = cf.CelestialFrame(axes_order=(0, 1), name='helioprojective',
+    sky_frame = cf.CelestialFrame(axes_order=(0, 1), name="helioprojective",
                                   reference_frame=Helioprojective(obstime="2018-01-01"),
                                   axes_names=("longitude", "latitude"),
                                   unit=(u.arcsec, u.arcsec),
@@ -118,7 +118,7 @@ def identity_gwcs_3d_temporal():
     identity = (TwoDScale(1 * u.arcsec / u.pixel) &
                 m.Multiply(1 * u.s / u.pixel))
 
-    sky_frame = cf.CelestialFrame(axes_order=(0, 1), name='helioprojective',
+    sky_frame = cf.CelestialFrame(axes_order=(0, 1), name="helioprojective",
                                   reference_frame=Helioprojective(obstime="2018-01-01"),
                                   axes_names=("longitude", "latitude"),
                                   unit=(u.arcsec, u.arcsec),
@@ -145,7 +145,7 @@ def identity_gwcs_4d():
     """
     identity = (TwoDScale(1 * u.arcsec / u.pixel) &
                 m.Multiply(1 * u.nm/u.pixel) & m.Multiply(1 * u.s/u.pixel))
-    sky_frame = cf.CelestialFrame(axes_order=(0, 1), name='helioprojective',
+    sky_frame = cf.CelestialFrame(axes_order=(0, 1), name="helioprojective",
                                   reference_frame=Helioprojective(obstime="2018-01-01"),
                                   unit=(u.arcsec, u.arcsec),
                                   axis_physical_types=("custom:pos.helioprojective.lon", "custom:pos.helioprojective.lat"))
@@ -168,17 +168,17 @@ def identity_gwcs_4d():
 
 
 # This function lives in dkist_inventory, but is copied here to avoid a test dep
-def generate_lookup_table(lookup_table, interpolation='linear', points_unit=u.pix, **kwargs):
+def generate_lookup_table(lookup_table, interpolation="linear", points_unit=u.pix, **kwargs):
     if not isinstance(lookup_table, u.Quantity):
         raise TypeError("lookup_table must be a Quantity.")
 
     # The integer location is at the centre of the pixel.
     points = (np.arange(lookup_table.size) - 0) * points_unit
 
     kwargs = {
-        'bounds_error': False,
-        'fill_value': np.nan,
-        'method': interpolation,
+        "bounds_error": False,
+        "fill_value": np.nan,
+        "method": interpolation,
         **kwargs
         }
 
@@ -188,9 +188,9 @@ def generate_lookup_table(lookup_table, interpolation='linear', points_unit=u.pi
 @pytest.fixture
 def identity_gwcs_5d_stokes(identity_gwcs_4d):
     stokes_frame = cf.StokesFrame(axes_order=(4,))
-    stokes_model = generate_lookup_table([1, 2, 3, 4] * u.one, interpolation='nearest')
+    stokes_model = generate_lookup_table([1, 2, 3, 4] * u.one, interpolation="nearest")
     transform = identity_gwcs_4d.forward_transform
-    frame = cf.CompositeFrame(identity_gwcs_4d.output_frame.frames + [stokes_frame])
+    frame = cf.CompositeFrame([*identity_gwcs_4d.output_frame.frames, stokes_frame])
 
     detector_frame = cf.CoordinateFrame(name="detector", naxes=5,
                                         axes_order=(0, 1, 2, 3, 4),
@@ -209,17 +209,17 @@ def identity_gwcs_5d_stokes(identity_gwcs_4d):
 @pytest.fixture
 def dataset(array, identity_gwcs):
     meta = {
-        'inventory': {
-            'bucket': 'data',
-            'datasetId': 'test_dataset',
-            'primaryProposalId': 'test_proposal',
-            'asdfObjectKey': 'test_proposal/test_dataset/test_dataset.asdf',
-            'browseMovieObjectKey': 'test_proposal/test_dataset/test_dataset.mp4',
-            'qualityReportObjectKey': 'test_proposal/test_dataset/test_dataset.pdf',
-            'wavelengthMin': 0,
-            'wavelengthMax': 0,
+        "inventory": {
+            "bucket": "data",
+            "datasetId": "test_dataset",
+            "primaryProposalId": "test_proposal",
+            "asdfObjectKey": "test_proposal/test_dataset/test_dataset.asdf",
+            "browseMovieObjectKey": "test_proposal/test_dataset/test_dataset.mp4",
+            "qualityReportObjectKey": "test_proposal/test_dataset/test_dataset.pdf",
+            "wavelengthMin": 0,
+            "wavelengthMax": 0,
         },
-        'headers': Table()
+        "headers": Table()
     }
 
     identity_gwcs.array_shape = array.shape
@@ -231,15 +231,15 @@ def dataset(array, identity_gwcs):
 
     # Construct the filename here as a scalar array to make sure that works as
     # it's what dkist-inventory does
-    ds._file_manager = FileManager.from_parts(np.array('test1.fits'), 0, 'float', array.shape,
+    ds._file_manager = FileManager.from_parts(np.array("test1.fits"), 0, "float", array.shape,
                                               loader=AstropyFITSLoader)
 
     return ds
 
 
 @pytest.fixture
 def empty_meta():
-    return {'inventory': {}, 'headers': {}}
+    return {"inventory": {}, "headers": {}}
 
 
 @pytest.fixture
@@ -270,16 +270,16 @@ def dataset_4d(identity_gwcs_4d, empty_meta):
 def eit_dataset():
     eitdir = Path(rootdir) / "EIT"
     with asdf.open(eitdir / "eit_test_dataset.asdf") as f:
-        return f.tree['dataset']
+        return f.tree["dataset"]
 
 
 @pytest.fixture
 def simple_tiled_dataset(dataset):
     datasets = [copy.deepcopy(dataset) for i in range(4)]
     for ds in datasets:
-        ds.meta['inventory'] = dataset.meta['inventory']
+        ds.meta["inventory"] = dataset.meta["inventory"]
     dataset_array = np.array(datasets).reshape((2,2))
-    return TiledDataset(dataset_array, dataset.meta['inventory'])
+    return TiledDataset(dataset_array, dataset.meta["inventory"])
 
 
 @pytest.fixture
@@ -303,7 +303,7 @@ def small_visp_dataset():
 
     vispdir = Path(rootdir) / "small_visp"
     with asdf.open(vispdir / "test_visp.asdf") as f:
-        return f.tree['dataset']
+        return f.tree["dataset"]
 
 
 @pytest.fixture(scope="session")

dkist/dataset/dataset.py

@@ -12,7 +12,7 @@
 
 from .utils import dataset_info_str
 
-__all__ = ['Dataset']
+__all__ = ["Dataset"]
 
 
 class FileManagerDescriptor(NDCubeLinkedDescriptor):
@@ -57,7 +57,7 @@ class Dataset(NDCube):
         Uncertainty in the dataset. Should have an attribute uncertainty_type
         that defines what kind of uncertainty is stored, for example "std"
         for standard deviation or "var" for variance. A metaclass defining such
-        an interface is `~astropy.nddata.NDUncertainty` - but isn’t mandatory.
+        an interface is `~astropy.nddata.NDUncertainty` - but isn't mandatory.
         If the uncertainty has no such attribute the uncertainty is stored as
         `~astropy.nddata.UnknownUncertainty`.
         Defaults to None.
@@ -152,7 +152,7 @@ def _slice_headers(self, slice_):
             file_idx.append(slc)
         grid = np.mgrid[tuple(file_idx)]
         file_idx = tuple(grid[i].ravel() for i in range(grid.shape[0]))
-        flat_idx = np.ravel_multi_index(file_idx[::-1], files_shape[::-1], order='F')
+        flat_idx = np.ravel_multi_index(file_idx[::-1], files_shape[::-1], order="F")
 
         # Explicitly create new header table to ensure consistency
         # Otherwise would return a reference sometimes and a new table others
@@ -193,7 +193,7 @@ def inventory(self):
         """
         Convenience attribute to access the inventory metadata.
         """
-        return self.meta['inventory']
+        return self.meta["inventory"]
 
     """
     Dataset loading and saving routines.
@@ -227,8 +227,7 @@ def __repr__(self):
         Overload the NDData repr because it does not play nice with the dask delayed io.
         """
         prefix = object.__repr__(self)
-        output = dedent(f"{prefix}\n{self.__str__()}")
-        return output
+        return dedent(f"{prefix}\n{self.__str__()}")
 
     def __str__(self):
         return dataset_info_str(self)