Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DM-47796: Deprecate some coadd-processing connections and configs. #1012

Merged
merged 3 commits into from
Dec 2, 2024
Merged

DM-47796: Deprecate some coadd-processing connections and configs. #1012

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
982ab2e
Deprecate coadd single-band deblending driver task.
TallJimbo Nov 25, 2024
850ee21
Deprecate input connection options for old deblender outputs.
TallJimbo Nov 25, 2024
b43ad54
Deprecate refcat matching in measureCoaddSources.
TallJimbo Nov 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 33 additions & 1 deletion python/lsst/pipe/tasks/deblendCoaddSourcesPipeline.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@

import numpy as np

from deprecated.sphinx import deprecated

from lsst.pipe.base import (Struct, PipelineTask, PipelineTaskConfig, PipelineTaskConnections)
import lsst.pipe.base.connectionTypes as cT

Expand Down Expand Up @@ -169,6 +171,13 @@ class DeblendCoaddSourcesMultiConfig(PipelineTaskConfig,
idGenerator = SkyMapIdGeneratorConfig.make_field()


# TODO[DM-47797] Remove this task.
@deprecated(
"Support for the old single-band deblender on coadds will be removed "
"after v29.",
version="v29",
category=FutureWarning
)
class DeblendCoaddSourcesBaseTask(PipelineTask):
def __init__(self, initInputs, **kwargs):
super().__init__(initInputs=initInputs, **kwargs)
Expand Down Expand Up @@ -196,6 +205,13 @@ def _makeSourceCatalog(self, mergedDetections, idFactory):
return sources


# TODO[DM-47797] Remove this task, its connections, and its config.
@deprecated(
"Support for the old single-band deblender on coadds will be removed "
"after v29.",
version="v29",
category=FutureWarning
)
class DeblendCoaddSourcesSingleTask(DeblendCoaddSourcesBaseTask):
ConfigClass = DeblendCoaddSourcesSingleConfig
_DefaultName = "deblendCoaddSourcesSingle"
Expand All @@ -213,12 +229,17 @@ def run(self, coadd, mergedDetections, idFactory):
return Struct(measureCatalog=sources)


class DeblendCoaddSourcesMultiTask(DeblendCoaddSourcesBaseTask):
class DeblendCoaddSourcesMultiTask(PipelineTask):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we want to rename this to DeblendCoaddSourcesTask now that there is no DeblendCoaddSouresSingleTask?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think it's worth the additional disruption right now (though if we did rename it, I'd want to get Sources out of the names of all of the coadd-processing tasks, too).

ConfigClass = DeblendCoaddSourcesMultiConfig
_DefaultName = "deblendCoaddSourcesMulti"

def __init__(self, initInputs, **kwargs):
super().__init__(initInputs=initInputs, **kwargs)
schema = initInputs["inputSchema"].schema
self.peakSchema = initInputs["peakSchema"].schema
self.schemaMapper = afwTable.SchemaMapper(schema)
self.schemaMapper.addMinimalSchema(schema)
self.schema = self.schemaMapper.getOutputSchema()
self.makeSubtask("multibandDeblend", schema=self.schema, peakSchema=self.peakSchema)
self.outputSchema = afwTable.SourceCatalog(self.schema)

Expand All @@ -240,3 +261,14 @@ def run(self, coadds, filters, mergedDetections, idFactory):
catalog, modelData = self.multibandDeblend.run(multiExposure, sources)
retStruct = Struct(deblendedCatalog=catalog, scarletModelData=modelData)
return retStruct

def _makeSourceCatalog(self, mergedDetections, idFactory):
# There may be gaps in the mergeDet catalog, which will cause the
# source ids to be inconsistent. So we update the id factory
# with the largest id already in the catalog.
maxId = np.max(mergedDetections["id"])
idFactory.notify(maxId)
table = afwTable.SourceTable.make(self.schema, idFactory)
sources = afwTable.SourceCatalog(table)
sources.extend(mergedDetections, self.schemaMapper)
return sources
49 changes: 41 additions & 8 deletions python/lsst/pipe/tasks/multiBand.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,11 +221,19 @@ def run(self, exposure, idFactory, expId):
return Struct(outputSources=sources, outputBackgrounds=backgrounds, outputExposure=exposure)


class MeasureMergedCoaddSourcesConnections(PipelineTaskConnections,
dimensions=("tract", "patch", "band", "skymap"),
defaultTemplates={"inputCoaddName": "deep",
"outputCoaddName": "deep",
"deblendedCatalog": "deblendedFlux"}):
class MeasureMergedCoaddSourcesConnections(
PipelineTaskConnections,
dimensions=("tract", "patch", "band", "skymap"),
defaultTemplates={
"inputCoaddName": "deep",
"outputCoaddName": "deep",
"deblendedCatalog": "deblendedFlux",
},
deprecatedTemplates={
# TODO[DM-47797]: remove this deprecated connection template.
"deblendedCatalog": "Support for old deblender outputs will be removed after v29."
},
):
inputSchema = cT.InitInput(
doc="Input schema for measure merged task produced by a deblender or detection task",
name="{inputCoaddName}Coadd_deblendedFlux_schema",
Expand All @@ -236,13 +244,15 @@ class MeasureMergedCoaddSourcesConnections(PipelineTaskConnections,
name="{inputCoaddName}Coadd_meas_schema",
storageClass="SourceCatalog"
)
# TODO[DM-47797]: remove this deprecated connection.
refCat = cT.PrerequisiteInput(
doc="Reference catalog used to match measured sources against known sources",
name="ref_cat",
storageClass="SimpleCatalog",
dimensions=("skypix",),
deferLoad=True,
multiple=True
multiple=True,
deprecated="Reference matching in measureCoaddSources will be removed after v29.",
)
exposure = cT.Input(
doc="Input coadd image",
Expand Down Expand Up @@ -284,6 +294,7 @@ class MeasureMergedCoaddSourcesConnections(PipelineTaskConnections,
multiple=True,
deferLoad=True,
)
# TODO[DM-47797]: remove this deprecated connection.
inputCatalog = cT.Input(
doc=("Name of the input catalog to use."
"If the single band deblender was used this should be 'deblendedFlux."
Expand All @@ -293,6 +304,7 @@ class MeasureMergedCoaddSourcesConnections(PipelineTaskConnections,
"be 'mergeDet'"),
name="{inputCoaddName}Coadd_{deblendedCatalog}",
storageClass="SourceCatalog",
deprecated="Support for old deblender outputs will be removed after v29.",
dimensions=("tract", "patch", "band", "skymap"),
)
scarletCatalog = cT.Input(
Expand All @@ -313,18 +325,22 @@ class MeasureMergedCoaddSourcesConnections(PipelineTaskConnections,
dimensions=("tract", "patch", "band", "skymap"),
storageClass="SourceCatalog",
)
# TODO[DM-47797]: remove this deprecated connection.
matchResult = cT.Output(
doc="Match catalog produced by configured matcher, optional on doMatchSources",
name="{outputCoaddName}Coadd_measMatch",
dimensions=("tract", "patch", "band", "skymap"),
storageClass="Catalog",
deprecated="Reference matching in measureCoaddSources will be removed after v29.",
)
# TODO[DM-47797]: remove this deprecated connection.
denormMatches = cT.Output(
doc="Denormalized Match catalog produced by configured matcher, optional on "
"doWriteMatchesDenormalized",
name="{outputCoaddName}Coadd_measMatchFull",
dimensions=("tract", "patch", "band", "skymap"),
storageClass="Catalog",
deprecated="Reference matching in measureCoaddSources will be removed after v29.",
)

def __init__(self, *, config=None):
Expand All @@ -339,6 +355,7 @@ def __init__(self, *, config=None):
del self.sourceTableHandles
if not config.propagateFlags.finalized_source_flags:
del self.finalizedSourceTableHandles
# TODO[DM-47797]: only the 'if' block contents here should survive.
if config.inputCatalog == "deblendedCatalog":
del self.inputCatalog
if not config.doAddFootprints:
Expand All @@ -347,6 +364,7 @@ def __init__(self, *, config=None):
del self.deblendedCatalog
del self.scarletModels

# TODO[DM-47797]: delete the conditionals below.
if not config.doMatchSources:
del self.refCat
del self.matchResult
Expand All @@ -368,6 +386,8 @@ class MeasureMergedCoaddSourcesConfig(PipelineTaskConfig,
"mergeDet": "The merged detections before deblending."
},
doc="The name of the input catalog.",
# TODO[DM-47797]: remove this config option and anything using it.
deprecated="Support for old deblender outputs will be removed after v29.",
)
doAddFootprints = Field(dtype=bool,
default=True,
Expand All @@ -389,13 +409,23 @@ class MeasureMergedCoaddSourcesConfig(PipelineTaskConfig,
doc="Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)"
)
propagateFlags = ConfigurableField(target=PropagateSourceFlagsTask, doc="Propagate source flags to coadd")
doMatchSources = Field(dtype=bool, default=True, doc="Match sources to reference catalog?")
match = ConfigurableField(target=DirectMatchTask, doc="Matching to reference catalog")
doMatchSources = Field(
dtype=bool,
default=False,
doc="Match sources to reference catalog?",
deprecated="Reference matching in measureCoaddSources will be removed after v29.",
)
match = ConfigurableField(
target=DirectMatchTask,
doc="Matching to reference catalog",
deprecated="Reference matching in measureCoaddSources will be removed after v29.",
)
doWriteMatchesDenormalized = Field(
dtype=bool,
default=False,
doc=("Write reference matches in denormalized format? "
"This format uses more disk space, but is more convenient to read."),
deprecated="Reference matching in measureCoaddSources will be removed after v29.",
)
coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
psfCache = Field(dtype=int, default=100, doc="Size of psfCache")
Expand Down Expand Up @@ -516,6 +546,7 @@ def __init__(self, schema=None, peakSchema=None, refObjLoader=None, initInputs=N
self.algMetadata = PropertyList()
self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
self.makeSubtask("setPrimaryFlags", schema=self.schema)
# TODO[DM-47797]: remove match subtask
if self.config.doMatchSources:
self.makeSubtask("match", refObjLoader=refObjLoader)
if self.config.doPropagateFlags:
Expand All @@ -531,6 +562,7 @@ def __init__(self, schema=None, peakSchema=None, refObjLoader=None, initInputs=N
def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)

# TODO[DM-47797]: remove this block
if self.config.doMatchSources:
refObjLoader = ReferenceObjectLoader([ref.datasetRef.dataId for ref in inputRefs.refCat],
inputs.pop('refCat'),
Expand Down Expand Up @@ -678,6 +710,7 @@ def run(self, exposure, sources, skyInfo, exposureId, ccdInputs=None,

results = Struct()

# TODO[DM-47797]: remove this block
if self.config.doMatchSources:
matchResult = self.match.run(sources, exposure.getInfo().getFilter().bandLabel)
matches = afwTable.packMatches(matchResult.matches)
Expand Down
Loading