Skip to content

DM-52435: Diffim bad mask plane handling #432

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
isullivan merged 6 commits into from
Sep 13, 2025
Merged

DM-52435: Diffim bad mask plane handling #432

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
516187d
Flag diaSources with SATURATED_TEMPLATE mask plane set
isullivan Sep 7, 2025
d7466dd
Catch diaSources whose footprint includes EDGE pixels
isullivan Sep 7, 2025
3b54d8d
Do not use detection.excludeMaskPlanes
isullivan Sep 7, 2025
8bad34f
Do not run sky source task if there are 0 sky sources
isullivan Sep 12, 2025
ea9bf47
Add fallback sky difference value for diffim metric
isullivan Sep 12, 2025
5757ba3
Fix unit tests to work with new bad mask plane handling
isullivan Sep 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions python/lsst/ip/diffim/detectAndMeasure.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,11 +286,13 @@ class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig,
dtype=str,
doc="Sources with any of these flags set are removed before writing the output catalog.",
default=("base_PixelFlags_flag_offimage",
"base_PixelFlags_flag_edge",
"base_PixelFlags_flag_interpolatedCenterAll",
"base_PixelFlags_flag_badCenterAll",
"base_PixelFlags_flag_edgeCenterAll",
"base_PixelFlags_flag_nodataCenterAll",
"base_PixelFlags_flag_saturatedCenterAll",
"base_PixelFlags_flag_saturated_templateCenterAll",
),
)
clearMaskPlanes = lsst.pex.config.ListField(
Expand Down Expand Up @@ -371,9 +373,7 @@ def setDefaults(self):
self.detection.thresholdValue = 5.0
self.detection.reEstimateBackground = False
self.detection.thresholdType = "pixel_stdev"
self.detection.excludeMaskPlanes = ["EDGE",
"BAD",
]
self.detection.excludeMaskPlanes = []

# Copy configs for binned streak detection from the base detection task
self.streakDetection.thresholdType = self.detection.thresholdType
Expand Down Expand Up @@ -416,9 +416,9 @@ def setDefaults(self):

# Keep track of which footprints contain streaks
self.measurement.plugins["base_PixelFlags"].masksFpAnywhere = [
"STREAK", "INJECTED", "INJECTED_TEMPLATE", "HIGH_VARIANCE"]
"STREAK", "INJECTED", "INJECTED_TEMPLATE", "HIGH_VARIANCE", "SATURATED_TEMPLATE"]
self.measurement.plugins["base_PixelFlags"].masksFpCenter = [
"STREAK", "INJECTED", "INJECTED_TEMPLATE", "HIGH_VARIANCE"]
"STREAK", "INJECTED", "INJECTED_TEMPLATE", "HIGH_VARIANCE", "SATURATED_TEMPLATE"]
self.skySources.avoidMask = ["DETECTED", "DETECTED_NEGATIVE", "BAD", "NO_DATA", "EDGE"]

def validate(self):
Expand Down Expand Up @@ -963,6 +963,9 @@ def addSkySources(self, diaSources, mask, seed,
"""
if subtask is None:
subtask = self.skySources
if subtask.config.nSources <= 0:
self.metadata[f"n_{subtask.getName()}"] = 0
return
skySourceFootprints = subtask.run(mask=mask, seed=seed, catalog=diaSources)
self.metadata[f"n_{subtask.getName()}"] = len(skySourceFootprints)

Expand Down
2 changes: 1 addition & 1 deletion python/lsst/ip/diffim/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -410,7 +410,7 @@ def footprint_mean(sources, sky=0):
else:
sky_mean = np.nan
sky_std = np.nan
sky_difference = 0
sky_difference = np.nanmedian(np.abs(difference.image.array))
science_footprints, difference_footprints, ratio = footprint_mean(selectStars, sky_difference)
return lsst.pipe.base.Struct(differenceFootprintRatioMean=ratio.mean(),
differenceFootprintRatioStdev=ratio.std(),
Expand Down
31 changes: 20 additions & 11 deletions tests/test_detectAndMeasure.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,8 @@ def _check_values(self, values, minValue=None, maxValue=None):
self.assertTrue(np.all(values <= maxValue))

def _setup_detection(self, doSkySources=True, nSkySources=5,
doSubtractBackground=False, run_sattle=False, **kwargs):
doSubtractBackground=False, run_sattle=False,
badSourceFlags=None, **kwargs):
"""Setup and configure the detection and measurement PipelineTask.

Parameters
Expand Down Expand Up @@ -143,12 +144,15 @@ def _setup_detection(self, doSkySources=True, nSkySources=5,
detector=12,
universe=lsst.daf.butler.DimensionUniverse(),
)
if badSourceFlags is None:
badSourceFlags = ["base_PixelFlags_flag_offimage", ]
config.idGenerator.packer.name = "observation"
config.idGenerator.packer["observation"].n_observations = 10000
config.idGenerator.packer["observation"].n_detectors = 99
config.idGenerator.n_releases = 8
config.idGenerator.release_id = 2
config.doSubtractBackground = doSubtractBackground
config.badSourceFlags = badSourceFlags
self.idGenerator = config.idGenerator.apply(dataId)

return self.detectionTask(config=config)
Expand Down Expand Up @@ -542,8 +546,12 @@ def test_exclude_mask_detections(self):
matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)

# Configure the detection Task
detectionTask = self._setup_detection()
excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes
badSourceFlags = ["base_PixelFlags_flag_offimage",
"base_PixelFlags_flag_edgeCenterAll",
"base_PixelFlags_flag_badCenterAll",
"base_PixelFlags_flag_saturatedCenterAll", ]
detectionTask = self._setup_detection(nSkySources=0, badSourceFlags=badSourceFlags)
excludeMaskPlanes = ["EDGE", "BAD", "SAT"]
nBad = len(excludeMaskPlanes)
self.assertGreater(nBad, 0)
kwargs["seed"] = transientSeed
Expand All @@ -563,10 +571,8 @@ def _detection_wrapper(setFlags=True):
srcBbox = lsst.geom.Box2I(lsst.geom.Point2I(srcX - radius, srcY - radius),
lsst.geom.Extent2I(2*radius + 1, 2*radius + 1))
difference[srcBbox].mask.array |= lsst.afw.image.Mask.getPlaneBitMask(badMask)
if setFlags:
with self.assertRaises(AlgorithmError):
output = detectionTask.run(science, matchedTemplate, difference, sources)
return
else:
output = detectionTask.run(science, matchedTemplate, difference, sources)
refIds = []
Expand Down Expand Up @@ -881,9 +887,7 @@ def test_detection_xy0(self):

self.assertImagesEqual(subtractedMeasuredExposure.image, difference.image)

# Not all of the sources will be detected: preconvolution results in
# a larger edge mask, so we miss an edge source.
self.assertEqual(len(output.diaSources), len(sources)-1)
self.assertEqual(len(output.diaSources), len(sources))
# no sources should be flagged as negative
self.assertEqual(len(~output.diaSources["is_negative"]), len(output.diaSources))
# TODO DM-41496: restore this block once we handle detections on edge
Expand Down Expand Up @@ -1103,13 +1107,19 @@ def test_exclude_mask_detections(self):
kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
science.getInfo().setVisitInfo(makeVisitInfo())
detector = DetectorWrapper(numAmps=1).detector
science.setDetector(detector)
matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)

subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
scienceKernel = science.psf.getKernel()
# Configure the detection Task
detectionTask = self._setup_detection()
excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes
badSourceFlags = ["base_PixelFlags_flag_offimage",
"base_PixelFlags_flag_edgeCenterAll",
"base_PixelFlags_flag_badCenterAll",
"base_PixelFlags_flag_saturatedCenterAll", ]
detectionTask = self._setup_detection(nSkySources=0, badSourceFlags=badSourceFlags)
excludeMaskPlanes = ["EDGE", "BAD", "SAT"]
nBad = len(excludeMaskPlanes)
self.assertGreater(nBad, 0)
kwargs["seed"] = transientSeed
Expand All @@ -1133,7 +1143,6 @@ def _detection_wrapper(setFlags=True):
if setFlags:
with self.assertRaises(AlgorithmError):
output = detectionTask.run(science, matchedTemplate, difference, score, sources)
return
else:
output = detectionTask.run(science, matchedTemplate, difference, score, sources)
refIds = []
Expand Down