From 516187ddc44b4c0e0c1e742e1181703ec8db4a81 Mon Sep 17 00:00:00 2001 From: Ian Sullivan Date: Sun, 7 Sep 2025 12:13:18 -0700 Subject: [PATCH 1/6] Flag diaSources with SATURATED_TEMPLATE mask plane set --- python/lsst/ip/diffim/detectAndMeasure.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/lsst/ip/diffim/detectAndMeasure.py b/python/lsst/ip/diffim/detectAndMeasure.py index 0b90d4778..ae4a51ae1 100644 --- a/python/lsst/ip/diffim/detectAndMeasure.py +++ b/python/lsst/ip/diffim/detectAndMeasure.py @@ -291,6 +291,7 @@ class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig, "base_PixelFlags_flag_edgeCenterAll", "base_PixelFlags_flag_nodataCenterAll", "base_PixelFlags_flag_saturatedCenterAll", + "base_PixelFlags_flag_saturated_templateCenterAll", ), ) clearMaskPlanes = lsst.pex.config.ListField( @@ -416,9 +417,9 @@ def setDefaults(self): # Keep track of which footprints contain streaks self.measurement.plugins["base_PixelFlags"].masksFpAnywhere = [ - "STREAK", "INJECTED", "INJECTED_TEMPLATE", "HIGH_VARIANCE"] + "STREAK", "INJECTED", "INJECTED_TEMPLATE", "HIGH_VARIANCE", "SATURATED_TEMPLATE"] self.measurement.plugins["base_PixelFlags"].masksFpCenter = [ - "STREAK", "INJECTED", "INJECTED_TEMPLATE", "HIGH_VARIANCE"] + "STREAK", "INJECTED", "INJECTED_TEMPLATE", "HIGH_VARIANCE", "SATURATED_TEMPLATE"] self.skySources.avoidMask = ["DETECTED", "DETECTED_NEGATIVE", "BAD", "NO_DATA", "EDGE"] def validate(self): From d7466dda839ecac971536dbce373c1e7a6f132f8 Mon Sep 17 00:00:00 2001 From: Ian Sullivan Date: Sun, 7 Sep 2025 12:23:03 -0700 Subject: [PATCH 2/6] Catch diaSources whose footprint includes EDGE pixels --- python/lsst/ip/diffim/detectAndMeasure.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/lsst/ip/diffim/detectAndMeasure.py b/python/lsst/ip/diffim/detectAndMeasure.py index ae4a51ae1..95cfd75f3 100644 --- a/python/lsst/ip/diffim/detectAndMeasure.py +++ b/python/lsst/ip/diffim/detectAndMeasure.py @@ -286,6 +286,7 @@ class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig, dtype=str, doc="Sources with any of these flags set are removed before writing the output catalog.", default=("base_PixelFlags_flag_offimage", + "base_PixelFlags_flag_edge", "base_PixelFlags_flag_interpolatedCenterAll", "base_PixelFlags_flag_badCenterAll", "base_PixelFlags_flag_edgeCenterAll", @@ -372,8 +373,7 @@ def setDefaults(self): self.detection.thresholdValue = 5.0 self.detection.reEstimateBackground = False self.detection.thresholdType = "pixel_stdev" - self.detection.excludeMaskPlanes = ["EDGE", - "BAD", + self.detection.excludeMaskPlanes = ["BAD", ] # Copy configs for binned streak detection from the base detection task From 3b54d8de4261af49f35c76acdcb416e88a17c601 Mon Sep 17 00:00:00 2001 From: Ian Sullivan Date: Sun, 7 Sep 2025 12:43:20 -0700 Subject: [PATCH 3/6] Do not use detection.excludeMaskPlanes --- python/lsst/ip/diffim/detectAndMeasure.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/lsst/ip/diffim/detectAndMeasure.py b/python/lsst/ip/diffim/detectAndMeasure.py index 95cfd75f3..9fe2a8768 100644 --- a/python/lsst/ip/diffim/detectAndMeasure.py +++ b/python/lsst/ip/diffim/detectAndMeasure.py @@ -373,8 +373,7 @@ def setDefaults(self): self.detection.thresholdValue = 5.0 self.detection.reEstimateBackground = False self.detection.thresholdType = "pixel_stdev" - self.detection.excludeMaskPlanes = ["BAD", - ] + self.detection.excludeMaskPlanes = [] # Copy configs for binned streak detection from the base detection task self.streakDetection.thresholdType = self.detection.thresholdType From 8bad34f4108e45dfa75137b79fb6a16aed5ba492 Mon Sep 17 00:00:00 2001 From: Ian Sullivan Date: Fri, 12 Sep 2025 12:50:22 -0700 Subject: [PATCH 4/6] Do not run sky source task if there are 0 sky sources Running the task with nSources=0 causes a division by zero error from the logger. --- python/lsst/ip/diffim/detectAndMeasure.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/lsst/ip/diffim/detectAndMeasure.py b/python/lsst/ip/diffim/detectAndMeasure.py index 9fe2a8768..352282285 100644 --- a/python/lsst/ip/diffim/detectAndMeasure.py +++ b/python/lsst/ip/diffim/detectAndMeasure.py @@ -963,6 +963,9 @@ def addSkySources(self, diaSources, mask, seed, """ if subtask is None: subtask = self.skySources + if subtask.config.nSources <= 0: + self.metadata[f"n_{subtask.getName()}"] = 0 + return skySourceFootprints = subtask.run(mask=mask, seed=seed, catalog=diaSources) self.metadata[f"n_{subtask.getName()}"] = len(skySourceFootprints) From ea9bf4783009ca47b7ea209bf8e44ef5bf397d05 Mon Sep 17 00:00:00 2001 From: Ian Sullivan Date: Fri, 12 Sep 2025 13:44:20 -0700 Subject: [PATCH 5/6] Add fallback sky difference value for diffim metric This makes the diffim metric have more reasonable values when sky sources are not provided. --- python/lsst/ip/diffim/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lsst/ip/diffim/utils.py b/python/lsst/ip/diffim/utils.py index f43103c4e..466fd6e1e 100644 --- a/python/lsst/ip/diffim/utils.py +++ b/python/lsst/ip/diffim/utils.py @@ -410,7 +410,7 @@ def footprint_mean(sources, sky=0): else: sky_mean = np.nan sky_std = np.nan - sky_difference = 0 + sky_difference = np.nanmedian(np.abs(difference.image.array)) science_footprints, difference_footprints, ratio = footprint_mean(selectStars, sky_difference) return lsst.pipe.base.Struct(differenceFootprintRatioMean=ratio.mean(), differenceFootprintRatioStdev=ratio.std(), From 5757ba346b14aa69513c0e295ab949a2741a9aea Mon Sep 17 00:00:00 2001 From: Ian Sullivan Date: Fri, 12 Sep 2025 13:46:07 -0700 Subject: [PATCH 6/6] Fix unit tests to work with new bad mask plane handling --- tests/test_detectAndMeasure.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/tests/test_detectAndMeasure.py b/tests/test_detectAndMeasure.py index 0c7744bcc..a23daaffb 100644 --- a/tests/test_detectAndMeasure.py +++ b/tests/test_detectAndMeasure.py @@ -111,7 +111,8 @@ def _check_values(self, values, minValue=None, maxValue=None): self.assertTrue(np.all(values <= maxValue)) def _setup_detection(self, doSkySources=True, nSkySources=5, - doSubtractBackground=False, run_sattle=False, **kwargs): + doSubtractBackground=False, run_sattle=False, + badSourceFlags=None, **kwargs): """Setup and configure the detection and measurement PipelineTask. Parameters @@ -143,12 +144,15 @@ def _setup_detection(self, doSkySources=True, nSkySources=5, detector=12, universe=lsst.daf.butler.DimensionUniverse(), ) + if badSourceFlags is None: + badSourceFlags = ["base_PixelFlags_flag_offimage", ] config.idGenerator.packer.name = "observation" config.idGenerator.packer["observation"].n_observations = 10000 config.idGenerator.packer["observation"].n_detectors = 99 config.idGenerator.n_releases = 8 config.idGenerator.release_id = 2 config.doSubtractBackground = doSubtractBackground + config.badSourceFlags = badSourceFlags self.idGenerator = config.idGenerator.apply(dataId) return self.detectionTask(config=config) @@ -542,8 +546,12 @@ def test_exclude_mask_detections(self): matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) # Configure the detection Task - detectionTask = self._setup_detection() - excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes + badSourceFlags = ["base_PixelFlags_flag_offimage", + "base_PixelFlags_flag_edgeCenterAll", + "base_PixelFlags_flag_badCenterAll", + "base_PixelFlags_flag_saturatedCenterAll", ] + detectionTask = self._setup_detection(nSkySources=0, badSourceFlags=badSourceFlags) + excludeMaskPlanes = ["EDGE", "BAD", "SAT"] nBad = len(excludeMaskPlanes) self.assertGreater(nBad, 0) kwargs["seed"] = transientSeed @@ -563,10 +571,8 @@ def _detection_wrapper(setFlags=True): srcBbox = lsst.geom.Box2I(lsst.geom.Point2I(srcX - radius, srcY - radius), lsst.geom.Extent2I(2*radius + 1, 2*radius + 1)) difference[srcBbox].mask.array |= lsst.afw.image.Mask.getPlaneBitMask(badMask) - if setFlags: with self.assertRaises(AlgorithmError): output = detectionTask.run(science, matchedTemplate, difference, sources) - return else: output = detectionTask.run(science, matchedTemplate, difference, sources) refIds = [] @@ -881,9 +887,7 @@ def test_detection_xy0(self): self.assertImagesEqual(subtractedMeasuredExposure.image, difference.image) - # Not all of the sources will be detected: preconvolution results in - # a larger edge mask, so we miss an edge source. - self.assertEqual(len(output.diaSources), len(sources)-1) + self.assertEqual(len(output.diaSources), len(sources)) # no sources should be flagged as negative self.assertEqual(len(~output.diaSources["is_negative"]), len(output.diaSources)) # TODO DM-41496: restore this block once we handle detections on edge @@ -1103,13 +1107,19 @@ def test_exclude_mask_detections(self): kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) science.getInfo().setVisitInfo(makeVisitInfo()) + detector = DetectorWrapper(numAmps=1).detector + science.setDetector(detector) matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() scienceKernel = science.psf.getKernel() # Configure the detection Task - detectionTask = self._setup_detection() - excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes + badSourceFlags = ["base_PixelFlags_flag_offimage", + "base_PixelFlags_flag_edgeCenterAll", + "base_PixelFlags_flag_badCenterAll", + "base_PixelFlags_flag_saturatedCenterAll", ] + detectionTask = self._setup_detection(nSkySources=0, badSourceFlags=badSourceFlags) + excludeMaskPlanes = ["EDGE", "BAD", "SAT"] nBad = len(excludeMaskPlanes) self.assertGreater(nBad, 0) kwargs["seed"] = transientSeed @@ -1133,7 +1143,6 @@ def _detection_wrapper(setFlags=True): if setFlags: with self.assertRaises(AlgorithmError): output = detectionTask.run(science, matchedTemplate, difference, score, sources) - return else: output = detectionTask.run(science, matchedTemplate, difference, score, sources) refIds = []