DM-32430: Create DECam DRP Processing Pipeline

config/apertures.py

@@ -0,0 +1,6 @@
+# Set up aperture photometry
+# 'config' should be a SourceMeasurementConfig
+
+# Use a large aperture to be independent of seeing in calibration
+# This config matches obs_subaru, to facilitate 1:1 comparisons between DECam and HSC
+config.plugins["base_CircularApertureFlux"].maxSincRadius = 12.0

config/calibrate.py

@@ -3,8 +3,12 @@
 """
 import os.path
 
+from lsst.meas.algorithms import ColorLimit
+from lsst.meas.astrom import MatchOptimisticBConfig
+
 obsConfigDir = os.path.join(os.path.dirname(__file__))
 
+# Astrometry/Photometry
 # This sets the reference catalog name for Gen2.
 for refObjLoader in (config.astromRefObjLoader,
                      config.photoRefObjLoader,
@@ -21,7 +25,22 @@
 # This sets up the reference catalog for Gen3.
 config.connections.astromRefCat = "ps1_pv3_3pi_20170110"
 config.connections.photoRefCat = "ps1_pv3_3pi_20170110"
+
+# Photometric calibration: use color terms
+config.photoCal.applyColorTerms = True
 config.photoCal.photoCatName = "ps1_pv3_3pi_20170110"
+colors = config.photoCal.match.referenceSelection.colorLimits
+# The following two color limits are adopted from obs_subaru for the HSC SSP survey
+colors["g-r"] = ColorLimit(primary="g_flux", secondary="r_flux", minimum=0.0)
+colors["r-i"] = ColorLimit(primary="r_flux", secondary="i_flux", maximum=0.5)
+config.photoCal.match.referenceSelection.doMagLimit = True
+config.photoCal.match.referenceSelection.magLimit.fluxField = "i_flux"
+config.photoCal.match.referenceSelection.magLimit.maximum = 22.0
+config.photoCal.colorterms.load(os.path.join(obsConfigDir, 'colorterms.py'))
+
+# Number of bright stars to use. Sets the max number of patterns that can be tested.
+# This config matches obs_subaru, to facilitate 1:1 comparisons between DECam and HSC
+config.astrometry.matcher.numBrightStars = 150
 
 # The Task default was reduced from 4 to 2 on RFC-577. We believe that 4 is
 # more appropriate for use with DECam data until a Jointcal-derived distortion
@@ -30,4 +49,38 @@
 # See Slack: https://lsstc.slack.com/archives/C2B6X08LS/p1586468459084600
 config.astrometry.wcsFitter.order = 4
 
+for matchConfig in (config.astrometry,
+                    ):
+    matchConfig.sourceFluxType = 'Psf'
+    matchConfig.sourceSelector.active.sourceFluxType = 'Psf'
+    matchConfig.matcher.maxOffsetPix = 250
+    if isinstance(matchConfig.matcher, MatchOptimisticBConfig):
+        matchConfig.matcher.allowedNonperpDeg = 0.2
+        matchConfig.matcher.maxMatchDistArcSec = 2.0
+        matchConfig.sourceSelector.active.excludePixelFlags = False
+
+# Set to match defaults currently used in HSC production runs (e.g. S15B+)
+config.catalogCalculation.plugins['base_ClassificationExtendedness'].fluxRatio = 0.95
+
+# Demand astrometry and photoCal succeed
+config.requireAstrometry = True
+config.requirePhotoCal = True
+
+config.doWriteMatchesDenormalized = True
+
+# Detection
+# This config matches obs_subaru, to facilitate 1:1 comparisons between DECam and HSC
+config.detection.isotropicGrow = True
+
+config.measurement.load(os.path.join(obsConfigDir, "apertures.py"))
+config.measurement.load(os.path.join(obsConfigDir, "kron.py"))
 config.measurement.load(os.path.join(obsConfigDir, "hsm.py"))
+
+# Deblender
+# These configs match obs_subaru, to facilitate 1:1 comparisons between DECam and HSC
+config.deblend.maxFootprintSize = 0
+config.deblend.maskLimits["NO_DATA"] = 0.25  # Ignore sources that are in the vignetted region
+config.deblend.maxFootprintArea = 10000
+
+config.measurement.plugins.names |= ["base_Jacobian", "base_FPPosition"]
+config.measurement.plugins["base_Jacobian"].pixelScale = 0.263

config/characterizeImage.py

@@ -1,8 +1,25 @@
 """
 DECam-specific overrides for CharacterizeImageTask
 """
+import os.path
+
+from lsst.meas.astrom import MatchOptimisticBConfig
+
+obsConfigDir = os.path.dirname(__file__)
+
+# Cosmic rays
+# These configs match obs_subaru, to facilitate 1:1 comparisons between DECam and HSC
 config.repair.cosmicray.nCrPixelMax = 100000
+config.repair.cosmicray.cond3_fac2 = 0.4
+
+# PSF determination
+# These configs match obs_subaru, to facilitate 1:1 comparisons between DECam and HSC
+config.measurePsf.reserve.fraction = 0.2
+config.measurePsf.starSelector["objectSize"].sourceFluxField = "base_PsfFlux_instFlux"
+config.measurePsf.starSelector["objectSize"].widthMin = 0.9
+config.measurePsf.starSelector["objectSize"].fluxMin = 4000
 
+# Astrometry/Photometry
 # This sets the reference catalog name for Gen2.
 # Note that in Gen3, we've stopped pretending (which is what Gen2 does,
 # for backwards compatibility) that charImage uses a reference catalog.
@@ -14,3 +31,62 @@
 config.refObjLoader.filterMap['N540'] = 'g'
 config.refObjLoader.filterMap['N708'] = 'i'
 config.refObjLoader.filterMap['N964'] = 'z'
+
+# Set to match defaults currently used in HSC production runs (e.g. S15B)
+config.catalogCalculation.plugins['base_ClassificationExtendedness'].fluxRatio = 0.95
+
+# Detection
+# This config matches obs_subaru, to facilitate 1:1 comparisons between DECam and HSC
+config.detection.isotropicGrow = True
+
+# Activate calibration of measurements: required for aperture corrections
+config.load(os.path.join(obsConfigDir, "cmodel.py"))
+config.measurement.load(os.path.join(obsConfigDir, "apertures.py"))
+config.measurement.load(os.path.join(obsConfigDir, "kron.py"))
+config.measurement.load(os.path.join(obsConfigDir, "convolvedFluxes.py"))
+config.measurement.load(os.path.join(obsConfigDir, "gaap.py"))
+config.measurement.load(os.path.join(obsConfigDir, "hsm.py"))
+if "ext_shapeHSM_HsmShapeRegauss" in config.measurement.plugins:
+    # no deblending has been done
+    config.measurement.plugins["ext_shapeHSM_HsmShapeRegauss"].deblendNChild = ""
+
+# Deblender
+config.deblend.maskLimits["NO_DATA"] = 0.25  # Ignore sources that are in the vignetted region
+config.deblend.maxFootprintArea = 10000
+
+config.measurement.plugins.names |= ["base_Jacobian", "base_FPPosition"]
+config.measurement.plugins["base_Jacobian"].pixelScale = 0.263
+
+# Convolved fluxes can fail for small target seeing if the observation seeing is larger
+if "ext_convolved_ConvolvedFlux" in config.measurement.plugins:
+    names = config.measurement.plugins["ext_convolved_ConvolvedFlux"].getAllResultNames()
+    config.measureApCorr.allowFailure += names
+
+if "ext_gaap_GaapFlux" in config.measurement.plugins:
+    names = config.measurement.plugins["ext_gaap_GaapFlux"].getAllGaapResultNames()
+    config.measureApCorr.allowFailure += names
+
+# For aperture correction modeling, only use objects that were used in the
+# PSF model and have psf flux signal-to-noise > 200.
+# These configs match obs_subaru, to facilitate 1:1 comparisons between DECam and HSC
+config.measureApCorr.sourceSelector['science'].doFlags = True
+config.measureApCorr.sourceSelector['science'].doUnresolved = False
+config.measureApCorr.sourceSelector['science'].doSignalToNoise = True
+config.measureApCorr.sourceSelector['science'].flags.good = ["calib_psf_used"]
+config.measureApCorr.sourceSelector['science'].flags.bad = []
+config.measureApCorr.sourceSelector['science'].signalToNoise.minimum = 200.0
+config.measureApCorr.sourceSelector['science'].signalToNoise.maximum = None
+config.measureApCorr.sourceSelector['science'].signalToNoise.fluxField = "base_PsfFlux_instFlux"
+config.measureApCorr.sourceSelector['science'].signalToNoise.errField = "base_PsfFlux_instFluxErr"
+config.measureApCorr.sourceSelector.name = "science"
+
+config.ref_match.sourceSelector.name = 'matcher'
+for matchConfig in (config.ref_match,
+                    ):
+    matchConfig.sourceFluxType = 'Psf'
+    matchConfig.sourceSelector.active.sourceFluxType = 'Psf'
+    matchConfig.matcher.maxOffsetPix = 250
+    if isinstance(matchConfig.matcher, MatchOptimisticBConfig):
+        matchConfig.matcher.allowedNonperpDeg = 0.2
+        matchConfig.matcher.maxMatchDistArcSec = 2.0
+        matchConfig.sourceSelector.active.excludePixelFlags = False

config/cmodel.py

@@ -0,0 +1,11 @@
+# Enable CModel mags (unsetup meas_modelfit to disable)
+# 'config' is a SourceMeasurementConfig.
+try:
+    import lsst.meas.modelfit
+    config.measurement.plugins.names |= ["modelfit_DoubleShapeletPsfApprox", "modelfit_CModel"]
+    config.measurement.slots.modelFlux = 'modelfit_CModel'
+    config.catalogCalculation.plugins['base_ClassificationExtendedness'].fluxRatio = 0.985
+except (KeyError, ImportError):
+    import logging
+    logging.getLogger("lsst.obs.decam.config").warning("Cannot import lsst.meas.modelfit:"
+                                                       " disabling CModel measurements")

config/colorAnalysis.py

@@ -0,0 +1,5 @@
+import os.path
+
+config.load(os.path.join(os.path.dirname(__file__), "extinctionCoeffs.py"))
+from lsst.obs.decam.decamFilters import DECAM_FILTER_DEFINITIONS
+config.physicalToBandFilterMap = DECAM_FILTER_DEFINITIONS.physical_to_band

config/colorterms.py

@@ -0,0 +1,52 @@
+from lsst.pipe.tasks.colorterms import Colorterm, ColortermDict
+
+# Average color terms for the DECam filters are calculated using the Pickles stellar spectra atlas.
+# The following values are provided by Song Huang (Tsinghua University; dr.guangtou@gmail.com)
+# Color terms for u, Y, N419 and N964 bands are not currently available.
+
+# The values for DECam g, r & z-bands are calculated independently and are different to the values at:
+# https://www.legacysurvey.org/dr9/description/
+
+# A Jupyter notebook to reproduce these values is available at:
+# https://github.com/MerianSurvey/caterpillar/blob/main/notebook/photocal/merian_filter_color_terms.ipynb 
+
+# In obs_subaru, colorterms.py is also used in:
+# - config/skyAnalysis.py
+# - config/compareCoaddAnalysis.py 
+# - config/coaddAnalysis.py
+# - config/visitAnalysis.py
+# - config/fgcmBuildStarsTable.py and config/fgcmBuildStars.py
+# These files are not currently available for obs_decam.
+
+config.data = {
+    "decam*": ColortermDict(data={
+        'g DECam SDSS c0001 4720.0 1520.0': Colorterm(primary="g", secondary="g"),
+        'r DECam SDSS c0002 6415.0 1480.0': Colorterm(primary="r", secondary="r"),
+        'i DECam SDSS c0003 7835.0 1470.0': Colorterm(primary="i", secondary="i"),
+        'z DECam SDSS c0004 9260.0 1520.0': Colorterm(primary="z", secondary="z"),
+    }),
+    "sdss*": ColortermDict(data={
+        'g DECam SDSS c0001 4720.0 1520.0': Colorterm(primary="g", secondary="r", c0=-0.008015, c1=-0.089869, c2=-0.018398), # For -1.0 < g-r < 1.8
+        'r DECam SDSS c0002 6415.0 1480.0': Colorterm(primary="r", secondary="i", c0=-0.0077434, c1=-0.202615, c2=0.016042), # For -1.0 < r-i < 2.2
+        'i DECam SDSS c0003 7835.0 1470.0': Colorterm(primary="i", secondary="z", c0=0.00018887, c1=-0.2748281, c2=-0.029417), # For -1.0 < i-z < 1.2
+        'z DECam SDSS c0004 9260.0 1520.0': Colorterm(primary="z", secondary="i", c0=-0.0059858, c1=0.1131192, c2=0.0156471), # For -1.0 < z-i < 0.5
+        'N540 DECam c0014 5403.2 210.0': Colorterm(primary="g", secondary="r", c0=-0.03359776, c1=-0.57665033, c2=-0.01510509), # For -1.0 < g-r < 1.5
+        'N708 DECam c0012 7080.0 400.0': Colorterm(primary="r", secondary="i", c0=-0.01428724, c1=-0.71755905, c2=0.08959316), # For -1.0 < r-i < 2.0
+    }),
+    "hsc*": ColortermDict(data={
+        'g DECam SDSS c0001 4720.0 1520.0': Colorterm(primary="g", secondary="r", c0=0.0000509, c1=-0.0152487, c2=-0.0029937), # For -1.0 < g-r < 1.4
+        'r DECam SDSS c0002 6415.0 1480.0': Colorterm(primary="r", secondary="i", c0=-0.0078995, c1=-0.1695323, c2=0.0251978), # For -1.0 < r-i < 2.2
+        'i DECam SDSS c0003 7835.0 1470.0': Colorterm(primary="i", secondary="z", c0=0.0010196, c1=-0.1314031, c2=0.0054605), # For -1.0 < i-z < 1.0
+        'z DECam SDSS c0004 9260.0 1520.0': Colorterm(primary="z", secondary="y", c0=-0.0013244, c1=-0.2846684, c2=-0.1229817), # For -1.0 < z-y < 0.5
+        'N540 DECam c0014 5403.2 210.0': Colorterm(primary="g", secondary="r", c0=-0.03015158, c1=-0.53715834, c2=-0.00202415), # For -1.0 < g-r < 1.5
+        'N708 DECam c0012 7080.0 400.0': Colorterm(primary="r", secondary="i", c0=-0.01978413, c1=-0.64368838, c2=0.09132738), # For -1.0 < r-i < 2.0
+    }),
+    "ps1*": ColortermDict(data={
+        'g DECam SDSS c0001 4720.0 1520.0': Colorterm(primary="g", secondary="r", c0=0.006388, c1=0.045875, c2=-0.004484), # For -1.0 < g-r < 1.2
+        'r DECam SDSS c0002 6415.0 1480.0': Colorterm(primary="r", secondary="i", c0=-0.006775, c1=-0.187304, c2=0.018928), # For -1.0 < r-i < 2.2
+        'i DECam SDSS c0003 7835.0 1470.0': Colorterm(primary="i", secondary="z", c0=0.0012204, c1=-0.282956, c2=-0.011321), # For -1.0 < i-z < 1.4
+        'z DECam SDSS c0004 9260.0 1520.0': Colorterm(primary="z", secondary="y", c0=-0.0087868, c1=-0.5204795, c2=-0.057955), # For -1.0 < z-y < 1.0
+        'N540 DECam c0014 5403.2 210.0': Colorterm(primary="g", secondary="r", c0=-0.02635164, c1=-0.50968428, c2=-0.00958104), # For -1.0 < g-r < 1.5
+        'N708 DECam c0012 7080.0 400.0': Colorterm(primary="r", secondary="i", c0=-0.01508987, c1=-0.69377675, c2=0.09079348), # For -1.0 < r-i < 2.0
+    }),
+}

config/convolvedFluxes.py

@@ -0,0 +1,11 @@
+# Enable measurement of convolved fluxes
+# 'config' is a SourceMeasurementConfig
+try:
+    import lsst.meas.extensions.convolved  # noqa: Load flux.convolved algorithm
+except ImportError as exc:
+    import logging
+    logging.getLogger("lsst.obs.decam.config").warning("Cannot import lsst.meas.extensions.convolved (%s):"
+                                                       " disabling convolved flux measurements", exc)
+else:
+    config.plugins.names.add("ext_convolved_ConvolvedFlux")
+    config.plugins["ext_convolved_ConvolvedFlux"].seeing.append(8.0)

config/extinctionCoeffs.py

@@ -0,0 +1,15 @@
+# Extinction coefficients for DECam filters for conversion from E(B-V) to extinction, A_filter.
+# Numbers initially provided by Song Huang (NAOJ).
+#
+# Band, A_filter/E(B-V)
+# Values for N419 and N964 filters are not currently available.
+config.extinctionCoeffs = {
+    "u": 3.994,
+    "g": 3.212,
+    "r": 2.164,
+    "i": 1.591,
+    "z": 1.211,
+    "y": 1.063,
+    "N540": 2.753,
+    "N708": 1.847,
+}

config/filterMap.py

@@ -0,0 +1,9 @@
+# Mapping of camera filter name: reference catalog filter name; each reference filter must exist in the refcat. 
+# Note that this does not perform any bandpass corrections: it is just a lookup.
+for source, target in [
+        ('N419', 'g'),
+        ('N540', 'g'),
+        ('N708', 'i'),
+        ('N964', 'z'),
+    ]:
+    config.filterMap[source] = target

config/forcedPhotCoadd.py

@@ -0,0 +1,46 @@
+import os.path
+
+from lsst.meas.base import CircularApertureFluxAlgorithm
+
+config.measurement.load(os.path.join(os.path.dirname(__file__), "apertures.py"))
+config.measurement.load(os.path.join(os.path.dirname(__file__), "kron.py"))
+config.measurement.load(os.path.join(os.path.dirname(__file__), "convolvedFluxes.py"))
+config.measurement.load(os.path.join(os.path.dirname(__file__), "gaap.py"))
+config.load(os.path.join(os.path.dirname(__file__), "cmodel.py"))
+
+# These configs match obs_subaru, to facilitate 1:1 comparisons between DECam and HSC
+config.catalogCalculation.plugins.names = ["base_ClassificationExtendedness"]
+config.measurement.slots.psfFlux = "base_PsfFlux"
+
+def doUndeblended(config, algName, fluxList=None):
+    """Activate undeblended measurements of algorithm
+
+    Parameters
+    ----------
+    algName : `str`
+        Algorithm name.
+    fluxList : `list` of `str`, or `None`
+        List of flux columns to register for aperture correction. If `None`,
+        then this will be the `algName` appended with `_instFlux`.
+    """
+    if algName not in config.measurement.plugins:
+        return
+    if fluxList is None:
+        fluxList = [algName + "_instFlux"]
+    config.measurement.undeblended.names.add(algName)
+    config.measurement.undeblended[algName] = config.measurement.plugins[algName]
+    for flux in fluxList:
+        config.applyApCorr.proxies["undeblended_" + flux] = flux
+
+
+# These configs match obs_subaru, to facilitate 1:1 comparisons between DECam and HSC
+doUndeblended(config, "base_PsfFlux")
+doUndeblended(config, "ext_photometryKron_KronFlux")
+doUndeblended(config, "base_CircularApertureFlux", [])  # No aperture correction for circular apertures
+doUndeblended(config, "ext_convolved_ConvolvedFlux",
+              config.measurement.plugins["ext_convolved_ConvolvedFlux"].getAllResultNames())
+doUndeblended(config, "ext_gaap_GaapFlux",
+              config.measurement.plugins["ext_gaap_GaapFlux"].getAllGaapResultNames())
+# Disable registration for apCorr of undeblended convolved; apCorr will be done through the deblended proxy
+config.measurement.undeblended["ext_convolved_ConvolvedFlux"].registerForApCorr = False
+config.measurement.undeblended["ext_gaap_GaapFlux"].registerForApCorr = False

config/gaap.py

@@ -0,0 +1,12 @@
+# Enable GAaP (Gaussian Aperture and PSF) colors
+# 'config' is typically a SourceMeasurementConfig
+try:
+    import lsst.meas.extensions.gaap  # noqa
+    config.plugins.names.add("ext_gaap_GaapFlux")
+    config.plugins["ext_gaap_GaapFlux"].sigmas = [0.5, 0.7, 1.0, 1.5, 2.5, 3.0]
+    # Enable PSF photometry after PSF-Gaussianization in the `ext_gaap_GaapFlux` plugin
+    config.plugins["ext_gaap_GaapFlux"].doPsfPhotometry = True
+except ImportError as exc:
+    import logging
+    logging.getLogger("lsst.obs.decam.config").warning("Cannot import lsst.meas.extensions.gaap (%s):"
+                                                       " disabling GAaP flux measurements", exc)

config/jointcal.py

@@ -0,0 +1,10 @@
+import os.path
+
+# `load()` appends to the filterMaps: we need them to be empty, so that
+# only the specified filter mappings are used.
+config.photometryRefObjLoader.filterMap = {}
+filterMapFile = os.path.join(os.path.dirname(__file__), "filterMap.py")
+config.photometryRefObjLoader.load(filterMapFile)
+# We have PS1 colorterms
+config.applyColorTerms = True
+config.colorterms.load(os.path.join(os.path.dirname(__file__), "colorterms.py"))

config/kron.py

@@ -0,0 +1,10 @@
+# Enable Kron mags
+# 'config' is a SourceMeasurementConfig
+
+try:
+    import lsst.meas.extensions.photometryKron
+    config.plugins.names |= ["ext_photometryKron_KronFlux"]
+except ImportError:
+    import logging
+    logging.getLogger("lsst.obs.decam.config").warning("Cannot import lsst.meas.extensions.photometryKron:"
+                                                       " disabling Kron measurements")

config/measureCoaddSources.py

@@ -1,8 +1,18 @@
+import os.path
+
+config.measurement.load(os.path.join(os.path.dirname(__file__), "apertures.py"))
+config.measurement.load(os.path.join(os.path.dirname(__file__), "kron.py"))
+config.measurement.load(os.path.join(os.path.dirname(__file__), "convolvedFluxes.py"))
+config.measurement.load(os.path.join(os.path.dirname(__file__), "hsm.py"))
+config.load(os.path.join(os.path.dirname(__file__), "cmodel.py"))
+
+# Set reference catalog for Gen2.
 config.match.refObjLoader.ref_dataset_name = "ps1_pv3_3pi_20170110"
-config.match.refObjLoader.filterMap['u'] = 'g'
-config.match.refObjLoader.filterMap['Y'] = 'y'
-config.match.refObjLoader.filterMap['N419'] = 'g'
-config.match.refObjLoader.filterMap['N540'] = 'g'
-config.match.refObjLoader.filterMap['N708'] = 'i'
-config.match.refObjLoader.filterMap['N964'] = 'z'
+# Set reference catalog for Gen3.
 config.connections.refCat = "ps1_pv3_3pi_20170110"
+
+config.match.refObjLoader.load(os.path.join(os.path.dirname(__file__), "filterMap.py"))
+
+config.doWriteMatchesDenormalized = True
+
+config.measurement.plugins.names |= ["base_InputCount"]

config/mergeDetections.py

@@ -1 +1 @@
-config.priorityList = ["r", "i", "z", "g", "y", "u"]
+config.priorityList = ["r", "i", "z", "g", "y", "u", "N708", "N540", "N964", "N419"]

config/mergeMeasurements.py

@@ -1 +1 @@
-config.priorityList = ["r", "i", "z", "g", "y", "u"]
+config.priorityList = ["r", "i", "z", "g", "y", "u", "N708", "N540", "N964", "N419"]