From 265192cef451fe7fdef560adbf1e3ea9cc33d6fc Mon Sep 17 00:00:00 2001 From: Ian Sullivan Date: Tue, 16 Dec 2025 10:26:37 -0800 Subject: [PATCH 1/4] Remove matchedTemplate from spatially sampled metrics --- pipelines/_ingredients/ApPipe.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pipelines/_ingredients/ApPipe.yaml b/pipelines/_ingredients/ApPipe.yaml index 47b69ea3..eab934d1 100644 --- a/pipelines/_ingredients/ApPipe.yaml +++ b/pipelines/_ingredients/ApPipe.yaml @@ -174,7 +174,6 @@ tasks: config: connections.coaddName: parameters.coaddName connections.science: preliminary_visit_image - connections.matchedTemplate: template_matched connections.template: template_detector connections.difference: difference_image connections.diaSources: dia_source_unstandardized @@ -299,9 +298,6 @@ contracts: - contract: subtractImages.connections.ConnectionsClass(config=subtractImages).science.name == associateApdb.connections.ConnectionsClass(config=associateApdb).exposure.name msg: "subtractImages.science != associateApdb.exposure" - - contract: subtractImages.connections.ConnectionsClass(config=subtractImages).matchedTemplate.name == - makeSampledImageSubtractionMetrics.connections.ConnectionsClass(config=makeSampledImageSubtractionMetrics).matchedTemplate.name - msg: "subtractImages.matchedTemplate != makeSampledImageSubtractionMetrics.matchedTemplate" - contract: detectAndMeasureDiaSource.connections.ConnectionsClass(config=detectAndMeasureDiaSource).diaSources.name == filterDiaSource.connections.ConnectionsClass(config=filterDiaSource).diaSourceCat.name msg: "detectAndMeasureDiaSource.diaSources != filterDiaSource.diaSourceCat" From ec1213b62c90b16025f2f94b955a432f25cd0efe Mon Sep 17 00:00:00 2001 From: Ian Sullivan Date: Tue, 16 Dec 2025 16:01:21 -0800 Subject: [PATCH 2/4] Switch sampled metrics to use post-reliability diaSource catalog Prompt Processing does not save `dia_source_unstandardized` so switch to an output catalog that will be available to the afterburner --- pipelines/_ingredients/ApPipe.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pipelines/_ingredients/ApPipe.yaml b/pipelines/_ingredients/ApPipe.yaml index eab934d1..3f20b580 100644 --- a/pipelines/_ingredients/ApPipe.yaml +++ b/pipelines/_ingredients/ApPipe.yaml @@ -176,7 +176,7 @@ tasks: connections.science: preliminary_visit_image connections.template: template_detector connections.difference: difference_image - connections.diaSources: dia_source_unstandardized + connections.diaSources: dia_source_detector connections.psfMatchingKernel: difference_kernel connections.spatiallySampledMetrics: difference_image_metrics @@ -319,9 +319,9 @@ contracts: - contract: filterDiaSource.connections.ConnectionsClass(config=filterDiaSource).filteredDiaSourceCat.name == filterDiaSourcePostReliability.connections.ConnectionsClass(config=filterDiaSourcePostReliability).diaSourceCat.name msg: "filterDiaSource.filteredDiaSourceCat != filterDiaSourcePostReliability.diaSourceCat" - - contract: filterDiaSource.connections.ConnectionsClass(config=filterDiaSource).filteredDiaSourceCat.name == + - contract: standardizeDiaSource.connections.ConnectionsClass(config=standardizeDiaSource).diaSourceTable.name == makeSampledImageSubtractionMetrics.connections.ConnectionsClass(config=makeSampledImageSubtractionMetrics).diaSources.name - msg: "filterDiaSource.filteredDiaSourceCat != makeSampledImageSubtractionMetrics.diaSources" + msg: "standardizeDiaSource.diaSourceTable != makeSampledImageSubtractionMetrics.diaSources" - contract: filterDiaSource.connections.ConnectionsClass(config=filterDiaSource).longTrailedSources.name == analyzeTrailedDiaSourceTable.connections.ConnectionsClass(config=analyzeTrailedDiaSourceTable).data.name msg: "filterDiaSource.longTrailedSources != analyzeTrailedDiaSourceTable.data" From 666f89b6b23f7c2f25ea14dcb7aca7f92defda69 Mon Sep 17 00:00:00 2001 From: Ian Sullivan Date: Tue, 30 Dec 2025 16:10:16 -0800 Subject: [PATCH 3/4] Add daytime processing pipeline and executable script --- bps/LSSTCam/bps_Daytime.yaml | 15 ++++++ bps/clustering/clustering_Daytime.yaml | 51 ++++++++++++++++++ scripts/LSSTCam/submit_ap_daytime.sh | 73 ++++++++++++++++++++++++++ 3 files changed, 139 insertions(+) create mode 100644 bps/LSSTCam/bps_Daytime.yaml create mode 100644 bps/clustering/clustering_Daytime.yaml create mode 100755 scripts/LSSTCam/submit_ap_daytime.sh diff --git a/bps/LSSTCam/bps_Daytime.yaml b/bps/LSSTCam/bps_Daytime.yaml new file mode 100644 index 00000000..17e7f83f --- /dev/null +++ b/bps/LSSTCam/bps_Daytime.yaml @@ -0,0 +1,15 @@ +pipelineYaml: $AP_PIPE_DIR/pipelines/LSSTCam/ApPipe.yaml + +project: ApPipe +campaign: AP-daytime + +payload: + # Use the prompt processing configuration for the embargo repo, which turns on compression. + # This can be changed to 'embargo' once compression is turned on by default + butlerConfig: "s3://embargo@rubin-summit-users/butler-prompt-processing.yaml" + # payloadName: Do not set here, must pass in from command line or script + # inCollection: Do not set here, must pass in from command line or script + # dataQuery: Do not set here, must pass in from command line or script + +includeConfigs: + - ${AP_PIPE_DIR}/bps/clustering/clustering_Daytime.yaml diff --git a/bps/clustering/clustering_Daytime.yaml b/bps/clustering/clustering_Daytime.yaml new file mode 100644 index 00000000..d1740f5b --- /dev/null +++ b/bps/clustering/clustering_Daytime.yaml @@ -0,0 +1,51 @@ +# This is a prescription for quantum clustering with BPS, suitable for any +# concrete pipeline based on the AP pipeline. Note that there are separate +# example files for pipelines with and without fakes. +# +# Use it by adding: +# +# includeConfigs: +# - ${AP_PIPE_DIR}/bps/clustering/clustering_ApPipe-daytime.yaml +# +# (with no outer indentation) to your BPS config file. If you are running +# fakes, use instead: +# +# includeConfigs: +# - ${AP_PIPE_DIR}/bps/clustering/clustering_ApPipeWithFakes-daytime.yaml +# +# again with no outer indentation. +# + +clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering +cluster: + preload: + pipetasks: loadDiaCatalogs,analyzeLoadDiaCatalogsMetrics,mpSkyEphemerisQuery,getRegionTimeFromVisit + dimensions: visit,detector + equalDimensions: visit:group + singleFrame: + pipetasks: isr,calibrateImage,analyzePreliminarySummaryStats + dimensions: visit,detector + equalDimensions: visit:exposure + diffim: + pipetasks: buildTemplate,subtractImages,detectAndMeasureDiaSource,analyzeImageDifferenceMetrics,analyzeDiaSourceDetectionMetrics,filterDiaSource,computeReliability,filterDiaSourcePostReliability,standardizeDiaSource + dimensions: visit,detector + association: + pipetasks: associateApdb,analyzeDiaSourceAssociationMetrics,analyzeAssociateDiaSourceTiming + dimensions: visit,detector + diaSrcDetectorAnalysis: + pipetasks: analyzeAssociatedDiaSourceTable,analyzeTrailedDiaSourceTable + dimensions: visit,detector + diffimMetrics: + pipetasks: makeSampledImageSubtractionMetrics,analyzeSampledImageSubtractionMetrics + dimensions: visit,detector + consolidateVisit: + pipetasks: consolidateVisitSummary,analyzeVisitSampledImageSubtractionMetrics + dimensions: visit + +ordering: + associationCluster: + ordering_type: sort + findDependencyMethod: sink + labels: association + dimensions: visit + blocking: False diff --git a/scripts/LSSTCam/submit_ap_daytime.sh b/scripts/LSSTCam/submit_ap_daytime.sh new file mode 100755 index 00000000..e5d06dfa --- /dev/null +++ b/scripts/LSSTCam/submit_ap_daytime.sh @@ -0,0 +1,73 @@ +#!/bin/sh +set -eu + +# An executable script that will prepare and submit the daytime Alert Production pipeline + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 YYYY-MM-DD" >&2 + exit 1 +fi + +DATE="$1" + +# POSIX-safe date normalization +DAY_OBS=$(printf '%s\n' "$DATE" | tr -d '-') + +# Create a temp file with the date in the name +TMP_APDB_REL=$(mktemp "apdb_config_${DATE}.XXXXXX.yaml") + +# Resolve to absolute path without readlink -f +case "$TMP_APDB_REL" in + /*) TMP_APDB="$TMP_APDB_REL" ;; + *) TMP_APDB="$(pwd)/$TMP_APDB_REL" ;; +esac + +# Copy APDB config from S3 using Singularity AWS CLI +singularity exec /sdf/sw/s3/aws-cli_latest.sif \ + aws --endpoint-url https://sdfembs3.sdf.slac.stanford.edu s3 \ + --profile embargo-s3 \ + cp s3://rubin-summit-users/apdb_config/cassandra/pp_apdb_lsstcam.yaml \ + "$TMP_APDB" + +# NOTE: +# No cleanup of TMP_APDB here since the job is launched with nohup +# and runtime duration is unknown. + +# Redirect Cassandra logs +export DAX_APDB_MONITOR_CONFIG="logging:lsst.dax.apdb.monitor" + +# Configure the filesystem to allow many open files +ulimit -n 65536 + +INSTRUMENT="LSSTCam" + +# List of detectors currently excluded from Prompt Processing +# These include the non-imaging wavefront sensors as well as some that are disabled in fan-out. +# See https://github.com/lsst-sqre/phalanx/blob/main/applications/next-visit-fan-out/values-usdfprod-prompt-processing.yaml +BAD_DETECTORS="120 122 0 20 27 65 123 161 168 188 1 19 30 68 158 169 187 \ +189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204" + +# Space-delimited list of observing blocks that generate science images +BLOCKS="BLOCK-407 BLOCK-408 BLOCK-416 BLOCK-417 BLOCK-419 BLOCK-421 BLOCK-T637" + +OUTPUT_COLLECTION="LSSTCam/prompt/output-${DATE}/daytime" + +# Convert lists to SQL IN() form +BAD_DETECTORS_SQL="($(printf '%s,' $BAD_DETECTORS | sed 's/,$//'))" +BLOCKS_SQL="($(printf "'%s'," $BLOCKS | sed 's/,$//'))" + +nohup bps submit "${AP_PIPE_DIR}/bps/LSSTCam/bps_Daytime.yaml" \ + --extra-qgraph-options "--skip-existing-in LSSTCam/prompt/output-${DATE} -c parameters:release_id=1 -c parameters:apdb_config=${TMP_APDB} --dataset-query-constraint template_coadd" \ + --extra-run-quantum-options "--no-raise-on-partial-outputs" \ + --input "LSSTCam/defaults,LSSTCam/templates,LSSTCam/prompt/output-${DATE}" \ + --output "$OUTPUT_COLLECTION" \ + -d "instrument='$INSTRUMENT' \ + AND skymap='lsst_cells_v1' \ + AND detector NOT IN $BAD_DETECTORS_SQL \ + AND day_obs=$DAY_OBS \ + AND exposure.science_program IN $BLOCKS_SQL" \ + > "output-${DATE}-step1.out" 2>&1 & + +echo "Submission started for date ${DATE}" +echo "Temporary APDB config: ${TMP_APDB}" +echo "Submission output log written to output-${DATE}.out" From df67dd1f50e9bc34741de771f227ba21e8815922 Mon Sep 17 00:00:00 2001 From: Ian Sullivan Date: Wed, 14 Jan 2026 12:34:00 -0800 Subject: [PATCH 4/4] Add update-dimension-regions to daytime finalJob --- bps/LSSTCam/bps_Daytime.yaml | 6 ++++++ scripts/LSSTCam/submit_ap_daytime.sh | 6 ++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/bps/LSSTCam/bps_Daytime.yaml b/bps/LSSTCam/bps_Daytime.yaml index 17e7f83f..168453dd 100644 --- a/bps/LSSTCam/bps_Daytime.yaml +++ b/bps/LSSTCam/bps_Daytime.yaml @@ -13,3 +13,9 @@ payload: includeConfigs: - ${AP_PIPE_DIR}/bps/clustering/clustering_Daytime.yaml + +# Once consolidateVisitSummary has been run, we can also update the butler records for the region of each image +# The initial region is just the predicted value from the nextVisit event, +# so this will replace that with the updated region after calibration. +finalJob: + command2: "${DAF_BUTLER_DIR}/bin/butler {finalPreCmdOpts} update-dimension-regions {butlerConfig} LSSTCam {outputRun} --batch-size 2000" diff --git a/scripts/LSSTCam/submit_ap_daytime.sh b/scripts/LSSTCam/submit_ap_daytime.sh index e5d06dfa..d3aa3998 100755 --- a/scripts/LSSTCam/submit_ap_daytime.sh +++ b/scripts/LSSTCam/submit_ap_daytime.sh @@ -52,6 +52,8 @@ BLOCKS="BLOCK-407 BLOCK-408 BLOCK-416 BLOCK-417 BLOCK-419 BLOCK-421 BLOCK-T637" OUTPUT_COLLECTION="LSSTCam/prompt/output-${DATE}/daytime" +LOG_FILE="output-${DATE}.out" + # Convert lists to SQL IN() form BAD_DETECTORS_SQL="($(printf '%s,' $BAD_DETECTORS | sed 's/,$//'))" BLOCKS_SQL="($(printf "'%s'," $BLOCKS | sed 's/,$//'))" @@ -66,8 +68,8 @@ nohup bps submit "${AP_PIPE_DIR}/bps/LSSTCam/bps_Daytime.yaml" \ AND detector NOT IN $BAD_DETECTORS_SQL \ AND day_obs=$DAY_OBS \ AND exposure.science_program IN $BLOCKS_SQL" \ - > "output-${DATE}-step1.out" 2>&1 & + > "${LOG_FILE}" 2>&1 & echo "Submission started for date ${DATE}" echo "Temporary APDB config: ${TMP_APDB}" -echo "Submission output log written to output-${DATE}.out" +echo "Submission output log written to ${LOG_FILE}"