DM-53634: Add daytime AP pipeline and scripts

bps/LSSTCam/bps_Daytime.yaml

@@ -0,0 +1,21 @@
+pipelineYaml: $AP_PIPE_DIR/pipelines/LSSTCam/ApPipe.yaml
+
+project: ApPipe
+campaign: AP-daytime
+
+payload:
+  # Use the prompt processing configuration for the embargo repo, which turns on compression.
+  # This can be changed to 'embargo' once compression is turned on by default
+  butlerConfig: "s3://embargo@rubin-summit-users/butler-prompt-processing.yaml"
+  # payloadName: Do not set here, must pass in from command line or script
+  # inCollection: Do not set here, must pass in from command line or script
+  # dataQuery: Do not set here, must pass in from command line or script
+
+includeConfigs:
+  - ${AP_PIPE_DIR}/bps/clustering/clustering_Daytime.yaml
+
+# Once consolidateVisitSummary has been run, we can also update the butler records for the region of each image
+# The initial region is just the predicted value from the nextVisit event,
+# so this will replace that with the updated region after calibration.
+finalJob:
+  command2: "${DAF_BUTLER_DIR}/bin/butler {finalPreCmdOpts} update-dimension-regions {butlerConfig} LSSTCam {outputRun} --batch-size 2000"

bps/clustering/clustering_Daytime.yaml

@@ -0,0 +1,51 @@
+# This is a prescription for quantum clustering with BPS, suitable for any
+# concrete pipeline based on the AP pipeline. Note that there are separate
+# example files for pipelines with and without fakes.
+#
+# Use it by adding:
+#
+#   includeConfigs:
+#     - ${AP_PIPE_DIR}/bps/clustering/clustering_ApPipe-daytime.yaml
+#
+# (with no outer indentation) to your BPS config file. If you are running
+# fakes, use instead:
+#
+#   includeConfigs:
+#     - ${AP_PIPE_DIR}/bps/clustering/clustering_ApPipeWithFakes-daytime.yaml
+#
+# again with no outer indentation.
+#
+
+clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering
+cluster:
+  preload:
+    pipetasks: loadDiaCatalogs,analyzeLoadDiaCatalogsMetrics,mpSkyEphemerisQuery,getRegionTimeFromVisit
+    dimensions: visit,detector
+    equalDimensions: visit:group
+  singleFrame:
+    pipetasks: isr,calibrateImage,analyzePreliminarySummaryStats
+    dimensions: visit,detector
+    equalDimensions: visit:exposure
+  diffim:
+    pipetasks: buildTemplate,subtractImages,detectAndMeasureDiaSource,analyzeImageDifferenceMetrics,analyzeDiaSourceDetectionMetrics,filterDiaSource,computeReliability,filterDiaSourcePostReliability,standardizeDiaSource
+    dimensions: visit,detector
+  association:
+    pipetasks: associateApdb,analyzeDiaSourceAssociationMetrics,analyzeAssociateDiaSourceTiming
+    dimensions: visit,detector
+  diaSrcDetectorAnalysis:
+    pipetasks: analyzeAssociatedDiaSourceTable,analyzeTrailedDiaSourceTable
+    dimensions: visit,detector
+  diffimMetrics:
+    pipetasks: makeSampledImageSubtractionMetrics,analyzeSampledImageSubtractionMetrics
+    dimensions: visit,detector
+  consolidateVisit:
+    pipetasks: consolidateVisitSummary,analyzeVisitSampledImageSubtractionMetrics
+    dimensions: visit
+
+ordering:
+  associationCluster:
+    ordering_type: sort
+    findDependencyMethod: sink
+    labels: association
+    dimensions: visit
+    blocking: False

pipelines/_ingredients/ApPipe.yaml

@@ -174,10 +174,9 @@ tasks:
     config:
       connections.coaddName: parameters.coaddName
       connections.science: preliminary_visit_image
-      connections.matchedTemplate: template_matched
       connections.template: template_detector
       connections.difference: difference_image
-      connections.diaSources: dia_source_unstandardized
+      connections.diaSources: dia_source_detector
       connections.psfMatchingKernel: difference_kernel
       connections.spatiallySampledMetrics: difference_image_metrics
 
@@ -299,9 +298,6 @@ contracts:
   - contract: subtractImages.connections.ConnectionsClass(config=subtractImages).science.name ==
               associateApdb.connections.ConnectionsClass(config=associateApdb).exposure.name
     msg: "subtractImages.science != associateApdb.exposure"
-  - contract: subtractImages.connections.ConnectionsClass(config=subtractImages).matchedTemplate.name ==
-              makeSampledImageSubtractionMetrics.connections.ConnectionsClass(config=makeSampledImageSubtractionMetrics).matchedTemplate.name
-    msg: "subtractImages.matchedTemplate != makeSampledImageSubtractionMetrics.matchedTemplate"
   - contract: detectAndMeasureDiaSource.connections.ConnectionsClass(config=detectAndMeasureDiaSource).diaSources.name ==
               filterDiaSource.connections.ConnectionsClass(config=filterDiaSource).diaSourceCat.name
     msg: "detectAndMeasureDiaSource.diaSources != filterDiaSource.diaSourceCat"
@@ -323,9 +319,9 @@ contracts:
   - contract: filterDiaSource.connections.ConnectionsClass(config=filterDiaSource).filteredDiaSourceCat.name ==
               filterDiaSourcePostReliability.connections.ConnectionsClass(config=filterDiaSourcePostReliability).diaSourceCat.name
     msg: "filterDiaSource.filteredDiaSourceCat != filterDiaSourcePostReliability.diaSourceCat"
-  - contract: filterDiaSource.connections.ConnectionsClass(config=filterDiaSource).filteredDiaSourceCat.name ==
+  - contract: standardizeDiaSource.connections.ConnectionsClass(config=standardizeDiaSource).diaSourceTable.name ==
               makeSampledImageSubtractionMetrics.connections.ConnectionsClass(config=makeSampledImageSubtractionMetrics).diaSources.name
-    msg: "filterDiaSource.filteredDiaSourceCat != makeSampledImageSubtractionMetrics.diaSources"
+    msg: "standardizeDiaSource.diaSourceTable != makeSampledImageSubtractionMetrics.diaSources"
   - contract: filterDiaSource.connections.ConnectionsClass(config=filterDiaSource).longTrailedSources.name ==
       analyzeTrailedDiaSourceTable.connections.ConnectionsClass(config=analyzeTrailedDiaSourceTable).data.name
     msg: "filterDiaSource.longTrailedSources != analyzeTrailedDiaSourceTable.data"

scripts/LSSTCam/submit_ap_daytime.sh

@@ -0,0 +1,75 @@
+#!/bin/sh
+set -eu
+
+# An executable script that will prepare and submit the daytime Alert Production pipeline
+
+if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 YYYY-MM-DD" >&2
+    exit 1
+fi
+
+DATE="$1"
+
+# POSIX-safe date normalization
+DAY_OBS=$(printf '%s\n' "$DATE" | tr -d '-')
+
+# Create a temp file with the date in the name
+TMP_APDB_REL=$(mktemp "apdb_config_${DATE}.XXXXXX.yaml")
+
+# Resolve to absolute path without readlink -f
+case "$TMP_APDB_REL" in
+    /*) TMP_APDB="$TMP_APDB_REL" ;;
+    *)  TMP_APDB="$(pwd)/$TMP_APDB_REL" ;;
+esac
+
+# Copy APDB config from S3 using Singularity AWS CLI
+singularity exec /sdf/sw/s3/aws-cli_latest.sif \
+  aws --endpoint-url https://sdfembs3.sdf.slac.stanford.edu s3 \
+  --profile embargo-s3 \
+  cp s3://rubin-summit-users/apdb_config/cassandra/pp_apdb_lsstcam.yaml \
+  "$TMP_APDB"
+
+# NOTE:
+# No cleanup of TMP_APDB here since the job is launched with nohup
+# and runtime duration is unknown.
+
+# Redirect Cassandra logs
+export DAX_APDB_MONITOR_CONFIG="logging:lsst.dax.apdb.monitor"
+
+# Configure the filesystem to allow many open files
+ulimit -n 65536
+
+INSTRUMENT="LSSTCam"
+
+# List of detectors currently excluded from Prompt Processing
+# These include the non-imaging wavefront sensors as well as some that are disabled in fan-out.
+# See https://github.com/lsst-sqre/phalanx/blob/main/applications/next-visit-fan-out/values-usdfprod-prompt-processing.yaml
+BAD_DETECTORS="120 122 0 20 27 65 123 161 168 188 1 19 30 68 158 169 187 \
+189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204"
+
+# Space-delimited list of observing blocks that generate science images
+BLOCKS="BLOCK-407 BLOCK-408 BLOCK-416 BLOCK-417 BLOCK-419 BLOCK-421 BLOCK-T637"
+
+OUTPUT_COLLECTION="LSSTCam/prompt/output-${DATE}/daytime"
+
+LOG_FILE="output-${DATE}.out"
+
+# Convert lists to SQL IN() form
+BAD_DETECTORS_SQL="($(printf '%s,' $BAD_DETECTORS | sed 's/,$//'))"
+BLOCKS_SQL="($(printf "'%s'," $BLOCKS | sed 's/,$//'))"
+
+nohup bps submit "${AP_PIPE_DIR}/bps/LSSTCam/bps_Daytime.yaml" \
+  --extra-qgraph-options "--skip-existing-in LSSTCam/prompt/output-${DATE} -c parameters:release_id=1 -c parameters:apdb_config=${TMP_APDB} --dataset-query-constraint template_coadd" \
+  --extra-run-quantum-options "--no-raise-on-partial-outputs" \
+  --input "LSSTCam/defaults,LSSTCam/templates,LSSTCam/prompt/output-${DATE}" \
+  --output "$OUTPUT_COLLECTION" \
+  -d "instrument='$INSTRUMENT' \
+      AND skymap='lsst_cells_v1' \
+      AND detector NOT IN $BAD_DETECTORS_SQL \
+      AND day_obs=$DAY_OBS \
+      AND exposure.science_program IN $BLOCKS_SQL" \
+  > "${LOG_FILE}" 2>&1 &
+
+echo "Submission started for date ${DATE}"
+echo "Temporary APDB config: ${TMP_APDB}"
+echo "Submission output log written to ${LOG_FILE}"