add script for running brickwise forced-photometry

bin/forced-motions.sh

@@ -0,0 +1,71 @@
+#! /bin/bash
+
+brick=$1
+
+#out_dir=$SCRATCH/forced-motions-3
+out_dir=$SCRATCH/forced-motions-4
+
+# On Perlmutter, RO-CFS is available on /dvs_ro/cfs both inside and outside shifter containers
+# (and on the login nodes too)
+export COSMO=/dvs_ro/cfs/cdirs/cosmo
+
+dr10=$COSMO/work/legacysurvey/dr10
+cat_dir=$dr10
+survey_dir=$dr10
+
+export SKY_TEMPLATE_DIR=$COSMO/work/legacysurvey/dr10/calib/sky_pattern
+export LARGEGALAXIES_CAT=$COSMO/staging/largegalaxies/v3.0/SGA-ellipse-v3.0.kd.fits
+#export LARGEGALAXIES_CAT=/global/cfs/cdirs/cosmo/staging/largegalaxies/v3.2/SGA-ellipse-v3.2.kd.fits
+export GAIA_CAT_DIR=$COSMO/data/gaia/edr3/healpix
+export GAIA_CAT_PREFIX=healpix
+export GAIA_CAT_SCHEME=nested
+export GAIA_CAT_VER=E
+
+# Don't add ~/.local/ to Python's sys.path
+export PYTHONNOUSERSITE=1
+
+export OMP_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export KMP_AFFINITY=disabled
+export MPICH_GNI_FORK_MODE=FULLCOPY
+
+mkdir -p ${out_dir}/logs-forced
+echo Logging to ${out_dir}/logs-forced/${brick}.log
+
+# # Config directory nonsense
+export TMPCACHE=$(mktemp -d)
+mkdir $TMPCACHE/cache
+mkdir $TMPCACHE/config
+# astropy
+export XDG_CACHE_HOME=$TMPCACHE/cache
+export XDG_CONFIG_HOME=$TMPCACHE/config
+mkdir $XDG_CACHE_HOME/astropy
+cp -r $HOME/.astropy/cache $XDG_CACHE_HOME/astropy
+mkdir $XDG_CONFIG_HOME/astropy
+cp -r $HOME/.astropy/config $XDG_CONFIG_HOME/astropy
+# matplotlib
+export MPLCONFIGDIR=$TMPCACHE/matplotlib
+mkdir $MPLCONFIGDIR
+cp -r $HOME/.config/matplotlib $MPLCONFIGDIR
+
+#python -O $LEGACYPIPE_DIR/legacypipe/forced_photom_brickwise.py \
+
+export PYTHONPATH=.:${PYTHONPATH}
+python -c "import legacypipe.forced_photom as x; print('forced_photom.py:', x.__file__)"
+python -O legacypipe/forced_photom_brickwise.py \
+       --brick $brick \
+       --survey-dir ${survey_dir} \
+       --catalog-dir ${cat_dir} \
+       --outdir ${out_dir} \
+       --bands g,r,i,z \
+       --derivs \
+       --threads 32 \
+       >> ${out_dir}/logs-forced/${brick}.log 2>&1
+#--do-calib \
+
+# Save the return value from the python command -- otherwise we
+# exit 0 because the rm succeeds!
+status=$?
+# /Config directory nonsense
+rm -R $TMPCACHE
+exit $status

bin/runbrick-perlmutter.sh

@@ -74,10 +74,11 @@ echo -e "\nStarting on $(hostname)\n" >> "$log"
 echo "-----------------------------------------------------------------------------------------" >> "$log"
 
 python -O $LEGACYPIPE_DIR/legacypipe/runbrick.py \
+       --zoom 100 300 100 300 \
      --brick "$brick" \
      --skip \
      --skip-calibs \
-     --bands g,r,z \
+     --bands g,r,i,z \
      --rgb-stretch 1.5 \
      --nsatur 2 \
      --survey-dir "$LEGACY_SURVEY_DIR" \
@@ -86,11 +87,11 @@ python -O $LEGACYPIPE_DIR/legacypipe/runbrick.py \
      --checkpoint-period 120 \
      --pickle "${outdir}/pickles/${bri}/runbrick-%(brick)s-%%(stage)s.pickle" \
      --write-stage srcs \
-     --stage image_coadds \
      --release 10011 \
      --threads 32 \
       >> "$log" 2>&1
 
+#     --stage image_coadds \
 #     --bands g,r,i,z \
 #    --cache-dir "$CACHE_DIR" \