From ff59929c86e3a90eabd7abad906d3ad5bd6f60ab Mon Sep 17 00:00:00 2001 From: JaysonAstro Date: Wed, 26 Nov 2025 22:04:45 +0000 Subject: [PATCH 1/9] Update miri to long alignment using internal catalogs --- config/alignment_testing/astronode_dirs.toml | 7 + .../config_half1_miri_to_long.toml | 579 ++++++++++++++++++ config/alignment_testing/run_reprocessing.py | 28 + .../astrometric_align_step.py | 67 +- 4 files changed, 673 insertions(+), 8 deletions(-) create mode 100644 config/alignment_testing/astronode_dirs.toml create mode 100644 config/alignment_testing/config_half1_miri_to_long.toml create mode 100755 config/alignment_testing/run_reprocessing.py diff --git a/config/alignment_testing/astronode_dirs.toml b/config/alignment_testing/astronode_dirs.toml new file mode 100644 index 0000000..b45f73b --- /dev/null +++ b/config/alignment_testing/astronode_dirs.toml @@ -0,0 +1,7 @@ +crds_path = '/nexus/posix0/MIA-astro-env/eschinner/jgonzalez/jwst_pipeline/crds_cache/jwst_ops' +raw_dir = '/nexus/posix0/MIA-astro-env/eschinner/jgonzalez/phangs/jwst_raw/3707' +reprocess_dir = '/nexus/posix0/MIA-astro-env/eschinner/jgonzalez/phangs/jwst_phangs_reprocessed' +alignment_dir = '/nexus/posix0/MIA-astro-env/eschinner/jgonzalez/pjpipe-dev/pjpipe/config/alignment_testing/alignment' +kernel_dir = '/nexus/posix0/MIA-astro-env/eschinner/jgonzalez/jwst_pipeline/kernels' +webb_psf_data = '/nexus/posix0/MIA-astro-env/eschinner/jgonzalez/jwst_pipeline/webbpsf-data' +processors = 20 diff --git a/config/alignment_testing/config_half1_miri_to_long.toml b/config/alignment_testing/config_half1_miri_to_long.toml new file mode 100644 index 0000000..7e08b78 --- /dev/null +++ b/config/alignment_testing/config_half1_miri_to_long.toml @@ -0,0 +1,579 @@ +# Original file from Ryan: latest cycle 2 version + +# List of targets + +# Version for the reprocessing +#version = 'v2p6_test_cy1_params_cy2_smallbatch' # original name +version = 'v2p6_cy1_params_cy2_test_alignment' +# ngc5643 +# ngc4298 +# ngc5248 +# ngc4548 +# ngc2997 +# ngc2903 +# ngc1808 + +# targets = ['ngc3239'] +# targets = ['ngc1546'] + +targets = [ + # 'ngc5530', + # 'ngc1808', + # 'ngc1068' + # 'ngc5530', # -- was WOPR in v0p3, but obs done in June 2025 + # 'ngc5248', # Eva's, but not yet done # NEW AGB -- was WOPR in v0p3, but obs done in June 2025 + # 'ngc1317', # AGB + # # 'ngc5643', # Eva's, done (2025-06-18) NEW AGB + # 'ngc4951', # NEW AGB + # 'ngc4654', # NEW AGB + # 'ngc4689', # NEW AGB + # 'ngc4569', # NEW AGB + # # # 'ngc4826', # Jiayi's program # NEW AGB + # 'ngc4298', # Eva's, but not yet done # NEW AGB + # 'ngc4536', # NEW AGB + # 'ngc4424', # NEW AGB # jay: done # jay: jessica says has very few point sources, try this one + # 'ngc3621', # NEW AGB + # 'ngc4540', + # 'ngc4781', + # # # 'ngc6300', # Eva's (done) # NEW AGB + # 'ngc5248', # Eva's, but not yet done # NEW AGB -- will need to re-download data because it's a WOPR (still as of 2 Dec 24) + # 'ngc4496a', + # 'ngc4548', # Eva's, but not yet done # NEW AGB + # 'ngc4571', # NEW AGB + # 'ngc4579', + # 'ngc4731', # NEW AGB + # 'ngc4941', + # 'ngc4457', + # 'ngc5042', + # 'ngc5134', + # 'ngc3596', # end half 1 + # 'ngc3626', # start half 2. # jay: alignment didn't go through, few gaia sources matched + # 'ngc3344', + # 'ngc3239', + #'ngc3511', # jay: done + # 'ngc3059', # NEW AGB + # 'ngc3137', + # 'ngc2997', # Eva's, but not yet done # NEW AGB + # 'ngc2903', # Eva's, but not yet done # NEW AGB + # 'ngc3521', + # 'ngc3507', + # 'ngc2775', # NEW AGB + # # # 'ngc1792', # agb. Eva's (done) + # 'ngc1511', + # 'ngc2283', + # # # 'ngc1637', # Eva's (done) # NEW AGB + # 'ngc1808', # Eva's, not yet done + # # # #'ngc1068', # run separately + # 'ic5273', + # 'ngc7456', + # 'ngc1809', + #'ngc1546', # jay: done + # 'ngc1559', # agb # jay: done + # 'ngc2090', + # # # 'ngc1097', # agb. Eva's (done) + # 'ic1954', # NEW AGB + # 'ngc0685', # has agb + # 'ngc3368', + 'ngc4694', # jay: jessica says has very few point sources, try this one + # 'ngc2566', # jay: done +] +# Eva's targets: +# Done (25 Oct): 'ngc1512', 'ngc1672', 'ngc1637', 'ngc1792', 'ngc1097', 'ngc6300' +# Not done: 'ngc5643', 'ngc1808', 'ngc5248', 'ngc4548', 'ngc4298', 'ngc3627', 'ngc3351', 'ngc2997', 'ngc2903', 'ngc1433' + +# 'ngc5643', # NEW AGB +# 'ngc1808', +# 'ngc5248', # NEW AGB +# 'ngc4548', # NEW AGB +# 'ngc4298', # NEW AGB +# 'ngc3627', +# 'ngc3351', +# 'ngc2997', # NEW AGB +# 'ngc2903', # NEW AGB +# 'ngc1433' +# ngc5643 # NEW AGB +# ngc1808 # NO AGB + + +# Bands to consider +bands = [ + 'F150W', + 'F187N', + 'F200W', + 'F300M', + 'F335M', + 'F770W', + 'F2100W', + 'F770W_bgr', + 'F2100W_bgr', +] + +# Steps. These can/should be different +# for NIRCam and MIRI, and we can distinguish them here + +steps = [ + 'download', + 'gaia_query', + 'lv1', + 'single_tile_destripe.nircam', + 'lv2', + 'get_wcs_adjust', + 'apply_wcs_adjust', + 'lyot_mask.miri', + # 'level_match', + 'multi_tile_destripe.nircam', + # 'psf_model', # TODO + 'lv3', + 'astrometric_catalog.miri', + #'astrometric_catalog', # test jay 19.11.2025 + 'astrometric_align', + # 'anchoring', + # 'psf_matching', + # 'release', + # 'regress_against_previous', +] + +# Parameters for downloading data. This just downloads +# the raw science +[parameters.download] + +prop_id = '3707' +product_type = ['SCIENCE'] +calib_level = [1] + +[parameters.gaia_query] + +radius = 20 + +[parameters.move_raw_obs] + +[parameters.move_raw_obs.obs_to_skip] + +# ngc3596 = 'jw02107007003_02105_00003*' +ngc1317 = ['jw03707005*', 'jw03707006*'] +ngc3239 = ['jw03707037*', 'jw03707038*'] +ngc3344 = ['jw03707123*', 'jw03707124*'] +ngc3596 = ['jw03707047*', 'jw03707048*'] +ngc4694 = ['jw03707093*', 'jw03707094*'] +ngc5248 = 'jw03707109003*' # NOTE: PARTIAL REPEAT (ONLY 109:3 FAILED). CAREFUL REDUCING THIS ONE. +ngc5530 = 'jw03707111001*' # NOTE: PARTIAL REPEAT (ONLY 111:1 FAILED). CAREFUL REDUCING THIS ONE. + +[parameters.lv1] + +jwst_parameters.save_results = true + +# Flag snowballs in NIRCam +#jwst_parameters.jump.expand_large_events.F187N = true +jwst_parameters.jump.expand_large_events.nircam = true + +jwst_parameters.rscd.skip = true +jwst_parameters.clean_flicker_noise.skip = true + +jwst_parameters.ramp_fit.suppress_one_group = false +jwst_parameters.refpix.use_side_ref_pixels = true + +[parameters.lv2] + +bgr_check_type = 'parallel_off' +bgr_background_name = 'off' +bgr_observation_types = ['miri'] + +jwst_parameters.save_results = true +jwst_parameters.bkg_subtract.save_combined_background = true +jwst_parameters.bkg_subtract.sigma = 1.5 + +[parameters.get_wcs_adjust] + +method.nircam = "tweakreg" +method.miri = "tweakreg" + +bands = ['F300M', 'F770W'] + +group_dithers = ['nircam', 'miri'] + +[parameters.get_wcs_adjust.tweakreg_parameters] + +# align_to_gaia = false # no longer a tweakreg parameter +starfinder = 'iraf' +brightest = 500 +snr_threshold = 3 +expand_refcat = true +fitgeometry = 'shift' +minobj = 3 +peakmax.nircam = 20 +roundlo.nircam = -0.5 +roundhi.nircam = 0.5 + +# Parameters to get wcs_adjust shifts +searchrad = 2 +separation.miri = 1 +tolerance.miri = 0.7 +separation.nircam = 2 +tolerance.nircam = 1 +use2dhist = true +nclip = 5 + +# Tweak boxsize, so we detect objects in diffuse emission +bkg_boxsize.nircam_short = 100 +bkg_boxsize.nircam_long = 100 +bkg_boxsize.miri = 25 +enforce_user_order = true + +[parameters.lyot_mask] +method = 'mask' + +[parameters.single_tile_destripe] +quadrants = true +vertical_subtraction = true +filter_diffuse = true +dilate_size = 7 +destriping_method = 'median_filter' +vertical_destriping_method = 'row_median' +filter_extend_mode = "reflect" +filter_scales = [3, 7, 15, 31, 63, 127, 255, 511] + +[parameters.multi_tile_destripe] +quadrants = true +sigma = 3 +weight_type = 'ivm' +do_large_scale = true +large_scale_filter_extend_mode = "reflect" + +[parameters.level_match] +do_sigma_clip = false +weight_method = 'rms' + +# fit_type_dithers = "level" +# fit_type_recombine_lyot = "level+slope" +# fit_type_mosaic_tiles.nircam = "level" +# fit_type_mosaic_tiles.miri = "level+slope" +# recombine_lyot = true +# combine_nircam_short = true + +[parameters.lv3] + +tweakreg_group_dithers = ['nircam_short', 'nircam_long', 'miri'] +skymatch_degroup_dithers = ['miri', 'nircam_short', 'nircam_long'] + +bgr_check_type = 'parallel_off' + +[parameters.lv3.jwst_parameters] +save_results = true + +[parameters.lv3.jwst_parameters.tweakreg] + +# Skip MIRI wavelengths, since we have already +# solved for this +skip.F770W = true +skip.F2100W = true + +# align_to_gaia = false # no longer a tweakreg parameter +starfinder = 'iraf' +brightest.nircam_short = 125 +brightest.nircam_long = 500 +brightest.miri = 500 +snr_threshold = 3 +expand_refcat = true +fitgeometry = 'shift' +minobj = 3 +peakmax.nircam = 20 +roundlo.nircam = -0.3#-0.5 # jay: 26.11 +roundhi.nircam = 0.3#0.5 # jay: 26.11 + +# Take relatively tight tolerances since we've already shifted +# close to the correct solution +separation.miri = 1 +separation.nircam = 2 +tolerance.nircam_short = 0.3 +tolerance.nircam_long = 0.3 +tolerance.miri = 1 +use2dhist = false + +# Tweak boxsize, so we detect objects in diffuse emission +bkg_boxsize.nircam_short = 100 +bkg_boxsize.nircam_long = 100 +bkg_boxsize.miri = 25 +enforce_user_order = true + +[parameters.lv3.jwst_parameters.skymatch] + +# Turn off matching since we've done it already +skymethod = 'local' +subtract = true +skystat = 'median' +nclip.nircam = 20 +nclip.miri = 10 +lsigma.nircam = 3 +lsigma.miri = 1.5 +usigma.nircam = 3 +usigma.miri = 1.5 + +[parameters.lv3.jwst_parameters.outlier_detection] +in_memory = true + +[parameters.lv3.jwst_parameters.resample] +rotation = 0.0 +in_memory = true + +[parameters.lv3.jwst_parameters.source_catalog] +snr_threshold = 2 +npixels = 5 +bkg_boxsize = 25 +deblend = true + +[parameters.astrometric_catalog] +snr = 10 +starfind_method = "iraf" + +# [parameters.astrometric_catalog.dao_parameters] +[parameters.astrometric_catalog.starfind_parameters] + +sharplo = 0.2 +sharphi = 1.0 +roundlo = -0.5 +roundhi = 0.5 + +[parameters.astrometric_align] + +overwrite = true + +[parameters.astrometric_align.align_mapping] +# Map F2100W to F770W +F2100W = 'F770W' +F2100W_bgr = 'F770W_bgr' + +# F150W = 'F300M' +# F187N = 'F335M' + +[parameters.astrometric_align.ref_long_filter] # Jay 19.11.2025 +F770W = 'F335M' # set to F1000W to F300M for cycle 1 + +[parameters.astrometric_align.catalogs] +# Ones w/ AGB catalogs +# NEW +ic5273 = 'ic5273_agb_cat.fits' +ngc1511 = 'ngc1511_agb_cat.fits' + +ngc1546.nircam = 'ngc1546_agb_cat.fits' +ngc1546.miri = 'ngc1546_nircam_lv3_f335m_cat.fits' + +ngc1559.nircam = 'ngc1559_agb_cat.fits' +ngc1559.miri = 'ngc1559_nircam_lv3_f335m_cat.fits' # test jay 19.11.2025 + +# ngc1637 = 'ngc1637_agb_cat.fits' +# ngc1792 = 'ngc1792_agb_cat.fits' +ngc1808 = 'ngc1808_agb_cat.fits' +ngc1809 = 'ngc1809_agb_cat.fits' +ngc2090 = 'ngc2090_agb_cat.fits' +# ngc2566 = 'ngc2566_agb_cat.fits' +# ngc2775 = 'ngc2775_agb_cat.fits' +# ngc2903 = 'ngc2903_agb_cat.fits' +ngc2997 = ['ngc2997e_agb_cat.fits', 'ngc2997w_agb_cat.fits'] +# ngc2997w = 'ngc2997w_agb_cat.fits' +# ngc3059 = 'ngc3059_agb_cat.fits' +ngc3507 = 'ngc3507_agb_cat.fits' + +ngc3511.nircam = 'ngc3511_agb_cat.fits' +ngc3511.miri = "ngc3511_nircam_lv3_f335m_cat.fits" + +ngc3521s = 'ngc3521s_agb_cat.fits' +ngc3596 = 'ngc3596_agb_cat.fits' +# ngc3621 = 'ngc3621_agb_cat.fits' +# ngc4298 = 'ngc4298_agb_cat.fits' +# ngc4424 = 'ngc4424_agb_cat.fits' +ngc4496a = 'ngc4496a_agb_cat.fits' +# ngc4536 = 'ngc4536_agb_cat.fits' +# ngc4548 = 'ngc4548_agb_cat.fits' +# ngc4569 = 'ngc4569_agb_cat.fits' +# ngc4571 = 'ngc4571_agb_cat.fits' +# ngc4654 = 'ngc4654_agb_cat.fits' +# ngc4689 = 'ngc4689_agb_cat.fits' +# ngc4694 = 'ngc4694_agb_cat.fits' # jay: jessica says has very few point sources, try this one +# ngc4731 = 'ngc4731_agb_cat.fits' +# ngc4826 = 'ngc4826_agb_cat.fits' +ngc4941 = 'ngc4941_agb_cat.fits' +# ngc4951 = 'ngc4951_agb_cat.fits' +ngc5042 = 'ngc5042_agb_cat.fits' +# ngc5248 = 'ngc5248_agb_cat.fits' +ngc5530 = 'ngc5530_agb_cat.fits' +# ngc5643 = 'ngc5643_agb_cat.fits' +# ngc6300 = 'ngc6300_agb_cat.fits' +ngc7456 = 'ngc7456_agb_cat.fits' + +# OLD + +ic1954 = 'ic1954_agb_cat.fits' +ngc0685 = 'ngc0685_agb_cat.fits' +ngc1097 = 'ngc1097_agb_cat.fits' +ngc1317 = 'ngc1317_agb_cat.fits' +#ngc1559 = 'ngc1559_agb_cat.fits' +ngc1792 = 'ngc1792_agb_cat.fits' +ngc2775 = 'ngc2775_agb_cat.fits' +ngc2903 = 'ngc2903_agb_cat.fits' +ngc3621 = 'ngc3621_agb_cat.fits' +ngc4298 = 'ngc4298_agb_cat.fits' +ngc4536 = 'ngc4536_agb_cat.fits' +ngc4548 = 'ngc4548_agb_cat.fits' +ngc4569 = 'ngc4569_agb_cat.fits' +ngc4571 = 'ngc4571_agb_cat.fits' +ngc4654 = 'ngc4654_agb_cat.fits' +ngc4689 = 'ngc4689_agb_cat.fits' +ngc4826 = 'ngc4826_agb_cat.fits' +ngc5248 = 'ngc5248_agb_cat.fits' +ngc5643 = 'ngc5643_agb_cat.fits' +ngc4951 = 'ngc4951_agb_cat.fits' + +ngc4424.nircam = 'ngc4424_agb_cat.fits' +ngc4424.miri = 'ngc4424_nircam_lv3_f335m_cat.fits' + +ngc6300 = 'ngc6300_agb_cat.fits' +ngc4731 = 'ngc4731_agb_cat.fits' +ngc3059 = 'ngc3059_agb_cat.fits' +# ngc2997 = 'ngc2997e_agb_cat.fits' # NOTE: the "e" means we have PSF photometry for the East pointing only (from Hwi Kim) +# ngc2903 = 'ngc2903_agb_cat.fits' +# ngc2775 = 'ngc2775_agb_cat.fits' +ngc1637 = 'ngc1637_agb_cat.fits' + +ngc4694.nircam = 'ngc4694_agb_cat.fits' +ngc4694.miri = 'ngc4694_nircam_lv3_f335m_cat.fits' + +ngc1068 = 'ngc1068_agb_cat.fits' + +ngc2566.nircam = 'ngc2566_agb_cat.fits' +ngc2566.miri = 'ngc2566_nircam_lv3_f335m_cat.fits' + +# Ones w/o AGB catalogs +# ngc5530 = 'Gaia_DR3_ngc5530.fits' +ngc3368 = 'Gaia_DR3_ngc3368.fits' +# ngc6300 = 'Gaia_DR3_ngc6300.fits' +# ngc5643 = 'Gaia_DR3_ngc5643.fits' +# ngc4951 = 'Gaia_DR3_ngc4951.fits' +# ngc4654 = 'Gaia_DR3_ngc4654.fits' +# ngc4689 = 'Gaia_DR3_ngc4689.fits' +# ngc4569 = 'Gaia_DR3_ngc4569.fits' +# ngc4826 = 'Gaia_DR3_ngc4826.fits' +# ngc4298 = 'Gaia_DR3_ngc4298.fits' +ngc4540 = 'Gaia_DR3_ngc4540.fits' +# ngc4496a = 'Gaia_DR3_ngc4496a.fits' +# ngc4548 = 'Gaia_DR3_ngc4548.fits' +# ngc4571 = 'Gaia_DR3_ngc4571.fits' +ngc4579 = 'Gaia_DR3_ngc4579.fits' +# ngc4731 = 'Gaia_DR3_ngc4731.fits' +# ngc4941 = 'Gaia_DR3_ngc4941.fits' +ngc4457 = 'Gaia_DR3_ngc4457.fits' +# ngc4694 = 'Gaia_DR3_ngc4694.fits' +ngc4781 = 'Gaia_DR3_ngc4781.fits' +# ngc5248 = 'Gaia_DR3_ngc5248.fits' +# ngc4536 = 'Gaia_DR3_ngc4536.fits' +# ngc4424 = 'Gaia_DR3_ngc4424.fits' +# ngc3621 = 'Gaia_DR3_ngc3621.fits' + +# ngc5042 = 'Gaia_DR3_ngc5042.fits' +ngc5134 = 'Gaia_DR3_ngc5134.fits' +# ngc3596 = 'Gaia_DR3_ngc3596.fits' + +ngc3626.nircam = 'Gaia_DR3_ngc3626.fits' +ngc3626.miri = 'ngc3626_nircam_lv3_f335m_cat.fits' # test jay 19.11.2025 +#ngc3626 = 'Gaia_DR3_ngc3626.fits' + +ngc3344 = 'Gaia_DR3_ngc3344.fits' +# ngc3511 = 'Gaia_DR3_ngc3511.fits' +# ngc3059 = 'Gaia_DR3_ngc3059.fits' +ngc3137 = 'Gaia_DR3_ngc3137.fits' +# ngc2997 = 'Gaia_DR3_ngc2997.fits' +ngc3521 = 'Gaia_DR3_ngc3521.fits' +# ngc2903 = 'Gaia_DR3_ngc2903.fits' +# ngc3507 = 'Gaia_DR3_ngc3507.fits' +ngc3239 = 'Gaia_DR3_ngc3239.fits' +# ngc2775 = 'Gaia_DR3_ngc2775.fits' +# ngc2566 = 'Gaia_DR3_ngc2566.fits' +# ic5273 = 'Gaia_DR3_ic5273.fits' +# ngc7456 = 'Gaia_DR3_ngc7456.fits' +# ngc1809 = 'Gaia_DR3_ngc1809.fits' +# ngc1546 = 'Gaia_DR3_ngc1546.fits' +# ngc1511 = 'Gaia_DR3_ngc1511.fits' +# ngc2090 = 'Gaia_DR3_ngc2090.fits' +# ngc1068 = 'Gaia_DR3_ngc1068.fits' +# ngc1808 = 'Gaia_DR3_ngc1808.fits' +# ngc1637 = 'Gaia_DR3_ngc1637.fits' +ngc2283 = 'Gaia_DR3_ngc2283.fits' + +# Cycle 1 targets +ic5332 = 'ic5332_agb_cat.fits' +ngc0628 = 'ngc0628_agb_cat.fits' +ngc1087 = 'ngc1087_agb_cat.fits' +ngc1300 = 'ngc1300_agb_cat.fits' +ngc1365 = 'ngc1365_agb_cat.fits' +ngc1385 = 'ngc1385_agb_cat.fits' +ngc1433 = 'ngc1433_agb_cat.fits' +ngc1512 = 'ngc1512_agb_cat.fits' +ngc1566 = 'ngc1566_agb_cat.fits' +ngc1672 = 'ngc1672_agb_cat.fits' +ngc2835 = 'ngc2835_agb_cat.fits' +ngc3351 = 'ngc3351_agb_cat.fits' +ngc3627 = 'ngc3627_agb_cat.fits' +ngc4254 = 'ngc4254_agb_cat.fits' +ngc4303 = 'ngc4303_agb_cat.fits' +ngc4321 = 'ngc4321_agb_cat.fits' +ngc4535 = 'ngc4535_agb_cat.fits' +ngc5068 = 'ngc5068_agb_cat.fits' +ngc7496 = 'ngc7496_agb_cat.fits' + + +# Initial pass to get decent shifts for absolute astrometry +[parameters.astrometric_align.tweakreg_parameters.iteration1] + +# Set quite a large search radius +searchrad.miri = 10 # trying larger radius (was 10 arcsec before) -- 7 May 24 +searchrad.nircam_long = 20 +searchrad.nircam_short = 40 +separation = 1 +tolerance = 2 # 14 Jul 25 -- default was 1. Tom's initial config had 2 for NGC 3627. +# tolerance = 1 +use2dhist = true # Changed to true on 13 Feb 24 -- to remove catastrophic fails +fitgeom = 'shift' +nclip = 5 +sigma = 3 + +# Second iteration with tightened up parameters to figure out any residual shifts left +[parameters.astrometric_align.tweakreg_parameters.iteration2] +searchrad = 2 +separation = 1 +tolerance = 0.5 +use2dhist = false +fitgeom = 'rshift' +nclip = 5 +sigma = 3 + + +[parameters.anchoring] +overwrite = true # true +internal_conv_band = 'F2100W' + +[parameters.anchoring.ref_band] +miri = 'F770W' +nircam = 'F300M' + +# List external bands in preference order +[parameters.anchoring.external_bands] +# miri = ['irac4_atgauss4', 'w3_atgauss15'] +# nircam = ['irac1_atgauss4', 'w1_atgauss7p5', 'irac2_atgauss4', 'w2_atgauss7p5'] +miri = ['w3_atgauss15', 'irac4_atgauss4'] +nircam = ['w1_atgauss7p5', 'w2_atgauss7p5', 'irac1_atgauss4', 'irac2_atgauss4'] + +[parameters.psf_matching] +skip_if_missing_kernels = true +target_bands = ['F770W', "gauss0p35", "gauss0p85"] +reproject_func = "interp" +overwrite = true + + +[parameters.release] +overwrite = true +move_tweakback = true +move_backgrounds = true +move_psf_matched = true +move_diagnostic_plots = true + +[parameters.regress_against_previous] +prev_version = 'v0p3' diff --git a/config/alignment_testing/run_reprocessing.py b/config/alignment_testing/run_reprocessing.py new file mode 100755 index 0000000..47aa7e2 --- /dev/null +++ b/config/alignment_testing/run_reprocessing.py @@ -0,0 +1,28 @@ +import os +os.environ["NUMEXPR_MAX_THREADS"] = "20" + +import pjpipe + +this_dir = os.path.dirname(os.path.realpath(__file__)) + +config_file = os.path.join(this_dir, "config_half1_miri_to_long.toml") +local_file = os.path.join(this_dir, "astronode_dirs.toml") + +# We need to set CRDS path +local = pjpipe.load_toml(local_file) +crds_path = local["crds_path"] +os.environ["CRDS_PATH"] = crds_path +if not os.path.exists(crds_path): + os.makedirs(crds_path) + +# If this is our first time running things, we need to pull some +# references to avoid fatal errors +crds_files = os.listdir(crds_path) +if "config" not in crds_files: + os.system("crds sync --jwst") + +pjp = pjpipe.PJPipeline( + config_file=config_file, + local_file=local_file, +) +pjp.do_pipeline() diff --git a/pjpipe/astrometric_align/astrometric_align_step.py b/pjpipe/astrometric_align/astrometric_align_step.py index 5eabeff..19855bd 100644 --- a/pjpipe/astrometric_align/astrometric_align_step.py +++ b/pjpipe/astrometric_align/astrometric_align_step.py @@ -18,6 +18,7 @@ import numpy as np from astropy.table import QTable, Table from astropy.wcs import WCS +from astropy.coordinates import SkyCoord from image_registration import cross_correlation_shifts from jwst.assign_wcs.util import update_fits_wcsinfo from reproject import reproject_interp, reproject_adaptive, reproject_exact @@ -200,6 +201,7 @@ def __init__( catalogs=None, align_mapping_mode="shift", align_mapping=None, + ref_long_filter=None, # jay 19.11.2025 tweakreg_parameters=None, reproject_func="interp", overwrite=False, @@ -230,6 +232,8 @@ def __init__( tweakreg solution from the existing file), or "cross-corr" (do some cross-correlation between the images) + ref_long_filter: Specify reference NIRCam band + if aligning MIRI to NIRCam. tweakreg_parameters: Dictionary of parameters to pass to tweakreg for the standard alignment reproject_func: Which reproject function to use. Defaults to 'interp', @@ -245,6 +249,8 @@ def __init__( catalogs = {} if align_mapping is None: align_mapping = {} + if ref_long_filter is None: + ref_long_filter = {} if tweakreg_parameters is None: tweakreg_parameters = {} @@ -260,6 +266,7 @@ def __init__( self.catalogs = catalogs self.align_mapping_mode = align_mapping_mode self.align_mapping = align_mapping + self.ref_long_filter = ref_long_filter self.tweakreg_parameters = tweakreg_parameters self.reproject_func = reproject_func self.overwrite = overwrite @@ -508,22 +515,66 @@ def tweakreg_align( log.warning("astrometric_alignment_table should be set!") return True - log.info("Aligning to external catalog") + current_band_type = get_band_type(self.band) - align_catalog = os.path.join( - self.catalog_dir, - self.catalogs[self.target], - ) + # test Jay 19.11.2025 -> use agb for nircam, pjpipe cat for miri + # TO DO: instead of counting the length of self.catalogs[self.target] use "if ref_long_filter is not None:" + + if len(self.catalogs[self.target]) == 2: # assumes that there has to be one for miri and one for nircam + if self.band in self.ref_long_filter: + log.info(f"Aligning to internal catalog from shorter wavelength ({self.ref_long_filter[self.band]})") + align_cat_dir = self.in_dir.replace(f"{self.band}", f"{self.ref_long_filter[self.band]}") + #align_cat_dir = align_cat_dir.replace(f"{current_band_type}", "nircam") + align_catalog = os.path.join( # when using internal catalog + align_cat_dir, + self.catalogs[self.target][current_band_type] + ) + elif self.band not in self.ref_long_filter: # only using external catalog + log.info("Aligning to external catalog") + align_catalog = os.path.join( + self.catalog_dir, + self.catalogs[self.target][current_band_type] + ) + else: # when only external catalog is used + log.info(f"Aligning to external catalog") + align_catalog = os.path.join( + self.catalog_dir, + self.catalogs[self.target] + ) if not os.path.exists(align_catalog): log.warning("Requested astrometric alignment table not found!") return True - + else: + log.info("Astrometric align table found") + align_table = QTable.read(align_catalog, format="fits") ref_tab = Table() - ref_tab["RA"] = align_table["ra"] - ref_tab["DEC"] = align_table["dec"] + if len(self.catalogs[self.target]) == 1: # when only using external AGB catalog + ref_tab["RA"] = align_table["ra"] + ref_tab["DEC"] = align_table["dec"] + + elif len(self.catalogs[self.target]) > 1: # when using external + internal catalog + if self.band in self.ref_long_filter: # miri -> ref long + # get wcs from ref_long aligned image + ref_long_dir = self.in_dir.replace(f"{self.band}", f"{self.ref_long_filter[self.band]}") + ref_long_img = os.path.join( + ref_long_dir, + f"{self.target}_nircam_lv3_{self.ref_long_filter[self.band].lower()}_i2d_align.fits" + ) + with fits.open(ref_long_img) as hdul: + ref_long_wcs = WCS(hdul[1].header) + + # use wcs on reference catalog x, y to get updated ra, dec + ref_x, ref_y = np.array(align_table["xcentroid"].data), np.array(align_table["ycentroid"].data) + align_coord = ref_long_wcs.pixel_to_world(ref_x, ref_y) + ref_tab["RA"] = align_coord.ra.deg + ref_tab["DEC"] = align_coord.dec.deg + + if self.band not in self.ref_long_filter: # nircam -> agb + ref_tab["RA"] = align_table["ra"] + ref_tab["DEC"] = align_table["dec"] if "xcentroid" in align_table.colnames: ref_tab["xcentroid"] = align_table["xcentroid"] From 2795dcba1a7df3b8ef266ed927d35bfdbd268e3e Mon Sep 17 00:00:00 2001 From: JaysonAstro Date: Thu, 27 Nov 2025 11:00:22 +0000 Subject: [PATCH 2/9] Update: Only need to specify ref_long_filter and not internal catalog name --- .../config_half1_miri_to_long.toml | 46 ++++++------------- .../astrometric_align_step.py | 41 ++++++++--------- 2 files changed, 33 insertions(+), 54 deletions(-) diff --git a/config/alignment_testing/config_half1_miri_to_long.toml b/config/alignment_testing/config_half1_miri_to_long.toml index 7e08b78..cab969f 100644 --- a/config/alignment_testing/config_half1_miri_to_long.toml +++ b/config/alignment_testing/config_half1_miri_to_long.toml @@ -4,7 +4,7 @@ # Version for the reprocessing #version = 'v2p6_test_cy1_params_cy2_smallbatch' # original name -version = 'v2p6_cy1_params_cy2_test_alignment' +version = 'v2p6_cy1_params_cy2_test_alignment_ngc4694_stable-version' # ngc5643 # ngc4298 # ngc5248 @@ -31,7 +31,7 @@ targets = [ # # # 'ngc4826', # Jiayi's program # NEW AGB # 'ngc4298', # Eva's, but not yet done # NEW AGB # 'ngc4536', # NEW AGB - # 'ngc4424', # NEW AGB # jay: done # jay: jessica says has very few point sources, try this one + # 'ngc4424', # NEW AGB # jay: done # 'ngc3621', # NEW AGB # 'ngc4540', # 'ngc4781', @@ -74,7 +74,7 @@ targets = [ # 'ic1954', # NEW AGB # 'ngc0685', # has agb # 'ngc3368', - 'ngc4694', # jay: jessica says has very few point sources, try this one + 'ngc4694', # jay: jessica says has very few point sources, try this one. indeed agb cat alignment fails # 'ngc2566', # jay: done ] # Eva's targets: @@ -277,8 +277,8 @@ expand_refcat = true fitgeometry = 'shift' minobj = 3 peakmax.nircam = 20 -roundlo.nircam = -0.3#-0.5 # jay: 26.11 -roundhi.nircam = 0.3#0.5 # jay: 26.11 +roundlo.nircam = -0.5 +roundhi.nircam = 0.5 # Take relatively tight tolerances since we've already shifted # close to the correct solution @@ -353,13 +353,8 @@ F770W = 'F335M' # set to F1000W to F300M for cycle 1 # NEW ic5273 = 'ic5273_agb_cat.fits' ngc1511 = 'ngc1511_agb_cat.fits' - -ngc1546.nircam = 'ngc1546_agb_cat.fits' -ngc1546.miri = 'ngc1546_nircam_lv3_f335m_cat.fits' - -ngc1559.nircam = 'ngc1559_agb_cat.fits' -ngc1559.miri = 'ngc1559_nircam_lv3_f335m_cat.fits' # test jay 19.11.2025 - +ngc1546 = 'ngc1546_agb_cat.fits' +ngc1559 = 'ngc1559_agb_cat.fits' # ngc1637 = 'ngc1637_agb_cat.fits' # ngc1792 = 'ngc1792_agb_cat.fits' ngc1808 = 'ngc1808_agb_cat.fits' @@ -372,10 +367,7 @@ ngc2997 = ['ngc2997e_agb_cat.fits', 'ngc2997w_agb_cat.fits'] # ngc2997w = 'ngc2997w_agb_cat.fits' # ngc3059 = 'ngc3059_agb_cat.fits' ngc3507 = 'ngc3507_agb_cat.fits' - -ngc3511.nircam = 'ngc3511_agb_cat.fits' -ngc3511.miri = "ngc3511_nircam_lv3_f335m_cat.fits" - +ngc3511 = 'ngc3511_agb_cat.fits' ngc3521s = 'ngc3521s_agb_cat.fits' ngc3596 = 'ngc3596_agb_cat.fits' # ngc3621 = 'ngc3621_agb_cat.fits' @@ -388,7 +380,7 @@ ngc4496a = 'ngc4496a_agb_cat.fits' # ngc4571 = 'ngc4571_agb_cat.fits' # ngc4654 = 'ngc4654_agb_cat.fits' # ngc4689 = 'ngc4689_agb_cat.fits' -# ngc4694 = 'ngc4694_agb_cat.fits' # jay: jessica says has very few point sources, try this one +# ngc4694 = 'ngc4694_agb_cat.fits' # ngc4731 = 'ngc4731_agb_cat.fits' # ngc4826 = 'ngc4826_agb_cat.fits' ngc4941 = 'ngc4941_agb_cat.fits' @@ -422,9 +414,7 @@ ngc4826 = 'ngc4826_agb_cat.fits' ngc5248 = 'ngc5248_agb_cat.fits' ngc5643 = 'ngc5643_agb_cat.fits' ngc4951 = 'ngc4951_agb_cat.fits' - -ngc4424.nircam = 'ngc4424_agb_cat.fits' -ngc4424.miri = 'ngc4424_nircam_lv3_f335m_cat.fits' +ngc4424 = 'ngc4424_agb_cat.fits' ngc6300 = 'ngc6300_agb_cat.fits' ngc4731 = 'ngc4731_agb_cat.fits' @@ -433,14 +423,9 @@ ngc3059 = 'ngc3059_agb_cat.fits' # ngc2903 = 'ngc2903_agb_cat.fits' # ngc2775 = 'ngc2775_agb_cat.fits' ngc1637 = 'ngc1637_agb_cat.fits' - -ngc4694.nircam = 'ngc4694_agb_cat.fits' -ngc4694.miri = 'ngc4694_nircam_lv3_f335m_cat.fits' - +ngc4694 = 'ngc4694_agb_cat.fits' ngc1068 = 'ngc1068_agb_cat.fits' - -ngc2566.nircam = 'ngc2566_agb_cat.fits' -ngc2566.miri = 'ngc2566_nircam_lv3_f335m_cat.fits' +ngc2566 = 'ngc2566_agb_cat.fits' # Ones w/o AGB catalogs # ngc5530 = 'Gaia_DR3_ngc5530.fits' @@ -467,15 +452,10 @@ ngc4781 = 'Gaia_DR3_ngc4781.fits' # ngc4536 = 'Gaia_DR3_ngc4536.fits' # ngc4424 = 'Gaia_DR3_ngc4424.fits' # ngc3621 = 'Gaia_DR3_ngc3621.fits' - # ngc5042 = 'Gaia_DR3_ngc5042.fits' ngc5134 = 'Gaia_DR3_ngc5134.fits' # ngc3596 = 'Gaia_DR3_ngc3596.fits' - -ngc3626.nircam = 'Gaia_DR3_ngc3626.fits' -ngc3626.miri = 'ngc3626_nircam_lv3_f335m_cat.fits' # test jay 19.11.2025 -#ngc3626 = 'Gaia_DR3_ngc3626.fits' - +ngc3626 = 'Gaia_DR3_ngc3626.fits' ngc3344 = 'Gaia_DR3_ngc3344.fits' # ngc3511 = 'Gaia_DR3_ngc3511.fits' # ngc3059 = 'Gaia_DR3_ngc3059.fits' diff --git a/pjpipe/astrometric_align/astrometric_align_step.py b/pjpipe/astrometric_align/astrometric_align_step.py index 19855bd..f4e25d5 100644 --- a/pjpipe/astrometric_align/astrometric_align_step.py +++ b/pjpipe/astrometric_align/astrometric_align_step.py @@ -249,8 +249,6 @@ def __init__( catalogs = {} if align_mapping is None: align_mapping = {} - if ref_long_filter is None: - ref_long_filter = {} if tweakreg_parameters is None: tweakreg_parameters = {} @@ -515,27 +513,26 @@ def tweakreg_align( log.warning("astrometric_alignment_table should be set!") return True - current_band_type = get_band_type(self.band) - - # test Jay 19.11.2025 -> use agb for nircam, pjpipe cat for miri - # TO DO: instead of counting the length of self.catalogs[self.target] use "if ref_long_filter is not None:" - - if len(self.catalogs[self.target]) == 2: # assumes that there has to be one for miri and one for nircam - if self.band in self.ref_long_filter: + # test Jay 19.11.2025 -> use external (agb/gaia) for nircam, internal cat for miri + # when using internal catalog + if self.ref_long_filter is not None: + if self.band in self.ref_long_filter: log.info(f"Aligning to internal catalog from shorter wavelength ({self.ref_long_filter[self.band]})") align_cat_dir = self.in_dir.replace(f"{self.band}", f"{self.ref_long_filter[self.band]}") - #align_cat_dir = align_cat_dir.replace(f"{current_band_type}", "nircam") - align_catalog = os.path.join( # when using internal catalog + align_cat_fn = f"{self.target}_nircam_lv3_{self.ref_long_filter[self.band].lower()}_cat.fits" + align_catalog = os.path.join( align_cat_dir, - self.catalogs[self.target][current_band_type] + align_cat_fn ) - elif self.band not in self.ref_long_filter: # only using external catalog + # using external catalog + elif self.band not in self.ref_long_filter: log.info("Aligning to external catalog") align_catalog = os.path.join( self.catalog_dir, - self.catalogs[self.target][current_band_type] + self.catalogs[self.target] ) - else: # when only external catalog is used + # when only external catalog is used for both + else: log.info(f"Aligning to external catalog") align_catalog = os.path.join( self.catalog_dir, @@ -551,12 +548,14 @@ def tweakreg_align( align_table = QTable.read(align_catalog, format="fits") ref_tab = Table() - if len(self.catalogs[self.target]) == 1: # when only using external AGB catalog + # when only using external AGB catalog + if self.ref_long_filter is None: ref_tab["RA"] = align_table["ra"] ref_tab["DEC"] = align_table["dec"] - - elif len(self.catalogs[self.target]) > 1: # when using external + internal catalog - if self.band in self.ref_long_filter: # miri -> ref long + # when using external + internal catalog + elif self.ref_long_filter is not None: + # miri -> ref long + if self.band in self.ref_long_filter: # get wcs from ref_long aligned image ref_long_dir = self.in_dir.replace(f"{self.band}", f"{self.ref_long_filter[self.band]}") ref_long_img = os.path.join( @@ -571,8 +570,8 @@ def tweakreg_align( align_coord = ref_long_wcs.pixel_to_world(ref_x, ref_y) ref_tab["RA"] = align_coord.ra.deg ref_tab["DEC"] = align_coord.dec.deg - - if self.band not in self.ref_long_filter: # nircam -> agb + # nircam -> agb + elif self.band not in self.ref_long_filter: ref_tab["RA"] = align_table["ra"] ref_tab["DEC"] = align_table["dec"] From 999e4786802a264c2c8a4f05984cd024e45b4663 Mon Sep 17 00:00:00 2001 From: JaysonAstro Date: Thu, 27 Nov 2025 11:04:57 +0000 Subject: [PATCH 3/9] Cleanup config file --- .../config_half1_miri_to_long.toml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/config/alignment_testing/config_half1_miri_to_long.toml b/config/alignment_testing/config_half1_miri_to_long.toml index cab969f..83f52da 100644 --- a/config/alignment_testing/config_half1_miri_to_long.toml +++ b/config/alignment_testing/config_half1_miri_to_long.toml @@ -124,13 +124,12 @@ steps = [ 'multi_tile_destripe.nircam', # 'psf_model', # TODO 'lv3', - 'astrometric_catalog.miri', - #'astrometric_catalog', # test jay 19.11.2025 + 'astrometric_catalog', 'astrometric_align', - # 'anchoring', - # 'psf_matching', - # 'release', - # 'regress_against_previous', + 'anchoring', + 'psf_matching', + 'release', + 'regress_against_previous', ] # Parameters for downloading data. This just downloads @@ -335,8 +334,6 @@ roundhi = 0.5 [parameters.astrometric_align] -overwrite = true - [parameters.astrometric_align.align_mapping] # Map F2100W to F770W F2100W = 'F770W' From abc54a5eb193d615e562c31df98bf6f88827abf4 Mon Sep 17 00:00:00 2001 From: Jay <70688790+JaysonAstro@users.noreply.github.com> Date: Mon, 1 Dec 2025 16:17:41 +0100 Subject: [PATCH 4/9] cleanup config file --- config/alignment_testing/config_half1_miri_to_long.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/alignment_testing/config_half1_miri_to_long.toml b/config/alignment_testing/config_half1_miri_to_long.toml index 83f52da..67e0175 100644 --- a/config/alignment_testing/config_half1_miri_to_long.toml +++ b/config/alignment_testing/config_half1_miri_to_long.toml @@ -4,7 +4,7 @@ # Version for the reprocessing #version = 'v2p6_test_cy1_params_cy2_smallbatch' # original name -version = 'v2p6_cy1_params_cy2_test_alignment_ngc4694_stable-version' +version = 'v2p6_cy1_params_cy2_test_alignment' # ngc5643 # ngc4298 # ngc5248 @@ -47,7 +47,7 @@ targets = [ # 'ngc5042', # 'ngc5134', # 'ngc3596', # end half 1 - # 'ngc3626', # start half 2. # jay: alignment didn't go through, few gaia sources matched + # 'ngc3626', # start half 2. # jay: alignment for nircam fails, few gaia sources matched # 'ngc3344', # 'ngc3239', #'ngc3511', # jay: done @@ -74,7 +74,7 @@ targets = [ # 'ic1954', # NEW AGB # 'ngc0685', # has agb # 'ngc3368', - 'ngc4694', # jay: jessica says has very few point sources, try this one. indeed agb cat alignment fails + 'ngc4694', # jay: jessica mentions has very few point sources. indeed agb cat alignment fails for nircam # 'ngc2566', # jay: done ] # Eva's targets: @@ -343,7 +343,7 @@ F2100W_bgr = 'F770W_bgr' # F187N = 'F335M' [parameters.astrometric_align.ref_long_filter] # Jay 19.11.2025 -F770W = 'F335M' # set to F1000W to F300M for cycle 1 +F770W = 'F335M' # set to F1000W = 'F300M' for cycle 1 [parameters.astrometric_align.catalogs] # Ones w/ AGB catalogs From b710c288e5e1df23c2158c273506689702d5801b Mon Sep 17 00:00:00 2001 From: JaysonAstro Date: Tue, 9 Dec 2025 19:30:47 +0000 Subject: [PATCH 5/9] cleanup files --- .../config_cy1_miri_to_long.toml | 529 ++++++++++++++++++ .../config_eva_4793_miri_to_long.toml | 520 +++++++++++++++++ .../config_half1_miri_to_long.toml | 115 ++-- config/alignment_testing/run_reprocessing.py | 4 +- .../astrometric_align_step.py | 6 +- 5 files changed, 1109 insertions(+), 65 deletions(-) create mode 100644 config/alignment_testing/config_cy1_miri_to_long.toml create mode 100644 config/alignment_testing/config_eva_4793_miri_to_long.toml diff --git a/config/alignment_testing/config_cy1_miri_to_long.toml b/config/alignment_testing/config_cy1_miri_to_long.toml new file mode 100644 index 0000000..b00d802 --- /dev/null +++ b/config/alignment_testing/config_cy1_miri_to_long.toml @@ -0,0 +1,529 @@ +# List of targets + +# Version for the reprocessing +version = 'v4p0_alignment_testing' +targets = [ + #'ic5332', + # # 'ngc0628', + #'ngc1087', + # #'ngc1300', + ##'ngc1365', + ##'ngc1385', + # # 'ngc1433', #Eva's + # # 'ngc1512', #Eva's + 'ngc1566', + # # # 'ngc1672', #Eva's + # 'ngc2835', + # 'ngc3351', + # # # 'ngc3627', + # 'ngc4254', + # 'ngc4303', + # 'ngc4321', + # 'ngc4535', + # 'ngc5068', + # 'ngc7496' +] +# Eva's targets: +# Done (25 Oct): 'ngc1512', 'ngc1672', 'ngc1637', 'ngc1792', 'ngc1097', 'ngc6300' +# Not done: 'ngc5643', 'ngc1808', 'ngc5248', 'ngc4548', 'ngc4298', 'ngc3627', 'ngc3351', 'ngc2997', 'ngc2903', 'ngc1433' + +# 'ngc5643', # NEW AGB +# 'ngc1808', +# 'ngc5248', # NEW AGB +# 'ngc4548', # NEW AGB +# 'ngc4298', # NEW AGB +# 'ngc3627', +# 'ngc3351', +# 'ngc2997', # NEW AGB +# 'ngc2903', # NEW AGB +# 'ngc1433' +# ngc5643 # NEW AGB +# ngc1808 # NO AGB + + +# Bands to consider +bands = [ + 'F200W', + 'F300M', + 'F335M', + 'F360M', + 'F1000W', + 'F770W', + 'F1130W', + 'F2100W', + 'F1000W_bgr', + 'F770W_bgr', + 'F1130W_bgr', + 'F2100W_bgr', +] + + +# Steps. These can/should be different +# for NIRCam and MIRI, and we can distinguish them here + +steps = [ + 'download', + 'gaia_query', + 'lv1', + 'single_tile_destripe.nircam', + 'lv2', + 'get_wcs_adjust', + 'apply_wcs_adjust', +# 'lyot_mask.miri', + 'lyot_separate.miri', + 'level_match', + 'multi_tile_destripe.nircam', +# 'psf_model', # TODO + 'lv3', + 'astrometric_catalog.miri', + 'astrometric_align', + 'anchoring', + 'psf_matching', + 'release', + 'regress_against_previous', +] + +# Parameters for downloading data. This just downloads +# the raw science +[parameters.download] + +prop_id = '2107' +product_type = ['SCIENCE'] +calib_level = [1] + +[parameters.gaia_query] + +radius = 20 + +[parameters.move_raw_obs] +# skip = true +overwrite = false + +[parameters.move_raw_obs.obs_to_skip] + +# Failed NGC1566 observation +ngc1566 = 'jw02107007003_02105_00003*' + +# New IC5332 MIRI offs, which we need to skip +ic5332.sci = 'jw02107042001_0?2*' + +# New NGC4303 MIRI offs, which we need to skip +ngc4303.sci = 'jw02107044001_0?2*' + +# New NGC4321 MIRI offs, which we need to skip +ngc4321.sci = 'jw02107043001_0?2*' + +# For IC5332, pull the backgrounds from NGC7496 but only for science images +[parameters.move_raw_obs.extra_obs_to_include] +ic5332.sci.ngc7496 = 'jw02107041001_0?2' + + +[parameters.lv1] +overwrite = false +# skip_raw_move = true +jwst_parameters.save_results = true + +# Flag snowballs in NIRCam +#jwst_parameters.jump.expand_large_events.F187N = true +jwst_parameters.jump.expand_large_events.nircam = true + +jwst_parameters.rscd.skip = true +jwst_parameters.clean_flicker_noise.skip = true + +jwst_parameters.ramp_fit.suppress_one_group = false +jwst_parameters.refpix.use_side_ref_pixels = true + +[parameters.lv2] +overwrite = false +bgr_check_type = 'parallel_off' +bgr_background_name = 'off' +bgr_observation_types = ['miri'] + +jwst_parameters.save_results = true +jwst_parameters.bkg_subtract.save_combined_background = true +jwst_parameters.bkg_subtract.sigma = 1.5 + +[parameters.get_wcs_adjust] + +method.nircam = "tweakreg" +method.miri = "tweakreg" + +bands = ['F300M', 'F1000W'] + +group_dithers = ['nircam', 'miri'] + +[parameters.get_wcs_adjust.tweakreg_parameters] + +# align_to_gaia = false # no longer a tweakreg parameter +starfinder = 'iraf' +brightest = 500 +snr_threshold = 3 +expand_refcat = true +fitgeometry = 'shift' +minobj = 3 +peakmax.nircam = 20 +roundlo.nircam = -0.5 +roundhi.nircam = 0.5 + +# Parameters to get wcs_adjust shifts +searchrad = 2 +separation.miri = 1 +tolerance.miri = 0.7 +separation.nircam = 2 +tolerance.nircam = 1 +use2dhist = true +nclip = 5 + +# Tweak boxsize, so we detect objects in diffuse emission +bkg_boxsize.nircam_short = 100 +bkg_boxsize.nircam_long = 100 +bkg_boxsize.miri = 25 +enforce_user_order = true + +[parameters.lyot_mask] +method = 'mask' + +[parameters.single_tile_destripe] +quadrants = true +vertical_subtraction = true +filter_diffuse = true +dilate_size = 7 +destriping_method = 'median_filter' +vertical_destriping_method = 'row_median' +filter_extend_mode = "reflect" +filter_scales = [3, 7, 15, 31, 63, 127, 255, 511] + +[parameters.multi_tile_destripe] +quadrants = true +sigma = 3 +weight_type = 'ivm' +do_large_scale = true +large_scale_filter_extend_mode = "reflect" + +[parameters.level_match] +do_sigma_clip = false +weight_method = 'rms' + +# fit_type_dithers = "level" +# fit_type_recombine_lyot = "level+slope" +# fit_type_mosaic_tiles.nircam = "level" +# fit_type_mosaic_tiles.miri = "level+slope" +# recombine_lyot = true +# combine_nircam_short = true + +[parameters.lv3] +overwrite = false +tweakreg_group_dithers = ['nircam_short', 'nircam_long', 'miri'] +skymatch_degroup_dithers = ['miri', 'nircam_short', 'nircam_long'] + +bgr_check_type = 'parallel_off' + +[parameters.lv3.jwst_parameters] +save_results = true + +[parameters.lv3.jwst_parameters.tweakreg] + +# Skip MIRI wavelengths, since we have already +# solved for this +skip.F770W = true +skip.F1000W = true +skip.F1130W = true +skip.F2100W = true + + +# align_to_gaia = false # no longer a tweakreg parameter +starfinder = 'iraf' +brightest.nircam_short = 125 +brightest.nircam_long = 500 +brightest.miri = 500 +snr_threshold = 3 +expand_refcat = true +fitgeometry = 'shift' +minobj = 3 +peakmax.nircam = 20 +roundlo.nircam = -0.5 +roundhi.nircam = 0.5 + +# Take relatively tight tolerances since we've already shifted +# close to the correct solution +separation.miri = 1 +separation.nircam = 2 +tolerance.nircam_short = 0.3 +tolerance.nircam_long = 0.3 +tolerance.miri = 1 +use2dhist = false + +# Tweak boxsize, so we detect objects in diffuse emission +bkg_boxsize.nircam_short = 100 +bkg_boxsize.nircam_long = 100 +bkg_boxsize.miri = 25 +enforce_user_order = true + +[parameters.lv3.jwst_parameters.skymatch] + +# Turn off matching since we've done it already +skymethod = 'local' +subtract = true +skystat = 'median' +nclip.nircam = 20 +nclip.miri = 10 +lsigma.nircam = 3 +lsigma.miri = 1.5 +usigma.nircam = 3 +usigma.miri = 1.5 + +[parameters.lv3.jwst_parameters.outlier_detection] +in_memory = true + +[parameters.lv3.jwst_parameters.resample] +rotation = 0.0 +in_memory = true + +[parameters.lv3.jwst_parameters.source_catalog] +snr_threshold = 2 +npixels = 5 +bkg_boxsize = 25 +deblend = true + +[parameters.astrometric_catalog] +snr = 10 +starfind_method = "iraf" + +# [parameters.astrometric_catalog.dao_parameters] +[parameters.astrometric_catalog.starfind_parameters] + +sharplo = 0.2 +sharphi = 1.0 +roundlo = -0.5 +roundhi = 0.5 + +[parameters.astrometric_align] + +overwrite = true + +[parameters.astrometric_align.align_mapping] +# Map everything to F1000W, since it has the most point sources +F770W = 'F1000W' +F1130W = 'F1000W' +F2100W = 'F1000W' + +F770W_bgr = 'F1000W_bgr' +F1130W_bgr = 'F1000W_bgr' +F2100W_bgr = 'F1000W_bgr' + +[parameters.astrometric_align.ref_long_filter] +# select reference nircam long filter to match miri internal catalogs +F1000W = 'F300M' + + +[parameters.astrometric_align.catalogs] +# Ones w/ AGB catalogs +# NEW +ic5273 = 'ic5273_agb_cat.fits' +ngc1511 = 'ngc1511_agb_cat.fits' +ngc1546 = 'ngc1546_agb_cat.fits' +# ngc1559 = 'ngc1559_agb_cat.fits' +# ngc1637 = 'ngc1637_agb_cat.fits' +# ngc1792 = 'ngc1792_agb_cat.fits' +ngc1808 = 'ngc1808_agb_cat.fits' +ngc1809 = 'ngc1809_agb_cat.fits' +ngc2090 = 'ngc2090_agb_cat.fits' +# ngc2566 = 'ngc2566_agb_cat.fits' +# ngc2775 = 'ngc2775_agb_cat.fits' +# ngc2903 = 'ngc2903_agb_cat.fits' +ngc2997e = 'ngc2997e_agb_cat.fits' +ngc2997w = 'ngc2997w_agb_cat.fits' +# ngc3059 = 'ngc3059_agb_cat.fits' +ngc3507 = 'ngc3507_agb_cat.fits' +ngc3511 = 'ngc3511_agb_cat.fits' +ngc3521s = 'ngc3521s_agb_cat.fits' +ngc3596 = 'ngc3596_agb_cat.fits' +# ngc3621 = 'ngc3621_agb_cat.fits' +# ngc4298 = 'ngc4298_agb_cat.fits' +# ngc4424 = 'ngc4424_agb_cat.fits' +ngc4496a = 'ngc4496a_agb_cat.fits' +# ngc4536 = 'ngc4536_agb_cat.fits' +# ngc4548 = 'ngc4548_agb_cat.fits' +# ngc4569 = 'ngc4569_agb_cat.fits' +# ngc4571 = 'ngc4571_agb_cat.fits' +# ngc4654 = 'ngc4654_agb_cat.fits' +# ngc4689 = 'ngc4689_agb_cat.fits' +# ngc4694 = 'ngc4694_agb_cat.fits' +# ngc4731 = 'ngc4731_agb_cat.fits' +# ngc4826 = 'ngc4826_agb_cat.fits' +ngc4941 = 'ngc4941_agb_cat.fits' +# ngc4951 = 'ngc4951_agb_cat.fits' +ngc5042 = 'ngc5042_agb_cat.fits' +# ngc5248 = 'ngc5248_agb_cat.fits' +ngc5530 = 'ngc5530_agb_cat.fits' +# ngc5643 = 'ngc5643_agb_cat.fits' +# ngc6300 = 'ngc6300_agb_cat.fits' +ngc7456 = 'ngc7456_agb_cat.fits' + +# OLD + +ic1954 = 'ic1954_agb_cat.fits' +ngc0685 = 'ngc0685_agb_cat.fits' +ngc1097 = 'ngc1097_agb_cat.fits' +ngc1317 = 'ngc1317_agb_cat.fits' +ngc1559 = 'ngc1559_agb_cat.fits' +ngc1792 = 'ngc1792_agb_cat.fits' +ngc2775 = 'ngc2775_agb_cat.fits' +ngc2903 = 'ngc2903_agb_cat.fits' +ngc3621 = 'ngc3621_agb_cat.fits' +ngc4298 = 'ngc4298_agb_cat.fits' +ngc4536 = 'ngc4536_agb_cat.fits' +ngc4548 = 'ngc4548_agb_cat.fits' +ngc4569 = 'ngc4569_agb_cat.fits' +ngc4571 = 'ngc4571_agb_cat.fits' +ngc4654 = 'ngc4654_agb_cat.fits' +ngc4689 = 'ngc4689_agb_cat.fits' +ngc4826 = 'ngc4826_agb_cat.fits' +ngc5248 = 'ngc5248_agb_cat.fits' +ngc5643 = 'ngc5643_agb_cat.fits' +ngc4951 = 'ngc4951_agb_cat.fits' +ngc4424 = 'ngc4424_agb_cat.fits' +ngc6300 = 'ngc6300_agb_cat.fits' +ngc4731 = 'ngc4731_agb_cat.fits' +ngc3059 = 'ngc3059_agb_cat.fits' +ngc2997 = 'ngc2997e_agb_cat.fits' # NOTE: the "e" means we have PSF photometry for the East pointing only (from Hwi Kim) +# ngc2903 = 'ngc2903_agb_cat.fits' +# ngc2775 = 'ngc2775_agb_cat.fits' +ngc1637 = 'ngc1637_agb_cat.fits' +ngc4694 = 'ngc4694_agb_cat.fits' +ngc1068 = 'ngc1068_agb_cat.fits' +ngc2566 = 'ngc2566_agb_cat.fits' + +# Ones w/o AGB catalogs +# ngc5530 = 'Gaia_DR3_ngc5530.fits' +ngc3368 = 'Gaia_DR3_ngc3368.fits' +# ngc6300 = 'Gaia_DR3_ngc6300.fits' +# ngc5643 = 'Gaia_DR3_ngc5643.fits' +# ngc4951 = 'Gaia_DR3_ngc4951.fits' +# ngc4654 = 'Gaia_DR3_ngc4654.fits' +# ngc4689 = 'Gaia_DR3_ngc4689.fits' +# ngc4569 = 'Gaia_DR3_ngc4569.fits' +# ngc4826 = 'Gaia_DR3_ngc4826.fits' +# ngc4298 = 'Gaia_DR3_ngc4298.fits' +ngc4540 = 'Gaia_DR3_ngc4540.fits' +# ngc4496a = 'Gaia_DR3_ngc4496a.fits' +# ngc4548 = 'Gaia_DR3_ngc4548.fits' +# ngc4571 = 'Gaia_DR3_ngc4571.fits' +ngc4579 = 'Gaia_DR3_ngc4579.fits' +# ngc4731 = 'Gaia_DR3_ngc4731.fits' +# ngc4941 = 'Gaia_DR3_ngc4941.fits' +ngc4457 = 'Gaia_DR3_ngc4457.fits' +# ngc4694 = 'Gaia_DR3_ngc4694.fits' +ngc4781 = 'Gaia_DR3_ngc4781.fits' +# ngc5248 = 'Gaia_DR3_ngc5248.fits' +# ngc4536 = 'Gaia_DR3_ngc4536.fits' +# ngc4424 = 'Gaia_DR3_ngc4424.fits' +# ngc3621 = 'Gaia_DR3_ngc3621.fits' + +# ngc5042 = 'Gaia_DR3_ngc5042.fits' +ngc5134 = 'Gaia_DR3_ngc5134.fits' +# ngc3596 = 'Gaia_DR3_ngc3596.fits' +ngc3626 = 'Gaia_DR3_ngc3626.fits' +ngc3344 = 'Gaia_DR3_ngc3344.fits' +# ngc3511 = 'Gaia_DR3_ngc3511.fits' +# ngc3059 = 'Gaia_DR3_ngc3059.fits' +ngc3137 = 'Gaia_DR3_ngc3137.fits' +# ngc2997 = 'Gaia_DR3_ngc2997.fits' +ngc3521 = 'Gaia_DR3_ngc3521.fits' +# ngc2903 = 'Gaia_DR3_ngc2903.fits' +# ngc3507 = 'Gaia_DR3_ngc3507.fits' +ngc3239 = 'Gaia_DR3_ngc3239.fits' +# ngc2775 = 'Gaia_DR3_ngc2775.fits' +# ngc2566 = 'Gaia_DR3_ngc2566.fits' +# ic5273 = 'Gaia_DR3_ic5273.fits' +# ngc7456 = 'Gaia_DR3_ngc7456.fits' +# ngc1809 = 'Gaia_DR3_ngc1809.fits' +# ngc1546 = 'Gaia_DR3_ngc1546.fits' +# ngc1511 = 'Gaia_DR3_ngc1511.fits' +# ngc2090 = 'Gaia_DR3_ngc2090.fits' +# ngc1068 = 'Gaia_DR3_ngc1068.fits' +# ngc1808 = 'Gaia_DR3_ngc1808.fits' +# ngc1637 = 'Gaia_DR3_ngc1637.fits' +ngc2283 = 'Gaia_DR3_ngc2283.fits' + +# Cycle 1 targets +ic5332 = 'ic5332_agb_cat.fits' +ngc0628 = 'ngc0628_agb_cat.fits' +#ngc1087 = 'ngc1087_agb_cat.fits' +ngc1087 = 'Gaia_DR3_ngc1087.fits' +ngc1300 = 'ngc1300_agb_cat.fits' +ngc1365 = 'ngc1365_agb_cat.fits' +ngc1385 = 'ngc1385_agb_cat.fits' +ngc1433 = 'ngc1433_agb_cat.fits' +ngc1512 = 'ngc1512_agb_cat.fits' +ngc1566 = 'ngc1566_agb_cat.fits' +ngc1672 = 'ngc1672_agb_cat.fits' +ngc2835 = 'ngc2835_agb_cat.fits' +ngc3351 = 'ngc3351_agb_cat.fits' +ngc3627 = 'ngc3627_agb_cat.fits' +ngc4254 = 'ngc4254_agb_cat.fits' +ngc4303 = 'ngc4303_agb_cat.fits' +ngc4321 = 'ngc4321_agb_cat.fits' +ngc4535 = 'ngc4535_agb_cat.fits' +ngc5068 = 'ngc5068_agb_cat.fits' +#ngc7496 = 'ngc7496_agb_cat.fits' +ngc7496 = 'Gaia_DR3_ngc7496.fits' + +# Initial pass to get decent shifts for absolute astrometry +[parameters.astrometric_align.tweakreg_parameters.iteration1] + +# Set quite a large search radius +searchrad.miri = 10 # trying larger radius (was 10 arcsec before) -- 7 May 24 +searchrad.nircam_long = 20 +searchrad.nircam_short = 40 +separation = 1 +tolerance = 2 # 14 Jul 25 -- default was 1. Tom's initial config had 2 for NGC 3627. +# tolerance = 1 +use2dhist = true # Changed to true on 13 Feb 24 -- to remove catastrophic fails +fitgeom = 'shift' +nclip = 5 +sigma = 3 + +# Second iteration with tightened up parameters to figure out any residual shifts left +[parameters.astrometric_align.tweakreg_parameters.iteration2] +searchrad = 2 +separation = 1 +tolerance = 0.5 +use2dhist = false +fitgeom = 'rshift' +nclip = 5 +sigma = 3 + + +[parameters.anchoring] +overwrite = false# true #true +internal_conv_band = 'F2100W' + +[parameters.anchoring.ref_band] +miri = 'F770W' +nircam = 'F300M' + +# List external bands in preference order +[parameters.anchoring.external_bands] +# miri = ['irac4_atgauss4', 'w3_atgauss15'] +# nircam = ['irac1_atgauss4', 'w1_atgauss7p5', 'irac2_atgauss4', 'w2_atgauss7p5'] +miri = ['w3_atgauss15', 'irac4_atgauss4'] +nircam = ['w1_atgauss7p5', 'w2_atgauss7p5', 'irac1_atgauss4', 'irac2_atgauss4'] + +[parameters.psf_matching] +#skip_if_missing_kernels = true +overwrite = true +target_bands = ['F200W', 'F300M', 'F770W', "gauss0p35", "gauss0p85"] +reproject_func = "interp" + + +[parameters.release] +overwrite = true +move_tweakback = true +move_backgrounds = true +move_psf_matched = true +move_diagnostic_plots = true + +[parameters.regress_against_previous] +overwrite = true +prev_version = 'v0p2p0' diff --git a/config/alignment_testing/config_eva_4793_miri_to_long.toml b/config/alignment_testing/config_eva_4793_miri_to_long.toml new file mode 100644 index 0000000..0a21ed4 --- /dev/null +++ b/config/alignment_testing/config_eva_4793_miri_to_long.toml @@ -0,0 +1,520 @@ +# List of targets + +targets = [ + # 'ngc1097', + # 'ngc1433', + # 'ngc1512', + # 'ngc1637', + 'ngc1672', + # 'ngc1792', + # 'ngc1808', # Doeesn't have 10um or 11.3um + # 'ngc2903', + # 'ngc2997', + # 'ngc3351', + # 'ngc3627', + # 'ngc4298', + # 'ngc4548', + # 'ngc5248', + # 'ngc5643' + # 'ngc6300', +] + +# Version for the reprocessing +version = 'v4p0_alignment_testing' + +# Bands to consider +bands = [ + #'F164N', + #'F212N', + #'F277W', + #'F360M', + #'F335M', + #'F444W', + 'F150W', + 'F187N', + 'F200W', + 'F300M', + #'F405N', + #'F430M', + #'F1130W', + #'F1000W', + #'F1000W_bgr', + #'F1130W_bgr', + #'F770W', + #'F2100W', + #'F770W_bgr', + #'F2100W_bgr', +] + +# Steps. These can/should be different +# for NIRCam and MIRI, and we can distinguish them here + +steps = [ + 'download', + 'gaia_query', + 'lv1', + 'single_tile_destripe.nircam', + 'lv2', + 'get_wcs_adjust', + 'apply_wcs_adjust', + 'lyot_mask.miri', + # 'level_match', + 'multi_tile_destripe.nircam', + # 'psf_model', # TODO + 'lv3', + 'astrometric_catalog.miri', + 'astrometric_align', + 'anchoring', + 'psf_matching', + 'release', + 'regress_against_previous', +] + +# Parameters for downloading data. This just downloads +# the raw science +[parameters.download] + +prop_id = ['4793', '3707', '2107'] +product_type = ['SCIENCE'] +calib_level = [1] + +[parameters.gaia_query] + +radius = 20 + +[parameters.move_raw_obs] + +[parameters.move_raw_obs.obs_to_skip] + +# ngc3596 = 'jw02107007003_02105_00003*' +ngc1317 = ['jw03707005*', 'jw03707006*'] +ngc3239 = ['jw03707037*', 'jw03707038*'] +ngc3344 = ['jw03707123*', 'jw03707124*'] +ngc3596 = ['jw03707047*', 'jw03707048*'] +ngc4694 = ['jw03707093*', 'jw03707094*'] +ngc1808 = 'jw04793007*' # had to repeat +ngc5643 = 'jw04793015*' +ngc5248 = 'jw03707109003*' # PARTIAL REPEAT (ONLY 109:3 FAILED). CAREFUL REDUCING THIS ONE. +ngc5530 = 'jw03707111001*' # NOTE: PARTIAL REPEAT (ONLY 111:1 FAILED). CAREFUL REDUCING THIS ONE. + + +[parameters.lv1] + +jwst_parameters.save_results = true + +# Flag snowballs in NIRCam +#jwst_parameters.jump.expand_large_events.F187N = true +jwst_parameters.jump.expand_large_events.nircam = true + +jwst_parameters.ramp_fit.suppress_one_group = false +jwst_parameters.refpix.use_side_ref_pixels = true + +jwst_parameters.rscd.skip = true +jwst_parameters.clean_flicker_noise.skip = true + +[parameters.lv2] + +bgr_check_type = 'parallel_off' +bgr_background_name = 'off' +bgr_observation_types = ['miri'] + +jwst_parameters.save_results = true +jwst_parameters.bkg_subtract.save_combined_background = true +jwst_parameters.bkg_subtract.sigma = 1.5 + +[parameters.get_wcs_adjust] + +method.nircam = "tweakreg" +method.miri = "tweakreg" + +bands = ['F300M', 'F770W'] + +group_dithers = ['nircam', 'miri'] + +[parameters.get_wcs_adjust.tweakreg_parameters] + +# align_to_gaia = false # no longer a tweakreg parameter +starfinder = 'iraf' +brightest = 500 +snr_threshold = 3 +expand_refcat = true +fitgeometry = 'shift' +minobj = 3 +peakmax.nircam = 20 +roundlo.nircam = -0.5 +roundhi.nircam = 0.5 + +# Parameters to get wcs_adjust shifts +searchrad = 2 +separation.miri = 1 +tolerance.miri = 0.7 +separation.nircam = 2 +tolerance.nircam = 1 +use2dhist = true +nclip = 5 + +# Tweak boxsize, so we detect objects in diffuse emission +bkg_boxsize.nircam_short = 100 +bkg_boxsize.nircam_long = 100 +bkg_boxsize.miri = 25 +enforce_user_order = true + +[parameters.lyot_mask] +method = 'mask' + +[parameters.single_tile_destripe] +quadrants = true +vertical_subtraction = true +filter_diffuse = true +dilate_size = 7 +destriping_method = 'median_filter' +vertical_destriping_method = 'row_median' +filter_extend_mode = "reflect" +filter_scales = [3, 7, 15, 31, 63, 127, 255, 511] + +[parameters.multi_tile_destripe] +quadrants = true +sigma = 3 +weight_type = 'ivm' +do_large_scale = true +large_scale_filter_extend_mode = "reflect" + +[parameters.level_match] +do_sigma_clip = false +weight_method = 'rms' + +# fit_type_dithers = "level" +# fit_type_recombine_lyot = "level+slope" +# fit_type_mosaic_tiles.nircam = "level" +# fit_type_mosaic_tiles.miri = "level+slope" +# recombine_lyot = true +# combine_nircam_short = true + +[parameters.lv3] + +tweakreg_group_dithers = ['nircam_short', 'nircam_long', 'miri'] +skymatch_degroup_dithers = ['miri', 'nircam_short', 'nircam_long'] + +bgr_check_type = 'parallel_off' + +[parameters.lv3.jwst_parameters] +save_results = true + +[parameters.lv3.jwst_parameters.tweakreg] + +# Skip MIRI wavelengths, since we have already +# solved for this +skip.F1000W = true +skip.F1130W = true +skip.F770W = true +skip.F2100W = true + +# align_to_gaia = false # no longer a tweakreg parameter +starfinder = 'iraf' +brightest.nircam_short = 125 +brightest.nircam_long = 500 +brightest.miri = 500 +snr_threshold = 3 +expand_refcat = true +fitgeometry = 'shift' +minobj = 3 +peakmax.nircam = 20 +roundlo.nircam = -0.5 +roundhi.nircam = 0.5 + +# Take relatively tight tolerances since we've already shifted +# close to the correct solution +separation.miri = 1 +separation.nircam = 2 +tolerance.nircam_short = 0.3 +tolerance.nircam_long = 0.3 +tolerance.miri = 1 +use2dhist = false + +# Tweak boxsize, so we detect objects in diffuse emission +bkg_boxsize.nircam_short = 100 +bkg_boxsize.nircam_long = 100 +bkg_boxsize.miri = 25 +enforce_user_order = true + +[parameters.lv3.jwst_parameters.skymatch] +# skip = true +# Turn off matching since we've done it already +skymethod = 'local' +# match_down = false +subtract = true +skystat = 'median' +nclip.nircam = 20 +# nclip.miri = 10 +lsigma.nircam = 3 +# lsigma.miri = 1.5 +usigma.nircam = 3 +# usigma.miri = 1.5 +# # in_memory = true +# # Turn off matching since we've done it already +# skymethod = 'global' +# subtract = true +# skystat = 'median' +# nclip.nircam = 20 +# nclip.miri = 10 +# lsigma.nircam = 3 +# lsigma.miri = 1.5 +# usigma.nircam = 3 +# usigma.miri = 1.5 + +[parameters.lv3.jwst_parameters.outlier_detection] +in_memory = true + +[parameters.lv3.jwst_parameters.resample] +rotation = 0.0 +in_memory = true + +[parameters.lv3.jwst_parameters.source_catalog] +snr_threshold = 2 +npixels = 5 +bkg_boxsize = 25 +deblend = true + +[parameters.astrometric_catalog] +snr = 10 +starfind_method = "iraf" + +# [parameters.astrometric_catalog.dao_parameters] +[parameters.astrometric_catalog.starfind_parameters] + +sharplo = 0.2 +sharphi = 1.0 +roundlo = -0.5 +roundhi = 0.5 + +[parameters.astrometric_align] +overwrite = false + +[parameters.astrometric_align.align_mapping] +# Map F2100W to F770W +# F1130W = 'F1000W' +# F1130W_bgr = 'F1000W_bgr' +F2100W = 'F770W' +F2100W_bgr = 'F770W_bgr' +# F1130W = 'F1000W' +# F1130W_bgr = 'F1000W_bgr' + +# F150W = 'F300M' +# F187N = 'F335M' + +[parameters.astrometric_align.ref_long_filter] +# select reference nircam long filter to match miri internal catalogs +F1000W = 'F300M' + +[parameters.astrometric_align.catalogs] +# Ones w/ AGB catalogs +# NEW +ic5273 = 'ic5273_agb_cat.fits' +ngc1511 = 'ngc1511_agb_cat.fits' +ngc1546 = 'ngc1546_agb_cat.fits' +# ngc1559 = 'ngc1559_agb_cat.fits' +# ngc1637 = 'ngc1637_agb_cat.fits' +# ngc1792 = 'ngc1792_agb_cat.fits' +ngc1808 = 'ngc1808_agb_cat.fits' +ngc1809 = 'ngc1809_agb_cat.fits' +ngc2090 = 'ngc2090_agb_cat.fits' +# ngc2566 = 'ngc2566_agb_cat.fits' +# ngc2775 = 'ngc2775_agb_cat.fits' +# ngc2903 = 'ngc2903_agb_cat.fits' +ngc2997e = 'ngc2997e_agb_cat.fits' +ngc2997w = 'ngc2997w_agb_cat.fits' +# ngc3059 = 'ngc3059_agb_cat.fits' +ngc3507 = 'ngc3507_agb_cat.fits' +ngc3511 = 'ngc3511_agb_cat.fits' +ngc3521s = 'ngc3521s_agb_cat.fits' +ngc3596 = 'ngc3596_agb_cat.fits' +# ngc3621 = 'ngc3621_agb_cat.fits' +# ngc4298 = 'ngc4298_agb_cat.fits' +# ngc4424 = 'ngc4424_agb_cat.fits' +ngc4496a = 'ngc4496a_agb_cat.fits' +# ngc4536 = 'ngc4536_agb_cat.fits' +# ngc4548 = 'ngc4548_agb_cat.fits' +# ngc4569 = 'ngc4569_agb_cat.fits' +# ngc4571 = 'ngc4571_agb_cat.fits' +# ngc4654 = 'ngc4654_agb_cat.fits' +# ngc4689 = 'ngc4689_agb_cat.fits' +# ngc4694 = 'ngc4694_agb_cat.fits' +# ngc4731 = 'ngc4731_agb_cat.fits' +# ngc4826 = 'ngc4826_agb_cat.fits' +ngc4941 = 'ngc4941_agb_cat.fits' +# ngc4951 = 'ngc4951_agb_cat.fits' +ngc5042 = 'ngc5042_agb_cat.fits' +# ngc5248 = 'ngc5248_agb_cat.fits' +ngc5530 = 'ngc5530_agb_cat.fits' +# ngc5643 = 'ngc5643_agb_cat.fits' +# ngc6300 = 'ngc6300_agb_cat.fits' +ngc7456 = 'ngc7456_agb_cat.fits' + +# OLD + +ic1954 = 'ic1954_agb_cat.fits' +ngc0685 = 'ngc0685_agb_cat.fits' +ngc1097 = 'ngc1097_agb_cat.fits' +ngc1317 = 'ngc1317_agb_cat.fits' +ngc1559 = 'ngc1559_agb_cat.fits' +ngc1792 = 'ngc1792_agb_cat.fits' +ngc2775 = 'ngc2775_agb_cat.fits' +ngc2903 = 'ngc2903_agb_cat.fits' +ngc3621 = 'ngc3621_agb_cat.fits' +ngc4298 = 'ngc4298_agb_cat.fits' +ngc4536 = 'ngc4536_agb_cat.fits' +ngc4548 = 'ngc4548_agb_cat.fits' +ngc4569 = 'ngc4569_agb_cat.fits' +ngc4571 = 'ngc4571_agb_cat.fits' +ngc4654 = 'ngc4654_agb_cat.fits' +ngc4689 = 'ngc4689_agb_cat.fits' +ngc4826 = 'ngc4826_agb_cat.fits' +ngc5248 = 'ngc5248_agb_cat.fits' +ngc5643 = 'ngc5643_agb_cat.fits' +ngc4951 = 'ngc4951_agb_cat.fits' +ngc4424 = 'ngc4424_agb_cat.fits' +ngc6300 = 'ngc6300_agb_cat.fits' +ngc4731 = 'ngc4731_agb_cat.fits' +ngc3059 = 'ngc3059_agb_cat.fits' +ngc2997 = 'ngc2997e_agb_cat.fits' # NOTE: the "e" means we have PSF photometry for the East pointing only (from Hwi Kim) +# ngc2903 = 'ngc2903_agb_cat.fits' +# ngc2775 = 'ngc2775_agb_cat.fits' +ngc1637 = 'ngc1637_agb_cat.fits' +ngc4694 = 'ngc4694_agb_cat.fits' +ngc1068 = 'ngc1068_agb_cat.fits' +ngc2566 = 'ngc2566_agb_cat.fits' + +# Ones w/o AGB catalogs +#ngc1672 = 'Gaia_DR3_ngc1672.fits' +# ngc5530 = 'Gaia_DR3_ngc5530.fits' +ngc3368 = 'Gaia_DR3_ngc3368.fits' +# ngc6300 = 'Gaia_DR3_ngc6300.fits' +# ngc5643 = 'Gaia_DR3_ngc5643.fits' +# ngc4951 = 'Gaia_DR3_ngc4951.fits' +# ngc4654 = 'Gaia_DR3_ngc4654.fits' +# ngc4689 = 'Gaia_DR3_ngc4689.fits' +# ngc4569 = 'Gaia_DR3_ngc4569.fits' +# ngc4826 = 'Gaia_DR3_ngc4826.fits' +# ngc4298 = 'Gaia_DR3_ngc4298.fits' +ngc4540 = 'Gaia_DR3_ngc4540.fits' +# ngc4496a = 'Gaia_DR3_ngc4496a.fits' +# ngc4548 = 'Gaia_DR3_ngc4548.fits' +# ngc4571 = 'Gaia_DR3_ngc4571.fits' +ngc4579 = 'Gaia_DR3_ngc4579.fits' +# ngc4731 = 'Gaia_DR3_ngc4731.fits' +# ngc4941 = 'Gaia_DR3_ngc4941.fits' +ngc4457 = 'Gaia_DR3_ngc4457.fits' +# ngc4694 = 'Gaia_DR3_ngc4694.fits' +ngc4781 = 'Gaia_DR3_ngc4781.fits' +# ngc5248 = 'Gaia_DR3_ngc5248.fits' +# ngc4536 = 'Gaia_DR3_ngc4536.fits' +# ngc4424 = 'Gaia_DR3_ngc4424.fits' +# ngc3621 = 'Gaia_DR3_ngc3621.fits' + +# ngc5042 = 'Gaia_DR3_ngc5042.fits' +ngc5134 = 'Gaia_DR3_ngc5134.fits' +# ngc3596 = 'Gaia_DR3_ngc3596.fits' +ngc3626 = 'Gaia_DR3_ngc3626.fits' +ngc3344 = 'Gaia_DR3_ngc3344.fits' +# ngc3511 = 'Gaia_DR3_ngc3511.fits' +# ngc3059 = 'Gaia_DR3_ngc3059.fits' +ngc3137 = 'Gaia_DR3_ngc3137.fits' +# ngc2997 = 'Gaia_DR3_ngc2997.fits' +ngc3521 = 'Gaia_DR3_ngc3521.fits' +# ngc2903 = 'Gaia_DR3_ngc2903.fits' +# ngc3507 = 'Gaia_DR3_ngc3507.fits' +ngc3239 = 'Gaia_DR3_ngc3239.fits' +# ngc2775 = 'Gaia_DR3_ngc2775.fits' +# ngc2566 = 'Gaia_DR3_ngc2566.fits' +# ic5273 = 'Gaia_DR3_ic5273.fits' +# ngc7456 = 'Gaia_DR3_ngc7456.fits' +# ngc1809 = 'Gaia_DR3_ngc1809.fits' +# ngc1546 = 'Gaia_DR3_ngc1546.fits' +# ngc1511 = 'Gaia_DR3_ngc1511.fits' +# ngc2090 = 'Gaia_DR3_ngc2090.fits' +# ngc1068 = 'Gaia_DR3_ngc1068.fits' +# ngc1808 = 'Gaia_DR3_ngc1808.fits' +# ngc1637 = 'Gaia_DR3_ngc1637.fits' +ngc2283 = 'Gaia_DR3_ngc2283.fits' + +# Cycle 1 targets +ic5332 = 'ic5332_agb_cat.fits' +ngc0628 = 'ngc0628_agb_cat.fits' +ngc1087 = 'ngc1087_agb_cat.fits' +ngc1300 = 'ngc1300_agb_cat.fits' +ngc1365 = 'ngc1365_agb_cat.fits' +ngc1385 = 'ngc1385_agb_cat.fits' +ngc1433 = 'ngc1433_agb_cat.fits' +ngc1512 = 'ngc1512_agb_cat.fits' +ngc1566 = 'ngc1566_agb_cat.fits' +ngc1672 = 'ngc1672_agb_cat.fits' +ngc2835 = 'ngc2835_agb_cat.fits' +ngc3351 = 'ngc3351_agb_cat.fits' +ngc3627 = 'ngc3627_agb_cat.fits' +ngc4254 = 'ngc4254_agb_cat.fits' +ngc4303 = 'ngc4303_agb_cat.fits' +ngc4321 = 'ngc4321_agb_cat.fits' +ngc4535 = 'ngc4535_agb_cat.fits' +ngc5068 = 'ngc5068_agb_cat.fits' +ngc7496 = 'ngc7496_agb_cat.fits' + + +# Initial pass to get decent shifts for absolute astrometry +[parameters.astrometric_align.tweakreg_parameters.iteration1] + +# Set quite a large search radius +searchrad.miri = 10 # trying larger radius (was 10 arcsec before) -- 7 May 24 +searchrad.nircam_long = 20 +searchrad.nircam_short = 40 +separation = 1 +tolerance = 2 # 14 Jul 25 -- default was 1. Tom's initial config had 2 for NGC 3627. +# tolerance = 1 +use2dhist = true # Changed to true on 13 Feb 24 -- to remove catastrophic fails +fitgeom = 'shift' +nclip = 5 +sigma = 3 + +# Second iteration with tightened up parameters to figure out any residual shifts left +[parameters.astrometric_align.tweakreg_parameters.iteration2] +searchrad = 2 +separation = 1 +tolerance = 0.5 +use2dhist = false +fitgeom = 'rshift' +nclip = 5 +sigma = 3 + + +[parameters.anchoring] +# band to use as reference for resolution +overwrite = false # true +internal_conv_band = 'F2100W' + +[parameters.anchoring.ref_band] +miri = 'F770W' +nircam = 'F300M' + +# List external bands in preference order +[parameters.anchoring.external_bands] +# miri = ['irac4_atgauss4', 'w3_atgauss15'] +# nircam = ['irac1_atgauss4', 'w1_atgauss7p5', 'irac2_atgauss4', 'w2_atgauss7p5'] +miri = ['w3_atgauss15', 'irac4_atgauss4'] +nircam = ['w1_atgauss7p5', 'w2_atgauss7p5', 'irac1_atgauss4', 'irac2_atgauss4'] + +[parameters.psf_matching] +#skip_if_missing_kernels = true +#target_bands = ['F200W', 'F300M', 'F770W', "gauss0p35", "gauss0p85"] +target_bands = ["gauss0p14", "gauss0p12"] # using safe gaussians of F300M from Williams+24 Table2 +reproject_func = "interp" +overwrite = true + +[parameters.release] +overwrite=true +move_tweakback = true +move_backgrounds = true +move_psf_matched = true +move_diagnostic_plots = true + +[parameters.regress_against_previous] +overwrite = true +prev_version = 'v3p0' diff --git a/config/alignment_testing/config_half1_miri_to_long.toml b/config/alignment_testing/config_half1_miri_to_long.toml index 83f52da..5677251 100644 --- a/config/alignment_testing/config_half1_miri_to_long.toml +++ b/config/alignment_testing/config_half1_miri_to_long.toml @@ -4,7 +4,7 @@ # Version for the reprocessing #version = 'v2p6_test_cy1_params_cy2_smallbatch' # original name -version = 'v2p6_cy1_params_cy2_test_alignment_ngc4694_stable-version' +version = 'v2p6_cy1_params_cy2_test_alignment' # ngc5643 # ngc4298 # ngc5248 @@ -17,65 +17,61 @@ version = 'v2p6_cy1_params_cy2_test_alignment_ngc4694_stable-version' # targets = ['ngc1546'] targets = [ - # 'ngc5530', - # 'ngc1808', - # 'ngc1068' - # 'ngc5530', # -- was WOPR in v0p3, but obs done in June 2025 - # 'ngc5248', # Eva's, but not yet done # NEW AGB -- was WOPR in v0p3, but obs done in June 2025 - # 'ngc1317', # AGB - # # 'ngc5643', # Eva's, done (2025-06-18) NEW AGB - # 'ngc4951', # NEW AGB - # 'ngc4654', # NEW AGB - # 'ngc4689', # NEW AGB - # 'ngc4569', # NEW AGB - # # # 'ngc4826', # Jiayi's program # NEW AGB - # 'ngc4298', # Eva's, but not yet done # NEW AGB - # 'ngc4536', # NEW AGB - # 'ngc4424', # NEW AGB # jay: done + # 'ic1954', # NEW AGB # start with paschen alpha + # 'ic5273', + # 'ngc0685', # has agb + # # # #'ngc1068', # run separately + # # # 'ngc1097', # agb. Eva's (done) + # 'ngc1511', + #'ngc1546', + # 'ngc1559', # agb + # # # 'ngc1637', # Eva's (done) # NEW AGB + # # # 'ngc1792', # agb. Eva's (done) + # 'ngc1808', # Eva's, not yet done + # 'ngc1809', + # 'ngc2090', + # 'ngc2283', + # 'ngc2775', # NEW AGB + # 'ngc2903', # Eva's, but not yet done # NEW AGB + # 'ngc2997', # Eva's, but not yet done # NEW AGB + # 'ngc3059', # NEW AGB + # 'ngc3137', + # 'ngc3239', + # 'ngc3344', + # 'ngc3368', + # 'ngc3507', + # 'ngc3511', + # 'ngc3521', + # 'ngc3596', # end half 1 # 'ngc3621', # NEW AGB + # 'ngc4298', # Eva's, but not yet done # NEW AGB + # 'ngc4424', # NEW AGB + # 'ngc4457', # 'ngc4540', + # 'ngc4548', # Eva's, but not yet done # NEW AGB + # 'ngc4569', # NEW AGB + # 'ngc4571', # NEW AGB + # 'ngc4654', # NEW AGB + # 'ngc4694', # 'ngc4781', + # # # 'ngc4826', # Jiayi's program # NEW AGB + # 'ngc4941', + # 'ngc4951', # NEW AGB + # 'ngc5042', + # 'ngc5248', # Eva's, but not yet done # NEW AGB -- was WOPR in v0p3, but obs done in June 2025 + # 'ngc5530', + # # 'ngc5643', # Eva's, done (2025-06-18) NEW AGB # # # 'ngc6300', # Eva's (done) # NEW AGB - # 'ngc5248', # Eva's, but not yet done # NEW AGB -- will need to re-download data because it's a WOPR (still as of 2 Dec 24) + # 'ngc7456', # end with paschen alpha + # 'ngc1317', # AGB # start without paschen alpha + # 'ngc2566', + # 'ngc3626', # start half 2. # 'ngc4496a', - # 'ngc4548', # Eva's, but not yet done # NEW AGB - # 'ngc4571', # NEW AGB + # 'ngc4536', # NEW AGB # 'ngc4579', + # 'ngc4689', # NEW AGB # 'ngc4731', # NEW AGB - # 'ngc4941', - # 'ngc4457', - # 'ngc5042', - # 'ngc5134', - # 'ngc3596', # end half 1 - # 'ngc3626', # start half 2. # jay: alignment didn't go through, few gaia sources matched - # 'ngc3344', - # 'ngc3239', - #'ngc3511', # jay: done - # 'ngc3059', # NEW AGB - # 'ngc3137', - # 'ngc2997', # Eva's, but not yet done # NEW AGB - # 'ngc2903', # Eva's, but not yet done # NEW AGB - # 'ngc3521', - # 'ngc3507', - # 'ngc2775', # NEW AGB - # # # 'ngc1792', # agb. Eva's (done) - # 'ngc1511', - # 'ngc2283', - # # # 'ngc1637', # Eva's (done) # NEW AGB - # 'ngc1808', # Eva's, not yet done - # # # #'ngc1068', # run separately - # 'ic5273', - # 'ngc7456', - # 'ngc1809', - #'ngc1546', # jay: done - # 'ngc1559', # agb # jay: done - # 'ngc2090', - # # # 'ngc1097', # agb. Eva's (done) - # 'ic1954', # NEW AGB - # 'ngc0685', # has agb - # 'ngc3368', - 'ngc4694', # jay: jessica says has very few point sources, try this one. indeed agb cat alignment fails - # 'ngc2566', # jay: done + # 'ngc5134', ] # Eva's targets: # Done (25 Oct): 'ngc1512', 'ngc1672', 'ngc1637', 'ngc1792', 'ngc1097', 'ngc6300' @@ -342,8 +338,9 @@ F2100W_bgr = 'F770W_bgr' # F150W = 'F300M' # F187N = 'F335M' -[parameters.astrometric_align.ref_long_filter] # Jay 19.11.2025 -F770W = 'F335M' # set to F1000W to F300M for cycle 1 +[parameters.astrometric_align.ref_long_filter] +# select reference nircam long filter to match miri internal catalogs +F770W = 'F335M' [parameters.astrometric_align.catalogs] # Ones w/ AGB catalogs @@ -524,7 +521,7 @@ sigma = 3 [parameters.anchoring] -overwrite = true # true +#overwrite = false # true internal_conv_band = 'F2100W' [parameters.anchoring.ref_band] @@ -539,10 +536,10 @@ miri = ['w3_atgauss15', 'irac4_atgauss4'] nircam = ['w1_atgauss7p5', 'w2_atgauss7p5', 'irac1_atgauss4', 'irac2_atgauss4'] [parameters.psf_matching] -skip_if_missing_kernels = true -target_bands = ['F770W', "gauss0p35", "gauss0p85"] -reproject_func = "interp" +#skip_if_missing_kernels = true overwrite = true +target_bands = ['F200W', 'F300M', 'F770W', "gauss0p35", "gauss0p85"] +reproject_func = "interp" [parameters.release] diff --git a/config/alignment_testing/run_reprocessing.py b/config/alignment_testing/run_reprocessing.py index 47aa7e2..8f57481 100755 --- a/config/alignment_testing/run_reprocessing.py +++ b/config/alignment_testing/run_reprocessing.py @@ -5,8 +5,8 @@ this_dir = os.path.dirname(os.path.realpath(__file__)) -config_file = os.path.join(this_dir, "config_half1_miri_to_long.toml") -local_file = os.path.join(this_dir, "astronode_dirs.toml") +config_file = os.path.join(this_dir, "config_eva_4793_miri_to_long.toml") +local_file = os.path.join(this_dir, "astronode_dirs_cy1.toml") # We need to set CRDS path local = pjpipe.load_toml(local_file) diff --git a/pjpipe/astrometric_align/astrometric_align_step.py b/pjpipe/astrometric_align/astrometric_align_step.py index f4e25d5..696ed01 100644 --- a/pjpipe/astrometric_align/astrometric_align_step.py +++ b/pjpipe/astrometric_align/astrometric_align_step.py @@ -201,7 +201,7 @@ def __init__( catalogs=None, align_mapping_mode="shift", align_mapping=None, - ref_long_filter=None, # jay 19.11.2025 + ref_long_filter=None, tweakreg_parameters=None, reproject_func="interp", overwrite=False, @@ -513,8 +513,6 @@ def tweakreg_align( log.warning("astrometric_alignment_table should be set!") return True - # test Jay 19.11.2025 -> use external (agb/gaia) for nircam, internal cat for miri - # when using internal catalog if self.ref_long_filter is not None: if self.band in self.ref_long_filter: log.info(f"Aligning to internal catalog from shorter wavelength ({self.ref_long_filter[self.band]})") @@ -548,7 +546,7 @@ def tweakreg_align( align_table = QTable.read(align_catalog, format="fits") ref_tab = Table() - # when only using external AGB catalog + # when only using external catalog if self.ref_long_filter is None: ref_tab["RA"] = align_table["ra"] ref_tab["DEC"] = align_table["dec"] From aabd02edd921efe390a1b65d2dde61b288978ae2 Mon Sep 17 00:00:00 2001 From: Jay <70688790+JaysonAstro@users.noreply.github.com> Date: Tue, 9 Dec 2025 20:33:15 +0100 Subject: [PATCH 6/9] clean format config file --- .../config_half1_miri_to_long.toml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/config/alignment_testing/config_half1_miri_to_long.toml b/config/alignment_testing/config_half1_miri_to_long.toml index 67e0175..e92ac7b 100644 --- a/config/alignment_testing/config_half1_miri_to_long.toml +++ b/config/alignment_testing/config_half1_miri_to_long.toml @@ -31,7 +31,7 @@ targets = [ # # # 'ngc4826', # Jiayi's program # NEW AGB # 'ngc4298', # Eva's, but not yet done # NEW AGB # 'ngc4536', # NEW AGB - # 'ngc4424', # NEW AGB # jay: done + # 'ngc4424', # NEW AGB # 'ngc3621', # NEW AGB # 'ngc4540', # 'ngc4781', @@ -47,10 +47,10 @@ targets = [ # 'ngc5042', # 'ngc5134', # 'ngc3596', # end half 1 - # 'ngc3626', # start half 2. # jay: alignment for nircam fails, few gaia sources matched + # 'ngc3626', # start half 2. # 'ngc3344', # 'ngc3239', - #'ngc3511', # jay: done + #'ngc3511', # 'ngc3059', # NEW AGB # 'ngc3137', # 'ngc2997', # Eva's, but not yet done # NEW AGB @@ -67,15 +67,15 @@ targets = [ # 'ic5273', # 'ngc7456', # 'ngc1809', - #'ngc1546', # jay: done - # 'ngc1559', # agb # jay: done + #'ngc1546', + # 'ngc1559', # agb # 'ngc2090', # # # 'ngc1097', # agb. Eva's (done) # 'ic1954', # NEW AGB # 'ngc0685', # has agb # 'ngc3368', - 'ngc4694', # jay: jessica mentions has very few point sources. indeed agb cat alignment fails for nircam - # 'ngc2566', # jay: done + 'ngc4694', + # 'ngc2566', ] # Eva's targets: # Done (25 Oct): 'ngc1512', 'ngc1672', 'ngc1637', 'ngc1792', 'ngc1097', 'ngc6300' @@ -342,8 +342,9 @@ F2100W_bgr = 'F770W_bgr' # F150W = 'F300M' # F187N = 'F335M' -[parameters.astrometric_align.ref_long_filter] # Jay 19.11.2025 -F770W = 'F335M' # set to F1000W = 'F300M' for cycle 1 +[parameters.astrometric_align.ref_long_filter] +# select reference nircam long filter to match miri internal catalogs +F770W = 'F335M' [parameters.astrometric_align.catalogs] # Ones w/ AGB catalogs From 246c6aa41c6724129b2c2b85690d88cf5e7a5b09 Mon Sep 17 00:00:00 2001 From: Jay <70688790+JaysonAstro@users.noreply.github.com> Date: Tue, 9 Dec 2025 20:34:59 +0100 Subject: [PATCH 7/9] cleanup comments Updated comments for clarity regarding catalog usage. --- pjpipe/astrometric_align/astrometric_align_step.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pjpipe/astrometric_align/astrometric_align_step.py b/pjpipe/astrometric_align/astrometric_align_step.py index f4e25d5..3f400d3 100644 --- a/pjpipe/astrometric_align/astrometric_align_step.py +++ b/pjpipe/astrometric_align/astrometric_align_step.py @@ -201,7 +201,7 @@ def __init__( catalogs=None, align_mapping_mode="shift", align_mapping=None, - ref_long_filter=None, # jay 19.11.2025 + ref_long_filter=None, tweakreg_parameters=None, reproject_func="interp", overwrite=False, @@ -513,9 +513,9 @@ def tweakreg_align( log.warning("astrometric_alignment_table should be set!") return True - # test Jay 19.11.2025 -> use external (agb/gaia) for nircam, internal cat for miri - # when using internal catalog + # use external catalog for nircam, internal catalog for miri if self.ref_long_filter is not None: + # when using internal catalog if self.band in self.ref_long_filter: log.info(f"Aligning to internal catalog from shorter wavelength ({self.ref_long_filter[self.band]})") align_cat_dir = self.in_dir.replace(f"{self.band}", f"{self.ref_long_filter[self.band]}") @@ -548,7 +548,7 @@ def tweakreg_align( align_table = QTable.read(align_catalog, format="fits") ref_tab = Table() - # when only using external AGB catalog + # when only using external catalog if self.ref_long_filter is None: ref_tab["RA"] = align_table["ra"] ref_tab["DEC"] = align_table["dec"] @@ -570,7 +570,7 @@ def tweakreg_align( align_coord = ref_long_wcs.pixel_to_world(ref_x, ref_y) ref_tab["RA"] = align_coord.ra.deg ref_tab["DEC"] = align_coord.dec.deg - # nircam -> agb + # nircam -> external elif self.band not in self.ref_long_filter: ref_tab["RA"] = align_table["ra"] ref_tab["DEC"] = align_table["dec"] From eb6088faa9800d12583510b0f5478b305ea9c252 Mon Sep 17 00:00:00 2001 From: JaysonAstro Date: Mon, 15 Dec 2025 12:19:10 +0000 Subject: [PATCH 8/9] Included QA plots for astrometric align step --- .../astrometric_align_step.py | 118 ++++++++++++++++++ pjpipe/utils/__init__.py | 2 + pjpipe/utils/utils.py | 6 + 3 files changed, 126 insertions(+) diff --git a/pjpipe/astrometric_align/astrometric_align_step.py b/pjpipe/astrometric_align/astrometric_align_step.py index 3f400d3..31c7474 100644 --- a/pjpipe/astrometric_align/astrometric_align_step.py +++ b/pjpipe/astrometric_align/astrometric_align_step.py @@ -26,12 +26,16 @@ from tqdm import tqdm from tweakwcs import fit_wcs, XYXYMatch from tweakwcs.correctors import FITSWCSCorrector, JWSTWCSCorrector +import matplotlib.pyplot as plt +import astropy.units as u from ..utils import ( get_band_type, parse_parameter_dict, recursive_setattr, get_default_args, + fwhms_pix, + jwst_pixel_scales, ) ALLOWED_REPROJECT_FUNCS = [ @@ -205,6 +209,7 @@ def __init__( tweakreg_parameters=None, reproject_func="interp", overwrite=False, + qa_plots=False, ): """Perform absolute astrometric alignment @@ -240,6 +245,8 @@ def __init__( but can also be 'exact' or 'adaptive' overwrite: Whether to overwrite or not. Defaults to False + qa_plots: Whether to create Quality Assurance plots to assess the + level of mismatch in astrometry. """ if reproject_func not in ALLOWED_REPROJECT_FUNCS: @@ -268,6 +275,7 @@ def __init__( self.tweakreg_parameters = tweakreg_parameters self.reproject_func = reproject_func self.overwrite = overwrite + self.qa_plots = qa_plots def do_step(self): """Run absolute astrometric alignment""" @@ -678,6 +686,116 @@ def tweakreg_align( tp_units="pix", ) + # -------- + # QA plots + # -------- + if self.qa_plots: + import matplotlib as mpl + from matplotlib.patches import Circle + mpl.rcParams.update({ + "axes.labelsize": 14, + "axes.titlesize": 14, + "xtick.labelsize": 12, + "ytick.labelsize": 12, + "legend.fontsize": 12, + "font.family": "sans-serif", + "font.size": 14, + "savefig.dpi": 150, + "figure.dpi": 150, + "lines.linewidth": 1.2, + "xtick.direction": "in", + "ytick.direction": "in", + "xtick.major.size": 5, + "ytick.major.size": 5, + "axes.linewidth": 0.6, + }) + + qa_outdir = os.path.join(self.in_dir, "qa_alignment") + os.makedirs(qa_outdir, exist_ok=True) + log.info(f"Saving alignment QA plots in {qa_outdir}") + + band_type, short_long = get_band_type(self.band, short_long_nircam=True) + pixscale_band = jwst_pixel_scales[short_long] + fwhm_pix_band = fwhms_pix[self.band] + fwhm_asec_band = fwhm_pix_band * pixscale_band + + # read all coordinates from catalogs + ref_ra_all = ref_tab['RA'].data + ref_dec_all = ref_tab['DEC'].data + tgt_ra_all = target_tab['ra'].data + tgt_dec_all = target_tab['dec'].data + + # read only matched pairs + if len(ref_idx) > 0 and len(target_idx) > 0: + # get all, and matched coordinates + ref_ra_matched = ref_ra_all[ref_idx] + ref_dec_matched = ref_dec_all[ref_idx] + tgt_ra_matched = tgt_ra_all[target_idx] + tgt_dec_matched = tgt_dec_all[target_idx] + + # compute projected separation between matched pairs + skycoord_ref = SkyCoord(ra=ref_ra_matched * u.deg, dec=ref_dec_matched * u.deg) + skycoord_tgt = SkyCoord(ra=tgt_ra_matched * u.deg, dec=tgt_dec_matched * u.deg) + sep_arcsec = skycoord_ref.separation(skycoord_tgt).to(u.arcsec) + + # compute ra-dec offsets between matched pairs + delta_ra, delta_dec = skycoord_ref.spherical_offsets_to(skycoord_tgt) + delta_ra_asec, delta_dec_asec = delta_ra.to(u.arcsec), delta_dec.to(u.arcsec) + median_delta_ra, median_delta_dec = np.nanmedian(delta_ra_asec.value), np.nanmedian(delta_dec_asec.value) + + # plot all reference coordinates and matched target sources + fig, ax = plt.subplots(figsize=(8, 6)) + fig.suptitle(f"{self.target}: Matched coordinates in {self.band} to {align_catalog.split('/')[-1]}") + ax.scatter(ref_ra_all, ref_dec_all, edgecolor='black', facecolor='none', marker='o', s=1.5, label="ref coords") + ax.scatter(tgt_ra_matched, tgt_dec_matched, color='red', marker='o', s=1, label="matched sources") + ax.set_xlabel("RA (deg)") + ax.set_ylabel("Dec (deg)") + ax.legend() + fig.tight_layout() + fig.savefig(os.path.join(qa_outdir, f"{self.target}_{self.band}_catalog_compare.png"), dpi=150) + plt.close(fig) + + # plot ra-dec offsets + fig = plt.figure(figsize=(15,5)) + fig.suptitle(f"{self.target}: Offsets between {self.band} to {align_catalog.split('/')[-1]}") + ax1 = fig.add_subplot(131) + ax1.hist(delta_ra_asec, edgecolor='black', color='gray') + ax1.axvline(median_delta_ra, color='red', lw=1, label=f'median {median_delta_ra:.4f}') + ax1.set_xlabel("RA offset arcsec") + ax1.set_ylabel("count") + ax1.legend() + ax2 = fig.add_subplot(132) + ax2.hist(delta_dec_asec, edgecolor='black', color='gray') + ax2.axvline(median_delta_dec, color='red', lw=1, label=f'median {median_delta_dec:.4f}') + ax2.set_xlabel("Dec offset arcsec") + ax2.set_ylabel("count") + ax2.legend() + # for scatterplot include pixel scale and fwhm in arcsec + # from Tom's scatterball plot example + pixelsquare_x = [pixscale_band / 2. , pixscale_band / 2., + -1. * pixscale_band / 2., -1. * pixscale_band / 2., + pixscale_band / 2.] + pixelsquare_y = [pixscale_band / 2., -1. * pixscale_band / 2., + -1. * pixscale_band / 2., pixscale_band / 2., + pixscale_band / 2.] + fwhm_circle = Circle((0., 0.), 0.5 * fwhm_asec_band, fill=False, color='red', + linestyle='dotted') + ax3 = fig.add_subplot(133) + ax3.axvline(0, color='gray', alpha=0.7, ls='--') + ax3.axhline(0, color='gray', alpha=0.7, ls='--') + ax3.plot(pixelsquare_x, pixelsquare_y, c='darkgrey') + ax3.add_patch(fwhm_circle) + ax3.axvline(median_delta_ra, color='red', lw=1) + ax3.axhline(median_delta_dec, color='red') + ax3.scatter(delta_ra_asec, delta_dec_asec, marker='o', edgecolor='black', color='gray', s=1) + ax3.set_xlabel("RA offset arcsec") + ax3.set_ylabel("Dec offset arcsec") + ax3.set_xlim(-0.05, 0.05) + ax3.set_ylim(-0.05, 0.05) + plt.tight_layout() + plt.savefig(os.path.join(qa_outdir, f"{self.target}_{self.band}_coord_offsets.png"), dpi=150) + plt.close(fig) + fit_wcs_args = get_default_args(fit_wcs) fit_wcs_kws = {} diff --git a/pjpipe/utils/__init__.py b/pjpipe/utils/__init__.py index 767e6d8..38ce1bf 100644 --- a/pjpipe/utils/__init__.py +++ b/pjpipe/utils/__init__.py @@ -11,6 +11,7 @@ parse_fits_to_table, get_dq_bit_mask, fwhms_pix, + jwst_pixel_scales, parse_parameter_dict, recursive_setattr, make_source_mask, @@ -36,6 +37,7 @@ "parse_fits_to_table", "get_dq_bit_mask", "fwhms_pix", + "jwst_pixel_scales", "parse_parameter_dict", "recursive_setattr", "make_source_mask", diff --git a/pjpipe/utils/utils.py b/pjpipe/utils/utils.py index 991b519..7d0486c 100644 --- a/pjpipe/utils/utils.py +++ b/pjpipe/utils/utils.py @@ -1002,6 +1002,9 @@ def do_jwst_convolution( image_hdu["ERR"].data = conv_err if output_grid is None: + dirpath = os.path.dirname(file_out) + if dirpath: + os.makedirs(dirpath, exist_ok=True) image_hdu.writeto(file_out, overwrite=True) else: # Reprojection to target wcs grid define in output_grid @@ -1030,6 +1033,9 @@ def do_jwst_convolution( header = image_hdu["ERR"].header hdulist_out.append(fits.ImageHDU(data=repr_err, header=header, name="ERR")) + dirpath = os.path.dirname(file_out) + if dirpath: + os.makedirs(dirpath, exist_ok=True) hdulist_out.writeto(file_out, overwrite=True) From c6a9e6f2094cc6f7ca58fa186a3fd37ccdad9190 Mon Sep 17 00:00:00 2001 From: JaysonAstro Date: Mon, 15 Dec 2025 12:23:27 +0000 Subject: [PATCH 9/9] included QA plots for astrometric align step --- .../astrometric_align_step.py | 118 ++++++++++++++++++ pjpipe/utils/__init__.py | 2 + 2 files changed, 120 insertions(+) diff --git a/pjpipe/astrometric_align/astrometric_align_step.py b/pjpipe/astrometric_align/astrometric_align_step.py index 3f400d3..31c7474 100644 --- a/pjpipe/astrometric_align/astrometric_align_step.py +++ b/pjpipe/astrometric_align/astrometric_align_step.py @@ -26,12 +26,16 @@ from tqdm import tqdm from tweakwcs import fit_wcs, XYXYMatch from tweakwcs.correctors import FITSWCSCorrector, JWSTWCSCorrector +import matplotlib.pyplot as plt +import astropy.units as u from ..utils import ( get_band_type, parse_parameter_dict, recursive_setattr, get_default_args, + fwhms_pix, + jwst_pixel_scales, ) ALLOWED_REPROJECT_FUNCS = [ @@ -205,6 +209,7 @@ def __init__( tweakreg_parameters=None, reproject_func="interp", overwrite=False, + qa_plots=False, ): """Perform absolute astrometric alignment @@ -240,6 +245,8 @@ def __init__( but can also be 'exact' or 'adaptive' overwrite: Whether to overwrite or not. Defaults to False + qa_plots: Whether to create Quality Assurance plots to assess the + level of mismatch in astrometry. """ if reproject_func not in ALLOWED_REPROJECT_FUNCS: @@ -268,6 +275,7 @@ def __init__( self.tweakreg_parameters = tweakreg_parameters self.reproject_func = reproject_func self.overwrite = overwrite + self.qa_plots = qa_plots def do_step(self): """Run absolute astrometric alignment""" @@ -678,6 +686,116 @@ def tweakreg_align( tp_units="pix", ) + # -------- + # QA plots + # -------- + if self.qa_plots: + import matplotlib as mpl + from matplotlib.patches import Circle + mpl.rcParams.update({ + "axes.labelsize": 14, + "axes.titlesize": 14, + "xtick.labelsize": 12, + "ytick.labelsize": 12, + "legend.fontsize": 12, + "font.family": "sans-serif", + "font.size": 14, + "savefig.dpi": 150, + "figure.dpi": 150, + "lines.linewidth": 1.2, + "xtick.direction": "in", + "ytick.direction": "in", + "xtick.major.size": 5, + "ytick.major.size": 5, + "axes.linewidth": 0.6, + }) + + qa_outdir = os.path.join(self.in_dir, "qa_alignment") + os.makedirs(qa_outdir, exist_ok=True) + log.info(f"Saving alignment QA plots in {qa_outdir}") + + band_type, short_long = get_band_type(self.band, short_long_nircam=True) + pixscale_band = jwst_pixel_scales[short_long] + fwhm_pix_band = fwhms_pix[self.band] + fwhm_asec_band = fwhm_pix_band * pixscale_band + + # read all coordinates from catalogs + ref_ra_all = ref_tab['RA'].data + ref_dec_all = ref_tab['DEC'].data + tgt_ra_all = target_tab['ra'].data + tgt_dec_all = target_tab['dec'].data + + # read only matched pairs + if len(ref_idx) > 0 and len(target_idx) > 0: + # get all, and matched coordinates + ref_ra_matched = ref_ra_all[ref_idx] + ref_dec_matched = ref_dec_all[ref_idx] + tgt_ra_matched = tgt_ra_all[target_idx] + tgt_dec_matched = tgt_dec_all[target_idx] + + # compute projected separation between matched pairs + skycoord_ref = SkyCoord(ra=ref_ra_matched * u.deg, dec=ref_dec_matched * u.deg) + skycoord_tgt = SkyCoord(ra=tgt_ra_matched * u.deg, dec=tgt_dec_matched * u.deg) + sep_arcsec = skycoord_ref.separation(skycoord_tgt).to(u.arcsec) + + # compute ra-dec offsets between matched pairs + delta_ra, delta_dec = skycoord_ref.spherical_offsets_to(skycoord_tgt) + delta_ra_asec, delta_dec_asec = delta_ra.to(u.arcsec), delta_dec.to(u.arcsec) + median_delta_ra, median_delta_dec = np.nanmedian(delta_ra_asec.value), np.nanmedian(delta_dec_asec.value) + + # plot all reference coordinates and matched target sources + fig, ax = plt.subplots(figsize=(8, 6)) + fig.suptitle(f"{self.target}: Matched coordinates in {self.band} to {align_catalog.split('/')[-1]}") + ax.scatter(ref_ra_all, ref_dec_all, edgecolor='black', facecolor='none', marker='o', s=1.5, label="ref coords") + ax.scatter(tgt_ra_matched, tgt_dec_matched, color='red', marker='o', s=1, label="matched sources") + ax.set_xlabel("RA (deg)") + ax.set_ylabel("Dec (deg)") + ax.legend() + fig.tight_layout() + fig.savefig(os.path.join(qa_outdir, f"{self.target}_{self.band}_catalog_compare.png"), dpi=150) + plt.close(fig) + + # plot ra-dec offsets + fig = plt.figure(figsize=(15,5)) + fig.suptitle(f"{self.target}: Offsets between {self.band} to {align_catalog.split('/')[-1]}") + ax1 = fig.add_subplot(131) + ax1.hist(delta_ra_asec, edgecolor='black', color='gray') + ax1.axvline(median_delta_ra, color='red', lw=1, label=f'median {median_delta_ra:.4f}') + ax1.set_xlabel("RA offset arcsec") + ax1.set_ylabel("count") + ax1.legend() + ax2 = fig.add_subplot(132) + ax2.hist(delta_dec_asec, edgecolor='black', color='gray') + ax2.axvline(median_delta_dec, color='red', lw=1, label=f'median {median_delta_dec:.4f}') + ax2.set_xlabel("Dec offset arcsec") + ax2.set_ylabel("count") + ax2.legend() + # for scatterplot include pixel scale and fwhm in arcsec + # from Tom's scatterball plot example + pixelsquare_x = [pixscale_band / 2. , pixscale_band / 2., + -1. * pixscale_band / 2., -1. * pixscale_band / 2., + pixscale_band / 2.] + pixelsquare_y = [pixscale_band / 2., -1. * pixscale_band / 2., + -1. * pixscale_band / 2., pixscale_band / 2., + pixscale_band / 2.] + fwhm_circle = Circle((0., 0.), 0.5 * fwhm_asec_band, fill=False, color='red', + linestyle='dotted') + ax3 = fig.add_subplot(133) + ax3.axvline(0, color='gray', alpha=0.7, ls='--') + ax3.axhline(0, color='gray', alpha=0.7, ls='--') + ax3.plot(pixelsquare_x, pixelsquare_y, c='darkgrey') + ax3.add_patch(fwhm_circle) + ax3.axvline(median_delta_ra, color='red', lw=1) + ax3.axhline(median_delta_dec, color='red') + ax3.scatter(delta_ra_asec, delta_dec_asec, marker='o', edgecolor='black', color='gray', s=1) + ax3.set_xlabel("RA offset arcsec") + ax3.set_ylabel("Dec offset arcsec") + ax3.set_xlim(-0.05, 0.05) + ax3.set_ylim(-0.05, 0.05) + plt.tight_layout() + plt.savefig(os.path.join(qa_outdir, f"{self.target}_{self.band}_coord_offsets.png"), dpi=150) + plt.close(fig) + fit_wcs_args = get_default_args(fit_wcs) fit_wcs_kws = {} diff --git a/pjpipe/utils/__init__.py b/pjpipe/utils/__init__.py index 767e6d8..38ce1bf 100644 --- a/pjpipe/utils/__init__.py +++ b/pjpipe/utils/__init__.py @@ -11,6 +11,7 @@ parse_fits_to_table, get_dq_bit_mask, fwhms_pix, + jwst_pixel_scales, parse_parameter_dict, recursive_setattr, make_source_mask, @@ -36,6 +37,7 @@ "parse_fits_to_table", "get_dq_bit_mask", "fwhms_pix", + "jwst_pixel_scales", "parse_parameter_dict", "recursive_setattr", "make_source_mask",