From 77cbd2e5689ee51e9038190ee912e8bcb44f684b Mon Sep 17 00:00:00 2001 From: Thomas WIlliams Date: Wed, 20 Aug 2025 11:58:27 +0100 Subject: [PATCH] Bugfixing - Fixed issue with background files in ``move_raw_obs_step`` - Fixed logging, now will properly save to file - Fixed issue with filtering guide stars in ``download_step`` - Fixed crash for ``astrometric_catalog`` for background observations --- CHANGES.rst | 4 ++++ pjpipe/__init__.py | 18 +++++++++++++++++- pjpipe/anchoring/anchoring_step.py | 3 +-- .../apply_wcs_adjust/apply_wcs_adjust_step.py | 3 +-- .../astrometric_align_step.py | 3 +-- .../astrometric_catalog_step.py | 9 +++++---- pjpipe/download/download_step.py | 5 ++--- pjpipe/gaia_query/gaia_query_step.py | 3 +-- pjpipe/get_wcs_adjust/custom_catalogs.py | 3 +-- pjpipe/get_wcs_adjust/get_wcs_adjust_step.py | 3 +-- pjpipe/level_match/level_match_step.py | 3 +-- pjpipe/lv1/lv1_step.py | 3 +-- pjpipe/lv2/lv2_step.py | 3 +-- pjpipe/lv3/lv3_step.py | 3 +-- pjpipe/lyot_mask/lyot_mask_step.py | 3 +-- pjpipe/lyot_separate/lyot_separate_step.py | 3 +-- .../mosaic_individual_fields_step.py | 3 +-- pjpipe/move_raw_obs/move_raw_obs_step.py | 5 ++--- .../multi_tile_destripe_step.py | 3 +-- pjpipe/pipeline.py | 3 +-- pjpipe/psf_matching/psf_matching_step.py | 3 +-- pjpipe/psf_model/psf_model_step.py | 3 +-- .../regress_against_previous_step.py | 3 +-- pjpipe/release/release_step.py | 3 +-- .../single_tile_destripe_step.py | 3 +-- pjpipe/utils/utils.py | 3 +-- 26 files changed, 51 insertions(+), 53 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 3d3c4d5..28f082f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,10 @@ 1.2.1 (Unreleased) ================== +- Fixed crash for ``astrometric_catalog`` for background observations +- Fixed crash when filtering out guide stars +- Fixed issue with background files in ``move_raw_obs_step`` +- Fixed logging, will now properly output to a log file - Fix crash for background observations in ``get_short_band_name`` - For background association, strip whitespace and lowercase for more robust matching - Imports update - now supports numpy>2.0! diff --git a/pjpipe/__init__.py b/pjpipe/__init__.py index a420636..fe59910 100644 --- a/pjpipe/__init__.py +++ b/pjpipe/__init__.py @@ -1,7 +1,9 @@ +import logging import os -import re import sys +from datetime import datetime from importlib.metadata import version +from logging.handlers import RotatingFileHandler # Set the CRDS server URL before any imports os.environ["CRDS_SERVER_URL"] = "https://jwst-crds.stsci.edu" @@ -12,6 +14,20 @@ # Get the version __version__ = version(__name__) +# Set up the logger +log = logging.getLogger(__name__) +dt = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") +log_file = f"pjpipe_{dt}.log" +log_level = "INFO" +log_format = "%(asctime)s - %(levelname)s - %(message)s" +logging.basicConfig(level=logging.INFO, + format=log_format, + handlers=[ + logging.FileHandler(log_file), + logging.StreamHandler(), + ] + ) + from .apply_wcs_adjust import ApplyWCSAdjustStep from .astrometric_align import AstrometricAlignStep from .astrometric_catalog import AstrometricCatalogStep diff --git a/pjpipe/anchoring/anchoring_step.py b/pjpipe/anchoring/anchoring_step.py index e4a6d45..9860fc0 100644 --- a/pjpipe/anchoring/anchoring_step.py +++ b/pjpipe/anchoring/anchoring_step.py @@ -34,8 +34,7 @@ matplotlib.rcParams['font.family'] = 'STIXGeneral' matplotlib.rcParams['font.size'] = 14 -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) def line_func_curvefit(x, b, m): diff --git a/pjpipe/apply_wcs_adjust/apply_wcs_adjust_step.py b/pjpipe/apply_wcs_adjust/apply_wcs_adjust_step.py index 54d0675..a1d9297 100644 --- a/pjpipe/apply_wcs_adjust/apply_wcs_adjust_step.py +++ b/pjpipe/apply_wcs_adjust/apply_wcs_adjust_step.py @@ -18,8 +18,7 @@ from ..utils import band_exts -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) class ApplyWCSAdjustStep: diff --git a/pjpipe/astrometric_align/astrometric_align_step.py b/pjpipe/astrometric_align/astrometric_align_step.py index 4cdf445..5eabeff 100644 --- a/pjpipe/astrometric_align/astrometric_align_step.py +++ b/pjpipe/astrometric_align/astrometric_align_step.py @@ -39,8 +39,7 @@ "exact", ] -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) def get_lv3_wcs(im): diff --git a/pjpipe/astrometric_catalog/astrometric_catalog_step.py b/pjpipe/astrometric_catalog/astrometric_catalog_step.py index c66ca39..f576d2c 100644 --- a/pjpipe/astrometric_catalog/astrometric_catalog_step.py +++ b/pjpipe/astrometric_catalog/astrometric_catalog_step.py @@ -10,12 +10,11 @@ from astropy.wcs import WCS from photutils.detection import DAOStarFinder, IRAFStarFinder -from ..utils import parse_parameter_dict, fwhms_pix, sigma_clip, recursive_setattr +from ..utils import parse_parameter_dict, fwhms_pix, sigma_clip, recursive_setattr, get_short_band_name from .constrained_diffusion import constrained_diffusion -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) ALLOWED_STARFIND_METHODS = [ "dao", @@ -143,7 +142,9 @@ def generate_astro_cat( mean, median, rms = sigma_clip(data, dq_mask=mask) threshold = median + snr * rms - kernel_fwhm = fwhms_pix[self.band] + # Make sure we only have the filter here + short_band = get_short_band_name(self.band) + kernel_fwhm = fwhms_pix[short_band] if self.starfind_method == "dao": finder = DAOStarFinder diff --git a/pjpipe/download/download_step.py b/pjpipe/download/download_step.py index 5b1a3c0..9b51f64 100644 --- a/pjpipe/download/download_step.py +++ b/pjpipe/download/download_step.py @@ -16,8 +16,7 @@ from stdatamodels.jwst import datamodels from tqdm import tqdm -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) def download( @@ -437,7 +436,7 @@ def get_products( # Filter out guide stars if requested if self.filter_gs: - mask = np.char.find(products["dataURI"], "_gs-") == -1 + mask = ["_gs-" not in x for x in products["dataURI"]] products = products[mask] # Perform filtering if requested diff --git a/pjpipe/gaia_query/gaia_query_step.py b/pjpipe/gaia_query/gaia_query_step.py index 5796827..bdc5e54 100644 --- a/pjpipe/gaia_query/gaia_query_step.py +++ b/pjpipe/gaia_query/gaia_query_step.py @@ -5,8 +5,7 @@ from astropy.coordinates import name_resolve from astroquery.gaia import Gaia -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) class GaiaQueryStep: diff --git a/pjpipe/get_wcs_adjust/custom_catalogs.py b/pjpipe/get_wcs_adjust/custom_catalogs.py index 40b0a7a..2c9a52c 100644 --- a/pjpipe/get_wcs_adjust/custom_catalogs.py +++ b/pjpipe/get_wcs_adjust/custom_catalogs.py @@ -6,8 +6,7 @@ from ..astrometric_catalog.constrained_diffusion import constrained_diffusion -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) def constrained_diffusion_catalog(datamodel, diff --git a/pjpipe/get_wcs_adjust/get_wcs_adjust_step.py b/pjpipe/get_wcs_adjust/get_wcs_adjust_step.py index 46f9810..979162a 100644 --- a/pjpipe/get_wcs_adjust/get_wcs_adjust_step.py +++ b/pjpipe/get_wcs_adjust/get_wcs_adjust_step.py @@ -26,8 +26,7 @@ from .custom_catalogs import constrained_diffusion_catalog -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) RAD_TO_ARCSEC = 3600 * np.rad2deg(1) diff --git a/pjpipe/level_match/level_match_step.py b/pjpipe/level_match/level_match_step.py index 45c2c39..d28a2ba 100644 --- a/pjpipe/level_match/level_match_step.py +++ b/pjpipe/level_match/level_match_step.py @@ -46,8 +46,7 @@ matplotlib.rcParams['font.family'] = 'STIXGeneral' matplotlib.rcParams['font.size'] = 14 -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) def get_dithers(files, diff --git a/pjpipe/lv1/lv1_step.py b/pjpipe/lv1/lv1_step.py index c36056d..b082690 100644 --- a/pjpipe/lv1/lv1_step.py +++ b/pjpipe/lv1/lv1_step.py @@ -15,8 +15,7 @@ from ..utils import attribute_setter, save_file -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) # Set OpenCV number of threads to 1 to avoid MP issues on newer Macbooks cv.setNumThreads(1) diff --git a/pjpipe/lv2/lv2_step.py b/pjpipe/lv2/lv2_step.py index 3914e50..83e243f 100644 --- a/pjpipe/lv2/lv2_step.py +++ b/pjpipe/lv2/lv2_step.py @@ -18,8 +18,7 @@ from ..utils import get_band_type, get_obs_table, attribute_setter, save_file -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) BGR_CHECK_TYPES = [ "parallel_off", diff --git a/pjpipe/lv3/lv3_step.py b/pjpipe/lv3/lv3_step.py index d258700..f9b8746 100644 --- a/pjpipe/lv3/lv3_step.py +++ b/pjpipe/lv3/lv3_step.py @@ -34,8 +34,7 @@ class ModelLibrary: get_short_band_name, ) -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) BGR_CHECK_TYPES = [ "parallel_off", diff --git a/pjpipe/lyot_mask/lyot_mask_step.py b/pjpipe/lyot_mask/lyot_mask_step.py index 3abe143..21f668d 100644 --- a/pjpipe/lyot_mask/lyot_mask_step.py +++ b/pjpipe/lyot_mask/lyot_mask_step.py @@ -15,8 +15,7 @@ from ..utils import get_dq_bit_mask -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) ALLOWED_METHODS = ["mask", "mask_overlap"] diff --git a/pjpipe/lyot_separate/lyot_separate_step.py b/pjpipe/lyot_separate/lyot_separate_step.py index 5c2b28a..784ca00 100644 --- a/pjpipe/lyot_separate/lyot_separate_step.py +++ b/pjpipe/lyot_separate/lyot_separate_step.py @@ -14,8 +14,7 @@ from ..utils import get_dq_bit_mask -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) # Rough lyot outline LYOT_I = slice(735, None) diff --git a/pjpipe/mosaic_individual_fields/mosaic_individual_fields_step.py b/pjpipe/mosaic_individual_fields/mosaic_individual_fields_step.py index e5df235..7d812e3 100644 --- a/pjpipe/mosaic_individual_fields/mosaic_individual_fields_step.py +++ b/pjpipe/mosaic_individual_fields/mosaic_individual_fields_step.py @@ -12,8 +12,7 @@ from ..utils import parse_parameter_dict, recursive_setattr -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) class MosaicIndividualFieldsStep: diff --git a/pjpipe/move_raw_obs/move_raw_obs_step.py b/pjpipe/move_raw_obs/move_raw_obs_step.py index 06aa78f..dca96f0 100644 --- a/pjpipe/move_raw_obs/move_raw_obs_step.py +++ b/pjpipe/move_raw_obs/move_raw_obs_step.py @@ -10,8 +10,7 @@ from ..utils import get_band_ext, get_band_type, save_file, get_short_band_name -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) class MoveRawObsStep: @@ -257,7 +256,7 @@ def move_raw_files( save_file(im=im, out_name=hdu_out_name, dr_version=self.dr_version) elif self.band_type == "miri": - if hdu_filter == self.band: + if hdu_filter == band_short: save_file(im=im, out_name=hdu_out_name, dr_version=self.dr_version) del im diff --git a/pjpipe/multi_tile_destripe/multi_tile_destripe_step.py b/pjpipe/multi_tile_destripe/multi_tile_destripe_step.py index 8eee9c2..6712818 100644 --- a/pjpipe/multi_tile_destripe/multi_tile_destripe_step.py +++ b/pjpipe/multi_tile_destripe/multi_tile_destripe_step.py @@ -31,8 +31,7 @@ matplotlib.rcParams['font.family'] = 'STIXGeneral' matplotlib.rcParams['font.size'] = 14 -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) ALLOWED_WEIGHT_METHODS = [ "mean", diff --git a/pjpipe/pipeline.py b/pjpipe/pipeline.py index 00d02bc..064d0f6 100644 --- a/pjpipe/pipeline.py +++ b/pjpipe/pipeline.py @@ -91,8 +91,7 @@ "release": None, } -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) class PJPipeline: diff --git a/pjpipe/psf_matching/psf_matching_step.py b/pjpipe/psf_matching/psf_matching_step.py index 82228a9..58b16cb 100644 --- a/pjpipe/psf_matching/psf_matching_step.py +++ b/pjpipe/psf_matching/psf_matching_step.py @@ -13,8 +13,7 @@ from ..utils import do_jwst_convolution -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) class PSFMatchingStep: diff --git a/pjpipe/psf_model/psf_model_step.py b/pjpipe/psf_model/psf_model_step.py index fb21555..eca598b 100644 --- a/pjpipe/psf_model/psf_model_step.py +++ b/pjpipe/psf_model/psf_model_step.py @@ -23,8 +23,7 @@ from ..utils import get_dq_bit_mask, make_source_mask matplotlib.use("agg") -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) ALLOWED_METHODS = ["replace", "subtract"] diff --git a/pjpipe/regress_against_previous/regress_against_previous_step.py b/pjpipe/regress_against_previous/regress_against_previous_step.py index 1bb3ebd..f84ba17 100644 --- a/pjpipe/regress_against_previous/regress_against_previous_step.py +++ b/pjpipe/regress_against_previous/regress_against_previous_step.py @@ -24,8 +24,7 @@ matplotlib.rcParams['font.family'] = 'STIXGeneral' matplotlib.rcParams['font.size'] = 14 -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) def get_diff_image( diff --git a/pjpipe/release/release_step.py b/pjpipe/release/release_step.py index bdb4bab..1b846b0 100644 --- a/pjpipe/release/release_step.py +++ b/pjpipe/release/release_step.py @@ -6,8 +6,7 @@ from astropy.io import fits from tqdm import tqdm -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) class ReleaseStep: diff --git a/pjpipe/single_tile_destripe/single_tile_destripe_step.py b/pjpipe/single_tile_destripe/single_tile_destripe_step.py index 6a7faa5..972bf19 100644 --- a/pjpipe/single_tile_destripe/single_tile_destripe_step.py +++ b/pjpipe/single_tile_destripe/single_tile_destripe_step.py @@ -33,8 +33,7 @@ matplotlib.rcParams['font.family'] = 'STIXGeneral' matplotlib.rcParams['font.size'] = 14 -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) DESTRIPING_METHODS = [ "row_median", diff --git a/pjpipe/utils/utils.py b/pjpipe/utils/utils.py index 6486a79..991b519 100644 --- a/pjpipe/utils/utils.py +++ b/pjpipe/utils/utils.py @@ -144,8 +144,7 @@ "miri": "mirimage", } -log = logging.getLogger("stpipe") -log.addHandler(logging.NullHandler()) +log = logging.getLogger(__name__) def get_pixscale(hdu):