From 65376ad38b81e012887eb423ffe711e0fd80c46f Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 15 Aug 2024 09:13:22 -0400 Subject: [PATCH 01/10] only load patch table when needed --- romancal/patch_match/patch_match.py | 5 ++--- romancal/pipeline/mosaic_pipeline.py | 4 ++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/romancal/patch_match/patch_match.py b/romancal/patch_match/patch_match.py index 1eeac86eb..d456a238d 100644 --- a/romancal/patch_match/patch_match.py +++ b/romancal/patch_match/patch_match.py @@ -83,6 +83,8 @@ def find_patch_matches(image_corners, image_shape=None): necessary information about the patches. """ + if PATCH_TABLE is None: + load_patch_table() if PATCH_TABLE is None: raise RuntimeError("No patch table has been loaded") if isinstance(image_corners, wcs.WCS): @@ -250,6 +252,3 @@ def veccoords_to_tangent_plane(vertices, tangent_point_vec): x_coords = np.dot(x_axis, avertices) * RAD_TO_ARCSEC y_coords = np.dot(y_axis, avertices) * RAD_TO_ARCSEC return x_coords, y_coords - - -load_patch_table() diff --git a/romancal/pipeline/mosaic_pipeline.py b/romancal/pipeline/mosaic_pipeline.py index 707faa0ac..6ea40e26f 100644 --- a/romancal/pipeline/mosaic_pipeline.py +++ b/romancal/pipeline/mosaic_pipeline.py @@ -88,6 +88,10 @@ def process(self, input): # if this is a valid skycell name load the database and get the skycell record if re.match(r"r\d{3}\w{2}\d{2}x\d{2}y\d{2}", skycell_name): + if patch_match.PATCH_TABLE is None: + patch_match.load_patch_table() + if patch_match.PATCH_TABLE is None: + raise RuntimeError("No patch table has been loaded") skycell_record = patch_match.PATCH_TABLE[ np.where(patch_match.PATCH_TABLE["name"][:] == skycell_name)[0][0] ] From 1993203d6faa5146d432ef3fe6910cac98297d99 Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 15 Aug 2024 09:16:40 -0400 Subject: [PATCH 02/10] add changelog --- CHANGES.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 2d3daac62..166b2a524 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,11 @@ +0.16.3 (unreleased) +=================== + +mosaic_pipeline +--------------- + +- Only load patch table when needed. [#1367] + 0.16.2 (2024-08-23) =================== From 6d25e71630399cc070966a2c796c3df6d4c95324 Mon Sep 17 00:00:00 2001 From: Eddie Schlafly Date: Wed, 28 Aug 2024 08:56:36 -0400 Subject: [PATCH 03/10] Populate segmentation model metadata for exposures and mosaics. --- romancal/source_catalog/source_catalog_step.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/romancal/source_catalog/source_catalog_step.py b/romancal/source_catalog/source_catalog_step.py index e1d0f32f6..8dd766688 100644 --- a/romancal/source_catalog/source_catalog_step.py +++ b/romancal/source_catalog/source_catalog_step.py @@ -146,9 +146,16 @@ def process(self, input_model): def save_base_results(self, segment_img, source_catalog_model): # save the segmentation map and output_filename = source_catalog_model.meta.filename - segmentation_model = maker_utils.mk_datamodel( - datamodels.MosaicSegmentationMapModel - ) + + if isinstance(source_catalog_model, datamodels.SourceCatalogModel): + seg_model = datamodels.SegmentationMapModel + else: + seg_model = datamodels.MosaicSegmentationMapModel + + segmentation_model = maker_utils.mk_datamodel(seg_model) + for key in segmentation_model.meta.keys(): + segmentation_model.meta[key] = source_catalog_model.meta[key] + if segment_img is not None: segmentation_model.data = segment_img.data.astype(np.uint32) self.save_model( From 9342692d0d4ac71c0c18e81c5af5b7902e7549a3 Mon Sep 17 00:00:00 2001 From: Eddie Schlafly Date: Wed, 28 Aug 2024 09:00:23 -0400 Subject: [PATCH 04/10] Add changelog. --- CHANGES.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 166b2a524..2dd32e56c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,6 +6,11 @@ mosaic_pipeline - Only load patch table when needed. [#1367] +source_catalog +-------------- + +- Populate segmentation image metadata. [#1391] + 0.16.2 (2024-08-23) =================== From 94fda06b135257d1bd17546d41967c01838fd8e9 Mon Sep 17 00:00:00 2001 From: Eddie Schlafly Date: Wed, 28 Aug 2024 10:42:35 -0400 Subject: [PATCH 05/10] Fix catalog and segementation file names from mosaic pipeline. Use association name for output meta.filename by default in resample. --- CHANGES.rst | 5 +++++ romancal/pipeline/mosaic_pipeline.py | 6 ++---- romancal/resample/resample_step.py | 13 ++++++++----- romancal/source_catalog/source_catalog_step.py | 4 +++- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2dd32e56c..fbfe17173 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -11,6 +11,11 @@ source_catalog - Populate segmentation image metadata. [#1391] +resample +-------- + +- Use association product name for output meta.filename by default [#1391] + 0.16.2 (2024-08-23) =================== diff --git a/romancal/pipeline/mosaic_pipeline.py b/romancal/pipeline/mosaic_pipeline.py index 6ea40e26f..5b0acc736 100644 --- a/romancal/pipeline/mosaic_pipeline.py +++ b/romancal/pipeline/mosaic_pipeline.py @@ -132,9 +132,7 @@ def process(self, input): wcs_file = asdf.open(self.resample.output_wcs) self.suffix = "i2d" result = self.resample(result) - self.output_file = input.asn["products"][0]["name"] - # force the SourceCatalogStep to save the results - self.sourcecatalog.save_results = True + self.sourcecatalog.output_file = input.asn["products"][0]["name"] result_catalog = self.sourcecatalog(result) else: log.info("resampling a mosaic file is not yet supported") @@ -144,7 +142,7 @@ def process(self, input): self.resample.suffix = "i2d" self.output_file = input.asn["products"][0]["name"] result = self.resample(result) - self.sourcecatalog.save_results = True + self.sourcecatalog.output_file = self.output_file result_catalog = self.sourcecatalog(result) # noqa: F841 self.suffix = "i2d" if input_filename: diff --git a/romancal/resample/resample_step.py b/romancal/resample/resample_step.py index 79b9587f4..5315f150d 100644 --- a/romancal/resample/resample_step.py +++ b/romancal/resample/resample_step.py @@ -83,11 +83,14 @@ def process(self, input): output = input_models.asn["products"][0]["name"] elif isinstance(input, ModelLibrary): input_models = input - # set output filename using the common prefix of all datamodels - output = f"{os.path.commonprefix([x['expname'] for x in input_models.asn['products'][0]['members']])}.asdf" - if len(output) == 0: - # set default filename if no common prefix can be determined - output = "resample_output.asdf" + if "name" in input_models.asn["products"][0]: + output = input_models.asn["products"][0]["name"] + else: + # set output filename using the common prefix of all datamodels + output = f"{os.path.commonprefix([x['expname'] for x in input_models.asn['products'][0]['members']])}.asdf" + if len(output) == 0: + # set default filename if no common prefix can be determined + output = "resample_output.asdf" else: raise TypeError( "Input must be an ASN filename, a ModelLibrary, " diff --git a/romancal/source_catalog/source_catalog_step.py b/romancal/source_catalog/source_catalog_step.py index 8dd766688..711df96cf 100644 --- a/romancal/source_catalog/source_catalog_step.py +++ b/romancal/source_catalog/source_catalog_step.py @@ -145,7 +145,9 @@ def process(self, input_model): def save_base_results(self, segment_img, source_catalog_model): # save the segmentation map and - output_filename = source_catalog_model.meta.filename + output_filename = (self.output_file + if self.output_file is not None + else source_catalog_model.meta.filename) if isinstance(source_catalog_model, datamodels.SourceCatalogModel): seg_model = datamodels.SegmentationMapModel From 4a3ef45024beb0984fa73e2ea304f1cdb4df5f6b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 14:43:06 +0000 Subject: [PATCH 06/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- romancal/pipeline/mosaic_pipeline.py | 4 +++- romancal/source_catalog/source_catalog_step.py | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/romancal/pipeline/mosaic_pipeline.py b/romancal/pipeline/mosaic_pipeline.py index 5b0acc736..de368f5cf 100644 --- a/romancal/pipeline/mosaic_pipeline.py +++ b/romancal/pipeline/mosaic_pipeline.py @@ -132,7 +132,9 @@ def process(self, input): wcs_file = asdf.open(self.resample.output_wcs) self.suffix = "i2d" result = self.resample(result) - self.sourcecatalog.output_file = input.asn["products"][0]["name"] + self.sourcecatalog.output_file = input.asn["products"][0][ + "name" + ] result_catalog = self.sourcecatalog(result) else: log.info("resampling a mosaic file is not yet supported") diff --git a/romancal/source_catalog/source_catalog_step.py b/romancal/source_catalog/source_catalog_step.py index 711df96cf..9946d037b 100644 --- a/romancal/source_catalog/source_catalog_step.py +++ b/romancal/source_catalog/source_catalog_step.py @@ -145,9 +145,11 @@ def process(self, input_model): def save_base_results(self, segment_img, source_catalog_model): # save the segmentation map and - output_filename = (self.output_file - if self.output_file is not None - else source_catalog_model.meta.filename) + output_filename = ( + self.output_file + if self.output_file is not None + else source_catalog_model.meta.filename + ) if isinstance(source_catalog_model, datamodels.SourceCatalogModel): seg_model = datamodels.SegmentationMapModel From 0543d782dbccb25b1674cbe67401e673098f0597 Mon Sep 17 00:00:00 2001 From: Eddie Schlafly Date: Wed, 28 Aug 2024 13:07:44 -0400 Subject: [PATCH 07/10] Keep output_file set appropriately for resample step. --- romancal/pipeline/mosaic_pipeline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/romancal/pipeline/mosaic_pipeline.py b/romancal/pipeline/mosaic_pipeline.py index 5b0acc736..36effdcbd 100644 --- a/romancal/pipeline/mosaic_pipeline.py +++ b/romancal/pipeline/mosaic_pipeline.py @@ -131,8 +131,9 @@ def process(self, input): ) wcs_file = asdf.open(self.resample.output_wcs) self.suffix = "i2d" + self.output_file = input.asn["products"][0]["name"] result = self.resample(result) - self.sourcecatalog.output_file = input.asn["products"][0]["name"] + self.sourcecatalog.output_file = self.output_file result_catalog = self.sourcecatalog(result) else: log.info("resampling a mosaic file is not yet supported") From 8fbf6b675c82db51dc2e4baa0ad8f92dcc983386 Mon Sep 17 00:00:00 2001 From: Eddie Schlafly Date: Wed, 28 Aug 2024 14:00:35 -0400 Subject: [PATCH 08/10] Change test to check for correct SegmentationModel type. --- romancal/source_catalog/tests/test_source_catalog.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/romancal/source_catalog/tests/test_source_catalog.py b/romancal/source_catalog/tests/test_source_catalog.py index f9e0b62d0..f1dfd15bf 100644 --- a/romancal/source_catalog/tests/test_source_catalog.py +++ b/romancal/source_catalog/tests/test_source_catalog.py @@ -14,6 +14,7 @@ MosaicModel, MosaicSegmentationMapModel, MosaicSourceCatalogModel, + SegmentationMapModel, SourceCatalogModel, ) from roman_datamodels.maker_utils import mk_level2_image, mk_level3_mosaic @@ -461,7 +462,7 @@ def test_do_psf_photometry_column_names(tmp_path, image_model, fit_psf): ImageModel, { "cat": SourceCatalogModel, - "segm": MosaicSegmentationMapModel, + "segm": SegmentationMapModel, "sourcecatalog": ImageModel, }, ), @@ -474,7 +475,7 @@ def test_do_psf_photometry_column_names(tmp_path, image_model, fit_psf): SourceCatalogModel, { "cat": SourceCatalogModel, - "segm": MosaicSegmentationMapModel, + "segm": SegmentationMapModel, }, ), ( @@ -486,7 +487,7 @@ def test_do_psf_photometry_column_names(tmp_path, image_model, fit_psf): ImageModel, { "cat": SourceCatalogModel, - "segm": MosaicSegmentationMapModel, + "segm": SegmentationMapModel, }, ), ( @@ -498,7 +499,7 @@ def test_do_psf_photometry_column_names(tmp_path, image_model, fit_psf): SourceCatalogModel, { "cat": SourceCatalogModel, - "segm": MosaicSegmentationMapModel, + "segm": SegmentationMapModel, }, ), ), From 590179f47e170f34db58c67a9b1d56f1a23db439 Mon Sep 17 00:00:00 2001 From: Eddie Schlafly Date: Thu, 29 Aug 2024 09:10:46 -0400 Subject: [PATCH 09/10] Only delete files with outlier_suffix at the end, rather than all suffix files. --- romancal/outlier_detection/outlier_detection_step.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/romancal/outlier_detection/outlier_detection_step.py b/romancal/outlier_detection/outlier_detection_step.py index 1441bc903..332791e4b 100644 --- a/romancal/outlier_detection/outlier_detection_step.py +++ b/romancal/outlier_detection/outlier_detection_step.py @@ -152,7 +152,7 @@ def process(self, input_models): intermediate_files_suffixes = ( "*blot.asdf", "*median.asdf", - f'*{pars.get("resample_suffix")}*.asdf', + f'*outlier_{pars.get("resample_suffix")}.asdf', ) for current_path in intermediate_files_paths: for suffix in intermediate_files_suffixes: From 71407e93d771f49280b93cf016f1baad577a5d11 Mon Sep 17 00:00:00 2001 From: zacharyburnett Date: Thu, 29 Aug 2024 10:33:16 -0400 Subject: [PATCH 10/10] freeze change log for 0.16.3 --- CHANGES.rst | 2 +- README.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index fbfe17173..a6c1b6845 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,4 +1,4 @@ -0.16.3 (unreleased) +0.16.3 (2024-08-29) =================== mosaic_pipeline diff --git a/README.md b/README.md index 8cf41a1dc..1ecbb1740 100644 --- a/README.md +++ b/README.md @@ -170,6 +170,7 @@ an [issue](https://github.com/spacetelescope/romancal/issues). | 0.16.0 | 24Q4_B15 | 063 | Aug 2024 | Release for Build 24Q3_B15 (Build 15) | | 0.16.1 | 24Q4_B15 | 063 | Aug 2024 | Release for Build 24Q3_B15 (Build 15) | | 0.16.2 | 24Q4_B15 | 063 | Aug 2024 | Release for Build 24Q3_B15 (Build 15) | +| 0.16.3 | 24Q4_B15 | 063 | Aug 2024 | Release for Build 24Q3_B15 (Build 15) | Note: CRDS_CONTEXT values flagged with an asterisk in the above table are estimates