diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8b58669b..8c1e33ca 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,7 @@ repos: - tomli - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 'v0.9.1' + rev: 'v0.9.2' hooks: - id: ruff args: ["--fix", "--show-fixes"] diff --git a/changes/453.feature.rst b/changes/453.feature.rst new file mode 100644 index 00000000..50c1f9e3 --- /dev/null +++ b/changes/453.feature.rst @@ -0,0 +1 @@ +Allow ``rdm.open`` to open file-like objects (like those returned by s3fs) diff --git a/src/roman_datamodels/datamodels/_core.py b/src/roman_datamodels/datamodels/_core.py index 4fd271ea..1b404af4 100644 --- a/src/roman_datamodels/datamodels/_core.py +++ b/src/roman_datamodels/datamodels/_core.py @@ -221,11 +221,11 @@ def save(self, path, dir_path=None, *args, **kwargs): return output_path def open_asdf(self, init=None, **kwargs): - from ._utils import _open_path_like + from ._utils import _open_asdf with validate.nuke_validation(): if isinstance(init, str): - return _open_path_like(init, **kwargs) + return _open_asdf(init, **kwargs) return asdf.AsdfFile(init, **kwargs) diff --git a/src/roman_datamodels/datamodels/_utils.py b/src/roman_datamodels/datamodels/_utils.py index 75745e5b..c4d7e3e4 100644 --- a/src/roman_datamodels/datamodels/_utils.py +++ b/src/roman_datamodels/datamodels/_utils.py @@ -23,16 +23,19 @@ class FilenameMismatchWarning(UserWarning): """ -def _open_path_like(init, lazy_tree=True, **kwargs): +def _open_asdf(init, lazy_tree=True, **kwargs): """ - Attempt to open init as if it was a path-like object. + Open init with `asdf.open`. + + If init is a path-like object the ``roman.meta.filename`` attribute + will be checked against ``Path.name`` and updated if they does not match. Parameters ---------- - init : str - Any path-like object that can be opened by asdf such as a valid string - memmap : bool - If we should open the file with memmap + init : str, ``Path`` or file-like + An object that can be opened by `asdf.open` + lazy_tree : bool + If we should open the file with a "lazy tree" **kwargs: Any additional arguments to pass to asdf.open @@ -43,7 +46,12 @@ def _open_path_like(init, lazy_tree=True, **kwargs): # asdf defaults to lazy_tree=False, this overwrites it to # lazy_tree=True for roman_datamodels kwargs["lazy_tree"] = lazy_tree - init = Path(init) + if isinstance(init, str): + path = Path(init) + elif isinstance(init, Path): + path = init + else: + path = None try: asdf_file = asdf.open(init, **kwargs) @@ -51,18 +59,19 @@ def _open_path_like(init, lazy_tree=True, **kwargs): raise TypeError("Open requires a filepath, file-like object, or Roman datamodel") from err if ( - "roman" in asdf_file + path is not None + and "roman" in asdf_file and isinstance(asdf_file["roman"], Mapping) # Fix issue for Python 3.10 and "meta" in asdf_file["roman"] and "filename" in asdf_file["roman"]["meta"] - and asdf_file["roman"]["meta"]["filename"] != init.name + and asdf_file["roman"]["meta"]["filename"] != path.name ): warnings.warn( - f"meta.filename: {asdf_file['roman']['meta']['filename']} does not match filename: {init.name}, updating the filename in memory!", + f"meta.filename: {asdf_file['roman']['meta']['filename']} does not match filename: {path.name}, updating the filename in memory!", FilenameMismatchWarning, stacklevel=2, ) - asdf_file["roman"]["meta"]["filename"] = init.name + asdf_file["roman"]["meta"]["filename"] = path.name return asdf_file @@ -75,11 +84,12 @@ def rdm_open(init, memmap=False, **kwargs): Parameters ---------- - init : str, `DataModel`, `asdf.AsdfFile` + init : str, ``Path``, `DataModel`, `asdf.AsdfFile`, file-like May be any one of the following types: - `asdf.AsdfFile` instance - - string indicating the path to an ASDF file + - string or ``Path`` indicating the path to an ASDF file - `DataModel` Roman data model instance + - file-like object compatible with `asdf.open` memmap : bool Open ASDF file binary data using memmap (default: False) @@ -104,7 +114,7 @@ def rdm_open(init, memmap=False, **kwargs): if "asn_n_members" in kwargs: del kwargs["asn_n_members"] - asdf_file = init if isinstance(init, asdf.AsdfFile) else _open_path_like(init, memmap=memmap, **kwargs) + asdf_file = init if isinstance(init, asdf.AsdfFile) else _open_asdf(init, memmap=memmap, **kwargs) if (model_type := type(asdf_file.tree["roman"])) in MODEL_REGISTRY: return MODEL_REGISTRY[model_type](asdf_file, **kwargs) diff --git a/tests/test_open.py b/tests/test_open.py index 5b94af4a..305a60d3 100644 --- a/tests/test_open.py +++ b/tests/test_open.py @@ -81,6 +81,17 @@ def test_model_input(tmp_path): reopened_model.close() +def test_file_input(tmp_path): + file_path = tmp_path / "test.asdf" + tree = utils.mk_level2_image(shape=(8, 8)) + with asdf.AsdfFile() as af: + af.tree = {"roman": tree} + af.write_to(file_path) + with open(file_path, "rb") as f: + with datamodels.open(f) as model: + assert model.meta.telescope == "ROMAN" + + def test_invalid_input(): with pytest.raises(TypeError): datamodels.open(fits.HDUList())