diff --git a/kerchunk/hdf.py b/kerchunk/hdf.py index 7d416f8..bc00517 100644 --- a/kerchunk/hdf.py +++ b/kerchunk/hdf.py @@ -2,7 +2,6 @@ import io import logging from typing import Union, BinaryIO -from packaging.version import Version import fsspec.core from fsspec.implementations.reference import LazyReferenceMapper @@ -10,10 +9,8 @@ import zarr import numcodecs -from kerchunk.zarr import dict_to_store - from .codecs import FillStringsCodec -from .utils import _encode_for_JSON, encode_fill_value +from .utils import _encode_for_JSON, encode_fill_value, dict_to_store try: import h5py diff --git a/kerchunk/tests/test_netcdf.py b/kerchunk/tests/test_netcdf.py index 0036c0a..755823d 100644 --- a/kerchunk/tests/test_netcdf.py +++ b/kerchunk/tests/test_netcdf.py @@ -1,5 +1,4 @@ import os -from typing import Any import fsspec @@ -8,7 +7,7 @@ import pytest from kerchunk import netCDF3 -import zarr +from kerchunk.utils import refs_as_store xr = pytest.importorskip("xarray") @@ -27,29 +26,19 @@ ) -def create_store(test_dict: dict, remote_options: Any = None): - if Version(zarr.__version__) < Version("3.0.0.a0"): - return fsspec.get_mapper( - "reference://", fo=test_dict, remote_protocol="s3", remote_options=remote_options - ) - else: - fs = fsspec.implementations.reference.ReferenceFileSystem(fo=test_dict, remote_options=remote_options) - return zarr.storage.RemoteStore(fs, mode="r") - - def test_one(m): m.pipe("data.nc3", bdata) h = netCDF3.netcdf_recording_file("memory://data.nc3") out = h.translate() - store = create_store(out, remote_options={"remote_protocol": "memory"}) + store = refs_as_store(out, remote_protocol="memory") ds = xr.open_dataset( store, engine="zarr", backend_kwargs={ "consolidated": False, - "zarr_format": "2", + "zarr_format": 2, }, ) assert (ds.data == data).all() diff --git a/kerchunk/utils.py b/kerchunk/utils.py index 59aad1a..c90f89f 100644 --- a/kerchunk/utils.py +++ b/kerchunk/utils.py @@ -1,6 +1,7 @@ import base64 import copy import itertools +from packaging.version import Version from typing import Any, cast import warnings @@ -10,8 +11,6 @@ import numpy as np import zarr -from kerchunk.zarr import fs_as_store - def refs_as_fs(refs, remote_protocol=None, remote_options=None, **kwargs): """Convert a reference set to an fsspec filesystem""" @@ -33,6 +32,38 @@ def refs_as_store(refs, remote_protocol=None, remote_options=None): return fs_as_store(fs) +def is_zarr3(): + """Check if the installed zarr version is version 3""" + return Version(zarr.__version__) >= Version("3.0.0.a0") + + +def dict_to_store(store_dict: dict): + """Create an in memory zarr store backed by the given dictionary""" + if is_zarr3(): + return zarr.storage.MemoryStore(mode="a", store_dict=store_dict) + else: + return zarr.storage.KVStore(store_dict) + + +def fs_as_store(fs, mode='r', remote_protocol=None, remote_options=None): + """Open the refs as a zarr store + + Parameters + ---------- + refs: dict-like + the references to open + mode: str + + Returns + ------- + zarr.storage.Store or zarr.storage.Mapper, fsspec.AbstractFileSystem + """ + if is_zarr3(): + return zarr.storage.RemoteStore(fs, mode=mode) + else: + return fs.get_mapper() + + def class_factory(func): """Experimental uniform API across function-based file scanners""" diff --git a/kerchunk/zarr.py b/kerchunk/zarr.py index 5560ea9..ea0612d 100644 --- a/kerchunk/zarr.py +++ b/kerchunk/zarr.py @@ -1,44 +1,9 @@ -from packaging.version import Version - import fsspec from fsspec.implementations.reference import LazyReferenceMapper -import zarr import kerchunk.utils -def is_zarr3(): - """Check if the installed zarr version is version 3""" - return Version(zarr.__version__) >= Version("3.0.0.a0") - - -def dict_to_store(store_dict: dict): - """Create an in memory zarr store backed by the given dictionary""" - if is_zarr3(): - return zarr.storage.MemoryStore(mode="a", store_dict=store_dict) - else: - return zarr.storage.KVStore(store_dict) - - -def fs_as_store(fs, mode='r', remote_protocol=None, remote_options=None): - """Open the refs as a zarr store - - Parameters - ---------- - refs: dict-like - the references to open - mode: str - - Returns - ------- - zarr.storage.Store or zarr.storage.Mapper, fsspec.AbstractFileSystem - """ - if is_zarr3(): - return zarr.storage.RemoteStore(fs, mode=mode) - else: - return fs.get_mapper() - - def single_zarr( uri_or_store, storage_options=None,