Skip to content

Commit

Permalink
Fix circular import
Browse files Browse the repository at this point in the history
  • Loading branch information
mpiannucci committed Oct 10, 2024
1 parent d556e52 commit b27e64c
Show file tree
Hide file tree
Showing 4 changed files with 37 additions and 55 deletions.
5 changes: 1 addition & 4 deletions kerchunk/hdf.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,15 @@
import io
import logging
from typing import Union, BinaryIO
from packaging.version import Version

import fsspec.core
from fsspec.implementations.reference import LazyReferenceMapper
import numpy as np
import zarr
import numcodecs

from kerchunk.zarr import dict_to_store

from .codecs import FillStringsCodec
from .utils import _encode_for_JSON, encode_fill_value
from .utils import _encode_for_JSON, encode_fill_value, dict_to_store

try:
import h5py
Expand Down
17 changes: 3 additions & 14 deletions kerchunk/tests/test_netcdf.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import os
from typing import Any


import fsspec
Expand All @@ -8,7 +7,7 @@
import pytest
from kerchunk import netCDF3

import zarr
from kerchunk.utils import refs_as_store

xr = pytest.importorskip("xarray")

Expand All @@ -27,29 +26,19 @@
)


def create_store(test_dict: dict, remote_options: Any = None):
if Version(zarr.__version__) < Version("3.0.0.a0"):
return fsspec.get_mapper(
"reference://", fo=test_dict, remote_protocol="s3", remote_options=remote_options
)
else:
fs = fsspec.implementations.reference.ReferenceFileSystem(fo=test_dict, remote_options=remote_options)
return zarr.storage.RemoteStore(fs, mode="r")


def test_one(m):
m.pipe("data.nc3", bdata)
h = netCDF3.netcdf_recording_file("memory://data.nc3")
out = h.translate()

store = create_store(out, remote_options={"remote_protocol": "memory"})
store = refs_as_store(out, remote_protocol="memory")

ds = xr.open_dataset(
store,
engine="zarr",
backend_kwargs={
"consolidated": False,
"zarr_format": "2",
"zarr_format": 2,
},
)
assert (ds.data == data).all()
Expand Down
35 changes: 33 additions & 2 deletions kerchunk/utils.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import base64
import copy
import itertools
from packaging.version import Version
from typing import Any, cast
import warnings

Expand All @@ -10,8 +11,6 @@
import numpy as np
import zarr

from kerchunk.zarr import fs_as_store


def refs_as_fs(refs, remote_protocol=None, remote_options=None, **kwargs):
"""Convert a reference set to an fsspec filesystem"""
Expand All @@ -33,6 +32,38 @@ def refs_as_store(refs, remote_protocol=None, remote_options=None):
return fs_as_store(fs)


def is_zarr3():
"""Check if the installed zarr version is version 3"""
return Version(zarr.__version__) >= Version("3.0.0.a0")


def dict_to_store(store_dict: dict):
"""Create an in memory zarr store backed by the given dictionary"""
if is_zarr3():
return zarr.storage.MemoryStore(mode="a", store_dict=store_dict)
else:
return zarr.storage.KVStore(store_dict)


def fs_as_store(fs, mode='r', remote_protocol=None, remote_options=None):
"""Open the refs as a zarr store
Parameters
----------
refs: dict-like
the references to open
mode: str
Returns
-------
zarr.storage.Store or zarr.storage.Mapper, fsspec.AbstractFileSystem
"""
if is_zarr3():
return zarr.storage.RemoteStore(fs, mode=mode)
else:
return fs.get_mapper()


def class_factory(func):
"""Experimental uniform API across function-based file scanners"""

Expand Down
35 changes: 0 additions & 35 deletions kerchunk/zarr.py
Original file line number Diff line number Diff line change
@@ -1,44 +1,9 @@
from packaging.version import Version

import fsspec
from fsspec.implementations.reference import LazyReferenceMapper
import zarr

import kerchunk.utils


def is_zarr3():
"""Check if the installed zarr version is version 3"""
return Version(zarr.__version__) >= Version("3.0.0.a0")


def dict_to_store(store_dict: dict):
"""Create an in memory zarr store backed by the given dictionary"""
if is_zarr3():
return zarr.storage.MemoryStore(mode="a", store_dict=store_dict)
else:
return zarr.storage.KVStore(store_dict)


def fs_as_store(fs, mode='r', remote_protocol=None, remote_options=None):
"""Open the refs as a zarr store
Parameters
----------
refs: dict-like
the references to open
mode: str
Returns
-------
zarr.storage.Store or zarr.storage.Mapper, fsspec.AbstractFileSystem
"""
if is_zarr3():
return zarr.storage.RemoteStore(fs, mode=mode)
else:
return fs.get_mapper()


def single_zarr(
uri_or_store,
storage_options=None,
Expand Down

0 comments on commit b27e64c

Please sign in to comment.