Save progress

fsspec · Oct 10, 2024 · 1f69a0b · 1f69a0b
1 parent 80fedcd
commit 1f69a0b
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 6 deletions.
diff --git a/kerchunk/netCDF3.py b/kerchunk/netCDF3.py
@@ -1,4 +1,5 @@
 from functools import reduce
+from packaging.version import Version
 from operator import mul
 
 import numpy as np
@@ -167,7 +168,13 @@ def translate(self):
         import zarr
 
         out = self.out
-        z = zarr.open(out, mode="w", zarr_format=2)
+        if Version(zarr.__version__) < Version("3.0.0.a0"):
+            store = zarr.storage.KVStore(out)
+            z = zarr.group(store=store, overwrite=True)
+        else:
+            store = zarr.storage.MemoryStore(mode="a", store_dict=out)
+            z = zarr.open(store, mode="w", zarr_format=2)
+
         for dim, var in self.variables.items():
             if dim in self.chunks:
                 shape = self.chunks[dim][-1]
@@ -197,7 +204,7 @@ def translate(self):
                     dtype=var.data.dtype,
                     fill_value=fill,
                     chunks=shape,
-                    compression=None,
+                    compressor=None,
                 )
                 part = ".".join(["0"] * len(shape)) or "0"
                 k = f"{dim}/{part}"
@@ -251,7 +258,7 @@ def translate(self):
                     dtype=base,
                     fill_value=fill,
                     chunks=(1,) + dtype.shape,
-                    compression=None,
+                    compressor=None,
                 )
                 arr.attrs.update(
                     {

diff --git a/kerchunk/tests/test_hdf.py b/kerchunk/tests/test_hdf.py
@@ -27,7 +27,7 @@ async def list_dir(store, path):
 def create_store(test_dict: dict, remote_options: Any = None):
     if Version(zarr.__version__) < Version("3.0.0.a0"):
         return fsspec.get_mapper(
-            "reference://", fo=test_dict, remote_protocol="s3", remote_options=so
+            "reference://", fo=test_dict, remote_protocol="s3", remote_options=remote_options
         )
     else:
         fs = fsspec.implementations.reference.ReferenceFileSystem(fo=test_dict, remote_options=remote_options)

diff --git a/kerchunk/tests/test_netcdf.py b/kerchunk/tests/test_netcdf.py
@@ -1,4 +1,5 @@
 import os
+from typing import Any
 
 
 import fsspec
@@ -7,6 +8,8 @@
 import pytest
 from kerchunk import netCDF3
 
+import zarr
+
 xr = pytest.importorskip("xarray")
 
 
@@ -24,16 +27,29 @@
 )
 
 
+def create_store(test_dict: dict, remote_options: Any = None):
+    if Version(zarr.__version__) < Version("3.0.0.a0"):
+        return fsspec.get_mapper(
+            "reference://", fo=test_dict, remote_protocol="s3", remote_options=remote_options
+        )
+    else:
+        fs = fsspec.implementations.reference.ReferenceFileSystem(fo=test_dict, remote_options=remote_options)
+        return zarr.storage.RemoteStore(fs, mode="r")
+
+
 def test_one(m):
     m.pipe("data.nc3", bdata)
     h = netCDF3.netcdf_recording_file("memory://data.nc3")
     out = h.translate()
+
+    store = create_store(out, remote_options={"remote_protocol": "memory"})
+
     ds = xr.open_dataset(
-        "reference://",
+        store,
         engine="zarr",
         backend_kwargs={
             "consolidated": False,
-            "storage_options": {"fo": out, "remote_protocol": "memory"},
+            "zarr_format": "2",
         },
     )
     assert (ds.data == data).all()