WIP: single_zarr_to_reference_file

kerchunk/tests/test_hdf.py

@@ -1,4 +1,5 @@
 import fsspec
+import fsspec.implementations.reference as reffs
 import os.path as osp
 
 import kerchunk.hdf
@@ -322,3 +323,11 @@ def test_embed():
         "0.004609216572327277",
         "0.01298182345556785",
     ]
+
+
+def test_hdf_to_reference_file(tmpdir):
+    uri = "s3://wrf-se-ak-ar5/ccsm/rcp85/daily/2060/WRFDS_2060-01-01.nc"
+    so = dict(anon=True, default_fill_cache=False, default_cache_type="none")
+    out = reffs.LazyReferenceMapper.create(f"{tmpdir}/out.parq", record_size=1)
+    with fsspec.open(uri, **so) as f:
+        _ = SingleHdf5ToZarr(f, uri, storage_options=so, out=out).translate()

kerchunk/tests/test_zarr.py

@@ -96,3 +96,12 @@ def test_zarr_json_dump_succeeds(tmpdir, ds):
         inline_threshold=0,
     ).translate()
     ujson.dumps(one)
+
+
+def test_single_zarr_to_reference_file(tmpdir, ds):
+    fn1 = f"{tmpdir}/test1.zarr"
+    ds.to_zarr(fn1)
+
+    fn = f"{tmpdir}/out.parq"
+    out = reffs.LazyReferenceMapper.create(fn, record_size=1)
+    kerchunk.zarr.ZarrToZarr(fn1, inline_threshold=0, **{"out": out}).translate()

kerchunk/zarr.py

@@ -49,9 +49,9 @@ def single_zarr(
     inline_threshold = inline or inline_threshold
     if inline_threshold:
         refs = do_inline(refs, inline_threshold, remote_options=storage_options)
+    refs = kerchunk.utils.consolidate(refs)
     if isinstance(refs, LazyReferenceMapper):
         refs.flush()
-    refs = kerchunk.utils.consolidate(refs)
     return refs