Skip to content

Commit

Permalink
Add parquet test too
Browse files Browse the repository at this point in the history
  • Loading branch information
martindurant committed Dec 15, 2023
1 parent da69e3b commit 2d2c38f
Show file tree
Hide file tree
Showing 2 changed files with 40 additions and 1 deletion.
2 changes: 1 addition & 1 deletion kerchunk/combine.py
Original file line number Diff line number Diff line change
Expand Up @@ -552,7 +552,7 @@ def translate(self, filename=None, storage_options=None):
if filename is not None:
with fsspec.open(filename, mode="wt", **(storage_options or {})) as f:
ujson.dump(out, f)
return out
return self.out


def _reorganise(coos):
Expand Down
39 changes: 39 additions & 0 deletions kerchunk/tests/test_combine.py
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,45 @@ def test_single_append(refs):
assert z.time.values.tolist() == [1, 2, 3]


def test_single_append_parquet(refs):
from fsspec.implementations.reference import LazyReferenceMapper

m = fsspec.filesystem("memory")
out = LazyReferenceMapper.create("memory://refs/out.parquet", fs=m)
mzz = MultiZarrToZarr(
[refs["single1"], refs["single2"]],
remote_protocol="memory",
concat_dims=["time"],
coo_dtypes={"time": "int16"},
out=out,
)
mzz.translate()

# reload here due to unknown bug after flush
out = LazyReferenceMapper("memory://refs/out.parquet", fs=m)
mzz = MultiZarrToZarr.append(
[refs["single3"]],
out,
remote_protocol="memory",
concat_dims=["time"],
coo_dtypes={"time": "int16"},
)
out = mzz.translate()

z = xr.open_dataset(
out,
backend_kwargs={
"storage_options": {"remote_protocol": "memory"},
},
engine="kerchunk",
decode_cf=False,
)
assert z.data.shape == (3, 10, 10)
assert out["data/1.0.0"] == ["memory:///single2.zarr/data/0.0.0"]
assert out["data/2.0.0"] == ["memory:///single3.zarr/data/0.0.0"]
assert z.time.values.tolist() == [1, 2, 3]


def test_lazy_filler(tmpdir, refs):
pd = pytest.importorskip("pandas")
pytest.importorskip("fastparquet")
Expand Down

0 comments on commit 2d2c38f

Please sign in to comment.