Skip to content

Commit

Permalink
adds lazy_reference_mapper_kwargs to refs_to_dataframe
Browse files Browse the repository at this point in the history
  • Loading branch information
norlandrhagen committed Sep 26, 2024
1 parent 0d5b2ae commit ecfb3d4
Showing 1 changed file with 5 additions and 0 deletions.
5 changes: 5 additions & 0 deletions kerchunk/df.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import fsspec
import zarr

from typing import Any, Dict

# example from preffs's README'
df = pd.DataFrame(
Expand Down Expand Up @@ -106,6 +107,7 @@ def refs_to_dataframe(
storage_options=None,
record_size=100_000,
categorical_threshold=10,
lazy_reference_mapper_kwargs: Dict[str, Any] = {},
):
"""Write references as a parquet files store.
Expand Down Expand Up @@ -134,6 +136,8 @@ def refs_to_dataframe(
Encode urls as pandas.Categorical to reduce memory footprint if the ratio
of the number of unique urls to total number of refs for each variable
is greater than or equal to this number. (default 10)
lazy_reference_mapper_kwargs : Dict[str, Any]
Optional kwargs to pass into LazyReferenceMapper
"""
from fsspec.implementations.reference import LazyReferenceMapper

Expand All @@ -156,6 +160,7 @@ def refs_to_dataframe(
root=url,
fs=fs,
categorical_threshold=categorical_threshold,
**lazy_reference_mapper_kwargs,
)

for k in sorted(refs):
Expand Down

0 comments on commit ecfb3d4

Please sign in to comment.