From cf6b793828a3e985b29f40d4214368452b66814c Mon Sep 17 00:00:00 2001 From: Yann Forget Date: Fri, 20 Dec 2024 20:56:36 +0100 Subject: [PATCH] fix(era5): use less memory at the cost of performance --- openhexa/toolbox/era5/aggregate.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/openhexa/toolbox/era5/aggregate.py b/openhexa/toolbox/era5/aggregate.py index bc04328..c2858d1 100644 --- a/openhexa/toolbox/era5/aggregate.py +++ b/openhexa/toolbox/era5/aggregate.py @@ -126,11 +126,12 @@ def merge(data_dir: Path | str) -> xr.Dataset: if isinstance(data_dir, str): data_dir = Path(data_dir) - datasets = [] - for fp in data_dir.glob("*.grib"): - datasets.append(xr.open_dataset(fp, engine="cfgrib")) + files = data_dir.glob("*.grib") + ds = xr.open_dataset(next(files), engine="cfgrib") + + for f in files: + ds = xr.concat([ds, xr.open_dataset(f, engine="cfgrib")], dim="tmp_dim").max(dim="tmp_dim") - ds = xr.concat(datasets, dim="tmp_dim").max(dim="tmp_dim") return ds