fix(era5): use less memory at the cost of performance

BLSQ · Dec 20, 2024 · cf6b793 · cf6b793
1 parent eb22fef
commit cf6b793
Showing 1 changed file with 5 additions and 4 deletions.
diff --git a/openhexa/toolbox/era5/aggregate.py b/openhexa/toolbox/era5/aggregate.py
@@ -126,11 +126,12 @@ def merge(data_dir: Path | str) -> xr.Dataset:
     if isinstance(data_dir, str):
         data_dir = Path(data_dir)
 
-    datasets = []
-    for fp in data_dir.glob("*.grib"):
-        datasets.append(xr.open_dataset(fp, engine="cfgrib"))
+    files = data_dir.glob("*.grib")
+    ds = xr.open_dataset(next(files), engine="cfgrib")
+
+    for f in files:
+        ds = xr.concat([ds, xr.open_dataset(f, engine="cfgrib")], dim="tmp_dim").max(dim="tmp_dim")
 
-    ds = xr.concat(datasets, dim="tmp_dim").max(dim="tmp_dim")
     return ds