MeteoSwiss-APN · cosunae · Oct 7, 2023 · Oct 7, 2023 · Oct 7, 2023 · Oct 7, 2023
diff --git a/jenkins/Jenkinsfile b/jenkins/Jenkinsfile
@@ -47,14 +47,12 @@ pipeline {
                 source $WORKSPACE/miniconda/etc/profile.d/conda.sh
                 conda activate no-deps
                 cd ${WORKSPACE}
-                export INPUT_DATA_DIR=/project/s83c/rz+/icon_data_processing_incubator/data/SWISS
                 pytest -m "not ifs" tests
                 '''
                 sh '''#!/usr/bin/env bash
                 source $WORKSPACE/miniconda/etc/profile.d/conda.sh
                 conda activate no-deps
                 cd ${WORKSPACE}
-                export INPUT_DATA_DIR=/project/s83c/rz+/icon_data_processing_incubator/data/SWISS
                 pytest -m ifs tests
                 '''
             }

diff --git a/src/idpi/system_definition.py b/src/idpi/system_definition.py
@@ -10,16 +10,6 @@
     print("The FIELDEXTRA_PATH is not set, exiting")
     raise
 
-try:
-    input_data_directory = os.environ["INPUT_DATA_DIR"]
-except KeyError:
-    input_data_directory = str(
-        pathlib.Path(__file__).parent.parent.parent.resolve() / "test_data"
-    )
-    print(f"The INPUT_DATA_DIR is not set, setting it to {input_data_directory}")
-
-
 FX_BINARY = fieldextra_executable
-INPUT_DATA_DIR = input_data_directory
 
 root_dir = (pathlib.Path(os.path.dirname(os.path.abspath(__file__))) / "..").resolve()
diff --git a/tests/test_idpi/conftest.py b/tests/test_idpi/conftest.py
@@ -1,7 +1,6 @@
 """Test configuration."""
 # Standard library
 import subprocess
-from collections.abc import Iterable
 from pathlib import Path
 
 # Third-party
@@ -13,7 +12,9 @@
 @pytest.fixture
 def data_dir():
     """Base data dir."""
-    return Path("/project/s83c/rz+/icon_data_processing_incubator/data/SWISS")
+    return Path(
+        "/project/s83c/rz+/icon_data_processing_incubator/datasets/32_39x45_51/"
+    )
 
 
 @pytest.fixture
@@ -34,25 +35,27 @@ def template_env():
 def fieldextra(tmp_path, data_dir, template_env, fieldextra_executable):
     """Run fieldextra on a given field."""
 
-    def f(field_name, hh: int | Iterable[int] | None = 0, **ctx):
-        default_conf_files = {
-            "inputi": data_dir / "lfff<DDHH>0000.ch",
-            "inputc": data_dir / "lfff00000000c.ch",
-            "output": f"<HH>_{field_name}.nc",
-        }
-        conf_files = ctx.pop("conf_files", default_conf_files)
-        template = template_env.get_template(f"test_{field_name}.nl")
-        nl_path = tmp_path / f"test_{field_name}.nl"
+    def f(
+        product: str,
+        conf_files: dict[str, str] | None = None,
+        load_output: str | list[str] = "00_outfile.nc",
+        **ctx,
+    ):
+        if not conf_files:
+            conf_files = {
+                "inputi": data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000",
+                "inputc": data_dir / "COSMO-1E/1h/const/000/lfff00000000c",
+                "output": "<HH>_outfile.nc",
+            }
+
+        template = template_env.get_template(f"test_{product}.nl")
+        nl_path = tmp_path / f"test_{product}.nl"
         nl_path.write_text(template.render(file=conf_files, **ctx))
 
         subprocess.run([fieldextra_executable, str(nl_path)], check=True, cwd=tmp_path)
 
-        if isinstance(hh, int):
-            return xr.open_dataset(tmp_path / f"{hh:02d}_{field_name}.nc")
-        if isinstance(hh, Iterable):
-            return [xr.open_dataset(tmp_path / f"{h:02d}_{field_name}.nc") for h in hh]
-        if hh is None:
-            return xr.open_dataset(tmp_path / f"{field_name}.nc")
-        raise TypeError("Unknown type for param hh")
+        if isinstance(load_output, str):
+            return xr.open_dataset(tmp_path / load_output)
+        return [xr.open_dataset(tmp_path / foutput) for foutput in load_output]
 
     return f
diff --git a/tests/test_idpi/test_brn.py b/tests/test_idpi/test_brn.py
@@ -7,8 +7,8 @@
 
 
 def test_brn(data_dir, fieldextra):
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     reader = GribReader([cdatafile, datafile])
     ds = reader.load_fieldnames(["P", "T", "QV", "U", "V", "HHL", "HSURF"])

diff --git a/tests/test_idpi/test_curl.py b/tests/test_idpi/test_curl.py
@@ -10,8 +10,8 @@
 
 
 def test_curl(data_dir):
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     reader = GribReader([cdatafile, datafile])
     ds = reader.load_fieldnames(["U", "V", "W", "HHL"])

diff --git a/tests/test_idpi/test_destagger.py b/tests/test_idpi/test_destagger.py
@@ -7,8 +7,8 @@
 
 
 def test_destagger(data_dir, fieldextra):
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     reader = GribReader([cdatafile, datafile])
     ds = reader.load_fieldnames(

diff --git a/tests/test_idpi/test_diff.py b/tests/test_idpi/test_diff.py
@@ -9,8 +9,7 @@
 
 
 def test_masspoint_field(data_dir):
-    datafile = data_dir / "lfff00000000.ch"
-
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
     reader = GribReader([datafile], ref_param="P")
 
     ds = reader.load_fieldnames(["P", "T"])
@@ -33,8 +32,7 @@ def test_masspoint_field(data_dir):
 
 
 def test_staggered_field(data_dir):
-    datafile = data_dir / "lfff00000000.ch"
-
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
     reader = GribReader([datafile], ref_param="W")
 
     ds = reader.load_fieldnames(["W"])

diff --git a/tests/test_idpi/test_flexpart.py b/tests/test_idpi/test_flexpart.py
@@ -52,7 +52,11 @@ def test_flexpart(data_dir, fieldextra):
         "output": "<HH>_flexpart.nc",
     }
 
-    fs_ds, *fs_ds_h = fieldextra("flexpart", hh=(0, 3, 6), conf_files=conf_files)
+    fs_ds, *fs_ds_h = fieldextra(
+        "flexpart",
+        load_output=[f"{i:02d}_flexpart.nc" for i in (0, 3, 6)],
+        conf_files=conf_files,
+    )
     fs_ds_o = {}
     for f in ("FIS", "FR_LAND", "SDOR"):
         fs_ds_o[f] = fs_ds[f].isel(y_1=slice(None, None, -1))

diff --git a/tests/test_idpi/test_gis.py b/tests/test_idpi/test_gis.py
@@ -42,8 +42,7 @@ def test_geolatlon2swiss(coords):
 
 
 def test_vref_rot2geolatlon(data_dir, fieldextra):
-    datafile = data_dir / "lfff00000000.ch"
-
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
     reader = grib_decoder.GribReader([datafile], ref_param="T")
     ds = reader.load_fieldnames(["U_10M", "V_10M"])
 

diff --git a/tests/test_idpi/test_hzerocl.py b/tests/test_idpi/test_hzerocl.py
@@ -9,8 +9,8 @@
 
 @pytest.mark.parametrize("extrapolate", [True, False])
 def test_hzerocl(data_dir, fieldextra, extrapolate):
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     reader = GribReader([cdatafile, datafile])
     ds = reader.load_fieldnames(

diff --git a/tests/test_idpi/test_integ_sfc2z.py b/tests/test_idpi/test_integ_sfc2z.py
@@ -14,8 +14,8 @@
     [("integral", "integ", 1e-4, 1e-6), ("normed_integral", "norm_integ", 1e-5, 1e-6)],
 )
 def test_integ_sfc2z(field, k_max, operator, fx_op, atol, rtol, data_dir, fieldextra):
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     # modes
     mode = "z2z"

diff --git a/tests/test_idpi/test_integ_z2z.py b/tests/test_idpi/test_integ_z2z.py
@@ -22,8 +22,8 @@ def test_integ_z2z(field, k_max, operator, fx_op, data_dir, fieldextra):
     k_top = 60
 
     # input data
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     # load input data set
     reader = GribReader([cdatafile, datafile])

diff --git a/tests/test_idpi/test_intpl_hk2p.py b/tests/test_idpi/test_intpl_hk2p.py
@@ -23,8 +23,8 @@ def test_intpl_hk2p(mode, fx_mode, rtol, data_dir, fieldextra):
     tc_units = "hPa"
 
     # input data
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     # load input data set
     reader = GribReader([cdatafile, datafile])

diff --git a/tests/test_idpi/test_intpl_k2p.py b/tests/test_idpi/test_intpl_k2p.py
@@ -22,7 +22,7 @@ def test_intpl_k2p(mode, fx_mode, atol, rtol, data_dir, fieldextra):
     tc_units = "hPa"
 
     # input data
-    datafile = data_dir / "lfff00000000.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
 
     # load input data set
     reader = GribReader([datafile], ref_param="P")

diff --git a/tests/test_idpi/test_intpl_k2theta.py b/tests/test_idpi/test_intpl_k2theta.py
@@ -18,8 +18,8 @@ def test_intpl_k2theta(mode, data_dir, fieldextra):
     tc_units = "K"
 
     # input data
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     # load input data set
     reader = GribReader([cdatafile, datafile])

diff --git a/tests/test_idpi/test_lat_ops.py b/tests/test_idpi/test_lat_ops.py
@@ -1,3 +1,6 @@
+# Standard library
+from pathlib import Path
+
 # Third-party
 import numpy as np
 import pytest
@@ -9,9 +12,14 @@
 from idpi.operators.hzerocl import fhzerocl
 
 
+@pytest.fixture
+def data_dir():
+    return Path("/project/s83c/rz+/icon_data_processing_incubator/datasets/original/")
+
+
 def test_fill_undef(data_dir, fieldextra):
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     reader = GribReader([cdatafile, datafile])
     ds = reader.load_fieldnames(["T", "HHL"])
@@ -27,8 +35,8 @@ def test_fill_undef(data_dir, fieldextra):
 
 
 def test_disk_avg(data_dir, fieldextra):
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     reader = GribReader([cdatafile, datafile])
 

diff --git a/tests/test_idpi/test_minmax_z2z.py b/tests/test_idpi/test_minmax_z2z.py
@@ -19,8 +19,8 @@ def test_minmax_z2z(operator, fx_op, field, layer, data_dir, fieldextra):
     k_top = 60
 
     # input data
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     # load input data set
     reader = GribReader([cdatafile, datafile])

diff --git a/tests/test_idpi/test_ninjo_k2th.py b/tests/test_idpi/test_ninjo_k2th.py
@@ -7,8 +7,8 @@
 
 
 def test_ninjo_k2th(data_dir, fieldextra):
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     reader = GribReader([cdatafile, datafile])
 
@@ -36,7 +36,7 @@ def test_ninjo_k2th(data_dir, fieldextra):
     assert_allclose(
         fs_ds["POT_VORTIC_AT_THETA"],
         observed_at_theta["pot_vortic"],
-        atol=1e-9,
+        atol=1e-8,
         rtol=1e-5,
     )
 
@@ -50,7 +50,7 @@ def test_ninjo_k2th(data_dir, fieldextra):
     assert_allclose(
         fs_ds["U"],
         observed_at_theta["u"],
-        atol=1e-9,
+        atol=1e-4,
         rtol=1e-5,
     )
 

diff --git a/tests/test_idpi/test_potvortic.py b/tests/test_idpi/test_potvortic.py
@@ -11,8 +11,8 @@
 
 
 def test_pv(data_dir, fieldextra):
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     reader = GribReader([cdatafile, datafile])
     ds = reader.load_fieldnames(["U", "V", "W", "P", "T", "QV", "QC", "QI", "HHL"])

diff --git a/tests/test_idpi/test_radiation.py b/tests/test_idpi/test_radiation.py
@@ -14,7 +14,10 @@
 
 @pytest.fixture
 def data_dir():
-    return Path("/project/s83c/rz+/icon_data_processing_incubator/data/temporal")
+    return Path(
+        "/project/s83c/rz+/icon_data_processing_incubator/datasets/"
+        "32_39x45_51/COSMO-1E_time/"
+    )
 
 
 def test_athd_s(data_dir, fieldextra):
@@ -30,7 +33,7 @@ def test_athd_s(data_dir, fieldextra):
 
     conf_files = {
         "inputi": data_dir / "lfff<DDHH>0000",
-        "output": "athd_s.nc",
+        "output": "00_outfile.nc",
     }
     fx_ds = fieldextra("athd_s", conf_files=conf_files, hh=None)
     expected = fx_ds["ATHD_S_TG"].transpose("epsd_1", "time", ...)

diff --git a/tests/test_idpi/test_regrid.py b/tests/test_idpi/test_regrid.py
@@ -1,4 +1,8 @@
+# Standard library
+from pathlib import Path
+
 # Third-party
+import pytest
 from numpy.testing import assert_allclose
 
 # First-party
@@ -7,9 +11,14 @@
 from idpi.operators.hzerocl import fhzerocl
 
 
+@pytest.fixture
+def data_dir():
+    return Path("/project/s83c/rz+/icon_data_processing_incubator/datasets/original/")
+
+
 def test_regrid(data_dir, fieldextra):
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     reader = grib_decoder.GribReader([cdatafile, datafile])
     ds = reader.load_fieldnames(["T", "HHL"])

diff --git a/tests/test_idpi/test_relhum.py b/tests/test_idpi/test_relhum.py
@@ -7,8 +7,8 @@
 
 
 def test_relhum(data_dir, fieldextra):
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     reader = GribReader([cdatafile, datafile], ref_param="P")
     ds = reader.load_fieldnames(["P", "T", "QV"])

diff --git a/tests/test_idpi/test_theta.py b/tests/test_idpi/test_theta.py
@@ -7,8 +7,7 @@
 
 
 def test_theta(data_dir, fieldextra):
-    datafile = data_dir / "lfff00000000.ch"
-
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
     reader = GribReader([datafile], ref_param="P")
 
     ds = reader.load_fieldnames(["P", "T"])

diff --git a/tests/test_idpi/test_thetav.py b/tests/test_idpi/test_thetav.py
@@ -7,8 +7,7 @@
 
 
 def test_thetav(data_dir, fieldextra):
-    datafile = data_dir / "lfff00000000.ch"
-
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
     reader = GribReader([datafile], ref_param="P")
 
     ds = reader.load_fieldnames(["P", "T", "QV"])