Reduced test data (#100)

## Purpose

Run tests on a reduced test dataset by default to speed up the test suite.

## Code changes:

- Added the custom pytest marker `data` 
- Remove the `system_defintion` module

Superseeds #88

jenkins/Jenkinsfile

@@ -50,7 +50,7 @@ pipeline {
                 source ${WORKSPACE}/miniconda/etc/profile.d/conda.sh
                 conda init bash --no-user --install --system
                 conda activate
-                bash tools/setup_env.sh -u -n no-deps -f /project/s83c/fieldextra/tsa/bin/fieldextra_gnu_opt_omp
+                bash tools/setup_env.sh -u -n no-deps
                 conda activate no-deps
                 pip install --no-deps .
                 '''
@@ -62,14 +62,12 @@ pipeline {
                 source $WORKSPACE/miniconda/etc/profile.d/conda.sh
                 conda activate no-deps
                 cd ${WORKSPACE}
-                export INPUT_DATA_DIR=/project/s83c/rz+/icon_data_processing_incubator/data/SWISS
                 pytest -m "not ifs" tests
                 '''
                 sh '''#!/usr/bin/env bash
                 source $WORKSPACE/miniconda/etc/profile.d/conda.sh
                 conda activate no-deps
                 cd ${WORKSPACE}
-                export INPUT_DATA_DIR=/project/s83c/rz+/icon_data_processing_incubator/data/SWISS
                 pytest -m ifs tests
                 '''
             }

pyproject.toml

@@ -55,6 +55,7 @@ addopts = [
 markers = [
     "ifs",   # marks tests that use IFS grib definitions
     "cosmo", # marks tests that use COSMO grib definitions
+    "data",  # marks tests that depend on specific test data
 ]
 
 [tool.black]

tests/test_idpi/conftest.py

@@ -2,7 +2,6 @@
 # Standard library
 import os
 import subprocess
-from collections.abc import Iterable
 from pathlib import Path
 
 # Third-party
@@ -27,12 +26,25 @@ def machine():
     return "unknown"
 
 
-@pytest.fixture(scope="session")
-def data_dir(machine):
+@pytest.fixture
+def data_dir(request, machine):
     """Base data dir."""
-    if machine == "tsa":
-        return Path("/project/s83c/rz+/icon_data_processing_incubator/data/SWISS")
-    raise RuntimeError("panic")
+    if machine != "tsa":
+        return None
+    base_dir = Path("/project/s83c/rz+/icon_data_processing_incubator/")
+    marker = request.node.get_closest_marker("data")
+    if marker is None:
+        return base_dir / "datasets/32_39x45_51"
+    match marker.args[0]:
+        case "original":
+            return base_dir / "datasets/original"
+        case "reduced":
+            return base_dir / "datasets/32_39x45_51"
+        case "reduced-time":
+            return base_dir / "datasets/32_39x45_51/COSMO-1E_time"
+        case "reduced-ens":
+            return base_dir / "datasets/32_39x45_51/COSMO-1E_ens"
+    raise RuntimeError(f"No match for data mark {marker.args[0]}")
 
 
 @pytest.fixture(scope="session")
@@ -87,28 +99,29 @@ def request_template():
 def fieldextra(tmp_path, data_dir, template_env, fieldextra_path):
     """Run fieldextra on a given field."""
 
-    def f(field_name, hh: int | Iterable[int] | None = 0, **ctx):
-        default_conf_files = {
-            "inputi": data_dir / "lfff<DDHH>0000.ch",
-            "inputc": data_dir / "lfff00000000c.ch",
-            "output": f"<HH>_{field_name}.nc",
-        }
-        conf_files = ctx.pop("conf_files", default_conf_files)
-        template = template_env.get_template(f"test_{field_name}.nl")
-        nl_path = tmp_path / f"test_{field_name}.nl"
+    def f(
+        product: str,
+        conf_files: dict[str, str] | None = None,
+        load_output: str | list[str] = "00_outfile.nc",
+        **ctx,
+    ):
+        if conf_files is None:
+            conf_files = {
+                "inputi": data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000",
+                "inputc": data_dir / "COSMO-1E/1h/const/000/lfff00000000c",
+                "output": "<HH>_outfile.nc",
+            }
+        template = template_env.get_template(f"test_{product}.nl")
+        nl_path = tmp_path / f"test_{product}.nl"
         ctx["file"] = conf_files
         ctx["resources"] = fieldextra_path / "resources"
         nl_path.write_text(template.render(**ctx))
 
         executable = str(fieldextra_path / "bin/fieldextra_gnu_opt_omp")
         subprocess.run([executable, str(nl_path)], check=True, cwd=tmp_path)
 
-        if isinstance(hh, int):
-            return xr.open_dataset(tmp_path / f"{hh:02d}_{field_name}.nc")
-        if isinstance(hh, Iterable):
-            return [xr.open_dataset(tmp_path / f"{h:02d}_{field_name}.nc") for h in hh]
-        if hh is None:
-            return xr.open_dataset(tmp_path / f"{field_name}.nc")
-        raise TypeError("Unknown type for param hh")
+        if isinstance(load_output, str):
+            return xr.open_dataset(tmp_path / load_output)
+        return [xr.open_dataset(tmp_path / filename) for filename in load_output]
 
     return f

tests/test_idpi/test_brn.py

@@ -7,8 +7,8 @@
 
 
 def test_brn(data_dir, fieldextra):
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     reader = GribReader.from_files([cdatafile, datafile])
     ds = reader.load_fieldnames(["P", "T", "QV", "U", "V", "HHL", "HSURF"])

tests/test_idpi/test_curl.py

@@ -10,8 +10,8 @@
 
 
 def test_curl(data_dir):
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     reader = GribReader.from_files([cdatafile, datafile])
     ds = reader.load_fieldnames(["U", "V", "W", "HHL"])

tests/test_idpi/test_destagger.py

@@ -7,8 +7,8 @@
 
 
 def test_destagger(data_dir, fieldextra):
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     reader = GribReader.from_files([cdatafile, datafile])
     ds = reader.load_fieldnames(

tests/test_idpi/test_diff.py

@@ -9,8 +9,7 @@
 
 
 def test_masspoint_field(data_dir):
-    datafile = data_dir / "lfff00000000.ch"
-
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
     reader = GribReader.from_files([datafile], ref_param="P")
 
     ds = reader.load_fieldnames(["P", "T"])
@@ -33,8 +32,7 @@ def test_masspoint_field(data_dir):
 
 
 def test_staggered_field(data_dir):
-    datafile = data_dir / "lfff00000000.ch"
-
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
     reader = GribReader.from_files([datafile], ref_param="W")
 
     ds = reader.load_fieldnames(["W"])

tests/test_idpi/test_flexpart.py

@@ -44,15 +44,19 @@ def test_flexpart(data_dir, fieldextra):
 
     with idpi.config.set_values(data_scope="ifs"):
         reader = GribReader.from_files(datafiles, ref_param="t")
-        ds = reader.load_fieldnames(inputf + constants, extract_pv="u")
+        ds = reader.load_fieldnames(list(inputf + constants), extract_pv="u")
 
     conf_files = {
         "inputi": str(data_dir / "efsf00<HH>0000"),
         "inputc": str(data_dir / "efsf00000000"),
         "output": "<HH>_flexpart.nc",
     }
 
-    fs_ds, *fs_ds_h = fieldextra("flexpart", hh=(0, 3, 6), conf_files=conf_files)
+    fs_ds, *fs_ds_h = fieldextra(
+        "flexpart",
+        load_output=[f"{i:02d}_flexpart.nc" for i in (0, 3, 6)],
+        conf_files=conf_files,
+    )
     fs_ds_o = {}
     for f in ("FIS", "FR_LAND", "SDOR"):
         fs_ds_o[f] = fs_ds[f].isel(y_1=slice(None, None, -1))

tests/test_idpi/test_gis.py

@@ -42,8 +42,7 @@ def test_geolatlon2swiss(coords):
 
 
 def test_vref_rot2geolatlon(data_dir, fieldextra):
-    datafile = data_dir / "lfff00000000.ch"
-
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
     reader = grib_decoder.GribReader.from_files([datafile], ref_param="T")
     ds = reader.load_fieldnames(["U_10M", "V_10M"])
 

tests/test_idpi/test_hzerocl.py

@@ -9,8 +9,8 @@
 
 @pytest.mark.parametrize("extrapolate", [True, False])
 def test_hzerocl(data_dir, fieldextra, extrapolate):
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     reader = GribReader.from_files([cdatafile, datafile])
     ds = reader.load_fieldnames(

tests/test_idpi/test_integ_sfc2z.py

@@ -14,8 +14,8 @@
     [("integral", "integ", 1e-4, 1e-6), ("normed_integral", "norm_integ", 1e-5, 1e-6)],
 )
 def test_integ_sfc2z(field, k_max, operator, fx_op, atol, rtol, data_dir, fieldextra):
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     # modes
     mode = "z2z"

tests/test_idpi/test_integ_z2z.py

@@ -22,8 +22,8 @@ def test_integ_z2z(field, k_max, operator, fx_op, data_dir, fieldextra):
     k_top = 60
 
     # input data
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     # load input data set
     reader = GribReader.from_files([cdatafile, datafile])

tests/test_idpi/test_intpl_hk2p.py

@@ -23,8 +23,8 @@ def test_intpl_hk2p(mode, fx_mode, rtol, data_dir, fieldextra):
     tc_units = "hPa"
 
     # input data
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     # load input data set
     reader = GribReader.from_files([cdatafile, datafile])

tests/test_idpi/test_intpl_k2any.py

@@ -12,8 +12,8 @@ def test_intpl_k2theta(data_dir, fieldextra):
     tc_values = [15.0]
 
     # input data
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     # load input data set
     reader = GribReader.from_files([cdatafile, datafile])

tests/test_idpi/test_intpl_k2p.py

@@ -22,7 +22,7 @@ def test_intpl_k2p(mode, fx_mode, atol, rtol, data_dir, fieldextra):
     tc_units = "hPa"
 
     # input data
-    datafile = data_dir / "lfff00000000.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
 
     # load input data set
     reader = GribReader.from_files([datafile], ref_param="P")

tests/test_idpi/test_intpl_k2theta.py

@@ -18,8 +18,8 @@ def test_intpl_k2theta(mode, data_dir, fieldextra):
     tc_units = "K"
 
     # input data
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     # load input data set
     reader = GribReader.from_files([cdatafile, datafile])

tests/test_idpi/test_lat_ops.py

@@ -9,9 +9,10 @@
 from idpi.operators.hzerocl import fhzerocl
 
 
+@pytest.mark.data("original")
 def test_fill_undef(data_dir, fieldextra):
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     reader = GribReader.from_files([cdatafile, datafile])
     ds = reader.load_fieldnames(["T", "HHL"])
@@ -26,9 +27,10 @@ def test_fill_undef(data_dir, fieldextra):
     assert_allclose(observed, expected, rtol=2e-6)
 
 
+@pytest.mark.data("original")
 def test_disk_avg(data_dir, fieldextra):
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     reader = GribReader.from_files([cdatafile, datafile])
 

tests/test_idpi/test_minmax_z2z.py

@@ -19,8 +19,8 @@ def test_minmax_z2z(operator, fx_op, field, layer, data_dir, fieldextra):
     k_top = 60
 
     # input data
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     # load input data set
     reader = GribReader.from_files([cdatafile, datafile])

tests/test_idpi/test_ninjo_k2th.py

@@ -7,8 +7,8 @@
 
 
 def test_ninjo_k2th(data_dir, fieldextra):
-    datafile = data_dir / "lfff00000000.ch"
-    cdatafile = data_dir / "lfff00000000c.ch"
+    datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
+    cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
     reader = GribReader.from_files([cdatafile, datafile])
 
@@ -36,7 +36,7 @@ def test_ninjo_k2th(data_dir, fieldextra):
     assert_allclose(
         fs_ds["POT_VORTIC_AT_THETA"],
         observed_at_theta["pot_vortic"],
-        atol=1e-9,
+        atol=1e-8,
         rtol=1e-5,
     )
 
@@ -50,7 +50,7 @@ def test_ninjo_k2th(data_dir, fieldextra):
     assert_allclose(
         fs_ds["U"],
         observed_at_theta["u"],
-        atol=1e-9,
+        atol=1e-4,
         rtol=1e-5,
     )
 

tests/test_idpi/test_potvortic.py

@@ -47,7 +47,7 @@ def test_pv(data, fieldextra):
 
     observed = pv.fpotvortic(ds["U"], ds["V"], ds["W"], theta, rho_tot, total_diff)
 
-    conf_files = cache.conf_files | {"output": "<hh>_POT_VORTIC.nc"}
+    conf_files = cache.conf_files | {"output": "<hh>_outfile.nc"}
     fs_ds = fieldextra("POT_VORTIC", conf_files=conf_files)
 
     assert_allclose(