Add more test data

tests/conftest.py

@@ -1,3 +1,4 @@
+from collections import Counter, namedtuple
 from pathlib import Path
 import shutil
 import tempfile
@@ -28,7 +29,8 @@ def merged_data_dir():
 
 @pytest.fixture(scope="session", autouse=True)
 def using_zarr_store():
-    return version("openghg") >= "0.8"
+    current_version = tuple(int(x) for x in version("openghg").split("."))
+    return current_version >= (0, 8)
 
 
 @pytest.fixture(scope="session")
@@ -88,39 +90,68 @@ def session_config_mocker(using_zarr_store) -> Iterator[None]:
 
 
 # TEST DATA
-obs_metadata = {
+TestData = namedtuple("TestData", ["func", "metadata", "path", "data_type"])
+test_data_list = []
+
+## Obs data
+tac_obs_metadata = {
     "source_format": "openghg",
     "network": "decc",
     "site": "tac",
     "inlet": "185m",
     "instrument": "picarro",
 }
+tac_obs_data_path = _raw_data_path / "obs_tac_ch4_185m_2019-01-01_2019-02-01_data.nc"
+test_data_list.append(TestData(standardise_surface, tac_obs_metadata, tac_obs_data_path, "surface"))
+
+mhd_obs_metadata = {
+    "source_format": "openghg",
+    "network": "agage",
+    "site": "mhd",
+    "inlet": "10m",
+    "instrument": "gcmd",
+    "calibration_scale": "WMO-x2004a",
+}
+mhd_obs_data_path = _raw_data_path / "obs_mhd_ch4_10m_2019-01-01_2019-01-07_data.nc"
+test_data_list.append(TestData(standardise_surface, mhd_obs_metadata, mhd_obs_data_path, "surface"))
+
+## BC data
 bc_metadata = {"species": "ch4", "bc_input": "cams", "domain": "europe", "store": "inversions_tests"}
-footprints_metadata = {
+bc_data_path = _raw_data_path / "bc_ch4_europe_cams_2019-01-01_2019-12-31_data.nc"
+test_data_list.append(TestData(standardise_bc, bc_metadata, bc_data_path, "boundary_conditions"))
+
+## Footprint data
+tac_footprints_metadata = {
     "site": "tac",
     "domain": "europe",
     "model": "name",
     "inlet": "185m",
     # "metmodel": "ukv",
 }
-flux_metadata = {"species": "ch4", "source": "total-ukghg-edgar7", "domain": "europe"}
-flux_dim_shuffle_metadata = {"species": "ch4", "source": "total-ukghg-edgar7-shuffled", "domain": "europe"}
+tac_footprints_data_path = _raw_data_path / "footprints_tac_europe_name_185m_2019-01-01_2019-01-07_data.nc"
+test_data_list.append(TestData(standardise_footprint, tac_footprints_metadata, tac_footprints_data_path, "footprints"))
 
-obs_data_path = _raw_data_path / "obs_tac_ch4_185m_2019-01-01_2019-02-01_data.nc"
-bc_data_path = _raw_data_path / "bc_ch4_europe_cams_2019-01-01_2019-12-31_data.nc"
-footprints_data_path = _raw_data_path / "footprints_tac_europe_name_185m_2019-01-01_2019-01-07_data.nc"
+mhd_footprints_metadata = {
+    "site": "mhd",
+    "domain": "europe",
+    "model": "name",
+    "inlet": "10m",
+    "source_format": "paris",
+    # "metmodel": "ukv",
+}
+mhd_footprints_data_path = _raw_data_path / "footprints_mhd_europe_name_10m_2019-01-01_2019-01-07_data.nc"
+test_data_list.append(TestData(standardise_footprint, mhd_footprints_metadata, mhd_footprints_data_path, "footprints"))
+
+## Flux data
+flux_metadata = {"species": "ch4", "source": "total-ukghg-edgar7", "domain": "europe"}
 flux_data_path = _raw_data_path / "flux_total_ch4_europe_edgar7_2019-01-01_2019-12-31_data.nc"
+test_data_list.append(TestData(standardise_flux, flux_metadata, flux_data_path, "flux"))
+
+flux_dim_shuffle_metadata = {"species": "ch4", "source": "total-ukghg-edgar7-shuffled", "domain": "europe"}
 flux_dim_shuffled_data_path = (
     _raw_data_path / "flux_total_ch4_europe_edgar7_2019-01-01_2019-12-31_data_dim_shuffled.nc"
 )
-
-data_info = {
-    "surface": [standardise_surface, obs_metadata, obs_data_path],
-    "boundary_conditions": [standardise_bc, bc_metadata, bc_data_path],
-    "flux": [standardise_flux, flux_metadata, flux_data_path],
-    "flux_dim_shuffle": [standardise_flux, flux_dim_shuffle_metadata, flux_dim_shuffled_data_path],
-    "footprints": [standardise_footprint, footprints_metadata, footprints_data_path],
-}
+test_data_list.append(TestData(standardise_flux, flux_dim_shuffle_metadata, flux_dim_shuffled_data_path, "flux"))
 
 
 @pytest.fixture(scope="session", autouse=True)
@@ -140,28 +171,22 @@ def session_object_store(session_config_mocker) -> None:
         results = search(store="inversions_tests")
     except ObjectStoreError:
         add_data = True
-        found_dtypes = []
     else:
-        add_data = results.results.shape[0] != 5
         try:
             found_dtypes = results.results["data_type"].to_list()
         except KeyError:
-            found_dtypes = []
+            add_data = True
+        else:
+            add_data = Counter([x.data_type for x in test_data_list]) != Counter(found_dtypes)
+
 
     # check if there are four pieces of data in the object store
     # if not, add the missing data
     if add_data:
-        all_data_types = ["surface", "boundary_conditions", "flux", "flux", "footprints"]
-        for dtype in found_dtypes:
-            all_data_types.remove(dtype)
-
-        if "flux" in all_data_types:
-            all_data_types.append("flux_dim_shuffle")
-
-        for dtype in all_data_types:
-            standardise_fn = data_info[dtype][0]
-            file_path = data_info[dtype][2]
-            metadata = data_info[dtype][1]
+        for test_data in test_data_list:
+            standardise_fn = test_data.func
+            file_path = test_data.path
+            metadata = test_data.metadata
             metadata["store"] = "inversions_tests"
             standardise_fn(filepath=file_path, **metadata)
 
@@ -211,3 +236,26 @@ def tac_ch4_data_args():
         "averaging_period": ["1H"],
     }
     return data_args
+
+
+@pytest.fixture(scope="module")
+def mhd_and_tac_ch4_data_args():
+    data_args = {
+        "species": "ch4",
+        "sites": ["MHD", "TAC"],
+        "start_date": "2019-01-01",
+        "end_date": "2019-01-02",
+        "bc_store": "inversions_tests",
+        "obs_store": "inversions_tests",
+        "footprint_store": "inversions_tests",
+        "emissions_store": "inversions_tests",
+        "inlet": ["10m", "185m"],
+        "instrument": ["gcmd", "picarro"],
+        "domain": "EUROPE",
+        "fp_height": ["10m", "185m"],
+        "fp_model": "NAME",
+        "emissions_name": ["total-ukghg-edgar7"],
+        # "met_model": "ukv",
+        "averaging_period": ["1h", "1h"],
+    }
+    return data_args

tests/test_full_inversion.py

@@ -125,6 +125,13 @@ def test_full_inversion_pollution_events_from_obs_no_bc(mcmc_args):
     fixedbasisMCMC(**mcmc_args)
 
 
+def test_full_inversion_two_sites(mcmc_args, mhd_and_tac_ch4_data_args):
+    mcmc_args.update(mhd_and_tac_ch4_data_args)
+    mcmc_args["reload_merged_data"] = False
+    print(mcmc_args)
+    fixedbasisMCMC(**mcmc_args)
+
+
 @pytest.mark.slow
 def test_full_inversion_long(mcmc_args):
     mcmc_args.update(