diff --git a/tests/conftest.py b/tests/conftest.py index 811277f..ccc0bd1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ +from collections import Counter, namedtuple from pathlib import Path import shutil import tempfile @@ -28,7 +29,8 @@ def merged_data_dir(): @pytest.fixture(scope="session", autouse=True) def using_zarr_store(): - return version("openghg") >= "0.8" + current_version = tuple(int(x) for x in version("openghg").split(".")) + return current_version >= (0, 8) @pytest.fixture(scope="session") @@ -88,39 +90,68 @@ def session_config_mocker(using_zarr_store) -> Iterator[None]: # TEST DATA -obs_metadata = { +TestData = namedtuple("TestData", ["func", "metadata", "path", "data_type"]) +test_data_list = [] + +## Obs data +tac_obs_metadata = { "source_format": "openghg", "network": "decc", "site": "tac", "inlet": "185m", "instrument": "picarro", } +tac_obs_data_path = _raw_data_path / "obs_tac_ch4_185m_2019-01-01_2019-02-01_data.nc" +test_data_list.append(TestData(standardise_surface, tac_obs_metadata, tac_obs_data_path, "surface")) + +mhd_obs_metadata = { + "source_format": "openghg", + "network": "agage", + "site": "mhd", + "inlet": "10m", + "instrument": "gcmd", + "calibration_scale": "WMO-x2004a", +} +mhd_obs_data_path = _raw_data_path / "obs_mhd_ch4_10m_2019-01-01_2019-01-07_data.nc" +test_data_list.append(TestData(standardise_surface, mhd_obs_metadata, mhd_obs_data_path, "surface")) + +## BC data bc_metadata = {"species": "ch4", "bc_input": "cams", "domain": "europe", "store": "inversions_tests"} -footprints_metadata = { +bc_data_path = _raw_data_path / "bc_ch4_europe_cams_2019-01-01_2019-12-31_data.nc" +test_data_list.append(TestData(standardise_bc, bc_metadata, bc_data_path, "boundary_conditions")) + +## Footprint data +tac_footprints_metadata = { "site": "tac", "domain": "europe", "model": "name", "inlet": "185m", # "metmodel": "ukv", } -flux_metadata = {"species": "ch4", "source": "total-ukghg-edgar7", "domain": "europe"} -flux_dim_shuffle_metadata = {"species": "ch4", "source": "total-ukghg-edgar7-shuffled", "domain": "europe"} +tac_footprints_data_path = _raw_data_path / "footprints_tac_europe_name_185m_2019-01-01_2019-01-07_data.nc" +test_data_list.append(TestData(standardise_footprint, tac_footprints_metadata, tac_footprints_data_path, "footprints")) -obs_data_path = _raw_data_path / "obs_tac_ch4_185m_2019-01-01_2019-02-01_data.nc" -bc_data_path = _raw_data_path / "bc_ch4_europe_cams_2019-01-01_2019-12-31_data.nc" -footprints_data_path = _raw_data_path / "footprints_tac_europe_name_185m_2019-01-01_2019-01-07_data.nc" +mhd_footprints_metadata = { + "site": "mhd", + "domain": "europe", + "model": "name", + "inlet": "10m", + "source_format": "paris", + # "metmodel": "ukv", +} +mhd_footprints_data_path = _raw_data_path / "footprints_mhd_europe_name_10m_2019-01-01_2019-01-07_data.nc" +test_data_list.append(TestData(standardise_footprint, mhd_footprints_metadata, mhd_footprints_data_path, "footprints")) + +## Flux data +flux_metadata = {"species": "ch4", "source": "total-ukghg-edgar7", "domain": "europe"} flux_data_path = _raw_data_path / "flux_total_ch4_europe_edgar7_2019-01-01_2019-12-31_data.nc" +test_data_list.append(TestData(standardise_flux, flux_metadata, flux_data_path, "flux")) + +flux_dim_shuffle_metadata = {"species": "ch4", "source": "total-ukghg-edgar7-shuffled", "domain": "europe"} flux_dim_shuffled_data_path = ( _raw_data_path / "flux_total_ch4_europe_edgar7_2019-01-01_2019-12-31_data_dim_shuffled.nc" ) - -data_info = { - "surface": [standardise_surface, obs_metadata, obs_data_path], - "boundary_conditions": [standardise_bc, bc_metadata, bc_data_path], - "flux": [standardise_flux, flux_metadata, flux_data_path], - "flux_dim_shuffle": [standardise_flux, flux_dim_shuffle_metadata, flux_dim_shuffled_data_path], - "footprints": [standardise_footprint, footprints_metadata, footprints_data_path], -} +test_data_list.append(TestData(standardise_flux, flux_dim_shuffle_metadata, flux_dim_shuffled_data_path, "flux")) @pytest.fixture(scope="session", autouse=True) @@ -140,28 +171,22 @@ def session_object_store(session_config_mocker) -> None: results = search(store="inversions_tests") except ObjectStoreError: add_data = True - found_dtypes = [] else: - add_data = results.results.shape[0] != 5 try: found_dtypes = results.results["data_type"].to_list() except KeyError: - found_dtypes = [] + add_data = True + else: + add_data = Counter([x.data_type for x in test_data_list]) != Counter(found_dtypes) + # check if there are four pieces of data in the object store # if not, add the missing data if add_data: - all_data_types = ["surface", "boundary_conditions", "flux", "flux", "footprints"] - for dtype in found_dtypes: - all_data_types.remove(dtype) - - if "flux" in all_data_types: - all_data_types.append("flux_dim_shuffle") - - for dtype in all_data_types: - standardise_fn = data_info[dtype][0] - file_path = data_info[dtype][2] - metadata = data_info[dtype][1] + for test_data in test_data_list: + standardise_fn = test_data.func + file_path = test_data.path + metadata = test_data.metadata metadata["store"] = "inversions_tests" standardise_fn(filepath=file_path, **metadata) @@ -211,3 +236,26 @@ def tac_ch4_data_args(): "averaging_period": ["1H"], } return data_args + + +@pytest.fixture(scope="module") +def mhd_and_tac_ch4_data_args(): + data_args = { + "species": "ch4", + "sites": ["MHD", "TAC"], + "start_date": "2019-01-01", + "end_date": "2019-01-02", + "bc_store": "inversions_tests", + "obs_store": "inversions_tests", + "footprint_store": "inversions_tests", + "emissions_store": "inversions_tests", + "inlet": ["10m", "185m"], + "instrument": ["gcmd", "picarro"], + "domain": "EUROPE", + "fp_height": ["10m", "185m"], + "fp_model": "NAME", + "emissions_name": ["total-ukghg-edgar7"], + # "met_model": "ukv", + "averaging_period": ["1h", "1h"], + } + return data_args diff --git a/tests/data/footprints_mhd_europe_name_10m_2019-01-01_2019-01-07_data.nc b/tests/data/footprints_mhd_europe_name_10m_2019-01-01_2019-01-07_data.nc new file mode 100644 index 0000000..6fb23c6 Binary files /dev/null and b/tests/data/footprints_mhd_europe_name_10m_2019-01-01_2019-01-07_data.nc differ diff --git a/tests/data/obs_mhd_ch4_10m_2019-01-01_2019-01-07_data.nc b/tests/data/obs_mhd_ch4_10m_2019-01-01_2019-01-07_data.nc new file mode 100644 index 0000000..7f33d61 Binary files /dev/null and b/tests/data/obs_mhd_ch4_10m_2019-01-01_2019-01-07_data.nc differ diff --git a/tests/data/obs_mhd_ch4_10m_2019-01-01_2019-01-07_data_old.nc b/tests/data/obs_mhd_ch4_10m_2019-01-01_2019-01-07_data_old.nc new file mode 100644 index 0000000..62e231a Binary files /dev/null and b/tests/data/obs_mhd_ch4_10m_2019-01-01_2019-01-07_data_old.nc differ diff --git a/tests/test_full_inversion.py b/tests/test_full_inversion.py index a026666..6f28f52 100644 --- a/tests/test_full_inversion.py +++ b/tests/test_full_inversion.py @@ -125,6 +125,13 @@ def test_full_inversion_pollution_events_from_obs_no_bc(mcmc_args): fixedbasisMCMC(**mcmc_args) +def test_full_inversion_two_sites(mcmc_args, mhd_and_tac_ch4_data_args): + mcmc_args.update(mhd_and_tac_ch4_data_args) + mcmc_args["reload_merged_data"] = False + print(mcmc_args) + fixedbasisMCMC(**mcmc_args) + + @pytest.mark.slow def test_full_inversion_long(mcmc_args): mcmc_args.update(