Added full inversion test with all required data

.gitignore

@@ -1,3 +1,7 @@
+# OpenGHG inversions specific
+bc_basis_functions/
+countries/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -152,3 +156,4 @@ dmypy.json
 # Cython debug symbols
 cython_debug/
 
+*.DS_Store

openghg_inversions/basis_functions.py

@@ -219,3 +219,4 @@ def qtoptim(x):
         if not os.path.exists(basisoutpath):
             os.makedirs(basisoutpath)
         newds.to_netcdf(os.path.join(basisoutpath,f"quadtree_{species}-{outputname}_{domain}_{start_date.split('-')[0]}.nc"), mode='w')
+        return outputdir

pytest.ini

@@ -0,0 +1,8 @@
+[pytest]
+testpaths = tests
+
+markers =
+        slow: tests that take a long time
+
+addopts =
+        -m 'not slow'

requirements-dev.txt

@@ -0,0 +1,3 @@
+# Required for testing
+flake8
+pytest >= 6.2.5

tests/conftest.py

@@ -0,0 +1,120 @@
+from pathlib import Path
+import shutil
+import tempfile
+from typing import Iterator
+from unittest.mock import patch
+
+import pytest
+
+from openghg.retrieve import search
+from openghg.standardise import standardise_surface, standardise_bc, standardise_flux, standardise_footprint
+from openghg.types import ObjectStoreError
+
+raw_data_path = Path(".").resolve() / "tests/data/"
+inversions_test_store_path = Path(tempfile.gettempdir(), "openghg_inversions_testing_store")
+bc_basis_function_path = Path(".").resolve() / "bc_basis_functions"
+countries_path = Path(".").resolve() / "countries"
+
+
+@pytest.fixture(scope="session", autouse=True)
+def session_config_mocker() -> Iterator[None]:
+    mock_config = {
+        "object_store": {
+            "inversions_tests": {"path": str(inversions_test_store_path), "permissions": "rw"},
+        },
+        "user_id": "test-id-123",
+        "config_version": "2",
+    }
+
+    with patch("openghg.objectstore._local_store.read_local_config", return_value=mock_config):
+        yield
+
+
+# TEST DATA
+obs_metadata = {
+    "source_format": "openghg",
+    "network": "decc",
+    "site": "tac",
+    "inlet": "185m",
+    "instrument": "picarro",
+}
+bc_metadata = {"species": "ch4", "bc_input": "cams", "domain": "europe", "store": "inversions_tests"}
+footprints_metadata = {
+    "site": "tac",
+    "domain": "europe",
+    "model": "name",
+    "inlet": "185m",
+    "metmodel": "ukv",
+}
+flux_metadata = {"species": "ch4", "source": "total-ukghg-edgar7", "domain": "europe"}
+
+obs_data_path = raw_data_path / "obs_tac_ch4_185m_2019-01-01_2019-02-01_data.nc"
+bc_data_path = raw_data_path / "bc_ch4_europe_cams_2019-01-01_2019-12-31_data.nc"
+footprints_data_path = raw_data_path / "footprints_tac_europe_name_185m_2019-01-01_2019-01-07_data.nc"
+flux_data_path = raw_data_path / "flux_total_ch4_europe_edgar7_2019-01-01_2019-12-31_data.nc"
+
+data_info = {
+    "surface": [standardise_surface, obs_metadata, obs_data_path],
+    "boundary_conditions": [standardise_bc, bc_metadata, bc_data_path],
+    "emissions": [standardise_flux, flux_metadata, flux_data_path],
+    "footprints": [standardise_footprint, footprints_metadata, footprints_data_path],
+}
+
+
+@pytest.fixture(scope="session", autouse=True)
+def session_object_store(session_config_mocker) -> None:
+    """Add data to test object.
+
+    Check first if there is enough data. A more specific
+    check for the data necessary for testing is carried out
+    in "test_conftest.py".
+
+    This fixture depends on `sesson_config_mocker` to make sure
+    that `session_config_mocker` runs first.
+    """
+    add_data = False  # flag, True if data needs to be added
+
+    try:
+        results = search(store="inversions_tests")
+    except ObjectStoreError:
+        add_data = True
+        found_dtypes = []
+    else:
+        add_data = results.results.shape[0] != 4
+        try:
+            found_dtypes = results.results["data_type"].to_list()
+        except KeyError:
+            found_dtypes = []
+
+    # check if there are four pieces of data in the object store
+    # if not, add the missing data
+    if add_data:
+        to_add = set(["surface", "boundary_conditions", "emissions", "footprints"]) - set(found_dtypes)
+
+        for dtype in to_add:
+            standardise_fn = data_info[dtype][0]
+            file_path = data_info[dtype][2]
+            metadata = data_info[dtype][1]
+            metadata["store"] = "inversions_tests"
+            standardise_fn(file_path, **metadata)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def session_ancilliary_files() -> None:
+    # Add bc basis function file
+    if not bc_basis_function_path.exists():
+        bc_basis_function_path.mkdir()
+    if not (bc_basis_function_path / "EUROPE").exists():
+        (bc_basis_function_path / "EUROPE").mkdir()
+
+    # copy basis file into default location if there isn't a file with the same name there
+    if not (bc_basis_function_path / "EUROPE" / "NESW_EUROPE_2019.nc").exists():
+        shutil.copy((raw_data_path / "bc_basis_NESW_EUROPE_2019.nc"), (bc_basis_function_path / "EUROPE" / "NESW_EUROPE_2019.nc"))
+
+    # Add country file
+    if not countries_path.exists():
+        countries_path.mkdir()
+
+    # copy country file into default location if there isn't a file with the same name there
+    if not (countries_path / "country_EUROPE.nc").exists():
+        shutil.copy((raw_data_path / "country_EUROPE.nc"), (countries_path / "country_EUROPE.nc"))

tests/test_conftest.py

@@ -0,0 +1,35 @@
+from openghg.retrieve import search
+
+
+def test_default_session_fixture():
+    """The default session fixture should mock `read_local_config`
+    so that the object store path is set to:
+
+    <temp dir>/openghg_inversions_testing_store
+    """
+    from openghg.objectstore._local_store import read_local_config
+    conf = read_local_config()
+
+    assert conf
+    assert 'inversions_tests' in conf['object_store']
+    assert 'openghg_inversions_testing_store' in conf['object_store']['inversions_tests']['path']
+
+
+def test_obs_in_test_store():
+    results = search(site='tac', species='ch4', data_type='surface', store="inversions_tests")
+    assert results
+
+
+def test_footprints_in_test_store():
+    results = search(site='tac', data_type='footprints')
+    assert results
+
+
+def test_bc_in_test_store():
+    results = search(species='ch4', data_type='boundary_conditions')
+    assert results
+
+
+def test_flux_in_test_store():
+    results = search(species='ch4', data_type='emissions')
+    assert results

tests/test_full_inversion.py

@@ -0,0 +1,58 @@
+import pytest
+
+from openghg_inversions.hbmcmc.hbmcmc import fixedbasisMCMC
+
+
+def test_full_inversion(tmp_path):
+    mcmc_args = {
+        "species": "ch4",
+        "sites": ["TAC"],
+        "start_date": "2019-01-01",
+        "end_date": "2019-01-02",
+        "inlet": ["185m"],
+        "instrument": ["picarro"],
+        "domain": "EUROPE",
+        "fp_height": ["185m"],
+        "fp_model": "NAME",
+        "emissions_name": ["total-ukghg-edgar7"],
+        "met_model": "ukv",
+        "meas_period": ["1H"],
+        "outputname": "test_run",
+        "outputpath": str(tmp_path),
+        "quadtree_basis": True,
+        "save_quadtree_to_outputpath": True,
+        "nit": 1,
+        "burn": 0,
+        "tune": 0,
+        "nbasis": 4,
+        "nchain": 1,
+    }
+    fixedbasisMCMC(**mcmc_args)
+
+
+@pytest.mark.slow
+def test_full_inversion_long(tmp_path):
+    mcmc_args = {
+        "species": "ch4",
+        "sites": ["TAC"],
+        "start_date": "2019-01-01",
+        "end_date": "2019-01-08",
+        "inlet": ["185m"],
+        "instrument": ["picarro"],
+        "domain": "EUROPE",
+        "fp_height": ["185m"],
+        "fp_model": "NAME",
+        "emissions_name": ["total-ukghg-edgar7"],
+        "met_model": "ukv",
+        "meas_period": ["1H"],
+        "outputname": "test_run",
+        "outputpath": str(tmp_path),
+        "quadtree_basis": True,
+        "save_quadtree_to_outputpath": True,
+        "nit": 5000,
+        "burn": 2000,
+        "tune": 1000,
+        "nbasis": 50,
+        "nchain": 4,
+    }
+    fixedbasisMCMC(**mcmc_args)