From 4fd2214c5f5f67bb371e1e2938cd63e6c252f026 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 21 Nov 2023 14:23:03 -0500 Subject: [PATCH] test: require dev exes, warn/skip if download/rebuilt not found (#1460) * allow testing without all downloaded/rebuilt binaries * factor out try_get_target for devtools compatibility * refactor targets setup in conftest.py --- autotest/conftest.py | 139 ++++++++++++++++++--------- autotest/test_gwf_csub_distypes.py | 6 +- autotest/test_gwf_returncodes.py | 2 +- autotest/test_gwt_mt3dms_p01.py | 112 +++++++++++---------- autotest/test_z03_examples.py | 3 +- autotest/test_z03_largetestmodels.py | 21 ++-- 6 files changed, 174 insertions(+), 109 deletions(-) diff --git a/autotest/conftest.py b/autotest/conftest.py index c4f3268cfb1..b360140c960 100644 --- a/autotest/conftest.py +++ b/autotest/conftest.py @@ -1,8 +1,9 @@ import platform import sys -from pathlib import Path from os import PathLike -from typing import Dict, Optional +from pathlib import Path +from typing import Optional +from warnings import warn import pytest from modflow_devtools.executables import Executables, get_suffixes @@ -38,60 +39,102 @@ def should_compare( return True +_lib_exts = { + "Darwin": ".dylib", + "Linux": ".so", + "Windows": ".dll", +} +_exe_ext, _lib_ext = get_suffixes(sys.platform) +_binaries_path = project_root_path / "bin" +_dl_bin_path = _binaries_path / "downloaded" +_rb_bin_path = _binaries_path / "rebuilt" +_binaries = { + "development": [ + ("mf6", _binaries_path / f"mf6{_exe_ext}"), + ("libmf6", _binaries_path / f"libmf6{_lib_ext}"), + ("mf5to6", _binaries_path / f"mf5to6{_exe_ext}"), + ("zbud6", _binaries_path / f"zbud6{_exe_ext}"), + ], + "downloaded": [ + ("mf2000", _dl_bin_path / f"mf2000{_exe_ext}"), + ("mf2005", _dl_bin_path / f"mf2005dbl{_exe_ext}"), + ("mfnwt", _dl_bin_path / f"mfnwtdbl{_exe_ext}"), + ("mfusg", _dl_bin_path / f"mfusgdbl{_exe_ext}"), + ("mflgr", _dl_bin_path / f"mflgrdbl{_exe_ext}"), + ("mf2005s", _dl_bin_path / f"mf2005{_exe_ext}"), + ("mt3dms", _dl_bin_path / f"mt3dms{_exe_ext}"), + ("crt", _dl_bin_path / f"crt{_exe_ext}"), + ("gridgen", _dl_bin_path / f"gridgen{_exe_ext}"), + ("mp6", _dl_bin_path / f"mp6{_exe_ext}"), + ("mp7", _dl_bin_path / f"mp7{_exe_ext}"), + ("swtv4", _dl_bin_path / f"swtv4{_exe_ext}"), + ("sutra", _dl_bin_path / f"sutra{_exe_ext}"), + ("triangle", _dl_bin_path / f"triangle{_exe_ext}"), + ("vs2dt", _dl_bin_path / f"vs2dt{_exe_ext}"), + ("zonbudusg", _dl_bin_path / f"zonbudusg{_exe_ext}"), + ], + "rebuilt": [ + ("mf6_regression", _rb_bin_path / f"mf6{_exe_ext}"), + ("libmf6_regression", _rb_bin_path / f"libmf6{_lib_ext}"), + ("mf5to6_regression", _rb_bin_path / f"mf5to6{_exe_ext}"), + ("zbud6_regression", _rb_bin_path / f"zbud6{_exe_ext}"), + ], +} + + @pytest.fixture(scope="session") def bin_path() -> Path: - return project_root_path / "bin" + return _binaries_path @pytest.fixture(scope="session") -def libmf6_path(bin_path) -> Path: - ext = { - "Darwin": ".dylib", - "Linux": ".so", - "Windows": ".dll", - }[platform.system()] - lib_name = bin_path / f"libmf6{ext}" - return lib_name +def libmf6_path() -> Path: + return _binaries_path / f"libmf6{_lib_exts[platform.system()]}" @pytest.fixture(scope="session") -def targets(bin_path) -> Executables: - exe_ext, lib_ext = get_suffixes(sys.platform) - dl_bin = bin_path / "downloaded" - rb_bin = bin_path / "rebuilt" - tgts = dict() - - # downloaded executables - tgts["mf2000"] = dl_bin / f"mf2000{exe_ext}" - tgts["mf2005"] = dl_bin / f"mf2005dbl{exe_ext}" - tgts["mfnwt"] = dl_bin / f"mfnwtdbl{exe_ext}" - tgts["mfusg"] = dl_bin / f"mfusgdbl{exe_ext}" - tgts["mflgr"] = dl_bin / f"mflgrdbl{exe_ext}" - tgts["mf2005s"] = dl_bin / f"mf2005{exe_ext}" - tgts["mt3dms"] = dl_bin / f"mt3dms{exe_ext}" - tgts["crt"] = dl_bin / f"crt{exe_ext}" - tgts["gridgen"] = dl_bin / f"gridgen{exe_ext}" - tgts["mp6"] = dl_bin / f"mp6{exe_ext}" - tgts["mp7"] = dl_bin / f"mp7{exe_ext}" - tgts["swtv4"] = dl_bin / f"swtv4{exe_ext}" - tgts["sutra"] = dl_bin / f"sutra{exe_ext}" - tgts["triangle"] = dl_bin / f"triangle{exe_ext}" - tgts["vs2dt"] = dl_bin / f"vs2dt{exe_ext}" - tgts["zonbudusg"] = dl_bin / f"zonbudusg{exe_ext}" - - # binaries rebuilt from last release - tgts["mf6_regression"] = rb_bin / f"mf6{exe_ext}" - tgts["libmf6_regression"] = rb_bin / f"libmf6{lib_ext}" - tgts["mf5to6_regression"] = rb_bin / f"mf5to6{exe_ext}" - tgts["zbud6_regression"] = rb_bin / f"zbud6{exe_ext}" - - # local development binaries - tgts["mf6"] = bin_path / f"mf6{exe_ext}" - tgts["libmf6"] = bin_path / f"libmf6{lib_ext}" - tgts["mf5to6"] = bin_path / f"mf5to6{exe_ext}" - tgts["zbud6"] = bin_path / f"zbud6{exe_ext}" - - return Executables(**tgts) +def targets() -> Executables: + d = dict() + + # require development binaries + for k, v in _binaries["development"]: + assert v.is_file(), f"Couldn't find binary '{k}' expected at: {v}" + d[k] = v + + # downloaded/rebuilt binaries are optional + for k, v in _binaries["downloaded"] + _binaries["rebuilt"]: + if v.is_file(): + d[k] = v + else: + warn(f"Couldn't find binary '{k}' expected at: {v}") + + return Executables(**d) + + +def try_get_target(targets: Executables, name: str) -> Path: + """Try to retrieve the path to a binary. If the binary is a development + target and can't be found, an error is raised. Otherwise (if the binary + is downloaded or rebuilt) the test is skipped. This is to allow testing + without downloaded or rebuilt binaries, e.g. if the network is down.""" + + try: + # modflow-devtools >= 1.3 + exe = targets.get(name) + if exe: + return exe + elif name in _binaries["development"]: + raise ValueError(f"Couldn't find binary '{name}'") + else: + pytest.skip(f"Couldn't find binary '{name}'") + except AttributeError: + # modflow-devtools < 1.3 + try: + return targets[name] + except: + if name in _binaries["development"]: + raise ValueError(f"Couldn't find binary '{name}'") + else: + pytest.skip(f"Couldn't find binary 'gridgen'") @pytest.fixture diff --git a/autotest/test_gwf_csub_distypes.py b/autotest/test_gwf_csub_distypes.py index bc5267dfdc0..ebe793c02f9 100644 --- a/autotest/test_gwf_csub_distypes.py +++ b/autotest/test_gwf_csub_distypes.py @@ -5,6 +5,7 @@ import pytest from flopy.utils.gridgen import Gridgen +from conftest import try_get_target from framework import TestFramework from simulation import TestSimulation @@ -425,17 +426,16 @@ def eval_zdis(sim): + f"z-displacement at time {totim}" ) - return - @pytest.mark.parametrize( "idx, name", list(enumerate(ex)), ) def test_mf6model(idx, name, function_tmpdir, targets): + gridgen = try_get_target(targets, "gridgen") ws = function_tmpdir test = TestFramework() - test.build(lambda i, w: build_model(i, w, targets.gridgen), idx, ws) + test.build(lambda i, w: build_model(i, w, gridgen), idx, ws) test.run( TestSimulation( name=name, diff --git a/autotest/test_gwf_returncodes.py b/autotest/test_gwf_returncodes.py index 3eda6b7ef76..e9195b00f3c 100644 --- a/autotest/test_gwf_returncodes.py +++ b/autotest/test_gwf_returncodes.py @@ -271,5 +271,5 @@ def compiler_argv(dir, exe): ), ) def test_main(fn, function_tmpdir, targets): - mf6 = targets.as_dict()["mf6"] + mf6 = targets.mf6 eval(fn)(function_tmpdir, mf6) diff --git a/autotest/test_gwt_mt3dms_p01.py b/autotest/test_gwt_mt3dms_p01.py index bd163a76202..a9b00d958cf 100644 --- a/autotest/test_gwt_mt3dms_p01.py +++ b/autotest/test_gwt_mt3dms_p01.py @@ -20,9 +20,14 @@ """ import os +from pathlib import Path +from typing import Tuple import flopy import numpy as np +import pytest + +from conftest import try_get_target testgroup = "mt3dms_p01" @@ -383,7 +388,6 @@ def p01mf6( theta_immobile = prsity2 porosity_immobile = theta_immobile / volfrac_immobile porosity_mobile = prsity / volfrac_mobile - first_order_decay = True if zero_order_decay: @@ -470,6 +474,14 @@ def p01mf6( return sim, conc +def get_binaries(targets) -> Tuple[Path, Path, Path]: + return ( + targets.mf6, + try_get_target(targets, "mf2005s"), + try_get_target(targets, "mt3dms"), + ) + + def test_mt3dmsp01a(function_tmpdir, targets): longitudinal_dispersivity = 0.0 retardation = 1.0 @@ -478,8 +490,10 @@ def test_mt3dmsp01a(function_tmpdir, targets): zeta = None prsity2 = None - mf6 = targets["mf6"] - mf6_ws = str(function_tmpdir / (testgroup + "a")) + mf6, mf2005, mt3dms = get_binaries(targets) + mf6_ws = function_tmpdir / f"{testgroup}a" + mt3d_ws = mf6_ws / "mt3d" + sim, conc_mf6 = p01mf6( mf6_ws, longitudinal_dispersivity, @@ -491,9 +505,6 @@ def test_mt3dmsp01a(function_tmpdir, targets): exe=mf6, ) - mf2005 = targets["mf2005s"] - mt3dms = targets["mt3dms"] - mt3d_ws = os.path.join(mf6_ws, "mt3d") mf, mt, conc_mt3d, cvt, mvt = p01mt3d( mt3d_ws, longitudinal_dispersivity, @@ -506,8 +517,9 @@ def test_mt3dmsp01a(function_tmpdir, targets): mt3dms=mt3dms, ) - msg = f"concentrations not equal {conc_mt3d} {conc_mf6}" - assert np.allclose(conc_mt3d, conc_mf6, atol=1e-4), msg + assert np.allclose( + conc_mt3d, conc_mf6, atol=1e-4 + ), f"concentrations not equal {conc_mt3d} {conc_mf6}" # load transport budget # budget text: @@ -543,8 +555,10 @@ def test_mt3dmsp01b(function_tmpdir, targets): zeta = None prsity2 = None - mf6 = targets["mf6"] - mf6_ws = str(function_tmpdir / (testgroup + "b")) + mf6, mf2005, mt3dms = get_binaries(targets) + mf6_ws = function_tmpdir / f"{testgroup}b" + mt3d_ws = mf6_ws / "mt3d" + sim, conc_mf6 = p01mf6( mf6_ws, longitudinal_dispersivity, @@ -556,9 +570,6 @@ def test_mt3dmsp01b(function_tmpdir, targets): exe=mf6, ) - mf2005 = targets["mf2005s"] - mt3dms = targets["mt3dms"] - mt3d_ws = os.path.join(mf6_ws, "mt3d") mf, mt, conc_mt3d, cvt, mvt = p01mt3d( mt3d_ws, longitudinal_dispersivity, @@ -571,8 +582,9 @@ def test_mt3dmsp01b(function_tmpdir, targets): mt3dms=mt3dms, ) - msg = f"concentrations not equal {conc_mt3d} {conc_mf6}" - assert np.allclose(conc_mt3d, conc_mf6, atol=1e-4), msg + assert np.allclose( + conc_mt3d, conc_mf6, atol=1e-4 + ), f"concentrations not equal {conc_mt3d} {conc_mf6}" def test_mt3dmsp01c(function_tmpdir, targets): @@ -583,8 +595,10 @@ def test_mt3dmsp01c(function_tmpdir, targets): zeta = None prsity2 = None - mf6 = targets["mf6"] - mf6_ws = str(function_tmpdir / (testgroup + "c")) + mf6, mf2005, mt3dms = get_binaries(targets) + mf6_ws = function_tmpdir / f"{testgroup}c" + mt3d_ws = mf6_ws / "mt3d" + sim, conc_mf6 = p01mf6( mf6_ws, longitudinal_dispersivity, @@ -596,9 +610,6 @@ def test_mt3dmsp01c(function_tmpdir, targets): exe=mf6, ) - mf2005 = targets["mf2005s"] - mt3dms = targets["mt3dms"] - mt3d_ws = os.path.join(mf6_ws, "mt3d") mf, mt, conc_mt3d, cvt, mvt = p01mt3d( mt3d_ws, longitudinal_dispersivity, @@ -611,8 +622,9 @@ def test_mt3dmsp01c(function_tmpdir, targets): mt3dms=mt3dms, ) - msg = f"concentrations not equal {conc_mt3d} {conc_mf6}" - assert np.allclose(conc_mt3d, conc_mf6, atol=1e-4), msg + assert np.allclose( + conc_mt3d, conc_mf6, atol=1e-4 + ), f"concentrations not equal {conc_mt3d} {conc_mf6}" def test_mt3dmsp01d(function_tmpdir, targets): @@ -623,8 +635,10 @@ def test_mt3dmsp01d(function_tmpdir, targets): zeta = None prsity2 = None - mf6 = targets["mf6"] - mf6_ws = str(function_tmpdir / (testgroup + "d")) + mf6, mf2005, mt3dms = get_binaries(targets) + mf6_ws = function_tmpdir / f"{testgroup}d" + mt3d_ws = mf6_ws / "mt3d" + sim, conc_mf6 = p01mf6( mf6_ws, longitudinal_dispersivity, @@ -636,9 +650,6 @@ def test_mt3dmsp01d(function_tmpdir, targets): exe=mf6, ) - mf2005 = targets["mf2005s"] - mt3dms = targets["mt3dms"] - mt3d_ws = os.path.join(mf6_ws, "mt3d") mf, mt, conc_mt3d, cvt, mvt = p01mt3d( mt3d_ws, longitudinal_dispersivity, @@ -651,8 +662,9 @@ def test_mt3dmsp01d(function_tmpdir, targets): mt3dms=mt3dms, ) - msg = f"concentrations not equal {conc_mt3d} {conc_mf6}" - assert np.allclose(conc_mt3d, conc_mf6, atol=1e-4), msg + assert np.allclose( + conc_mt3d, conc_mf6, atol=1e-4 + ), f"concentrations not equal {conc_mt3d} {conc_mf6}" def test_mt3dmsp01e(function_tmpdir, targets): @@ -663,8 +675,10 @@ def test_mt3dmsp01e(function_tmpdir, targets): zeta = 0.1 prsity2 = 0.05 - mf6 = targets["mf6"] - mf6_ws = str(function_tmpdir / (testgroup + "e")) + mf6, mf2005, mt3dms = get_binaries(targets) + mf6_ws = function_tmpdir / f"{testgroup}e" + mt3d_ws = mf6_ws / "mt3d" + sim, conc_mf6 = p01mf6( mf6_ws, longitudinal_dispersivity, @@ -676,9 +690,6 @@ def test_mt3dmsp01e(function_tmpdir, targets): exe=mf6, ) - mf2005 = targets["mf2005s"] - mt3dms = targets["mt3dms"] - mt3d_ws = os.path.join(mf6_ws, "mt3d") mf, mt, conc_mt3d, cvt, mvt = p01mt3d( mt3d_ws, longitudinal_dispersivity, @@ -691,8 +702,9 @@ def test_mt3dmsp01e(function_tmpdir, targets): mt3dms=mt3dms, ) - msg = f"concentrations not equal {conc_mt3d} {conc_mf6}" - assert np.allclose(conc_mt3d, conc_mf6, atol=1e-1), msg + assert np.allclose( + conc_mt3d, conc_mf6, atol=1e-1 + ), f"concentrations not equal {conc_mt3d} {conc_mf6}" def test_mt3dmsp01f(function_tmpdir, targets): @@ -703,8 +715,10 @@ def test_mt3dmsp01f(function_tmpdir, targets): zeta = 0.1 prsity2 = 0.05 - mf6 = targets["mf6"] - mf6_ws = str(function_tmpdir / (testgroup + "f")) + mf6, mf2005, mt3dms = get_binaries(targets) + mf6_ws = function_tmpdir / f"{testgroup}f" + mt3d_ws = mf6_ws / "mt3d" + sim, conc_mf6 = p01mf6( mf6_ws, longitudinal_dispersivity, @@ -717,9 +731,6 @@ def test_mt3dmsp01f(function_tmpdir, targets): exe=mf6, ) - mf2005 = targets["mf2005s"] - mt3dms = targets["mt3dms"] - mt3d_ws = os.path.join(mf6_ws, "mt3d") mf, mt, conc_mt3d, cvt, mvt = p01mt3d( mt3d_ws, longitudinal_dispersivity, @@ -732,8 +743,9 @@ def test_mt3dmsp01f(function_tmpdir, targets): mt3dms=mt3dms, ) - msg = f"concentrations not equal {conc_mt3d} {conc_mf6}" - assert np.allclose(conc_mt3d, conc_mf6, atol=1e-1), msg + assert np.allclose( + conc_mt3d, conc_mf6, atol=1e-1 + ), f"concentrations not equal {conc_mt3d} {conc_mf6}" def test_mt3dmsp01g(function_tmpdir, targets): @@ -744,8 +756,10 @@ def test_mt3dmsp01g(function_tmpdir, targets): zeta = None prsity2 = None - mf6 = targets["mf6"] - mf6_ws = str(function_tmpdir / (testgroup + "g")) + mf6, mf2005, mt3dms = get_binaries(targets) + mf6_ws = function_tmpdir / f"{testgroup}g" + mt3d_ws = mf6_ws / "mt3d" + sim, conc_mf6 = p01mf6( mf6_ws, longitudinal_dispersivity, @@ -758,9 +772,6 @@ def test_mt3dmsp01g(function_tmpdir, targets): exe=mf6, ) - mf2005 = targets["mf2005s"] - mt3dms = targets["mt3dms"] - mt3d_ws = os.path.join(mf6_ws, "mt3d") mf, mt, conc_mt3d, cvt, mvt = p01mt3d( mt3d_ws, longitudinal_dispersivity, @@ -775,5 +786,6 @@ def test_mt3dmsp01g(function_tmpdir, targets): mt3dms=mt3dms, ) - msg = f"concentrations not equal {conc_mt3d} {conc_mf6}" - assert np.allclose(conc_mt3d, conc_mf6, atol=1.0e-4), msg + assert np.allclose( + conc_mt3d, conc_mf6, atol=1.0e-4 + ), f"concentrations not equal {conc_mt3d} {conc_mf6}" diff --git a/autotest/test_z03_examples.py b/autotest/test_z03_examples.py index 124d8f9229a..83b06cc418f 100644 --- a/autotest/test_z03_examples.py +++ b/autotest/test_z03_examples.py @@ -1,4 +1,5 @@ import pytest + from conftest import should_compare from simulation import TestSimulation @@ -62,7 +63,7 @@ def test_scenario(function_tmpdir, example_scenario, targets): workspace = function_tmpdir / model_name sim = TestSimulation( name=model_name, - exe_dict=targets.as_dict(), + exe_dict=targets, mf6_regression=True, cmp_verbose=False, make_comparison=should_compare( diff --git a/autotest/test_z03_largetestmodels.py b/autotest/test_z03_largetestmodels.py index c5f9175579a..7a7c17ea2c1 100644 --- a/autotest/test_z03_largetestmodels.py +++ b/autotest/test_z03_largetestmodels.py @@ -1,13 +1,22 @@ import pytest + from conftest import should_compare from simulation import TestSimulation excluded_models = [] excluded_comparisons = { - "test1004_mvlake_laksfr_tr": ["6.4.1",], - "test1004_mvlake_lak_tr": ["6.4.1",], - "test1003_MNW2_Fig28": ["6.2.1",], - "test1001_Peterson": ["6.2.1",], + "test1004_mvlake_laksfr_tr": [ + "6.4.1", + ], + "test1004_mvlake_lak_tr": [ + "6.4.1", + ], + "test1003_MNW2_Fig28": [ + "6.2.1", + ], + "test1001_Peterson": [ + "6.2.1", + ], } @@ -23,13 +32,13 @@ def test_model( if name in excluded_models: pytest.skip(f"Excluding large mf6 model '{name}'") - + if "dev" in name and "not developmode" in markers: pytest.skip(f"Skipping large mf6 model '{name}' (develop mode only)") sim = TestSimulation( name=name, - exe_dict=targets.as_dict(), + exe_dict=targets, mf6_regression=not original_regression, cmp_verbose=False, make_comparison=should_compare(name, excluded_comparisons, targets),