diff --git a/easybuild/base/testing.py b/easybuild/base/testing.py index 71700794d3..ccef36a1ac 100644 --- a/easybuild/base/testing.py +++ b/easybuild/base/testing.py @@ -221,6 +221,16 @@ def mocked_stdout_stderr(self, mock_stdout=True, mock_stderr=True): if mock_stderr: self.mock_stderr(False) + @contextmanager + def saved_env(self): + """Context manager to reset environment to state when it was entered""" + orig_env = os.environ.copy() + try: + yield + finally: + os.environ.clear() + os.environ.update(orig_env) + def tearDown(self): """Cleanup after running a test.""" self.mock_stdout(False) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index dd7f65fc37..c5339ac283 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -208,7 +208,7 @@ def __init__(self, ec, logfile=None): self.skip = None self.module_extra_extensions = '' # extra stuff for module file required by extensions - # indicates whether or not this instance represents an extension or not; + # indicates whether or not this instance represents an extension # may be set to True by ExtensionEasyBlock self.is_extension = False @@ -670,12 +670,11 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): 'name': ext_name, 'version': ext_version, 'options': ext_options, + # if a particular easyblock is specified, make sure it's used + # (this is picked up by init_ext_instances) + 'easyblock': ext_options.get('easyblock', None), } - # if a particular easyblock is specified, make sure it's used - # (this is picked up by init_ext_instances) - ext_src['easyblock'] = ext_options.get('easyblock', None) - # construct dictionary with template values; # inherited from parent, except for name/version templates which are specific to this extension template_values = copy.deepcopy(self.cfg.template_values) @@ -1810,7 +1809,7 @@ def inject_module_extra_paths(self): msg += f"and paths='{env_var}'" self.log.debug(msg) - def expand_module_search_path(self, search_path, path_type=ModEnvVarType.PATH_WITH_FILES): + def expand_module_search_path(self, *_, **__): """ REMOVED in EasyBuild 5.1, use EasyBlock.module_load_environment.expand_paths instead """ @@ -2773,7 +2772,7 @@ def check_checksums_for(self, ent, sub='', source_cnt=None): # if the filename is a dict, the actual source file name is the "filename" element if isinstance(fn, dict): fn = fn["filename"] - if fn in checksums_from_json.keys(): + if fn in checksums_from_json: checksums += [checksums_from_json[fn]] if source_cnt is None: @@ -2964,7 +2963,8 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True): if os.path.isabs(self.rpath_wrappers_dir): _log.info(f"Using {self.rpath_wrappers_dir} to store/use RPATH wrappers") else: - raise EasyBuildError(f"Path used for rpath_wrappers_dir is not an absolute path: {path}") + raise EasyBuildError("Path used for rpath_wrappers_dir is not an absolute path: %s", + self.rpath_wrappers_dir) if self.iter_idx > 0: # reset toolchain for iterative runs before preparing it again @@ -3360,6 +3360,16 @@ def post_processing_step(self): def _dispatch_sanity_check_step(self, *args, **kwargs): """Decide whether to run the dry-run or the real version of the sanity-check step""" + if 'extension' in kwargs: + extension = kwargs.pop('extension') + self.log.deprecated( + "Passing `extension` to `sanity_check_step` is no longer necessary and will be ignored " + f"(Easyblock: {self.__class__.__name__}).", + '6.0', + ) + if extension != self.is_extension: + raise EasyBuildError('Unexpected value for `extension` argument. ' + f'Should be: {self.is_extension}, got: {extension}') if self.dry_run: self._sanity_check_step_dry_run(*args, **kwargs) else: @@ -4076,8 +4086,10 @@ def _sanity_check_step_common(self, custom_paths, custom_commands): paths = {} for key in path_keys_and_check: paths.setdefault(key, []) - paths.update({SANITY_CHECK_PATHS_DIRS: ['bin', ('lib', 'lib64')]}) - self.log.info("Using default sanity check paths: %s", paths) + # Default paths for extensions are handled in the parent easyconfig if desired + if not self.is_extension: + paths.update({SANITY_CHECK_PATHS_DIRS: ['bin', ('lib', 'lib64')]}) + self.log.info("Using default sanity check paths: %s", paths) # if enhance_sanity_check is enabled *and* sanity_check_paths are specified in the easyconfig, # those paths are used to enhance the paths provided by the easyblock @@ -4097,9 +4109,11 @@ def _sanity_check_step_common(self, custom_paths, custom_commands): # verify sanity_check_paths value: only known keys, correct value types, at least one non-empty value only_list_values = all(isinstance(x, list) for x in paths.values()) only_empty_lists = all(not x for x in paths.values()) - if sorted_keys != known_keys or not only_list_values or only_empty_lists: + if sorted_keys != known_keys or not only_list_values or (only_empty_lists and not self.is_extension): error_msg = "Incorrect format for sanity_check_paths: should (only) have %s keys, " - error_msg += "values should be lists (at least one non-empty)." + error_msg += "values should be lists" + if not self.is_extension: + error_msg += " (at least one non-empty)." raise EasyBuildError(error_msg % ', '.join("'%s'" % k for k in known_keys)) # Resolve arch specific entries @@ -4256,7 +4270,7 @@ def sanity_check_load_module(self, extension=False, extra_modules=None): return self.fake_mod_data - def _sanity_check_step(self, custom_paths=None, custom_commands=None, extension=False, extra_modules=None): + def _sanity_check_step(self, custom_paths=None, custom_commands=None, extra_modules=None): """ Real version of sanity_check_step method. @@ -4322,7 +4336,7 @@ def xs2str(xs): trace_msg("%s %s found: %s" % (typ, xs2str(xs), ('FAILED', 'OK')[found])) if not self.sanity_check_module_loaded: - self.sanity_check_load_module(extension=extension, extra_modules=extra_modules) + self.sanity_check_load_module(extension=self.is_extension, extra_modules=extra_modules) # allow oversubscription of P processes on C cores (P>C) for software installed on top of Open MPI; # this is useful to avoid failing of sanity check commands that involve MPI @@ -4351,26 +4365,27 @@ def xs2str(xs): trace_msg(f"result for command '{cmd}': {cmd_result_str}") # also run sanity check for extensions (unless we are an extension ourselves) - if not extension: + if not self.is_extension: if build_option('skip_extensions'): self.log.info("Skipping sanity check for extensions since skip-extensions is enabled...") else: self._sanity_check_step_extensions() - linked_shared_lib_fails = self.sanity_check_linked_shared_libs() - if linked_shared_lib_fails: - self.log.warning("Check for required/banned linked shared libraries failed!") - self.sanity_check_fail_msgs.append(linked_shared_lib_fails) - - # software installed with GCCcore toolchain should not have Fortran module files (.mod), - # unless that's explicitly allowed - if self.toolchain.name in ('GCCcore',) and not self.cfg['skip_mod_files_sanity_check']: - mod_files_found_msg = self.sanity_check_mod_files() - if mod_files_found_msg: - if build_option('fail_on_mod_files_gcccore'): - self.sanity_check_fail_msgs.append(mod_files_found_msg) - else: - print_warning(mod_files_found_msg) + # Do not do those checks for extensions, only in the main easyconfig + linked_shared_lib_fails = self.sanity_check_linked_shared_libs() + if linked_shared_lib_fails: + self.log.warning("Check for required/banned linked shared libraries failed!") + self.sanity_check_fail_msgs.append(linked_shared_lib_fails) + + # software installed with GCCcore toolchain should not have Fortran module files (.mod), + # unless that's explicitly allowed + if self.toolchain.name in ('GCCcore',) and not self.cfg['skip_mod_files_sanity_check']: + mod_files_found_msg = self.sanity_check_mod_files() + if mod_files_found_msg: + if build_option('fail_on_mod_files_gcccore'): + self.sanity_check_fail_msgs.append(mod_files_found_msg) + else: + print_warning(mod_files_found_msg) # cleanup if self.fake_mod_data: @@ -4400,14 +4415,14 @@ def xs2str(xs): self.log.debug("Skipping CUDA sanity check: CUDA is not in dependencies") # pass or fail - if self.sanity_check_fail_msgs: + if not self.sanity_check_fail_msgs: + self.log.debug("Sanity check passed!") + elif not self.is_extension: raise EasyBuildError( "Sanity check failed: " + '\n'.join(self.sanity_check_fail_msgs), exit_code=EasyBuildExit.FAIL_SANITY_CHECK, ) - self.log.debug("Sanity check passed!") - def _set_module_as_default(self, fake=False): """ Sets the default module version except if we are in dry run diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 22f2b9f42e..d0497333b9 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -294,8 +294,6 @@ def sanity_check_step(self): """ Sanity check to run after installing extension """ - res = (True, '') - if os.path.isdir(self.installdir): change_dir(self.installdir) @@ -330,9 +328,6 @@ def sanity_check_step(self): fail_msg = 'command "%s" failed' % cmd fail_msg += "; output:\n%s" % cmd_res.output.strip() self.log.warning("Sanity check for '%s' extension failed: %s", self.name, fail_msg) - res = (False, fail_msg) # keep track of all reasons of failure # (only relevant when this extension is installed stand-alone via ExtensionEasyBlock) self.sanity_check_fail_msgs.append(fail_msg) - - return res diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index 9fb7f1563c..f778204063 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -72,8 +72,6 @@ def extra_options(extra_vars=None): def __init__(self, *args, **kwargs): """Initialize either as EasyBlock or as Extension.""" - self.is_extension = False - if isinstance(args[0], EasyBlock): # make sure that extra custom easyconfig parameters are known extra_params = self.__class__.extra_options() @@ -187,20 +185,20 @@ def sanity_check_step(self, exts_filter=None, custom_paths=None, custom_commands self.log.info(info_msg) trace_msg(info_msg) # perform sanity check for stand-alone extension - (sanity_check_ok, fail_msg) = Extension.sanity_check_step(self) + Extension.sanity_check_step(self) else: # perform single sanity check for extension - (sanity_check_ok, fail_msg) = Extension.sanity_check_step(self) + Extension.sanity_check_step(self) - if custom_paths or custom_commands or not self.is_extension: - super().sanity_check_step(custom_paths=custom_paths, - custom_commands=custom_commands, - extension=self.is_extension) + super().sanity_check_step(custom_paths=custom_paths, + custom_commands=custom_commands) # pass or fail sanity check - if sanity_check_ok: + if not self.sanity_check_fail_msgs: + sanity_check_ok = True self.log.info("Sanity check for %s successful!", self.name) else: + sanity_check_ok = False if not self.is_extension: msg = "Sanity check for %s failed: %s" % (self.name, '; '.join(self.sanity_check_fail_msgs)) raise EasyBuildError(msg) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 78825bde6c..13f074f417 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1095,6 +1095,8 @@ def load(self, modules, mod_paths=None, purge=False, init_env=None, allow_reload :param init_env: original environment to restore after running 'module purge' :param allow_reload: allow reloading an already loaded module """ + if not any((modules, mod_paths, purge)): + return # Avoid costly module paths if nothing to do if mod_paths is None: mod_paths = [] diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 35af52cbcc..baba2c13d1 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -30,13 +30,13 @@ @author: Maxime Boissonneault (Compute Canada) @author: Jan Andre Reuter (Juelich Supercomputing Centre) """ -import copy import fileinput import os import re import shutil import sys import tempfile +import textwrap from inspect import cleandoc from test.framework.github import requires_github_access from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config @@ -52,7 +52,6 @@ from easybuild.tools import LooseVersion, config from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import get_module_syntax, update_build_option -from easybuild.tools.environment import modify_env from easybuild.tools.filetools import change_dir, copy_dir, copy_file, mkdir, read_file, remove_dir, remove_file from easybuild.tools.filetools import symlink, verify_checksum, write_file from easybuild.tools.module_generator import module_generator @@ -1289,14 +1288,14 @@ def test_post_processing_step(self): eb.silent = True depr_msg = r"EasyBlock.post_install_step\(\) is deprecated, use EasyBlock.post_processing_step\(\) instead" expected_error = r"DEPRECATED \(since v6.0\).*" + depr_msg - with self.mocked_stdout_stderr(): + with self.mocked_stdout_stderr(), self.saved_env(): self.assertErrorRegex(EasyBuildError, expected_error, eb.run_all_steps, True) change_dir(cwd) toy_ec = EasyConfig(toy_ec_fn) eb = EB_toy(toy_ec) eb.silent = True - with self.mocked_stdout_stderr() as (_, stderr): + with self.mocked_stdout_stderr() as (_, stderr), self.saved_env(): eb.run_all_steps(True) # no deprecation warning stderr = stderr.getvalue() @@ -1308,14 +1307,13 @@ def test_post_processing_step(self): # check again with toy easyblock that still uses post_install_step, # to verify that the expected file is being created when deprecated functionality is allow remove_file(libtoy_post_a) - modify_env(os.environ, self.orig_environ, verbose=False) change_dir(cwd) self.allow_deprecated_behaviour() toy_ec = EasyConfig(toy_ec_fn) eb = EB_toy_deprecated(toy_ec) eb.silent = True - with self.mocked_stdout_stderr() as (stdout, stderr): + with self.mocked_stdout_stderr() as (stdout, stderr), self.saved_env(): eb.run_all_steps(True) regex = re.compile(depr_msg, re.M) @@ -2598,14 +2596,13 @@ def test_extensions_sanity_check(self): exts_list = toy_ec['exts_list'] exts_list[-1][2]['exts_filter'] = ("thisshouldfail", '') toy_ec['exts_list'] = exts_list - toy_ec['exts_defaultclass'] = 'DummyExtension' eb = EB_toy(toy_ec) eb.silent = True error_pattern = r"Sanity check failed: extensions sanity check failed for 1 extensions: toy\n" error_pattern += r"failing sanity check for 'toy' extension: " error_pattern += r'command "thisshouldfail" failed; output:\n.* thisshouldfail: command not found' - with self.mocked_stdout_stderr(): + with self.mocked_stdout_stderr(), self.saved_env(): self.assertErrorRegex(EasyBuildError, error_pattern, eb.run_all_steps, True) # purposely put sanity check command in place that breaks the build, @@ -2613,12 +2610,58 @@ def test_extensions_sanity_check(self): # sanity check commands are checked after checking sanity check paths, so this should work toy_ec = EasyConfig(toy_ec_fn) toy_ec.update('sanity_check_commands', [("%(installdir)s/bin/toy && rm %(installdir)s/bin/toy", '')]) - toy_ec['exts_defaultclass'] = 'DummyExtension' eb = EB_toy(toy_ec) eb.silent = True - with self.mocked_stdout_stderr(): + with self.mocked_stdout_stderr(), self.saved_env(): eb.run_all_steps(True) + # Verify custom paths and commands of extensions are checked + toy_ec_fn = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') + toy_ec_txt = read_file(toy_ec_fn) + + self.contents = toy_ec_txt + textwrap.dedent(""" + exts_defaultclass = 'DummyExtension' + exts_filter = ('true', '') + exts_list = [ + ('bar', '0.0', { + 'sanity_check_commands': ['echo "%(name)s extension output"'], + 'sanity_check_paths': {'dirs': ['.'], 'files': ['any_file']}, + }), + ('barbar', '0.0', {'sanity_check_commands': ['echo "%(name)s extension output"']}), + ] + sanity_check_commands = ['echo "%(name)s output"'] + sanity_check_paths = { + 'dirs': ['.'], + 'files': [], + } + """) + + self.writeEC() + eb = EB_toy(EasyConfig(self.eb_file)) + eb.silent = True + write_file(os.path.join(eb.installdir, 'any_file'), '') + with self.mocked_stdout_stderr(), self.saved_env(): + eb.sanity_check_step() + logtxt = read_file(eb.logfile) + self.assertRegex(logtxt, 'Running .*command.*echo "toy output"') + self.assertRegex(logtxt, 'Running .*command.*echo "bar extension output"') + self.assertRegex(logtxt, 'Using .*sanity check paths .*any_file') + self.assertRegex(logtxt, 'Running .*command.*echo "barbar extension output"') + self.assertRegex(logtxt, 'Running .*command.*echo "barbar extension output"') + # Only do this once, not for every extension which would be redundant + self.assertEqual(logtxt.count('Checking for banned/required linked shared libraries'), 1) + + # Verify that sanity_check_paths are actually verified + self.contents += "\nexts_list[-1][2]['sanity_check_paths'] = {'dirs': [], 'files': ['nosuchfile']}" + self.writeEC() + eb = EB_toy(EasyConfig(self.eb_file)) + eb.silent = True + write_file(os.path.join(eb.installdir, 'any_file'), '') + with self.mocked_stdout_stderr(): + self.assertRaisesRegex(EasyBuildError, + "extensions sanity check failed for 1 extensions: barbar.*nosuchfile", + eb.sanity_check_step) + def test_parallel(self): """Test defining of parallelism.""" topdir = os.path.abspath(os.path.dirname(__file__)) @@ -2975,23 +3018,19 @@ def test_extension_patch_step(self): cwd = os.getcwd() self.assertExists(cwd) - # Take environment with test-specific variable set up - orig_environ = copy.deepcopy(os.environ) def run_extension_step(): - try: - change_dir(cwd) - eb = EasyBlock(ec) - # Cleanup build directory - if os.path.exists(eb.builddir): - remove_dir(eb.builddir) + change_dir(cwd) + eb = EasyBlock(ec) + # Cleanup build directory + if os.path.exists(eb.builddir): + remove_dir(eb.builddir) + # restore original environment to continue testing with a clean slate + with self.saved_env(): eb.make_builddir() eb.update_config_template_run_step() eb.extensions_step(fetch=True, install=True) - return os.path.join(eb.builddir) - finally: - # restore original environment to continue testing with a clean slate - modify_env(os.environ, orig_environ, verbose=False) + return os.path.join(eb.builddir) ec['exts_defaultclass'] = 'DummyExtension' ec['exts_list'] = [('toy', '0.0', {'easyblock': 'DummyExtension'})] diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 7a07eab449..83db63750a 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1490,7 +1490,6 @@ def test_start_dir_template(self): preconfigopts = 'echo start_dir in configure is %(start_dir)s && ' prebuildopts = 'echo start_dir in build is %(start_dir)s && ' - exts_defaultclass = 'EB_Toy' exts_list = [ ('bar', '0.0', { 'sources': ['bar-0.0-local.tar.gz'], diff --git a/test/framework/modules.py b/test/framework/modules.py index bfd21778ba..b69bb69c3e 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -44,7 +44,6 @@ from easybuild.framework.easyconfig.easyconfig import EasyConfig from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.environment import modify_env from easybuild.tools.filetools import adjust_permissions, copy_file, copy_dir, mkdir from easybuild.tools.filetools import read_file, remove_dir, remove_file, symlink, write_file from easybuild.tools.modules import EnvironmentModules, EnvironmentModulesC, EnvironmentModulesTcl, Lmod, NoModulesTool @@ -103,11 +102,10 @@ def test_run_module(self): os.environ.pop(key, None) # arguments can be passed in two ways: multiple arguments, or just 1 list argument - self.modtool.run_module('load', 'GCC/6.4.0-2.28') - self.assertEqual(os.environ['EBROOTGCC'], '/prefix/software/GCC/6.4.0-2.28') + with self.saved_env(): + self.modtool.run_module('load', 'GCC/6.4.0-2.28') + self.assertEqual(os.environ['EBROOTGCC'], '/prefix/software/GCC/6.4.0-2.28') - # restore original environment - modify_env(os.environ, self.orig_environ, verbose=False) self.reset_modulepath([os.path.join(testdir, 'modules')]) self.assertNotIn('EBROOTGCC', os.environ) diff --git a/test/framework/options.py b/test/framework/options.py index 35d70b84f1..757e878c53 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -53,7 +53,6 @@ from easybuild.tools.build_log import EasyBuildError, EasyBuildLog from easybuild.tools.config import DEFAULT_MODULECLASSES, BuildOptions, ConfigurationVariables from easybuild.tools.config import build_option, find_last_log, get_build_log_path, get_module_syntax, module_classes -from easybuild.tools.environment import modify_env from easybuild.tools.filetools import adjust_permissions, change_dir, copy_dir, copy_file, download_file from easybuild.tools.filetools import is_patch_file, mkdir, move_file, parse_http_header_fields_urlpat from easybuild.tools.filetools import read_file, remove_dir, remove_file, which, write_file @@ -4322,7 +4321,6 @@ def check_tmpdir(tmpdir): # cleanup os.close(fd) shutil.rmtree(mytmpdir) - modify_env(os.environ, self.orig_environ) tempfile.tempdir = None orig_tmpdir = tempfile.gettempdir() @@ -4333,7 +4331,8 @@ def check_tmpdir(tmpdir): os.path.join(orig_tmpdir, '[ab @cd]%/#*'), ] for tmpdir in cand_tmpdirs: - check_tmpdir(tmpdir) + with self.saved_env(): + check_tmpdir(tmpdir) def test_minimal_toolchains(self): """End-to-end test for --minimal-toolchains.""" @@ -6597,7 +6596,6 @@ def test_sanity_check_only(self): test_ec_txt += '\n' + '\n'.join([ "sanity_check_commands = ['barbar', 'toy']", "sanity_check_paths = {'files': ['bin/barbar', 'bin/toy'], 'dirs': ['bin']}", - "exts_defaultclass = 'DummyExtension'", "exts_list = [", " ('barbar', '0.0', {", " 'start_dir': 'src',", diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index da631b2390..393598e9d9 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -81,9 +81,10 @@ def prepare_for_extensions(self): """ Prepare for installing toy extensions. """ - # insert new packages by building them with RPackage - self.cfg['exts_defaultclass'] = "Toy_Extension" - self.cfg['exts_filter'] = ("%(ext_name)s", "") + if not self.cfg.get('exts_defaultclass', resolve=False): + self.cfg['exts_defaultclass'] = "Toy_Extension" + if not self.cfg.get('exts_filter', resolve=False): + self.cfg['exts_filter'] = ("%(ext_name)s", "") def run_all_steps(self, *args, **kwargs): """ diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 73831b6bdb..18dd5edcf1 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -28,7 +28,6 @@ @author: Kenneth hoste (Ghent University) @author: Ward Poelmans (Ghent University) """ -import copy import ctypes import logging import os @@ -41,7 +40,7 @@ import easybuild.tools.systemtools as st from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.environment import modify_env, setvar +from easybuild.tools.environment import setvar from easybuild.tools.filetools import adjust_permissions, mkdir, read_file, symlink, which, write_file from easybuild.tools.run import RunShellCmdResult, run_shell_cmd from easybuild.tools.systemtools import CPU_ARCHITECTURES, AARCH32, AARCH64, POWER, X86_64 @@ -740,9 +739,9 @@ def test_cpu_architecture(self): 'x86_64': X86_64, 'some_fancy_arch': UNKNOWN, } - for name in machine_names: + for name, arch in machine_names.items(): MACHINE_NAME = name - self.assertEqual(get_cpu_architecture(), machine_names[name]) + self.assertEqual(get_cpu_architecture(), arch) def test_cpu_arch_name_native(self): """Test getting CPU arch name.""" @@ -1323,9 +1322,6 @@ def test_find_library_path(self): def test_get_cuda_object_dump_raw(self): """Test get_cuda_object_dump_raw function""" - # This test modifies environment, make sure we can revert the changes: - start_env = copy.deepcopy(os.environ) - # Mock the shell command for certain known commands st.run_shell_cmd = mocked_run_shell_cmd @@ -1373,14 +1369,8 @@ def test_get_cuda_object_dump_raw(self): # Test case 7: call on CUDA static lib, which only contains device code self.assertEqual(get_cuda_object_dump_raw('mock_cuda_staticlib'), CUOBJDUMP_DEVICE_CODE_ONLY) - # Restore original environment - modify_env(os.environ, start_env, verbose=False) - def test_get_cuda_architectures(self): """Test get_cuda_architectures function""" - # This test modifies environment, make sure we can revert the changes: - start_env = copy.deepcopy(os.environ) - # Mock the shell command for certain known commands st.run_shell_cmd = mocked_run_shell_cmd @@ -1450,9 +1440,6 @@ def test_get_cuda_architectures(self): self.assertTrue(warning_regex_elf.search(logtxt), fail_msg) self.assertIsNone(res_elf) - # Restore original environment - modify_env(os.environ, start_env, verbose=False) - def test_get_linked_libs_raw(self): """ Test get_linked_libs_raw function. diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 9045494bc7..91864784a6 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -29,7 +29,6 @@ @author: Kenneth Hoste (Ghent University) @author: Damian Alvarez (Forschungszentrum Juelich GmbH) """ -import copy import glob import grp import os @@ -54,7 +53,7 @@ from easybuild.main import main_with_hooks from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import get_module_syntax, get_repositorypath -from easybuild.tools.environment import modify_env, setvar +from easybuild.tools.environment import setvar from easybuild.tools.filetools import adjust_permissions, change_dir, copy_file, mkdir, move_file from easybuild.tools.filetools import read_file, remove_dir, remove_file, which, write_file from easybuild.tools.module_generator import ModuleGeneratorTcl @@ -1356,7 +1355,6 @@ def test_toy_extension_patches_postinstallcmds(self): test_ec = os.path.join(self.test_prefix, 'test.eb') test_ec_txt = f"{toy_ec_txt}\n" + textwrap.dedent(""" - exts_defaultclass = "DummyExtension" exts_list = [ ("bar", "0.0", { "buildopts": " && ls -l test.txt", @@ -1422,7 +1420,6 @@ def test_toy_extension_sources(self): # test use of single-element list in 'sources' with just the filename test_ec_txt = '\n'.join([ toy_ec_txt, - 'exts_defaultclass = "DummyExtension"', 'exts_list = [', ' ("bar", "0.0", {', ' "sources": %s,' % bar_sources_spec, @@ -1450,7 +1447,6 @@ def test_toy_extension_sources(self): test_ec_txt = '\n'.join([ toy_ec_txt, - 'exts_defaultclass = "DummyExtension"', 'exts_list = [', ' ("bar", "0.0", {', ' "source_urls": ["file://%s"],' % test_source_path, @@ -1466,7 +1462,6 @@ def test_toy_extension_sources(self): # check that checksums are picked up and verified test_ec_txt = '\n'.join([ toy_ec_txt, - 'exts_defaultclass = "DummyExtension"', 'exts_list = [', ' ("bar", "0.0", {', ' "source_urls": ["file://%s"],' % test_source_path, @@ -1490,7 +1485,6 @@ def test_toy_extension_sources(self): # test again with correct checksum for bar-0.0.tar.gz, but faulty checksum for patch file test_ec_txt = '\n'.join([ toy_ec_txt, - 'exts_defaultclass = "DummyExtension"', 'exts_list = [', ' ("bar", "0.0", {', ' "source_urls": ["file://%s"],' % test_source_path, @@ -1514,7 +1508,6 @@ def test_toy_extension_sources(self): # test again with correct checksums test_ec_txt = '\n'.join([ toy_ec_txt, - 'exts_defaultclass = "DummyExtension"', 'exts_list = [', ' ("bar", "0.0", {', ' "source_urls": ["file://%s"],' % test_source_path, @@ -1540,7 +1533,6 @@ def test_toy_extension_extract_cmd(self): test_ec = os.path.join(self.test_prefix, 'test.eb') test_ec_txt = '\n'.join([ toy_ec_txt, - 'exts_defaultclass = "DummyExtension"', 'exts_list = [', ' ("bar", "0.0", {', # deliberately incorrect custom extract command, just to verify that it's picked up @@ -1579,7 +1571,6 @@ def test_toy_extension_sources_git_config(self): test_ec_txt = '\n'.join([ toy_ec_txt, 'prebuildopts = "echo \\\"%s\\\" > %s && ",' % (ext_code, ext_cfile), - 'exts_defaultclass = "DummyExtension"', 'exts_list = [', ' ("exts-git", "0.0", {', ' "buildopts": "&& ls -l %s %s",' % (ext_tarball, ext_tarfile), @@ -1759,18 +1750,14 @@ def test_external_dependencies(self): installed_test_modules = os.path.join(self.test_installpath, 'modules', 'all') self.reset_modulepath([modulepath, installed_test_modules]) - start_env = copy.deepcopy(os.environ) - with self.mocked_stdout_stderr(): self._test_toy_build(ec_file=toy_ec, versionsuffix='-external-deps', verbose=True, raise_error=True) - self.modtool.load(['toy/0.0-external-deps']) - # note build dependency is not loaded - mods = ['intel/2018a', 'GCC/6.4.0-2.28', 'foobar/1.2.3', 'toy/0.0-external-deps'] - self.assertEqual([x['mod_name'] for x in self.modtool.list()], mods) - - # restore original environment (to undo 'module load' done above) - modify_env(os.environ, start_env, verbose=False) + with self.saved_env(): + self.modtool.load(['toy/0.0-external-deps']) + # note build dependency is not loaded + mods = ['intel/2018a', 'GCC/6.4.0-2.28', 'foobar/1.2.3', 'toy/0.0-external-deps'] + self.assertEqual([x['mod_name'] for x in self.modtool.list()], mods) # check behaviour when a non-existing external (build) dependency is included extraectxt = "\nbuilddependencies = [('nosuchbuilddep/0.0.0', EXTERNAL_MODULE)]" @@ -1946,7 +1933,6 @@ def test_module_only_extensions(self): test_ec_txt += '\n' + '\n'.join([ "sanity_check_commands = ['barbar', 'toy']", "sanity_check_paths = {'files': ['bin/barbar', 'bin/toy'], 'dirs': ['bin']}", - "exts_defaultclass = 'DummyExtension'", "exts_list = [", " ('barbar', '0.0', {", " 'start_dir': 'src',", @@ -1983,7 +1969,7 @@ def test_module_only_extensions(self): move_file(libbarbar, libbarbar + '.foobar') # check whether sanity check fails now when using --module-only - error_pattern = 'Sanity check failed: command "ls -l lib/libbarbar.a" failed' + error_pattern = 'Sanity check failed: .*command "ls -l lib/libbarbar.a" failed' for extra_args in (['--module-only'], ['--module-only', '--rebuild']): with self.mocked_stdout_stderr(): self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, [test_ec] + extra_args, @@ -2016,7 +2002,6 @@ def test_toy_exts_parallel(self): test_ec = os.path.join(self.test_prefix, 'test.eb') test_ec_txt = read_file(toy_ec) test_ec_txt += '\n' + '\n'.join([ - "exts_defaultclass = 'DummyExtension'", "exts_list = [", " ('ls'),", " ('bar', '0.0'),", @@ -2397,7 +2382,6 @@ def test_reproducibility_ext_easyblocks(self): ec1 = os.path.join(self.test_prefix, 'toy1.eb') ec1_txt = '\n'.join([ toy_ec_txt, - "exts_defaultclass = 'DummyExtension'", "exts_list = [('barbar', '1.2', {'start_dir': 'src'})]", "", ]) @@ -3147,10 +3131,6 @@ def test_toy_filter_rpath_sanity_libs(self): def test_toy_cuda_sanity_check(self): """Test the CUDA sanity check""" - # We need to mock a cuobjdump executable and prepend in on the PATH - # First, make sure we can restore environment at the end of this test - start_env = copy.deepcopy(os.environ) - # Define the toy_ec file we want to use topdir = os.path.dirname(os.path.abspath(__file__)) toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') @@ -3540,9 +3520,6 @@ def assert_cuda_report(missing_cc, additional_cc, missing_ptx, log, stdout=None, self.assertTrue(expected_result.search(outtxt), msg) assert_cuda_report(missing_cc=0, additional_cc=0, missing_ptx=0, log=outtxt, stdout=stdout, num_checked=0) - # Restore original environment - modify_env(os.environ, start_env, verbose=False) - def test_toy_modaltsoftname(self): """Build two dependent toys as in test_toy_toy but using modaltsoftname""" topdir = os.path.dirname(os.path.abspath(__file__)) @@ -3654,7 +3631,6 @@ def test_toy_build_hooks(self): test_ec = os.path.join(self.test_prefix, 'test.eb') test_ec_txt = read_file(toy_ec) + '\n'.join([ "exts_list = [('bar', '0.0'), ('toy', '0.0')]", - "exts_defaultclass = 'DummyExtension'", ]) write_file(test_ec, test_ec_txt) @@ -3818,7 +3794,6 @@ def test_toy_multi_deps(self): test_ec = os.path.join(self.test_prefix, 'test.eb') # also inject (minimal) list of extensions to test iterative installation of extensions - test_ec_txt += "\nexts_defaultclass = 'DummyExtension'" test_ec_txt += "\nexts_list = [('barbar', '1.2', {'start_dir': 'src'})]" test_ec_txt += "\nmulti_deps = {'GCC': ['4.6.3', '7.3.0-2.30']}" @@ -3953,7 +3928,8 @@ def check_toy_load(depends_on=False): # just undo self.modtool.unload(['toy/0.0', 'GCC/4.6.3']) - check_toy_load() + with self.saved_env(): + check_toy_load() # this behaviour can be disabled via "multi_dep_load_defaults = False" write_file(test_ec, test_ec_txt + "\nmulti_deps_load_default = False") @@ -3965,8 +3941,9 @@ def check_toy_load(depends_on=False): self.assertNotIn(expected, toy_mod_txt) - self.modtool.load(['toy/0.0']) - loaded_mod_names = [x['mod_name'] for x in self.modtool.list()] + with self.saved_env(): + self.modtool.load(['toy/0.0']) + loaded_mod_names = [x['mod_name'] for x in self.modtool.list()] self.assertIn('toy/0.0', loaded_mod_names) self.assertNotIn('GCC/4.6.3', loaded_mod_names) self.assertNotIn('GCC/7.3.0-2.30', loaded_mod_names) @@ -3989,10 +3966,6 @@ def check_toy_load(depends_on=False): error_msg_whatis = "Pattern '%s' should be found in: %s" % (expected_whatis_no_default, toy_mod_txt) self.assertIn(expected_whatis_no_default, toy_mod_txt, error_msg_whatis) - # restore original environment to continue testing with a clean slate - modify_env(os.environ, self.orig_environ, verbose=False) - self.modtool.use(test_mod_path) - # disable showing of progress bars (again), doesn't make sense when running tests os.environ['EASYBUILD_DISABLE_SHOW_PROGRESS_BAR'] = '1' diff --git a/test/framework/utilities.py b/test/framework/utilities.py index eb93170c87..16a441f655 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -501,9 +501,9 @@ def init_config(args=None, build_options=None, with_include=True, clear_caches=T 'valid_module_classes': module_classes(), 'valid_stops': [x[0] for x in EasyBlock.get_steps()], } - for key in default_build_options: + for key, def_option in default_build_options.items(): if key not in build_options: - build_options[key] = default_build_options[key] + build_options[key] = def_option config.init_build_options(build_options=build_options)