diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 20d69ac066..8af5fda069 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -42,8 +42,10 @@ * Caspar van Leeuwen (SURF) * Jan Andre Reuter (Juelich Supercomputing Centre) * Jasper Grimm (UoY) +* Alexander Grund (TU Dresden) """ import concurrent +import contextlib import copy import functools import glob @@ -593,7 +595,7 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None): if not isinstance(patch_specs, tuple) or len(patch_specs) != 2: raise EasyBuildError('Patch specs must be a tuple of (patches, post-install patches) or a list') post_install_patches = patch_specs[1] - patch_specs = itertools.chain(*patch_specs) + patch_specs = itertools.chain.from_iterable(patch_specs) patches = [] for index, patch_spec in enumerate(patch_specs): @@ -683,6 +685,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): source_urls = resolve_template(ext_options.get('source_urls', []), template_values) checksums = resolve_template(ext_options.get('checksums', []), template_values) + ext_src['checksums'] = checksums download_instructions = resolve_template(ext_options.get('download_instructions'), template_values) @@ -725,6 +728,11 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): # copy 'path' entry to 'src' for use with extensions 'src': src['path'], }) + filename = src['name'] + else: + filename = source.get('filename') + if filename is not None: + ext_src['sources'] = [filename] else: @@ -738,6 +746,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): raise EasyBuildError(error_msg, type(src_fn).__name__, src_fn) src_fn = resolve_template(src_fn, template_values) + ext_src['sources'] = [src_fn] if fetch_files: src_path = self.obtain_file(src_fn, extension=True, urls=source_urls, @@ -781,7 +790,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): if fetch_files: ext_patches = self.fetch_patches(patch_specs=ext_patch_specs, extension=True) else: - ext_patches = [create_patch_info(p) for p in itertools.chain(*ext_patch_specs)] + ext_patches = [create_patch_info(p) for p in itertools.chain.from_iterable(ext_patch_specs)] if ext_patches: self.log.debug('Found patches for extension %s: %s', ext_name, ext_patches) @@ -2748,10 +2757,11 @@ def check_checksums_for(self, ent, sub='', source_cnt=None): checksums = ent.get('checksums', []) except EasyBuildError: if isinstance(ent, EasyConfig): - sources = ent.get_ref('sources') - data_sources = ent.get_ref('data_sources') - patches = ent.get_ref('patches') + ent.get_ref('postinstallpatches') - checksums = ent.get_ref('checksums') + with ent.disable_templating(): + sources = ent['sources'] + data_sources = ent['data_sources'] + patches = ent['patches'] + ent['postinstallpatches'] + checksums = ent['checksums'] # Single source should be re-wrapped as a list, and checksums with it if isinstance(sources, dict): @@ -2761,25 +2771,30 @@ def check_checksums_for(self, ent, sub='', source_cnt=None): if isinstance(checksums, str): checksums = [checksums] - sources = sources + data_sources + def get_name(fn, key): + # if the filename is a tuple, the actual source file name is the first element + if isinstance(fn, tuple): + fn = fn[0] + # if the filename is a dict, the actual source file name is inside + if isinstance(fn, dict): + fn = fn[key] + return fn + + sources = [get_name(src, 'filename') for src in itertools.chain(sources, data_sources)] + patches = [get_name(patch, 'name') for patch in patches] + + if source_cnt is None: + source_cnt = len(sources) + patch_cnt = len(patches) - if not checksums: + if not checksums and (source_cnt + patch_cnt) > 0: checksums_from_json = self.get_checksums_from_json() # recreate a list of checksums. If each filename is found, the generated list of checksums should match # what is expected in list format - for fn in sources + patches: - # if the filename is a tuple, the actual source file name is the first element - if isinstance(fn, tuple): - fn = fn[0] - # if the filename is a dict, the actual source file name is the "filename" element - if isinstance(fn, dict): - fn = fn["filename"] - if fn in checksums_from_json.keys(): - checksums += [checksums_from_json[fn]] + with contextlib.suppress(KeyError): + checksums.extend(checksums_from_json[fn] for fn in sources + patches) - if source_cnt is None: - source_cnt = len(sources) - patch_cnt, checksum_cnt = len(patches), len(checksums) + checksum_cnt = len(checksums) if (source_cnt + patch_cnt) != checksum_cnt: if sub: @@ -2835,17 +2850,9 @@ def check_checksums(self): checksum_issues.extend(self.check_checksums_for(self.cfg)) # also check checksums for extensions - for ext in self.cfg.get_ref('exts_list'): - # just skip extensions for which only a name is specified - # those are just there to check for things that are in the "standard library" - if not isinstance(ext, str): - ext_name = ext[0] - # take into account that extension may be a 2-tuple with just name/version - ext_opts = ext[2] if len(ext) == 3 else {} - # only a single source per extension is supported (see source_tmpl) - source_cnt = 1 if not ext_opts.get('nosource') else 0 - res = self.check_checksums_for(ext_opts, sub="of extension %s" % ext_name, source_cnt=source_cnt) - checksum_issues.extend(res) + for ext in self.collect_exts_file_info(fetch_files=False, verify_checksums=False): + res = self.check_checksums_for(ext, sub=f"of extension {ext['name']}") + checksum_issues.extend(res) return checksum_issues diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 35af52cbcc..cf8b08830e 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -29,14 +29,17 @@ @author: Kenneth Hoste (Ghent University) @author: Maxime Boissonneault (Compute Canada) @author: Jan Andre Reuter (Juelich Supercomputing Centre) +@author: Alexander Grund (TU Dresden) """ import copy import fileinput +import itertools import os import re import shutil import sys import tempfile +import textwrap from inspect import cleandoc from test.framework.github import requires_github_access from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config @@ -2350,17 +2353,25 @@ def test_collect_exts_file_info(self): toy_sources = os.path.join(testdir, 'sandbox', 'sources', 'toy') toy_ext_sources = os.path.join(toy_sources, 'extensions') toy_ec_file = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') - toy_ec = process_easyconfig(toy_ec_file)[0] + + test_ec = os.path.join(self.test_prefix, 'test.eb') + new_ext_txt = "('baz', '0.0', {'nosource': True})," # With nosource option + new_ext_txt += "('barbar', '0.0', {'sources': [SOURCE_TAR_GZ]})," # With sources containing a list + test_ectxt = re.sub(r'\(name, version', new_ext_txt+r"\g<0>", read_file(toy_ec_file)) + write_file(test_ec, test_ectxt) + + toy_ec = process_easyconfig(test_ec)[0] toy_eb = EasyBlock(toy_ec['ec']) exts_file_info = toy_eb.collect_exts_file_info() self.assertIsInstance(exts_file_info, list) - self.assertEqual(len(exts_file_info), 4) + self.assertEqual(len(exts_file_info), 6) self.assertEqual(exts_file_info[0], {'name': 'ulimit'}) self.assertEqual(exts_file_info[1]['name'], 'bar') + self.assertEqual(exts_file_info[1]['sources'], ['bar-0.0.tar.gz']) self.assertEqual(exts_file_info[1]['src'], os.path.join(toy_ext_sources, 'bar-0.0.tar.gz')) bar_patch1 = 'bar-0.0_fix-silly-typo-in-printf-statement.patch' self.assertEqual(exts_file_info[1]['patches'][0]['name'], bar_patch1) @@ -2368,24 +2379,41 @@ def test_collect_exts_file_info(self): bar_patch2 = 'bar-0.0_fix-very-silly-typo-in-printf-statement.patch' self.assertEqual(exts_file_info[1]['patches'][1]['name'], bar_patch2) self.assertEqual(exts_file_info[1]['patches'][1]['path'], os.path.join(toy_ext_sources, bar_patch2)) + self.assertEqual(len(exts_file_info[1]['checksums']), 1) self.assertEqual(exts_file_info[2]['name'], 'barbar') + self.assertEqual(exts_file_info[2]['sources'], ['barbar-1.2.tar.gz']) self.assertEqual(exts_file_info[2]['src'], os.path.join(toy_ext_sources, 'barbar-1.2.tar.gz')) self.assertNotIn('patches', exts_file_info[2]) + self.assertEqual(len(exts_file_info[2]['checksums']), 0) - self.assertEqual(exts_file_info[3]['name'], 'toy') - self.assertEqual(exts_file_info[3]['src'], os.path.join(toy_sources, 'toy-0.0.tar.gz')) + self.assertEqual(exts_file_info[3]['name'], 'baz') + self.assertNotIn('sources', exts_file_info[3]) + self.assertNotIn('sources', exts_file_info[3]['options']) + self.assertNotIn('src', exts_file_info[3]) self.assertNotIn('patches', exts_file_info[3]) + self.assertEqual(len(exts_file_info[3]['checksums']), 0) + + self.assertEqual(exts_file_info[4]['name'], 'barbar') + self.assertEqual(exts_file_info[4]['sources'], ['barbar-0.0.tar.gz']) + self.assertEqual(exts_file_info[4]['src'], os.path.join(toy_ext_sources, 'barbar-0.0.tar.gz')) + self.assertNotIn('patches', exts_file_info[4]) + + self.assertEqual(exts_file_info[5]['name'], 'toy') + self.assertEqual(exts_file_info[5]['sources'], ['toy-0.0.tar.gz']) + self.assertEqual(exts_file_info[5]['src'], os.path.join(toy_sources, 'toy-0.0.tar.gz')) + self.assertNotIn('patches', exts_file_info[5]) # location of files is missing when fetch_files is set to False exts_file_info = toy_eb.collect_exts_file_info(fetch_files=False, verify_checksums=False) self.assertIsInstance(exts_file_info, list) - self.assertEqual(len(exts_file_info), 4) + self.assertEqual(len(exts_file_info), 6) self.assertEqual(exts_file_info[0], {'name': 'ulimit'}) self.assertEqual(exts_file_info[1]['name'], 'bar') + self.assertEqual(exts_file_info[1]['sources'], ['bar-0.0.tar.gz']) self.assertNotIn('src', exts_file_info[1]) self.assertEqual(exts_file_info[1]['patches'][0]['name'], bar_patch1) self.assertNotIn('path', exts_file_info[1]['patches'][0]) @@ -2393,13 +2421,26 @@ def test_collect_exts_file_info(self): self.assertNotIn('path', exts_file_info[1]['patches'][1]) self.assertEqual(exts_file_info[2]['name'], 'barbar') + self.assertEqual(exts_file_info[2]['sources'], ['barbar-1.2.tar.gz']) self.assertNotIn('src', exts_file_info[2]) self.assertNotIn('patches', exts_file_info[2]) - self.assertEqual(exts_file_info[3]['name'], 'toy') + self.assertEqual(exts_file_info[3]['name'], 'baz') + self.assertNotIn('sources', exts_file_info[3]) + self.assertNotIn('sources', exts_file_info[3]['options']) self.assertNotIn('src', exts_file_info[3]) self.assertNotIn('patches', exts_file_info[3]) + self.assertEqual(exts_file_info[4]['name'], 'barbar') + self.assertEqual(exts_file_info[4]['sources'], ['barbar-0.0.tar.gz']) + self.assertNotIn('src', exts_file_info[4]) + self.assertNotIn('patches', exts_file_info[4]) + + self.assertEqual(exts_file_info[5]['name'], 'toy') + self.assertEqual(exts_file_info[5]['sources'], ['toy-0.0.tar.gz']) + self.assertNotIn('src', exts_file_info[5]) + self.assertNotIn('patches', exts_file_info[5]) + error_msg = "Can't verify checksums for extension files if they are not being fetched" self.assertErrorRegex(EasyBuildError, error_msg, toy_eb.collect_exts_file_info, fetch_files=False) @@ -3337,21 +3378,21 @@ def run_checks(): self.assertEqual(res[0], expected) self.assertTrue(res[1].startswith("Non-SHA256 checksum(s) found for toy-0.0.tar.gz:")) + ext_error_tmpl = "Checksums missing for one or more sources/patches of extension %s in " + # check for main sources/patches should reveal two issues with checksums res = eb.check_checksums_for(eb.cfg) self.assertEqual(len(res), 2) run_checks() # full check also catches checksum issues with extensions + eb.json_checksums = {} # Avoid picking up checksums from JSON file res = eb.check_checksums() self.assertEqual(len(res), 4) run_checks() - idx = 2 - for ext in ['bar', 'barbar']: - expected = "Checksums missing for one or more sources/patches of extension %s in " % ext - self.assertTrue(res[idx].startswith(expected)) - idx += 1 + for ext, line in zip(('bar', 'barbar'), res[2:]): + self.assertIn(ext_error_tmpl % ext, line) # check whether tuple of alternative SHA256 checksums is correctly recognized toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') @@ -3414,6 +3455,24 @@ def run_checks(): )] self.assertEqual(eb.check_checksums(), []) + self.contents = textwrap.dedent(""" + easyblock = "ConfigureMake" + name = "Uniq_1" + version = "3.14" + homepage = "http://example.com" + description = "test" + toolchain = SYSTEM + # Templates of parent used in extensions + exts_list = [ + ('%(namelower)s', version), + ] + """) + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + res = eb.check_checksums() + self.assertEqual(len(res), 1) + self.assertIn(ext_error_tmpl % 'uniq_1', res[0]) + # no checksums in easyconfig, then picked up from checksums.json next to easyconfig file test_ec = os.path.join(self.test_prefix, 'test.eb') copy_file(toy_ec, test_ec) @@ -3426,14 +3485,84 @@ def run_checks(): expected += "found 1 sources + 2 patches vs 0 checksums" self.assertEqual(res[0], expected) - # all is fine is checksums.json is also copied + # all is fine if checksums.json is also copied copy_file(os.path.join(os.path.dirname(toy_ec), 'checksums.json'), self.test_prefix) eb.json_checksums = None self.assertEqual(eb.check_checksums(), []) + self.contents = textwrap.dedent(""" + easyblock = "ConfigureMake" + name = "Uniq_1" + version = "3" + homepage = "http://example.com" + description = "test" + toolchain = SYSTEM + # Different ways of specifying sources, patches, template usages to make sure they are resolved correctly + exts_list = [ + 'ulimit', # extension that is part of "standard library" + ('ext1', '0.0'), # Default source filename + ('ext2', '1.2', { + 'source_tmpl': "%(name)s.zip", + 'patches': ['%(name)s.patch'], + }), + ('ext3', '1.2', { + 'sources': "%(name)s.zip", + 'postinstallpatches': ['%(name)s.patch'], + }), + ('ext-%(namelower)s', version + '.14', { + 'sources': {'filename': '%(name)s-%(version)s.zip', 'download_filename': 'foo.tar'}, + 'patches': [{'name': '%(name)s.patch'}], + }), + ('ext-ok1', version, { + 'checksums': '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc' + }), + ('ext-ok2', version, { + 'data_sources': {'filename': '%(name)s-%(version)s.zip', 'download_filename': 'bar.tar'}, + 'checksums': '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc' + }), + ('ext-ok3', version, { + 'nosource': True + }), + ('ext-ok1', version, { + 'checksums': ['44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc'] + }), + ('ext-ok2', version, { + 'nosource': True + }), + ] + """) + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + res = eb.check_checksums() + self.assertEqual(len(res), 4) + extensions = ['ext1', 'ext2', 'ext3', 'ext-uniq_1'] + for ext, line in zip(extensions, res): + self.assertIn(ext_error_tmpl % ext, line) + + # Gradually add checksums to JSON dict and test that the associated extension checksums are now fine + sha256_cs = '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487' # Valid format only + eb.json_checksums = {'ext1-0.0.tar.gz': sha256_cs} + for ext, line in itertools.zip_longest(extensions[1:], eb.check_checksums(), fillvalue=''): + self.assertIn(ext_error_tmpl % ext, line) + eb.json_checksums['ext2.zip'] = sha256_cs + for ext, line in itertools.zip_longest(extensions[1:], eb.check_checksums(), fillvalue=''): + self.assertIn(ext_error_tmpl % ext, line) + eb.json_checksums['ext2.patch'] = sha256_cs + for ext, line in itertools.zip_longest(extensions[2:], eb.check_checksums(), fillvalue=''): + self.assertIn(ext_error_tmpl % ext, line) + eb.json_checksums['ext3.zip'] = sha256_cs + for ext, line in itertools.zip_longest(extensions[2:], eb.check_checksums(), fillvalue=''): + self.assertIn(ext_error_tmpl % ext, line) + eb.json_checksums['ext3.patch'] = sha256_cs + for ext, line in itertools.zip_longest(extensions[3:], eb.check_checksums(), fillvalue=''): + self.assertIn(ext_error_tmpl % ext, line) + eb.json_checksums['ext-uniq_1-3.14.zip'] = sha256_cs + eb.json_checksums['ext-uniq_1.patch'] = sha256_cs + self.assertEqual(eb.check_checksums(), []) + # more checks for check_checksums_for method, which also takes regular dict as input self.assertEqual(eb.check_checksums_for({}), []) - expected = "Checksums missing for one or more sources/patches in test.eb: " + expected = f"Checksums missing for one or more sources/patches in {os.path.basename(self.eb_file)}: " expected += "found 1 sources + 0 patches vs 0 checksums" self.assertEqual(eb.check_checksums_for({'sources': ['test.tar.gz']}), [expected])