Merge branch 'develop' into 5.0.x

RELEASE_NOTES

@@ -6,6 +6,31 @@ These release notes can also be consulted at http://easybuild.readthedocs.org/en
 The latest version of easybuild-easyblocks provides 251 software-specific easyblocks and 42 generic easyblocks.
 
 
+v4.8.2 (29 October 2023)
+------------------------
+
+update/bugfix release
+
+- minor enhancements and updates, including:
+  - allow use of `test_cmd` without `runtest` for `ConfigureMake` (#2837)
+  - enhance `CMakeMake` easyblock to run `ctest` command if `runtest` is `True` (#2838)
+  - enhance `Conda` easyblock: add support for using custom `conda` command (like `mamba`), and clean up after installation (#2996)
+  - update WRF easyblock to correctly determine `WRF*` subdirectory for WRF versions >= 4.5.1 (#2997)
+  - update numpy easyblock to be able to use FlexiBLAS with MKL as dependency (#2999)
+  - add error regexp for PyTorch tests run in subprocess and enhance error message (#3003, #3005)
+  - don't blindly overwrite `-Dccflags` + honour `preconfigopts` in Perl easyblock (#3010)
+  - use more test programs in sanity check step of OpenMPI easyblock (#3016)
+  - fix `CMAKE_PREFIX_PATH` in imkl easyblock (#3021)
+  - update sanity check in wxPython easyblock for version 4.2.0 (#3023)
+- various bug fixes, including:
+  - fix creation of symlink for libraries in TBB easyblock (#2927)
+  - fix `--sanity-check-only` and `--module-only` for UCX plugins (#3007)
+  - replace umlaut in docstring of PerlBundle easyblock (to fix Non-ASCII character error when using Python 2.7) (#3008)
+  - enhance TensorFlow easyblock to avoid use of `-mcpu=native` for XNNPACK component when building on aarch64 (#3011)
+  - only use `-DCMAKE_SKIP_RPATH=ON` for CMake < 3.5.0 (#3012)
+  - fix the blas/lapack name passed to meson when building recent scipy versions (>= 1.9.0) on top of Intel MKL (#3024)
+
+
 v4.8.1 (11 September 2023)
 --------------------------
 

easybuild/easyblocks/generic/cmakemake.py

@@ -267,9 +267,10 @@ def configure_step(self, srcdir=None, builddir=None):
                     self.log.info("Using absolute path to compiler command: %s", value)
                 options[option] = value
 
-        if build_option('rpath'):
+        if build_option('rpath') and LooseVersion(self.cmake_version) < LooseVersion('3.5.0'):
             # instruct CMake not to fiddle with RPATH when --rpath is used, since it will undo stuff on install...
-            # https://github.com/LLNL/spack/blob/0f6a5cd38538e8969d11bd2167f11060b1f53b43/lib/spack/spack/build_environment.py#L416
+            # this is only required for CMake < 3.5.0, since newer version are more careful w.r.t. RPATH,
+            # see https://github.com/Kitware/CMake/commit/3ec9226779776811240bde88a3f173c29aa935b5
             options['CMAKE_SKIP_RPATH'] = 'ON'
 
         # show what CMake is doing by default

easybuild/easyblocks/generic/modulerc.py

@@ -30,6 +30,7 @@
 import os
 
 from easybuild.framework.easyblock import EasyBlock
+from easybuild.framework.easyconfig import CUSTOM
 from easybuild.framework.easyconfig.easyconfig import ActiveMNS
 from easybuild.tools.build_log import EasyBuildError, print_msg
 from easybuild.tools.config import install_path
@@ -41,6 +42,16 @@ class ModuleRC(EasyBlock):
     Generic easyblock to create a software-specific .modulerc file
     """
 
+    @staticmethod
+    def extra_options(extra_vars=None):
+        """Define extra easyconfig parameters specific to ModuleRC"""
+        if extra_vars is None:
+            extra_vars = {}
+        extra_vars.update({
+            'check_version': [True, "Check version is prefix of dependency", CUSTOM],
+        })
+        return EasyBlock.extra_options(extra_vars)
+
     def configure_step(self):
         """Do nothing."""
         pass
@@ -67,7 +78,8 @@ def make_module_step(self, fake=False):
             raise EasyBuildError("Name does not match dependency name: %s vs %s", self.name, deps[0]['name'])
 
         # ensure version to alias to is a prefix of the version of the dependency
-        if not deps[0]['version'].startswith(self.version) and not self.version == "default":
+        if self.cfg['check_version'] and \
+           not deps[0]['version'].startswith(self.version) and not self.version == "default":
             raise EasyBuildError("Version is not 'default' and not a prefix of dependency version: %s vs %s",
                                  self.version, deps[0]['version'])
 
@@ -85,7 +97,7 @@ def make_module_step(self, fake=False):
 
         module_version_specs = {
             'modname': alias_modname,
-            'sym_version': self.version,
+            'sym_version': self.version + self.cfg['versionsuffix'],
             'version': deps[0]['version'],
         }
         self.module_generator.modulerc(module_version=module_version_specs, filepath=modulerc)

easybuild/easyblocks/i/imkl.py

@@ -546,11 +546,13 @@ def make_module_req_guess(self):
                     os.path.join(self.mkl_basedir, 'include'),
                     os.path.join(self.mkl_basedir, 'include', 'fftw'),
                 ]
+                cmake_prefix_path = [self.mkl_basedir]
                 guesses.update({
                     'PATH': [],
                     'LD_LIBRARY_PATH': library_path,
                     'LIBRARY_PATH': library_path,
                     'CPATH': cpath,
+                    'CMAKE_PREFIX_PATH': cmake_prefix_path,
                     'PKG_CONFIG_PATH': pkg_config_path,
                 })
                 if self.cfg['flexiblas']:

easybuild/easyblocks/o/openmpi.py

@@ -211,8 +211,18 @@ def sanity_check_step(self):
         mpi_cmd_tmpl, params = get_mpi_cmd_template(toolchain.OPENMPI, dict(), mpi_version=self.version)
         # Limit number of ranks to 8 to avoid it failing due to hyperthreading
         ranks = min(8, self.cfg['parallel'])
-        for src, compiler in (('hello_c.c', 'mpicc'), ('hello_mpifh.f', 'mpifort'), ('hello_usempi.f90', 'mpif90')):
-            src_path = os.path.join(self.cfg['start_dir'], 'examples', src)
+        for srcdir, src, compiler in (
+            ('examples', 'hello_c.c', 'mpicc'),
+            ('examples', 'hello_mpifh.f', 'mpifort'),
+            ('examples', 'hello_usempi.f90', 'mpif90'),
+            ('examples', 'ring_c.c', 'mpicc'),
+            ('examples', 'ring_mpifh.f', 'mpifort'),
+            ('examples', 'ring_usempi.f90', 'mpif90'),
+            ('test/simple', 'thread_init.c', 'mpicc'),
+            ('test/simple', 'intercomm1.c', 'mpicc'),
+            ('test/simple', 'mpi_barrier.c', 'mpicc'),
+        ):
+            src_path = os.path.join(self.cfg['start_dir'], srcdir, src)
             if os.path.exists(src_path):
                 test_exe = os.path.join(self.builddir, 'mpi_test_' + os.path.splitext(src)[0])
                 self.log.info("Adding minimal MPI test program to sanity checks: %s", test_exe)

easybuild/easyblocks/p/palm.py

@@ -0,0 +1,76 @@
+##
+# Copyright 2023 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+EasyBuild support for building and installing PALM, implemented as an easyblock
+
+@author: Viktor Rehnberg (Chalmers University of Technology)
+"""
+import os
+
+from easybuild.framework.easyblock import EasyBlock
+from easybuild.tools.filetools import find_glob_pattern
+from easybuild.tools.run import run_cmd
+
+
+class EB_PALM(EasyBlock):
+    """Support for building/installing PALM."""
+
+    def __init__(self, *args, **kwargs):
+        """Initialise PALM easyblock."""
+        super().__init__(*args, **kwargs)
+
+    def configure_step(self):
+        """No configuration procedure for PALM."""
+        pass
+
+    def build_step(self):
+        """No build procedure for PALM."""
+        pass
+
+    def install_step(self):
+        """Custom install procedure for PALM."""
+
+        install_script_pattern = "install"
+        if self.dry_run:
+            install_script = install_script_pattern
+        else:
+            install_script = find_glob_pattern(install_script_pattern)
+
+        cmd = ' '.join([
+            self.cfg['preinstallopts'],
+            "bash",
+            install_script,
+            "-p %s" % self.installdir,
+            self.cfg['installopts'],
+        ])
+        run_cmd(cmd, log_all=True, simple=True)
+
+    def sanity_check_step(self):
+        """Custom sanity check for PALM."""
+        custom_paths = {
+            'files': [os.path.join(self.installdir, 'bin', 'palmrun')],
+            'dirs': [],
+        }
+        super().sanity_check_step(custom_paths=custom_paths)

easybuild/easyblocks/p/perl.py

@@ -74,7 +74,7 @@ def configure_step(self):
         configopts = [
             self.cfg['configopts'],
             '-Dcc="{0}"'.format(os.getenv('CC')),
-            '-Dccflags="{0}"'.format(os.getenv('CFLAGS')),
+            '-Dccflags="{0}"'.format(os.getenv('CFLAGS')) if '-Dccflags' not in self.cfg['configopts'] else '',
             '-Dinc_version_list=none',
             '-Dprefix=%(installdir)s',
             # guarantee that scripts are installed in /bin in the installation directory (and not in a guessed path)
@@ -115,7 +115,7 @@ def configure_step(self):
         if os.getenv('COLUMNS', None) == '0':
             unset_env_vars(['COLUMNS'])
 
-        cmd = './Configure -de %s' % configopts
+        cmd = '%s ./Configure -de %s' % (self.cfg['preconfigopts'], configopts)
         run_cmd(cmd, log_all=True, simple=True)
 
     def test_step(self):

easybuild/easyblocks/p/pytorch.py

@@ -318,7 +318,7 @@ def get_count_for_pattern(regex, text):
             return 0
 
         # Create clear summary report
-        failure_report = ""
+        failure_report = []
         failure_cnt = 0
         error_cnt = 0
         failed_test_suites = []
@@ -337,9 +337,9 @@ def get_count_for_pattern(regex, text):
             # E.g. 'failures=3, errors=10, skipped=190, expected failures=6'
             failure_summary = m.group('failure_summary')
             total, test_suite = m.group('test_cnt', 'failed_test_suite_name')
-            failure_report += "{test_suite} ({total} total tests, {failure_summary})\n".format(
+            failure_report.append("{test_suite} ({total} total tests, {failure_summary})".format(
                     test_suite=test_suite, total=total, failure_summary=failure_summary
-                )
+                ))
             failure_cnt += get_count_for_pattern(r"(?<!expected )failures=([0-9]+)", failure_summary)
             error_cnt += get_count_for_pattern(r"errors=([0-9]+)", failure_summary)
             failed_test_suites.append(test_suite)
@@ -362,9 +362,9 @@ def get_count_for_pattern(regex, text):
             # E.g. '2 failed, 128 passed, 2 skipped, 2 warnings'
             failure_summary = m.group('failure_summary')
             test_suite = m.group('failed_test_suite_name')
-            failure_report += "{test_suite} ({failure_summary})\n".format(
+            failure_report.append("{test_suite} ({failure_summary})".format(
                     test_suite=test_suite, failure_summary=failure_summary
-                )
+                ))
             failure_cnt += get_count_for_pattern(r"([0-9]+) failed", failure_summary)
             error_cnt += get_count_for_pattern(r"([0-9]+) error", failure_summary)
             failed_test_suites.append(test_suite)
@@ -389,9 +389,9 @@ def get_count_for_pattern(regex, text):
             # E.g. '2 unit test(s) failed'
             failure_summary = m.group('failure_summary')
             test_suite = m.group('failed_test_suite_name')
-            failure_report += "{test_suite} ({failure_summary})\n".format(
+            failure_report.append("{test_suite} ({failure_summary})".format(
                     test_suite=test_suite, failure_summary=failure_summary
-                )
+                ))
             failure_cnt += get_count_for_pattern(r"([0-9]+) unit test\(s\) failed", failure_summary)
             failed_test_suites.append(test_suite)
 
@@ -404,12 +404,13 @@ def get_count_for_pattern(regex, text):
         )
         # If we missed any test suites prepend a list of all failed test suites
         if failed_test_suites != all_failed_test_suites:
-            failure_report = ['Failed tests (suites/files):'] + ([failure_report] if failure_report else [])
+            failure_report = ['Failed tests (suites/files):'] + failure_report
             # Test suites where we didn't match a specific regexp and hence likely didn't count the failures
             failure_report.extend('+ %s' % t for t in sorted(all_failed_test_suites - failed_test_suites))
             # Test suites not included in the catch-all regexp but counted. Should be empty.
             failure_report.extend('? %s' % t for t in sorted(failed_test_suites - all_failed_test_suites))
-            failure_report = '\n'.join(failure_report)
+
+        failure_report = '\n'.join(failure_report)
 
         # Calculate total number of unsuccesful and total tests
         failed_test_cnt = failure_cnt + error_cnt

easybuild/easyblocks/s/scipy.py

@@ -124,7 +124,7 @@ def configure_step(self):
             if lapack_lib == toolchain.FLEXIBLAS:
                 blas_lapack = 'flexiblas'
             elif lapack_lib == toolchain.INTELMKL:
-                blas_lapack = 'mkl'
+                blas_lapack = 'mkl-dynamic-lp64-seq'
             elif lapack_lib == toolchain.OPENBLAS:
                 blas_lapack = 'openblas'
             else:

easybuild/easyblocks/t/tensorflow.py

@@ -48,7 +48,7 @@
 from easybuild.tools.filetools import is_readable, read_file, which, write_file, remove_file
 from easybuild.tools.modules import get_software_root, get_software_version, get_software_libdir
 from easybuild.tools.run import run_cmd
-from easybuild.tools.systemtools import X86_64, get_cpu_architecture, get_os_name, get_os_version
+from easybuild.tools.systemtools import AARCH64, X86_64, get_cpu_architecture, get_os_name, get_os_version
 
 
 CPU_DEVICE = 'cpu'
@@ -687,6 +687,19 @@ def configure_step(self):
         cmd = self.cfg['preconfigopts'] + './configure ' + self.cfg['configopts']
         run_cmd(cmd, log_all=True, simple=True)
 
+        # when building on Arm 64-bit we can't just use --copt=-mcpu=native (or likewise for any -mcpu=...),
+        # because it breaks the build of XNNPACK;
+        # see also https://github.com/easybuilders/easybuild-easyconfigs/issues/18899
+        if get_cpu_architecture() == AARCH64:
+            tf_conf_bazelrc = os.path.join(self.start_dir, '.tf_configure.bazelrc')
+            regex_subs = [
+                # use --per_file_copt instead of --copt to selectively use -mcpu=native (not for XNNPACK),
+                # the leading '-' ensures that -mcpu=native is *not* used when building XNNPACK;
+                # see https://github.com/google/XNNPACK/issues/5566 + https://bazel.build/docs/user-manual#per-file-copt
+                ('--copt=-mcpu=', '--per_file_copt=-.*XNNPACK/.*@-mcpu='),
+            ]
+            apply_regex_substitutions(tf_conf_bazelrc, regex_subs)
+
     def patch_crosstool_files(self):
         """Patches the CROSSTOOL files to include EasyBuild provided compiler paths"""
         inc_paths, lib_paths = [], []

easybuild/easyblocks/u/ucx_plugins.py

@@ -29,6 +29,7 @@
 @author: Mikael Öhman (Chalmers University of Techonology)
 """
 from collections import defaultdict
+from itertools import chain
 import os
 
 from easybuild.easyblocks.generic.configuremake import ConfigureMake
@@ -42,16 +43,37 @@ class EB_UCX_Plugins(ConfigureMake):
     """Support for building additional plugins for a existing UCX module"""
 
     def __init__(self, *args, **kwargs):
-        """Custom initialization for UCX: set correct module name."""
+        """Custom initialization for UCX-Plugins."""
         super(EB_UCX_Plugins, self).__init__(*args, **kwargs)
-        self.plugins = {}
+        self._plugins = None
         self.makefile_dirs = []
 
+    @property
+    def plugins(self):
+        """Property to determine list of plugins based on loaded dependencies, or return cached list of plugins."""
+        if self._plugins is None:
+            plugins = defaultdict(list)
+            dep_names = self.cfg.dependency_names()
+
+            if 'CUDAcore' in dep_names or 'CUDA' in dep_names:
+                for key in ('ucm', 'uct', 'ucx_perftest'):
+                    plugins[key].append('cuda')
+
+                if 'GDRCopy' in dep_names:
+                    plugins['uct_cuda'].append('gdrcopy')
+
+            if 'ROCm' in dep_names:
+                for key in ('ucm', 'uct', 'ucx_perftest'):
+                    plugins[key].append('rocm')
+
+            self._plugins = dict(plugins)
+            self.log.info("Creating plugins for %s", ", ".join(sorted(set(chain(*plugins.values())))))
+        return self._plugins
+
     def configure_step(self):
         """Customize configuration for building requested plugins."""
         # make sure that required dependencies are loaded
-        ucxroot = get_software_root('UCX')
-        if not ucxroot:
+        if not get_software_root('UCX'):
             raise EasyBuildError("UCX is a required dependency")
 
         self.cfg['configure_cmd'] = 'contrib/configure-release'
@@ -61,30 +83,21 @@ def configure_step(self):
         # omit the lib subdirectory since we are just installing plugins
         configopts += '--libdir=%(installdir)s '
 
-        plugins = defaultdict(list)
         cudaroot = get_software_root('CUDAcore') or get_software_root('CUDA')
         if cudaroot:
             configopts += '--with-cuda=%s ' % cudaroot
-            for key in ('ucm', 'uct', 'ucx_perftest'):
-                plugins[key].append('cuda')
 
             gdrcopyroot = get_software_root('GDRCopy')
             if gdrcopyroot:
-                plugins['uct_cuda'].append('gdrcopy')
                 configopts += '--with-gdrcopy=%s ' % gdrcopyroot
 
             self.makefile_dirs.extend(os.path.join(x, 'cuda') for x in ('uct', 'ucm', 'tools/perf'))
 
-        # To be supported in the future:
         rocmroot = get_software_root('ROCm')
         if rocmroot:
-            for key in ('ucm', 'uct', 'ucx_perftest'):
-                plugins[key].append('rocm')
             configopts += '--with-rocm=%s ' % rocmroot
             self.makefile_dirs.extend(os.path.join(x, 'rocm') for x in ('uct', 'ucm', 'tools/perf'))
 
-        self.plugins = dict(plugins)
-
         self.cfg.update('configopts', configopts)
 
         super(EB_UCX_Plugins, self).configure_step()