From 17048288c7e6f9f0d0cf907400b8e32721cdcfdd Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Wed, 9 Apr 2025 22:42:47 -0700 Subject: [PATCH 01/37] add new system: perlmutter --- .../hardware_description.yaml | 47 +++ systems/lbnl-perlmutter/system.py | 307 ++++++++++++++++++ 2 files changed, 354 insertions(+) create mode 100644 systems/all_hardware_descriptions/HPECray-zen3-A100-Slingshot/hardware_description.yaml create mode 100644 systems/lbnl-perlmutter/system.py diff --git a/systems/all_hardware_descriptions/HPECray-zen3-A100-Slingshot/hardware_description.yaml b/systems/all_hardware_descriptions/HPECray-zen3-A100-Slingshot/hardware_description.yaml new file mode 100644 index 000000000..af09b4687 --- /dev/null +++ b/systems/all_hardware_descriptions/HPECray-zen3-A100-Slingshot/hardware_description.yaml @@ -0,0 +1,47 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +system_definition: + name: HPECray-zen3-A100-Slingshot + integrator: + vendor: HPECray + name: EX235a + processor: + vendor: AMD + name: EPYC-Zen3 + ISA: x86_64 + uArch: zen3 + accelerator: + vendor: NVIDIA + name: A100 + ISA: PTX + uArch: sm_80 + interconnect: + vendor: HPECray + name: Slingshot11 + systems-tested: + csc-lumi: + os: HPECrayOS + scheduler: slurm + compiler: cce + runtime: rocm + mpi: cray-mpich + installation-year: 2023 + llnl-elcapitan: + os: TOSS + scheduler: flux + compiler: cce + runtime: rocm + mpi: cray-mpich + top500-system-instances: + Frontier: + benchpark_system: + top500: + Lumi: + benchpark_system: csc-lumi + top500: + Tioga: + benchpark_system: llnl-elcapitan + top500: diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py new file mode 100644 index 000000000..01c7552ce --- /dev/null +++ b/systems/lbnl-perlmutter/system.py @@ -0,0 +1,307 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +from packaging.version import Version + +from benchpark.directives import variant, maintainers +from benchpark.system import System +from benchpark.paths import hardware_descriptions + + +class LbnlPerlmutter(System): + + maintainers("slabasan") + + id_to_resources = { + "perlmutter": { + "cuda_arch": "80", + "sys_cores_per_node": 64, + "sys_gpus_per_node": 4, + "system_site": "lbnl", + "hardware_key": str(hardware_descriptions) + + "/HPECray-zen3-A100-Slingshot/hardware_description.yaml", + }, + } + + variant( + "compiler", + default="cce", + values=("cce", "gcc"), + description="Which compiler to use", + ) + + def __init__(self, spec): + super().__init__(spec) + + if self.spec.satisfies("compiler=gcc"): + self.gcc_version = Version("12.2.0") + self.mpi_version = Version("8.1.26") + + self.scheduler = "slurm" + attrs = self.id_to_resources.get("perlmutter") + for k, v in attrs.items(): + setattr(self, k, v) + + def compute_packages_section(self): + selections = { + "packages": { + "all": {"require": "target=x86_64:"}, + "tar": {"externals": [{"spec": "tar@1.30", "prefix": "/usr"}]}, + "coreutils": { + "externals": [{"spec": "coreutils@8.30", "prefix": "/usr"}] + }, + "libtool": {"externals": [{"spec": "libtool@2.4.6", "prefix": "/usr"}]}, + "flex": {"externals": [{"spec": "flex@2.6.1+lex", "prefix": "/usr"}]}, + "openssl": { + "externals": [{"spec": "openssl@1.1.1k", "prefix": "/usr"}] + }, + "m4": {"externals": [{"spec": "m4@1.4.18", "prefix": "/usr"}]}, + "groff": {"externals": [{"spec": "groff@1.22.3", "prefix": "/usr"}]}, + "cmake": { + "externals": [ + {"spec": "cmake@3.20.2", "prefix": "/usr"}, + {"spec": "cmake@3.23.1", "prefix": "/usr/tce"}, + {"spec": "cmake@3.24.2", "prefix": "/usr/tce"}, + ], + "buildable": False, + }, + "elfutils": { + "externals": [{"spec": "elfutils@0.190", "prefix": "/usr"}], + "buildable": False, + }, + "papi": { + "externals": [{"spec": "papi@5.6.0.0", "prefix": "/usr"}], + "buildable": False, + }, + "unwind": { + "externals": [{"spec": "unwind@8.0.1", "prefix": "/usr"}], + "buildable": False, + }, + "pkgconf": {"externals": [{"spec": "pkgconf@1.4.2", "prefix": "/usr"}]}, + "curl": { + "externals": [ + {"spec": "curl@7.61.1+gssapi+ldap+nghttp2", "prefix": "/usr"} + ] + }, + "gmake": {"externals": [{"spec": "gmake@4.2.1", "prefix": "/usr"}]}, + "subversion": { + "externals": [{"spec": "subversion@1.10.2", "prefix": "/usr"}] + }, + "diffutils": { + "externals": [{"spec": "diffutils@3.6", "prefix": "/usr"}] + }, + "swig": {"externals": [{"spec": "swig@3.0.12", "prefix": "/usr"}]}, + "gawk": {"externals": [{"spec": "gawk@4.2.1", "prefix": "/usr"}]}, + "binutils": { + "externals": [{"spec": "binutils@2.30.113", "prefix": "/usr"}] + }, + "findutils": { + "externals": [{"spec": "findutils@4.6.0", "prefix": "/usr"}] + }, + "git-lfs": { + "externals": [{"spec": "git-lfs@2.11.0", "prefix": "/usr/tce"}] + }, + "ccache": {"externals": [{"spec": "ccache@3.7.7", "prefix": "/usr"}]}, + "automake": { + "externals": [{"spec": "automake@1.16.1", "prefix": "/usr"}] + }, + "cvs": {"externals": [{"spec": "cvs@1.11.23", "prefix": "/usr"}]}, + "git": { + "externals": [ + {"spec": "git@2.31.1+tcltk", "prefix": "/usr"}, + {"spec": "git@2.29.1+tcltk", "prefix": "/usr/tce"}, + ] + }, + "openssh": {"externals": [{"spec": "openssh@8.0p1", "prefix": "/usr"}]}, + "autoconf": { + "externals": [{"spec": "autoconf@2.69", "prefix": "/usr"}] + }, + "texinfo": {"externals": [{"spec": "texinfo@6.5", "prefix": "/usr"}]}, + "bison": {"externals": [{"spec": "bison@3.0.4", "prefix": "/usr"}]}, + "python": { + "externals": [ + { + "spec": "python@3.9.12", + "prefix": "/usr/tce/packages/python/python-3.9.12", + "buildable": False, + } + ] + }, + "unzip": { + "buildable": False, + "externals": [{"spec": "unzip@6.0", "prefix": "/usr"}], + }, + "hypre": {"variants": "amdgpu_target=gfx90a"}, + "hwloc": { + "externals": [ + {"spec": "hwloc@2.9.1", "prefix": "/usr", "buildable": False} + ] + }, + "fftw": {"buildable": False}, + "intel-oneapi-mkl": { + "externals": [ + { + "spec": "intel-oneapi-mkl@2023.2.0", + "prefix": "/opt/intel/oneapi", + } + ], + "buildable": False, + }, + "mpi": {"buildable": False}, + "libfabric": { + "externals": [ + {"spec": "libfabric@2.1", "prefix": "/opt/cray/libfabric/2.1"} + ], + "buildable": False, + }, + } + } + + selections["packages"] |= self.mpi_config()["packages"] + + if self.spec.satisfies("compiler=cce"): + selections["packages"] |= { + "cray-libsci": { + "externals": [ + { + "spec": "cray-libsci@23.05.1.4%cce", + "prefix": "/opt/cray/pe/libsci/23.05.1.4/cray/12.0/x86_64/", + } + ] + } + } + elif self.spec.satisfies("compiler=gcc"): + selections["packages"] |= { + "cray-libsci": { + "externals": [ + { + "spec": "cray-libsci@23.05.1.4%gcc", + "prefix": "/opt/cray/pe/libsci/23.05.1.4/gnu/10.3/x86_64/", + } + ] + } + } + + selections["packages"] |= self.compiler_weighting_cfg()["packages"] + + return selections + + def compiler_weighting_cfg(self): + compiler = self.spec.variants["compiler"][0] + + if compiler == "cce": + return {"packages": {"all": {"require": [{"one_of": ["%cce", "%gcc"]}]}}} + elif compiler == "gcc": + return {"packages": {}} + else: + raise ValueError(f"Unexpected value for compiler: {compiler}") + + def compute_compilers_section(self): + selections = { + "compilers": [ + { + "compiler": { + "spec": "gcc@12.2.0", + "paths": { + "cc": "/opt/cray/pe/gcc/12.2.0/bin/gcc", + "cxx": "/opt/cray/pe/gcc/12.2.0/bin/g++", + "f77": "/opt/cray/pe/gcc/12.2.0/bin/gfortran", + "fc": "/opt/cray/pe/gcc/12.2.0/bin/gfortran", + }, + "flags": {}, + "operating_system": "rhel8", + "target": "x86_64", + "modules": [], + "environment": {}, + "extra_rpaths": [], + } + } + ] + } + + return selections + + def mpi_config(self): + gtl = self.spec.variants["gtl"][0] + + if self.spec.satisfies("compiler=cce"): + dont_use_gtl = { + "gtl_lib_path": f"/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", + "ldflags": f"-L/opt/cray/pe/mpich/{self.mpi_version}/ofi/crayclang/{self.short_cce_version}/lib -lmpi -L/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", + } + + use_gtl = { + "gtl_flags": "$MV2_COMM_WORLD_LOCAL_RANK", + "gtl_cutoff_size": 4096, + "fi_cxi_ats": 0, + "gtl_lib_path": f"/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", + "gtl_libs": ["libmpi_gtl_hsa"], + "ldflags": f"-L/opt/cray/pe/mpich/{self.mpi_version}/ofi/crayclang/{self.short_cce_version}/lib -lmpi -L/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -lmpi_gtl_hsa", + } + + if gtl: + gtl_spec = "+gtl" + gtl_cfg = use_gtl + else: + gtl_spec = "~gtl" + gtl_cfg = dont_use_gtl + + return { + "packages": { + "cray-mpich": { + "externals": [ + { + "spec": f"cray-mpich@{self.mpi_version}%cce@{self.cce_version} {gtl_spec} +wrappers", + "prefix": f"/opt/cray/pe/mpich/{self.mpi_version}/ofi/crayclang/{self.short_cce_version}", + "extra_attributes": gtl_cfg, # Assuming `gtl_cfg` is already defined elsewhere + } + ] + } + } + } + + elif self.spec.satisfies("compiler=gcc"): + return { + "packages": { + "cray-mpich": { + "externals": [ + { + "spec": f"cray-mpich@{self.mpi_version}%gcc@{self.gcc_version} ~gtl +wrappers", + "prefix": f"/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/10.3", + "extra_attributes": { + "gtl_lib_path": f"/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", + "ldflags": f"-L/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/10.3/lib -lmpi -L/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", + }, + } + ] + } + } + } + + def compute_software_section(self): + """This is somewhat vestigial: for the Tioga config that is committed + to the repo, multiple instances of mpi/compilers are stored and + and these variables were used to choose consistent dependencies. + The configs generated by this class should only ever have one + instance of MPI etc., so there is no need for that. The experiments + will fail if these variables are not defined though, so for now + they are still generated (but with more-generic values). + """ + return { + "software": { + "packages": { + "default-compiler": { + "pkg_spec": f"{self.spec.variants['compiler'][0]}" + }, + "default-mpi": {"pkg_spec": "cray-mpich"}, + "compiler-amdclang": {"pkg_spec": "clang"}, + "compiler-gcc": {"pkg_spec": "gcc"}, + "mpi-gcc": {"pkg_spec": "cray-mpich~gtl"}, + "blas": {"pkg_spec": f"{self.spec.variants['blas'][0]}"}, + "lapack": {"pkg_spec": f"{self.spec.variants['lapack'][0]}"}, + "lapack-oneapi": {"pkg_spec": "intel-oneapi-mkl"}, + } + } + } From 23309741c93f674245162603ad309e41825c0947 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Thu, 24 Apr 2025 12:00:05 -0700 Subject: [PATCH 02/37] update externals with benchpark system external --- systems/lbnl-perlmutter/system.py | 141 +++++++++++++++--------------- 1 file changed, 71 insertions(+), 70 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 01c7552ce..d7ff09375 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -36,8 +36,8 @@ def __init__(self, spec): super().__init__(spec) if self.spec.satisfies("compiler=gcc"): - self.gcc_version = Version("12.2.0") - self.mpi_version = Version("8.1.26") + self.gcc_version = Version("13.2.1") + self.mpi_version = Version("8.1.30") self.scheduler = "slurm" attrs = self.id_to_resources.get("perlmutter") @@ -48,116 +48,117 @@ def compute_packages_section(self): selections = { "packages": { "all": {"require": "target=x86_64:"}, - "tar": {"externals": [{"spec": "tar@1.30", "prefix": "/usr"}]}, + "tar": {"externals": [{"spec": "tar@1.34", "prefix": "/usr"}]}, "coreutils": { - "externals": [{"spec": "coreutils@8.30", "prefix": "/usr"}] + "externals": [{"spec": "coreutils@8.32", "prefix": "/usr"}] }, "libtool": {"externals": [{"spec": "libtool@2.4.6", "prefix": "/usr"}]}, - "flex": {"externals": [{"spec": "flex@2.6.1+lex", "prefix": "/usr"}]}, + "flex": {"externals": [{"spec": "flex@2.6.4+lex", "prefix": "/usr"}]}, "openssl": { - "externals": [{"spec": "openssl@1.1.1k", "prefix": "/usr"}] + "externals": [{"spec": "openssl@1.1.1l-fips", "prefix": "/usr"}] }, "m4": {"externals": [{"spec": "m4@1.4.18", "prefix": "/usr"}]}, - "groff": {"externals": [{"spec": "groff@1.22.3", "prefix": "/usr"}]}, + "groff": {"externals": [{"spec": "groff@1.22.4", "prefix": "/usr"}]}, "cmake": { - "externals": [ - {"spec": "cmake@3.20.2", "prefix": "/usr"}, - {"spec": "cmake@3.23.1", "prefix": "/usr/tce"}, - {"spec": "cmake@3.24.2", "prefix": "/usr/tce"}, - ], - "buildable": False, - }, - "elfutils": { - "externals": [{"spec": "elfutils@0.190", "prefix": "/usr"}], + "externals": [{"spec": "cmake@3.20.4", "prefix": "/usr"}], "buildable": False, }, - "papi": { - "externals": [{"spec": "papi@5.6.0.0", "prefix": "/usr"}], - "buildable": False, - }, - "unwind": { - "externals": [{"spec": "unwind@8.0.1", "prefix": "/usr"}], - "buildable": False, - }, - "pkgconf": {"externals": [{"spec": "pkgconf@1.4.2", "prefix": "/usr"}]}, "curl": { "externals": [ - {"spec": "curl@7.61.1+gssapi+ldap+nghttp2", "prefix": "/usr"} + {"spec": "curl@8.0.1+gssapi+ldap+nghttp2", "prefix": "/usr"} ] }, "gmake": {"externals": [{"spec": "gmake@4.2.1", "prefix": "/usr"}]}, "subversion": { - "externals": [{"spec": "subversion@1.10.2", "prefix": "/usr"}] + "externals": [{"spec": "subversion@1.14.1", "prefix": "/usr"}], + "buildable": False, }, "diffutils": { - "externals": [{"spec": "diffutils@3.6", "prefix": "/usr"}] + "externals": [{"spec": "diffutils@3.6", "prefix": "/usr"}], + "buildable": False, }, - "swig": {"externals": [{"spec": "swig@3.0.12", "prefix": "/usr"}]}, "gawk": {"externals": [{"spec": "gawk@4.2.1", "prefix": "/usr"}]}, "binutils": { - "externals": [{"spec": "binutils@2.30.113", "prefix": "/usr"}] + "externals": [ + {"spec": "binutils@2.40~gold~headers", "prefix": "/opt/cray/pe/cce/18.0.0/binutils/x86_64/x86_64-pc-linux-gnu"} + {"spec": "binutils@2.43.1~gold~headers", "prefix": "/usr"} + ], + "buildable": False, }, "findutils": { - "externals": [{"spec": "findutils@4.6.0", "prefix": "/usr"}] + "externals": [{"spec": "findutils@4.8.0", "prefix": "/usr"}], + "buildable": False, }, - "git-lfs": { - "externals": [{"spec": "git-lfs@2.11.0", "prefix": "/usr/tce"}] + "ccache": { + "externals": [{"spec": "ccache@3.4.7", "prefix": "/usr"}]}, + "buildable": False, }, - "ccache": {"externals": [{"spec": "ccache@3.7.7", "prefix": "/usr"}]}, "automake": { - "externals": [{"spec": "automake@1.16.1", "prefix": "/usr"}] + "externals": [{"spec": "automake@1.15.1", "prefix": "/usr"}], + "buildable": False, }, - "cvs": {"externals": [{"spec": "cvs@1.11.23", "prefix": "/usr"}]}, "git": { - "externals": [ - {"spec": "git@2.31.1+tcltk", "prefix": "/usr"}, - {"spec": "git@2.29.1+tcltk", "prefix": "/usr/tce"}, - ] + "externals": [{"spec": "git@2.35.3~tcltk", "prefix": "/usr"}], + "buildable": False, + }, + "openssh": { + "externals": [{"spec": "openssh@8.4p1", "prefix": "/usr"}], + "buildable": False, }, - "openssh": {"externals": [{"spec": "openssh@8.0p1", "prefix": "/usr"}]}, "autoconf": { - "externals": [{"spec": "autoconf@2.69", "prefix": "/usr"}] + "externals": [{"spec": "autoconf@2.69", "prefix": "/usr"}], + "buildable": False, + }, + "bison": { + "externals": [{"spec": "bison@3.0.4", "prefix": "/usr"}], + "buildable": False, }, - "texinfo": {"externals": [{"spec": "texinfo@6.5", "prefix": "/usr"}]}, - "bison": {"externals": [{"spec": "bison@3.0.4", "prefix": "/usr"}]}, "python": { "externals": [ { - "spec": "python@3.9.12", - "prefix": "/usr/tce/packages/python/python-3.9.12", - "buildable": False, - } - ] + "prefix": "/usr", + "spec": "python@2.7.18+bz2+crypt+ctypes~dbm~lzma+nis~pyexpat+pythoncmd~readline~sqlite3~ssl~tkinter+uuid+zlib", + }, + { + "prefix": "/usr", + "spec": "python@3.6.15+bz2+crypt+ctypes~dbm+lzma+nis+pyexpat~pythoncmd+readline+sqlite3+ssl~tkinter+uuid+zlib", + }, + ], + "buildable": False, }, - "unzip": { + "doxygen": { + "externals": [{"spec": "doxygen@1.8.14~graphviz~mscgen", "prefix": "/usr"}], "buildable": False, - "externals": [{"spec": "unzip@6.0", "prefix": "/usr"}], }, - "hypre": {"variants": "amdgpu_target=gfx90a"}, - "hwloc": { - "externals": [ - {"spec": "hwloc@2.9.1", "prefix": "/usr", "buildable": False} - ] + "gettext": { + "externals": [{"spec": "gettext@0.20.2", "prefix": "/usr"}], + "buildable": False, }, - "fftw": {"buildable": False}, - "intel-oneapi-mkl": { - "externals": [ - { - "spec": "intel-oneapi-mkl@2023.2.0", - "prefix": "/opt/intel/oneapi", - } - ], + "ninja": { + "externals": [{"spec": "ninja@1.10.0", "prefix": "/usr"}], "buildable": False, }, - "mpi": {"buildable": False}, - "libfabric": { - "externals": [ - {"spec": "libfabric@2.1", "prefix": "/opt/cray/libfabric/2.1"} - ], + "perl": { + "externals": [{"spec": "perl@5.26.1~cpanm+opcode+open+shared+threads", "prefix": "/usr"}], + "buildable": False, + }, + "pkg-config": { + "externals": [{"spec": "pkg-config@0.29.2", "prefix": "/usr"}], + "buildable": False, + }, + "sed": { + "externals": [{"spec": "sed@4.4", "prefix": "/usr"}], + "buildable": False, + }, + "tar": { + "externals": [{"spec": "tar@1.34", "prefix": "/usr"}], + "buildable": False, + }, + "zlib": { + "externals": [{"spec": "zlib@1.2.13", "prefix": "/usr"}], "buildable": False, }, } - } selections["packages"] |= self.mpi_config()["packages"] From 8a087e4c428f6b8f3a4ca37322aef58157b4fb86 Mon Sep 17 00:00:00 2001 From: Stephanie Labasan Date: Thu, 8 May 2025 11:11:53 -0700 Subject: [PATCH 03/37] remove cce compiler for now --- systems/lbnl-perlmutter/system.py | 59 +++---------------------------- 1 file changed, 5 insertions(+), 54 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index d7ff09375..797d4f23f 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -27,8 +27,8 @@ class LbnlPerlmutter(System): variant( "compiler", - default="cce", - values=("cce", "gcc"), + default="gcc", + values=("gcc"), description="Which compiler to use", ) @@ -162,18 +162,7 @@ def compute_packages_section(self): selections["packages"] |= self.mpi_config()["packages"] - if self.spec.satisfies("compiler=cce"): - selections["packages"] |= { - "cray-libsci": { - "externals": [ - { - "spec": "cray-libsci@23.05.1.4%cce", - "prefix": "/opt/cray/pe/libsci/23.05.1.4/cray/12.0/x86_64/", - } - ] - } - } - elif self.spec.satisfies("compiler=gcc"): + if self.spec.satisfies("compiler=gcc"): selections["packages"] |= { "cray-libsci": { "externals": [ @@ -192,9 +181,7 @@ def compute_packages_section(self): def compiler_weighting_cfg(self): compiler = self.spec.variants["compiler"][0] - if compiler == "cce": - return {"packages": {"all": {"require": [{"one_of": ["%cce", "%gcc"]}]}}} - elif compiler == "gcc": + if compiler == "gcc": return {"packages": {}} else: raise ValueError(f"Unexpected value for compiler: {compiler}") @@ -227,43 +214,7 @@ def compute_compilers_section(self): def mpi_config(self): gtl = self.spec.variants["gtl"][0] - if self.spec.satisfies("compiler=cce"): - dont_use_gtl = { - "gtl_lib_path": f"/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", - "ldflags": f"-L/opt/cray/pe/mpich/{self.mpi_version}/ofi/crayclang/{self.short_cce_version}/lib -lmpi -L/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", - } - - use_gtl = { - "gtl_flags": "$MV2_COMM_WORLD_LOCAL_RANK", - "gtl_cutoff_size": 4096, - "fi_cxi_ats": 0, - "gtl_lib_path": f"/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", - "gtl_libs": ["libmpi_gtl_hsa"], - "ldflags": f"-L/opt/cray/pe/mpich/{self.mpi_version}/ofi/crayclang/{self.short_cce_version}/lib -lmpi -L/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -lmpi_gtl_hsa", - } - - if gtl: - gtl_spec = "+gtl" - gtl_cfg = use_gtl - else: - gtl_spec = "~gtl" - gtl_cfg = dont_use_gtl - - return { - "packages": { - "cray-mpich": { - "externals": [ - { - "spec": f"cray-mpich@{self.mpi_version}%cce@{self.cce_version} {gtl_spec} +wrappers", - "prefix": f"/opt/cray/pe/mpich/{self.mpi_version}/ofi/crayclang/{self.short_cce_version}", - "extra_attributes": gtl_cfg, # Assuming `gtl_cfg` is already defined elsewhere - } - ] - } - } - } - - elif self.spec.satisfies("compiler=gcc"): + if self.spec.satisfies("compiler=gcc"): return { "packages": { "cray-mpich": { From 937a7c03ff22d472bec869e6a70c81afb3e00a50 Mon Sep 17 00:00:00 2001 From: Stephanie Labasan Date: Thu, 8 May 2025 12:02:37 -0700 Subject: [PATCH 04/37] checkpoint --- systems/lbnl-perlmutter/system.py | 81 ++++++++++++++----------------- 1 file changed, 36 insertions(+), 45 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 797d4f23f..4c3986217 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -28,7 +28,6 @@ class LbnlPerlmutter(System): variant( "compiler", default="gcc", - values=("gcc"), description="Which compiler to use", ) @@ -68,6 +67,18 @@ def compute_packages_section(self): {"spec": "curl@8.0.1+gssapi+ldap+nghttp2", "prefix": "/usr"} ] }, + "cusolver": { + "externals": [{"spec": "cusolver@12.4", "prefix": "/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/cuda/12.4"}], + "buildable": False, + }, + "cublas": { + "externals": [{"spec": "cublas@12.4", "prefix": "/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/cuda/12.4"}], + "buildable": False, + }, + "automake": { + "externals": [{"spec": "automake@1.15.1", "prefix": "/usr"}], + "buildable": False, + }, "gmake": {"externals": [{"spec": "gmake@4.2.1", "prefix": "/usr"}]}, "subversion": { "externals": [{"spec": "subversion@1.14.1", "prefix": "/usr"}], @@ -80,7 +91,7 @@ def compute_packages_section(self): "gawk": {"externals": [{"spec": "gawk@4.2.1", "prefix": "/usr"}]}, "binutils": { "externals": [ - {"spec": "binutils@2.40~gold~headers", "prefix": "/opt/cray/pe/cce/18.0.0/binutils/x86_64/x86_64-pc-linux-gnu"} + {"spec": "binutils@2.40~gold~headers", "prefix": "/opt/cray/pe/cce/18.0.0/binutils/x86_64/x86_64-pc-linux-gnu"}, {"spec": "binutils@2.43.1~gold~headers", "prefix": "/usr"} ], "buildable": False, @@ -93,10 +104,6 @@ def compute_packages_section(self): "externals": [{"spec": "ccache@3.4.7", "prefix": "/usr"}]}, "buildable": False, }, - "automake": { - "externals": [{"spec": "automake@1.15.1", "prefix": "/usr"}], - "buildable": False, - }, "git": { "externals": [{"spec": "git@2.35.3~tcltk", "prefix": "/usr"}], "buildable": False, @@ -160,32 +167,22 @@ def compute_packages_section(self): }, } - selections["packages"] |= self.mpi_config()["packages"] + #selections["packages"] |= self.mpi_config() - if self.spec.satisfies("compiler=gcc"): - selections["packages"] |= { - "cray-libsci": { - "externals": [ - { - "spec": "cray-libsci@23.05.1.4%gcc", - "prefix": "/opt/cray/pe/libsci/23.05.1.4/gnu/10.3/x86_64/", - } - ] - } - } - - selections["packages"] |= self.compiler_weighting_cfg()["packages"] +# if self.spec.satisfies("compiler=gcc"): +# selections["packages"] |= { +# "cray-libsci": { +# "externals": [ +# { +# "spec": "cray-libsci@23.05.1.4%gcc", +# "prefix": "/opt/cray/pe/libsci/23.05.1.4/gnu/10.3/x86_64/", +# } +# ] +# } +# } return selections - def compiler_weighting_cfg(self): - compiler = self.spec.variants["compiler"][0] - - if compiler == "gcc": - return {"packages": {}} - else: - raise ValueError(f"Unexpected value for compiler: {compiler}") - def compute_compilers_section(self): selections = { "compilers": [ @@ -212,23 +209,19 @@ def compute_compilers_section(self): return selections def mpi_config(self): - gtl = self.spec.variants["gtl"][0] - if self.spec.satisfies("compiler=gcc"): return { - "packages": { - "cray-mpich": { - "externals": [ - { - "spec": f"cray-mpich@{self.mpi_version}%gcc@{self.gcc_version} ~gtl +wrappers", - "prefix": f"/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/10.3", - "extra_attributes": { - "gtl_lib_path": f"/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", - "ldflags": f"-L/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/10.3/lib -lmpi -L/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", - }, - } - ] - } + "cray-mpich": { + "externals": [ + { + "spec": f"cray-mpich@{self.mpi_version}%gcc@{self.gcc_version} ~gtl +wrappers", + "prefix": f"/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/10.3", + "extra_attributes": { + "gtl_lib_path": f"/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", + "ldflags": f"-L/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/10.3/lib -lmpi -L/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", + }, + } + ] } } @@ -251,8 +244,6 @@ def compute_software_section(self): "compiler-amdclang": {"pkg_spec": "clang"}, "compiler-gcc": {"pkg_spec": "gcc"}, "mpi-gcc": {"pkg_spec": "cray-mpich~gtl"}, - "blas": {"pkg_spec": f"{self.spec.variants['blas'][0]}"}, - "lapack": {"pkg_spec": f"{self.spec.variants['lapack'][0]}"}, "lapack-oneapi": {"pkg_spec": "intel-oneapi-mkl"}, } } From f1eded496efdefa8e8f71c610c43e2bc4a9bee7f Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Thu, 8 May 2025 13:05:39 -0700 Subject: [PATCH 05/37] fix system.py --- systems/lbnl-perlmutter/system.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 4c3986217..8faede6b3 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -101,7 +101,7 @@ def compute_packages_section(self): "buildable": False, }, "ccache": { - "externals": [{"spec": "ccache@3.4.7", "prefix": "/usr"}]}, + "externals": [{"spec": "ccache@3.4.7", "prefix": "/usr"}], "buildable": False, }, "git": { @@ -166,6 +166,7 @@ def compute_packages_section(self): "buildable": False, }, } + } #selections["packages"] |= self.mpi_config() From 8e8cd714cd3b517fc70bdfa3bebffd8222968b8d Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Thu, 8 May 2025 14:39:05 -0700 Subject: [PATCH 06/37] fix path to cublas --- systems/lbnl-perlmutter/system.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 8faede6b3..a5cc3cca5 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -28,6 +28,7 @@ class LbnlPerlmutter(System): variant( "compiler", default="gcc", + values=("gcc",), description="Which compiler to use", ) @@ -68,11 +69,11 @@ def compute_packages_section(self): ] }, "cusolver": { - "externals": [{"spec": "cusolver@12.4", "prefix": "/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/cuda/12.4"}], + "externals": [{"spec": "cusolver@12.4", "prefix": "/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/math_libs/12.4"}], "buildable": False, }, "cublas": { - "externals": [{"spec": "cublas@12.4", "prefix": "/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/cuda/12.4"}], + "externals": [{"spec": "cublas@12.4", "prefix": "/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/math_libs/12.4"}], "buildable": False, }, "automake": { From 1ab226a659b6ae285a8a1fe8ea9e0a7eb14e96b0 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Thu, 25 Sep 2025 14:34:15 -0700 Subject: [PATCH 07/37] making progress --- systems/lbnl-perlmutter/system.py | 34 +++++++++++++++---------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index a5cc3cca5..0022038fc 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -36,7 +36,7 @@ def __init__(self, spec): super().__init__(spec) if self.spec.satisfies("compiler=gcc"): - self.gcc_version = Version("13.2.1") + self.gcc_version = Version("12.3.0") self.mpi_version = Version("8.1.30") self.scheduler = "slurm" @@ -171,17 +171,17 @@ def compute_packages_section(self): #selections["packages"] |= self.mpi_config() -# if self.spec.satisfies("compiler=gcc"): -# selections["packages"] |= { -# "cray-libsci": { -# "externals": [ -# { -# "spec": "cray-libsci@23.05.1.4%gcc", -# "prefix": "/opt/cray/pe/libsci/23.05.1.4/gnu/10.3/x86_64/", -# } -# ] -# } -# } + if self.spec.satisfies("compiler=gcc"): + selections["packages"] |= { + "cray-libsci": { + "externals": [ + { + "spec": "cray-libsci@23.05.1.4%gcc", + "prefix": "/opt/cray/pe/libsci/23.05.1.4/gnu/10.3/x86_64/", + } + ] + } + } return selections @@ -190,12 +190,12 @@ def compute_compilers_section(self): "compilers": [ { "compiler": { - "spec": "gcc@12.2.0", + "spec": "gcc@12.3.0", "paths": { - "cc": "/opt/cray/pe/gcc/12.2.0/bin/gcc", - "cxx": "/opt/cray/pe/gcc/12.2.0/bin/g++", - "f77": "/opt/cray/pe/gcc/12.2.0/bin/gfortran", - "fc": "/opt/cray/pe/gcc/12.2.0/bin/gfortran", + "cc": "/opt/cray/pe/gcc-native/12/bin/gcc", + "cxx": "/opt/cray/pe/gcc-native/12/bin/g++", + "f77": "/opt/cray/pe/gcc-native/12/bin/gfortran", + "fc": "/opt/cray/pe/gcc-native/12/bin/gfortran", }, "flags": {}, "operating_system": "rhel8", From 28cd29285d44b76251d75f7ed6587044ae3679c9 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Thu, 25 Sep 2025 15:23:45 -0700 Subject: [PATCH 08/37] update libsci version --- systems/lbnl-perlmutter/system.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 0022038fc..530867c4c 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -166,6 +166,9 @@ def compute_packages_section(self): "externals": [{"spec": "zlib@1.2.13", "prefix": "/usr"}], "buildable": False, }, + "mpi": { + "buildable": False, + }, } } @@ -176,8 +179,8 @@ def compute_packages_section(self): "cray-libsci": { "externals": [ { - "spec": "cray-libsci@23.05.1.4%gcc", - "prefix": "/opt/cray/pe/libsci/23.05.1.4/gnu/10.3/x86_64/", + "spec": "cray-libsci@24.07.0%gcc", + "prefix": "/opt/cray/pe/libsci/24.07.0/gnu/12.3/x86_64/", } ] } From d4b4253e90fc78f739d59c9d61aaee2a71a6ed17 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Thu, 25 Sep 2025 16:41:05 -0700 Subject: [PATCH 09/37] update cmake version --- systems/lbnl-perlmutter/system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 530867c4c..1d7f3b164 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -60,7 +60,7 @@ def compute_packages_section(self): "m4": {"externals": [{"spec": "m4@1.4.18", "prefix": "/usr"}]}, "groff": {"externals": [{"spec": "groff@1.22.4", "prefix": "/usr"}]}, "cmake": { - "externals": [{"spec": "cmake@3.20.4", "prefix": "/usr"}], + "externals": [{"spec": "cmake@3.30.2", "prefix": "/global/common/software/nersc9/cmake/3.30.2"}], "buildable": False, }, "curl": { From ab5a169f10b708bb02c1a40d8c6bff74117a84e5 Mon Sep 17 00:00:00 2001 From: Stephanie Lam Date: Mon, 29 Sep 2025 10:49:40 -0700 Subject: [PATCH 10/37] updated to Spack 1.0 version; getting error regarding gcc --- systems/lbnl-perlmutter/system.py | 124 +++++++++++++++++++----------- 1 file changed, 81 insertions(+), 43 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 1d7f3b164..b3d5df25b 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -6,7 +6,7 @@ from packaging.version import Version from benchpark.directives import variant, maintainers -from benchpark.system import System +from benchpark.system import System, compiler_section_for, compiler_def from benchpark.paths import hardware_descriptions @@ -121,19 +121,19 @@ def compute_packages_section(self): "externals": [{"spec": "bison@3.0.4", "prefix": "/usr"}], "buildable": False, }, - "python": { - "externals": [ - { - "prefix": "/usr", - "spec": "python@2.7.18+bz2+crypt+ctypes~dbm~lzma+nis~pyexpat+pythoncmd~readline~sqlite3~ssl~tkinter+uuid+zlib", - }, - { - "prefix": "/usr", - "spec": "python@3.6.15+bz2+crypt+ctypes~dbm+lzma+nis+pyexpat~pythoncmd+readline+sqlite3+ssl~tkinter+uuid+zlib", - }, - ], - "buildable": False, - }, + # "python": { + # "externals": [ + # { + # "prefix": "/usr", + # "spec": "python@2.7.18+bz2+crypt+ctypes~dbm~lzma+nis~pyexpat+pythoncmd~readline~sqlite3~ssl~tkinter+uuid+zlib", + # }, + # { + # "prefix": "/usr", + # "spec": "python@3.6.15+bz2+crypt+ctypes~dbm+lzma+nis+pyexpat~pythoncmd+readline+sqlite3+ssl~tkinter+uuid+zlib", + # }, + # ], + # "buildable": False, + # }, "doxygen": { "externals": [{"spec": "doxygen@1.8.14~graphviz~mscgen", "prefix": "/usr"}], "buildable": False, @@ -166,7 +166,22 @@ def compute_packages_section(self): "externals": [{"spec": "zlib@1.2.13", "prefix": "/usr"}], "buildable": False, }, - "mpi": { + "cray-mpich": { + "externals": [ + { + "spec": "cray-mpich@8.1.30+wrappers", + "prefix": "/opt/cray/pe/mpich/8.1.30/ofi/gnu/12.3", + } + ], + "buildable": False, + }, + "cray-libsci": { + "externals": [ + { + "spec": "cray-libsci@23.02.1.1", + "prefix": "/opt/cray/pe/libsci/23.02.1.1/cray/9.0/x86_64", + } + ], "buildable": False, }, } @@ -189,29 +204,41 @@ def compute_packages_section(self): return selections def compute_compilers_section(self): - selections = { - "compilers": [ - { - "compiler": { - "spec": "gcc@12.3.0", - "paths": { - "cc": "/opt/cray/pe/gcc-native/12/bin/gcc", - "cxx": "/opt/cray/pe/gcc-native/12/bin/g++", - "f77": "/opt/cray/pe/gcc-native/12/bin/gfortran", - "fc": "/opt/cray/pe/gcc-native/12/bin/gfortran", - }, - "flags": {}, - "operating_system": "rhel8", - "target": "x86_64", - "modules": [], - "environment": {}, - "extra_rpaths": [], - } - } - ] - } + # selections = { + # "compilers": [ + # { + # "compiler": { + # "spec": "gcc@12.3.0", + # "paths": { + # "cc": "/opt/cray/pe/gcc-native/12/bin/gcc", + # "cxx": "/opt/cray/pe/gcc-native/12/bin/g++", + # "f77": "/opt/cray/pe/gcc-native/12/bin/gfortran", + # "fc": "/opt/cray/pe/gcc-native/12/bin/gfortran", + # }, + # "flags": {}, + # "operating_system": "rhel8", + # "target": "x86_64", + # "modules": [], + # "environment": {}, + # "extra_rpaths": [], + # } + # } + # ] + # } - return selections + # return selections + if self.spec.satisfies("compiler=gcc"): + cfg = compiler_section_for( + "gcc", + [ + compiler_def( + "gcc@12.3.0 languages:=c,c++,fortran", + "/opt/cray/pe/gcc-native/12/bin/gcc", + {"c": "gcc", "cxx": "g++", "fortran": "gfortran"}, + ) + ], + ) + return cfg def mpi_config(self): if self.spec.satisfies("compiler=gcc"): @@ -239,16 +266,27 @@ def compute_software_section(self): will fail if these variables are not defined though, so for now they are still generated (but with more-generic values). """ + # return { + # "software": { + # "packages": { + # "default-compiler": { + # "pkg_spec": f"{self.spec.variants['compiler'][0]}" + # }, + # "default-mpi": {"pkg_spec": "cray-mpich"}, + # "compiler-amdclang": {"pkg_spec": "clang"}, + # "compiler-gcc": {"pkg_spec": "gcc"}, + # "mpi-gcc": {"pkg_spec": "cray-mpich~gtl"}, + # "lapack-oneapi": {"pkg_spec": "intel-oneapi-mkl"}, + # } + # } + # } return { "software": { "packages": { - "default-compiler": { - "pkg_spec": f"{self.spec.variants['compiler'][0]}" - }, + #"default-compiler": {"pkg_spec": default_compiler}, "default-mpi": {"pkg_spec": "cray-mpich"}, - "compiler-amdclang": {"pkg_spec": "clang"}, - "compiler-gcc": {"pkg_spec": "gcc"}, - "mpi-gcc": {"pkg_spec": "cray-mpich~gtl"}, + #"compiler-gcc": {"pkg_spec": "gcc"}, + "mpi-gcc": {"pkg_spec": "cray-mpich"}, "lapack-oneapi": {"pkg_spec": "intel-oneapi-mkl"}, } } From 9533de5f1569a937dc3752938ba5060b0192997c Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Wed, 1 Oct 2025 10:31:42 -0700 Subject: [PATCH 11/37] changes --- experiments/amg2023/experiment.py | 1 + systems/lbnl-perlmutter/system.py | 33 ++++++++++++++++++------------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/experiments/amg2023/experiment.py b/experiments/amg2023/experiment.py index c62352efc..51c0baa9b 100644 --- a/experiments/amg2023/experiment.py +++ b/experiments/amg2023/experiment.py @@ -143,3 +143,4 @@ def compute_package_section(self): self.name, [f"amg2023{self.determine_version()} {mixedint}"] ) self.add_package_spec("hypre", ["hypre+lapack"]) + #self.add_package_spec("hypre", ["hypre+lapack+mpi"]) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index b3d5df25b..34538e549 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -166,6 +166,9 @@ def compute_packages_section(self): "externals": [{"spec": "zlib@1.2.13", "prefix": "/usr"}], "buildable": False, }, + "mpi": { + "buildable": False, + }, "cray-mpich": { "externals": [ { @@ -173,7 +176,6 @@ def compute_packages_section(self): "prefix": "/opt/cray/pe/mpich/8.1.30/ofi/gnu/12.3", } ], - "buildable": False, }, "cray-libsci": { "externals": [ @@ -187,7 +189,7 @@ def compute_packages_section(self): } } - #selections["packages"] |= self.mpi_config() + selections["packages"] |= self.mpi_config()["packages"] if self.spec.satisfies("compiler=gcc"): selections["packages"] |= { @@ -233,27 +235,30 @@ def compute_compilers_section(self): [ compiler_def( "gcc@12.3.0 languages:=c,c++,fortran", - "/opt/cray/pe/gcc-native/12/bin/gcc", + "/opt/cray/pe/gcc-native/12", {"c": "gcc", "cxx": "g++", "fortran": "gfortran"}, ) ], ) + return cfg def mpi_config(self): if self.spec.satisfies("compiler=gcc"): return { - "cray-mpich": { - "externals": [ - { - "spec": f"cray-mpich@{self.mpi_version}%gcc@{self.gcc_version} ~gtl +wrappers", - "prefix": f"/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/10.3", - "extra_attributes": { - "gtl_lib_path": f"/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", - "ldflags": f"-L/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/10.3/lib -lmpi -L/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", - }, - } - ] + "packages": { + "cray-mpich": { + "externals": [ + { + "spec": f"cray-mpich@{self.mpi_version}%gcc@{self.gcc_version} ~gtl +wrappers", + "prefix": f"/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/10.3", + "extra_attributes": { + "gtl_lib_path": f"/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", + "ldflags": f"-L/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/10.3/lib -lmpi -L/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", + }, + } + ] + } } } From c050a0cffbf347af57422673bf6180e7e2c4e564 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Wed, 1 Oct 2025 10:58:49 -0700 Subject: [PATCH 12/37] successfully builds stream, running has errors with job script sbatch: error: Job request does not match any supported policy. sbatch: error: Batch job submission failed: Unspecified error --- systems/lbnl-perlmutter/system.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 34538e549..db62a708f 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -250,11 +250,12 @@ def mpi_config(self): "cray-mpich": { "externals": [ { - "spec": f"cray-mpich@{self.mpi_version}%gcc@{self.gcc_version} ~gtl +wrappers", - "prefix": f"/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/10.3", + "spec": f"cray-mpich@{self.mpi_version}+gtl+wrappers %gcc@{self.gcc_version}", + "prefix": f"/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/12.3", "extra_attributes": { "gtl_lib_path": f"/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", - "ldflags": f"-L/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/10.3/lib -lmpi -L/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", + "gtl_libs": "libmpi_gtl_cuda", + "ldflags": f"-L/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/12.3/lib -lmpi -L/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", }, } ] From aacba7c8c34b32b023c16c9b9437a9fbb65daa96 Mon Sep 17 00:00:00 2001 From: Stephanie Lam Date: Thu, 2 Oct 2025 23:27:18 -0700 Subject: [PATCH 13/37] added constraint variant --- systems/lbnl-perlmutter/system.py | 38 +++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index db62a708f..ba118bb6d 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -32,6 +32,13 @@ class LbnlPerlmutter(System): description="Which compiler to use", ) + variant( + "constraint", + default="cpu", + values=("cpu", "gpu", "gpu&hbmg40", "gpu&hbmg80"), + description="Which constraint to use" + ) + def __init__(self, spec): super().__init__(spec) @@ -263,6 +270,37 @@ def mpi_config(self): } } + def system_specific_variables(self): + opts = super().system_specific_variables() + if self.spec.satisfies("constraint=cpu"): + opts.update( + { + "extra_batch_opts":f"--constraint={self.spec.variants['constraint'][0]}", + } + ) + elif self.spec.satisfies("constraint=gpu"): + opts.update( + { + "extra_batch_opts":f"--constraint={self.spec.variants['constraint'][0]}", + } + ) + elif self.spec.satisfies("constraint=gpu&hbmg40"): + opts.update( + { + "extra_batch_opts":f"--constraint={self.spec.variants['constraint'][0]}", + } + ) + elif self.spec.satisfies("constraint=gpu&hbmg80"): + opts.update( + { + "extra_batch_opts":f"--constraint={self.spec.variants['constraint'][0]}", + } + ) + + return opts + + + def compute_software_section(self): """This is somewhat vestigial: for the Tioga config that is committed to the repo, multiple instances of mpi/compilers are stored and From 79388322a1c0fe810dd5fc2a4aafb011822500c9 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Thu, 9 Oct 2025 13:47:44 -0700 Subject: [PATCH 14/37] add queue variant --- systems/lbnl-perlmutter/system.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index ba118bb6d..363c8251c 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -6,7 +6,12 @@ from packaging.version import Version from benchpark.directives import variant, maintainers -from benchpark.system import System, compiler_section_for, compiler_def +from benchpark.system import ( + System, + compiler_section_for, + compiler_def, + JobQueue, +) from benchpark.paths import hardware_descriptions @@ -22,6 +27,7 @@ class LbnlPerlmutter(System): "system_site": "lbnl", "hardware_key": str(hardware_descriptions) + "/HPECray-zen3-A100-Slingshot/hardware_description.yaml", + "queues": [JobQueue("regular", 2880, 3072), JobQueue("debug", 30, 8)], }, } @@ -39,6 +45,14 @@ class LbnlPerlmutter(System): description="Which constraint to use" ) + variant( + "queue", + default="regular", + values=("none", "regular", "debug"), + multi=False, + description="Submit to queue", + ) + def __init__(self, spec): super().__init__(spec) From 2214ee5ff493ea5ababd6c0d588e1477b197a6f7 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Fri, 10 Oct 2025 12:28:53 -0700 Subject: [PATCH 15/37] remove intel oneapi, add mpi spec to cray-libsci --- systems/lbnl-perlmutter/system.py | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 363c8251c..998f332ac 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -190,21 +190,13 @@ def compute_packages_section(self): "mpi": { "buildable": False, }, - "cray-mpich": { - "externals": [ - { - "spec": "cray-mpich@8.1.30+wrappers", - "prefix": "/opt/cray/pe/mpich/8.1.30/ofi/gnu/12.3", - } - ], - }, "cray-libsci": { "externals": [ - { - "spec": "cray-libsci@23.02.1.1", - "prefix": "/opt/cray/pe/libsci/23.02.1.1/cray/9.0/x86_64", - } - ], + { + "spec": "cray-libsci@23.02.1.1", + "prefix": "/opt/cray/pe/libsci/23.02.1.1/cray/9.0/x86_64", + } + ], "buildable": False, }, } @@ -217,7 +209,7 @@ def compute_packages_section(self): "cray-libsci": { "externals": [ { - "spec": "cray-libsci@24.07.0%gcc", + "spec": "cray-libsci@24.07.0%gcc+mpi", "prefix": "/opt/cray/pe/libsci/24.07.0/gnu/12.3/x86_64/", } ] @@ -334,7 +326,6 @@ def compute_software_section(self): # "compiler-amdclang": {"pkg_spec": "clang"}, # "compiler-gcc": {"pkg_spec": "gcc"}, # "mpi-gcc": {"pkg_spec": "cray-mpich~gtl"}, - # "lapack-oneapi": {"pkg_spec": "intel-oneapi-mkl"}, # } # } # } @@ -345,7 +336,6 @@ def compute_software_section(self): "default-mpi": {"pkg_spec": "cray-mpich"}, #"compiler-gcc": {"pkg_spec": "gcc"}, "mpi-gcc": {"pkg_spec": "cray-mpich"}, - "lapack-oneapi": {"pkg_spec": "intel-oneapi-mkl"}, } } } From dc6cad20e22c90f0fcd94f4c8cc69d514e07a06d Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Fri, 10 Oct 2025 16:12:14 -0700 Subject: [PATCH 16/37] add blas and lapack --- systems/lbnl-perlmutter/system.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 998f332ac..efab6cc8a 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -190,6 +190,12 @@ def compute_packages_section(self): "mpi": { "buildable": False, }, + "lapac": { + "buildable": False, + }, + "blas": { + "buildable": False, + }, "cray-libsci": { "externals": [ { From cdd65070c5a5c393934f27a6b37dabfc8dd76194 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Fri, 10 Oct 2025 16:32:02 -0700 Subject: [PATCH 17/37] add mpi to amg2023 experiment, update libcray-sci with mpi --- systems/lbnl-perlmutter/system.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index efab6cc8a..754a0bf53 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -215,7 +215,7 @@ def compute_packages_section(self): "cray-libsci": { "externals": [ { - "spec": "cray-libsci@24.07.0%gcc+mpi", + "spec": "cray-libsci@24.07.0 +mpi %gcc", "prefix": "/opt/cray/pe/libsci/24.07.0/gnu/12.3/x86_64/", } ] @@ -344,4 +344,4 @@ def compute_software_section(self): "mpi-gcc": {"pkg_spec": "cray-mpich"}, } } - } + } From cce42e0355713ace852ab16302b6af87695d86a3 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Sun, 19 Oct 2025 23:17:26 -0700 Subject: [PATCH 18/37] new system.py, seeded from cscs-eiger system.py --- systems/lbnl-perlmutter/system.py | 293 ++++-------------------------- 1 file changed, 36 insertions(+), 257 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 754a0bf53..f38e05e93 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -3,8 +3,6 @@ # # SPDX-License-Identifier: Apache-2.0 -from packaging.version import Version - from benchpark.directives import variant, maintainers from benchpark.system import ( System, @@ -12,6 +10,8 @@ compiler_def, JobQueue, ) +from benchpark.openmpsystem import OpenMPCPUOnlySystem +from packaging.version import Version from benchpark.paths import hardware_descriptions @@ -34,7 +34,6 @@ class LbnlPerlmutter(System): variant( "compiler", default="gcc", - values=("gcc",), description="Which compiler to use", ) @@ -55,264 +54,58 @@ class LbnlPerlmutter(System): def __init__(self, spec): super().__init__(spec) + self.programming_models = [OpenMPCPUOnlySystem()] - if self.spec.satisfies("compiler=gcc"): - self.gcc_version = Version("12.3.0") - self.mpi_version = Version("8.1.30") + self.gcc_version = Version("12.3.0") + self.mpi_version = Version("8.1.30") + self.cce_version = Version("16.0.0") self.scheduler = "slurm" attrs = self.id_to_resources.get("perlmutter") for k, v in attrs.items(): setattr(self, k, v) + def compute_compilers_section(self): + return compiler_section_for( + "gcc", + [ + compiler_def( + "gcc@12.3.0 languages:=c,c++,fortran", + f"/opt/cray/pe/gcc-native/12/", + {"c": "cc", "cxx": "CC", "fortran": "ftn"}, + modules=["PrgEnv-gnu", "gcc/12.3.0"], + compilers_use_relative_paths=True, + env={ + "append_path": { + "LD_LIBRARY_PATH": "/opt/cray/libfabric/1.22.0/lib64/:/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/cuda/12.4/lib64" + } + }, + ) + ], + ) + def compute_packages_section(self): + selections = { "packages": { - "all": {"require": "target=x86_64:"}, - "tar": {"externals": [{"spec": "tar@1.34", "prefix": "/usr"}]}, - "coreutils": { - "externals": [{"spec": "coreutils@8.32", "prefix": "/usr"}] - }, - "libtool": {"externals": [{"spec": "libtool@2.4.6", "prefix": "/usr"}]}, - "flex": {"externals": [{"spec": "flex@2.6.4+lex", "prefix": "/usr"}]}, - "openssl": { - "externals": [{"spec": "openssl@1.1.1l-fips", "prefix": "/usr"}] - }, - "m4": {"externals": [{"spec": "m4@1.4.18", "prefix": "/usr"}]}, - "groff": {"externals": [{"spec": "groff@1.22.4", "prefix": "/usr"}]}, - "cmake": { - "externals": [{"spec": "cmake@3.30.2", "prefix": "/global/common/software/nersc9/cmake/3.30.2"}], - "buildable": False, - }, - "curl": { + "all": {"providers": {"mpi": ["cray-mpich"]}}, + "cray-mpich": { "externals": [ - {"spec": "curl@8.0.1+gssapi+ldap+nghttp2", "prefix": "/usr"} + { + "spec": "cray-mpich@8.1.28", + "prefix": f"/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/12.3", + } ] }, - "cusolver": { - "externals": [{"spec": "cusolver@12.4", "prefix": "/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/math_libs/12.4"}], - "buildable": False, - }, - "cublas": { - "externals": [{"spec": "cublas@12.4", "prefix": "/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/math_libs/12.4"}], - "buildable": False, - }, - "automake": { - "externals": [{"spec": "automake@1.15.1", "prefix": "/usr"}], - "buildable": False, - }, - "gmake": {"externals": [{"spec": "gmake@4.2.1", "prefix": "/usr"}]}, - "subversion": { - "externals": [{"spec": "subversion@1.14.1", "prefix": "/usr"}], - "buildable": False, - }, - "diffutils": { - "externals": [{"spec": "diffutils@3.6", "prefix": "/usr"}], - "buildable": False, - }, - "gawk": {"externals": [{"spec": "gawk@4.2.1", "prefix": "/usr"}]}, - "binutils": { - "externals": [ - {"spec": "binutils@2.40~gold~headers", "prefix": "/opt/cray/pe/cce/18.0.0/binutils/x86_64/x86_64-pc-linux-gnu"}, - {"spec": "binutils@2.43.1~gold~headers", "prefix": "/usr"} - ], - "buildable": False, - }, - "findutils": { - "externals": [{"spec": "findutils@4.8.0", "prefix": "/usr"}], - "buildable": False, - }, - "ccache": { - "externals": [{"spec": "ccache@3.4.7", "prefix": "/usr"}], - "buildable": False, - }, - "git": { - "externals": [{"spec": "git@2.35.3~tcltk", "prefix": "/usr"}], - "buildable": False, - }, - "openssh": { - "externals": [{"spec": "openssh@8.4p1", "prefix": "/usr"}], - "buildable": False, - }, - "autoconf": { - "externals": [{"spec": "autoconf@2.69", "prefix": "/usr"}], - "buildable": False, - }, - "bison": { - "externals": [{"spec": "bison@3.0.4", "prefix": "/usr"}], - "buildable": False, - }, - # "python": { - # "externals": [ - # { - # "prefix": "/usr", - # "spec": "python@2.7.18+bz2+crypt+ctypes~dbm~lzma+nis~pyexpat+pythoncmd~readline~sqlite3~ssl~tkinter+uuid+zlib", - # }, - # { - # "prefix": "/usr", - # "spec": "python@3.6.15+bz2+crypt+ctypes~dbm+lzma+nis+pyexpat~pythoncmd+readline+sqlite3+ssl~tkinter+uuid+zlib", - # }, - # ], - # "buildable": False, - # }, - "doxygen": { - "externals": [{"spec": "doxygen@1.8.14~graphviz~mscgen", "prefix": "/usr"}], - "buildable": False, - }, - "gettext": { - "externals": [{"spec": "gettext@0.20.2", "prefix": "/usr"}], - "buildable": False, - }, - "ninja": { - "externals": [{"spec": "ninja@1.10.0", "prefix": "/usr"}], - "buildable": False, - }, - "perl": { - "externals": [{"spec": "perl@5.26.1~cpanm+opcode+open+shared+threads", "prefix": "/usr"}], - "buildable": False, - }, - "pkg-config": { - "externals": [{"spec": "pkg-config@0.29.2", "prefix": "/usr"}], - "buildable": False, - }, - "sed": { - "externals": [{"spec": "sed@4.4", "prefix": "/usr"}], - "buildable": False, - }, - "tar": { - "externals": [{"spec": "tar@1.34", "prefix": "/usr"}], - "buildable": False, - }, "zlib": { "externals": [{"spec": "zlib@1.2.13", "prefix": "/usr"}], "buildable": False, }, - "mpi": { - "buildable": False, - }, - "lapac": { - "buildable": False, - }, - "blas": { - "buildable": False, - }, - "cray-libsci": { - "externals": [ - { - "spec": "cray-libsci@23.02.1.1", - "prefix": "/opt/cray/pe/libsci/23.02.1.1/cray/9.0/x86_64", - } - ], - "buildable": False, - }, } } - selections["packages"] |= self.mpi_config()["packages"] - - if self.spec.satisfies("compiler=gcc"): - selections["packages"] |= { - "cray-libsci": { - "externals": [ - { - "spec": "cray-libsci@24.07.0 +mpi %gcc", - "prefix": "/opt/cray/pe/libsci/24.07.0/gnu/12.3/x86_64/", - } - ] - } - } - return selections - def compute_compilers_section(self): - # selections = { - # "compilers": [ - # { - # "compiler": { - # "spec": "gcc@12.3.0", - # "paths": { - # "cc": "/opt/cray/pe/gcc-native/12/bin/gcc", - # "cxx": "/opt/cray/pe/gcc-native/12/bin/g++", - # "f77": "/opt/cray/pe/gcc-native/12/bin/gfortran", - # "fc": "/opt/cray/pe/gcc-native/12/bin/gfortran", - # }, - # "flags": {}, - # "operating_system": "rhel8", - # "target": "x86_64", - # "modules": [], - # "environment": {}, - # "extra_rpaths": [], - # } - # } - # ] - # } - - # return selections - if self.spec.satisfies("compiler=gcc"): - cfg = compiler_section_for( - "gcc", - [ - compiler_def( - "gcc@12.3.0 languages:=c,c++,fortran", - "/opt/cray/pe/gcc-native/12", - {"c": "gcc", "cxx": "g++", "fortran": "gfortran"}, - ) - ], - ) - - return cfg - - def mpi_config(self): - if self.spec.satisfies("compiler=gcc"): - return { - "packages": { - "cray-mpich": { - "externals": [ - { - "spec": f"cray-mpich@{self.mpi_version}+gtl+wrappers %gcc@{self.gcc_version}", - "prefix": f"/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/12.3", - "extra_attributes": { - "gtl_lib_path": f"/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", - "gtl_libs": "libmpi_gtl_cuda", - "ldflags": f"-L/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/12.3/lib -lmpi -L/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", - }, - } - ] - } - } - } - - def system_specific_variables(self): - opts = super().system_specific_variables() - if self.spec.satisfies("constraint=cpu"): - opts.update( - { - "extra_batch_opts":f"--constraint={self.spec.variants['constraint'][0]}", - } - ) - elif self.spec.satisfies("constraint=gpu"): - opts.update( - { - "extra_batch_opts":f"--constraint={self.spec.variants['constraint'][0]}", - } - ) - elif self.spec.satisfies("constraint=gpu&hbmg40"): - opts.update( - { - "extra_batch_opts":f"--constraint={self.spec.variants['constraint'][0]}", - } - ) - elif self.spec.satisfies("constraint=gpu&hbmg80"): - opts.update( - { - "extra_batch_opts":f"--constraint={self.spec.variants['constraint'][0]}", - } - ) - - return opts - - - def compute_software_section(self): """This is somewhat vestigial: for the Tioga config that is committed to the repo, multiple instances of mpi/compilers are stored and @@ -322,26 +115,12 @@ def compute_software_section(self): will fail if these variables are not defined though, so for now they are still generated (but with more-generic values). """ - # return { - # "software": { - # "packages": { - # "default-compiler": { - # "pkg_spec": f"{self.spec.variants['compiler'][0]}" - # }, - # "default-mpi": {"pkg_spec": "cray-mpich"}, - # "compiler-amdclang": {"pkg_spec": "clang"}, - # "compiler-gcc": {"pkg_spec": "gcc"}, - # "mpi-gcc": {"pkg_spec": "cray-mpich~gtl"}, - # } - # } - # } return { "software": { "packages": { - #"default-compiler": {"pkg_spec": default_compiler}, - "default-mpi": {"pkg_spec": "cray-mpich"}, - #"compiler-gcc": {"pkg_spec": "gcc"}, - "mpi-gcc": {"pkg_spec": "cray-mpich"}, + "default-compiler": {"pkg_spec": "gcc@12.3.0"}, + "default-mpi": {"pkg_spec": "cray-mpich@8.1.28"}, + "compiler-gcc": {"pkg_spec": "gcc@12.3.0"}, } } - } + } From 9146f2340cd5c5b17d95dba88f2591aeff5257cc Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Wed, 22 Oct 2025 16:09:01 -0700 Subject: [PATCH 19/37] builds stream successfully --- systems/lbnl-perlmutter/system.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index f38e05e93..c2be8a233 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -72,7 +72,7 @@ def compute_compilers_section(self): compiler_def( "gcc@12.3.0 languages:=c,c++,fortran", f"/opt/cray/pe/gcc-native/12/", - {"c": "cc", "cxx": "CC", "fortran": "ftn"}, + {"c": "gcc", "cxx": "g++", "fortran": "gfortran"}, modules=["PrgEnv-gnu", "gcc/12.3.0"], compilers_use_relative_paths=True, env={ @@ -92,7 +92,7 @@ def compute_packages_section(self): "cray-mpich": { "externals": [ { - "spec": "cray-mpich@8.1.28", + "spec": "cray-mpich@8.1.30", "prefix": f"/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/12.3", } ] @@ -101,6 +101,14 @@ def compute_packages_section(self): "externals": [{"spec": "zlib@1.2.13", "prefix": "/usr"}], "buildable": False, }, + "gmake": { + "externals": [{"spec": "gmake@4.2.1", "prefix": "/usr"}], + "buildable": False, + }, + "cmake": { + "externals": [{"spec": "cmake@3.30.2", "prefix": "/global/common/software/nersc9/cmake/3.30.2"}], + "buildable": False, + }, } } From 21946f9d7906a08241801d3ecb03cf2c97fa3094 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Mon, 27 Oct 2025 13:26:22 -0700 Subject: [PATCH 20/37] checkpoint --- systems/lbnl-perlmutter/system.py | 62 ++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index c2be8a233..2d31d356d 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -9,6 +9,7 @@ compiler_section_for, compiler_def, JobQueue, + merge_dicts, ) from benchpark.openmpsystem import OpenMPCPUOnlySystem from packaging.version import Version @@ -66,23 +67,57 @@ def __init__(self, spec): setattr(self, k, v) def compute_compilers_section(self): - return compiler_section_for( - "gcc", + nvhpc_cfg = compiler_section_for( + "nvhpc", [ compiler_def( - "gcc@12.3.0 languages:=c,c++,fortran", - f"/opt/cray/pe/gcc-native/12/", - {"c": "gcc", "cxx": "g++", "fortran": "gfortran"}, - modules=["PrgEnv-gnu", "gcc/12.3.0"], - compilers_use_relative_paths=True, - env={ - "append_path": { - "LD_LIBRARY_PATH": "/opt/cray/libfabric/1.22.0/lib64/:/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/cuda/12.4/lib64" - } - }, + "nvhpc@25.5", + "/opt/cray/pe/craype/2.7.32/", + {"c": "cc", "cxx": "CC", "fortran": "ftn"}, + modules=["PrgEnv-nvhpc/8.5.0", "nvhpc/25.5"], ) ], ) + if self.spec.satisfies("compiler=gcc"): + gcc_cfg = compiler_section_for( + "gcc", + [ + compiler_def( + "gcc@12.3.0 languages:=c,c++,fortran", + f"/opt/cray/pe/gcc-native/12/", + {"c": "gcc", "cxx": "g++", "fortran": "gfortran"}, + modules=["PrgEnv-gnu", "gcc/12.3.0"], + compilers_use_relative_paths=True, + env={ + "append_path": { + "LD_LIBRARY_PATH": "/opt/cray/libfabric/1.22.0/lib64/:/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/cuda/12.4/lib64" + } + }, + ) + ], + ) + cfg = merge_dicts(nvhpc_cfg, gcc_cfg) + else: + cfg = nvhpc_cfg + + return cfg +# return compiler_section_for( +# "gcc", +# [ +# compiler_def( +# "gcc@12.3.0 languages:=c,c++,fortran", +# f"/opt/cray/pe/gcc-native/12/", +# {"c": "gcc", "cxx": "g++", "fortran": "gfortran"}, +# modules=["PrgEnv-gnu", "gcc/12.3.0"], +# compilers_use_relative_paths=True, +# env={ +# "append_path": { +# "LD_LIBRARY_PATH": "/opt/cray/libfabric/1.22.0/lib64/:/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/cuda/12.4/lib64" +# } +# }, +# ) +# ], +# ) def compute_packages_section(self): @@ -95,7 +130,8 @@ def compute_packages_section(self): "spec": "cray-mpich@8.1.30", "prefix": f"/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/12.3", } - ] + ], + "buildable": False, }, "zlib": { "externals": [{"spec": "zlib@1.2.13", "prefix": "/usr"}], From 8198b781e57066607f5415575bde607f49c9301b Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Fri, 31 Oct 2025 13:38:26 -0700 Subject: [PATCH 21/37] checkpoint --- systems/lbnl-perlmutter/system.py | 103 ++++++++++++++++-------------- 1 file changed, 56 insertions(+), 47 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 2d31d356d..24692c73b 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -67,57 +67,57 @@ def __init__(self, spec): setattr(self, k, v) def compute_compilers_section(self): - nvhpc_cfg = compiler_section_for( - "nvhpc", - [ - compiler_def( - "nvhpc@25.5", - "/opt/cray/pe/craype/2.7.32/", - {"c": "cc", "cxx": "CC", "fortran": "ftn"}, - modules=["PrgEnv-nvhpc/8.5.0", "nvhpc/25.5"], - ) - ], - ) - if self.spec.satisfies("compiler=gcc"): - gcc_cfg = compiler_section_for( - "gcc", - [ - compiler_def( - "gcc@12.3.0 languages:=c,c++,fortran", - f"/opt/cray/pe/gcc-native/12/", - {"c": "gcc", "cxx": "g++", "fortran": "gfortran"}, - modules=["PrgEnv-gnu", "gcc/12.3.0"], - compilers_use_relative_paths=True, - env={ - "append_path": { - "LD_LIBRARY_PATH": "/opt/cray/libfabric/1.22.0/lib64/:/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/cuda/12.4/lib64" - } - }, - ) - ], - ) - cfg = merge_dicts(nvhpc_cfg, gcc_cfg) - else: - cfg = nvhpc_cfg - - return cfg -# return compiler_section_for( -# "gcc", +# nvhpc_cfg = compiler_section_for( +# "nvhpc", # [ # compiler_def( -# "gcc@12.3.0 languages:=c,c++,fortran", -# f"/opt/cray/pe/gcc-native/12/", -# {"c": "gcc", "cxx": "g++", "fortran": "gfortran"}, -# modules=["PrgEnv-gnu", "gcc/12.3.0"], -# compilers_use_relative_paths=True, -# env={ -# "append_path": { -# "LD_LIBRARY_PATH": "/opt/cray/libfabric/1.22.0/lib64/:/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/cuda/12.4/lib64" -# } -# }, +# "cray-mpich@8.1.30", +# "/opt/cray/pe/craype/2.7.32/", +# {"c": "cc", "cxx": "CC", "fortran": "ftn"}, +# modules=["PrgEnv-cray/8.5.0", "craype/2.7.32"], # ) # ], # ) +# if self.spec.satisfies("compiler=gcc"): +# gcc_cfg = compiler_section_for( +# "gcc", +# [ +# compiler_def( +# "gcc@12.3.0 languages:=c,c++,fortran", +# f"/opt/cray/pe/gcc-native/12/", +# {"c": "gcc", "cxx": "g++", "fortran": "gfortran"}, +# modules=["PrgEnv-gnu", "gcc/12.3.0"], +# compilers_use_relative_paths=True, +# env={ +# "append_path": { +# "LD_LIBRARY_PATH": "/opt/cray/libfabric/1.22.0/lib64/:/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/cuda/12.4/lib64" +# } +# }, +# ) +# ], +# ) +# cfg = merge_dicts(nvhpc_cfg, gcc_cfg) +# else: +# cfg = nvhpc_cfg +# +# return cfg + return compiler_section_for( + "gcc", + [ + compiler_def( + "gcc@12.3.0 languages:=c,c++,fortran", + f"/opt/cray/pe/gcc-native/12/", + {"c": "gcc", "cxx": "g++", "fortran": "gfortran"}, + modules=["PrgEnv-gnu", "gcc/12.3.0"], + compilers_use_relative_paths=True, + env={ + "append_path": { + "LD_LIBRARY_PATH": "/opt/cray/libfabric/1.22.0/lib64/:/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/cuda/12.4/lib64" + } + }, + ) + ], + ) def compute_packages_section(self): @@ -128,7 +128,7 @@ def compute_packages_section(self): "externals": [ { "spec": "cray-mpich@8.1.30", - "prefix": f"/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/12.3", + "prefix": f"/opt/cray/pe/craype/2.7.32/", } ], "buildable": False, @@ -145,6 +145,15 @@ def compute_packages_section(self): "externals": [{"spec": "cmake@3.30.2", "prefix": "/global/common/software/nersc9/cmake/3.30.2"}], "buildable": False, }, + "cray-libsci": { + "externals": [ + { + "spec": "cray-libsci@24.07.0", + "prefix": "/opt/cray/pe/libsci/24.07.0/CRAYCLANG/17.0/x86_64/", + } + ], + "buildable": False, + }, } } From 68b7aca2964b89f753cb44af3e9e43527912f922 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Fri, 31 Oct 2025 14:48:18 -0700 Subject: [PATCH 22/37] amg builds successfully! update libsci path to GNU for now --- systems/lbnl-perlmutter/system.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 24692c73b..e941e3be5 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -128,7 +128,7 @@ def compute_packages_section(self): "externals": [ { "spec": "cray-mpich@8.1.30", - "prefix": f"/opt/cray/pe/craype/2.7.32/", + "prefix": f"/opt/cray/pe/mpich/8.1.30/ofi/gnu/12.3/", } ], "buildable": False, @@ -149,11 +149,12 @@ def compute_packages_section(self): "externals": [ { "spec": "cray-libsci@24.07.0", - "prefix": "/opt/cray/pe/libsci/24.07.0/CRAYCLANG/17.0/x86_64/", + "prefix": "/opt/cray/pe/libsci/24.07.0/GNU/12.3/x86_64/", } ], "buildable": False, }, + "mpi": {"buildable": False}, } } From d9a8030d00812e6f1bcecbaae2599664a6cbae10 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Fri, 31 Oct 2025 15:14:45 -0700 Subject: [PATCH 23/37] toggle between craysci --- systems/lbnl-perlmutter/system.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index e941e3be5..0290eba82 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -145,6 +145,12 @@ def compute_packages_section(self): "externals": [{"spec": "cmake@3.30.2", "prefix": "/global/common/software/nersc9/cmake/3.30.2"}], "buildable": False, }, + "mpi": {"buildable": False}, + } + } + + if self.spec.satisfies("compiler=gcc"): + selections["packages"] |= { "cray-libsci": { "externals": [ { @@ -153,10 +159,20 @@ def compute_packages_section(self): } ], "buildable": False, - }, - "mpi": {"buildable": False}, + } + } + else: + selections["packages"] |= { + "cray-libsci": { + "externals": [ + { + "spec": "cray-libsci@24.07.0", + "prefix": "/opt/cray/pe/libsci/24.07.0/CRAY/17.0/x86_64/", + } + ], + "buildable": False, + } } - } return selections From 7fa07fd20e9c9c21081450641bf772b9743c68d4 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Mon, 3 Nov 2025 11:37:44 -0800 Subject: [PATCH 24/37] add back contraint flag --- systems/lbnl-perlmutter/system.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 0290eba82..0bb4b3d91 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -176,6 +176,35 @@ def compute_packages_section(self): return selections + def system_specific_variables(self): + opts = super().system_specific_variables() + if self.spec.satisfies("constraint=cpu"): + opts.update( + { + "extra_batch_opts":f"--constraint={self.spec.variants['constraint'][0]}", + } + ) + elif self.spec.satisfies("constraint=gpu"): + opts.update( + { + "extra_batch_opts":f"--constraint={self.spec.variants['constraint'][0]}", + } + ) + elif self.spec.satisfies("constraint=gpu&hbmg40"): + opts.update( + { + "extra_batch_opts":f"--constraint={self.spec.variants['constraint'][0]}", + } + ) + elif self.spec.satisfies("constraint=gpu&hbmg80"): + opts.update( + { + "extra_batch_opts":f"--constraint={self.spec.variants['constraint'][0]}", + } + ) + + return opts + def compute_software_section(self): """This is somewhat vestigial: for the Tioga config that is committed to the repo, multiple instances of mpi/compilers are stored and From a6c549a7385f6699ec8b313d7beb5746d1dcbf56 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Mon, 3 Nov 2025 12:10:15 -0800 Subject: [PATCH 25/37] undo adding of mpi to amg experiment --- experiments/amg2023/experiment.py | 1 - 1 file changed, 1 deletion(-) diff --git a/experiments/amg2023/experiment.py b/experiments/amg2023/experiment.py index 51c0baa9b..c62352efc 100644 --- a/experiments/amg2023/experiment.py +++ b/experiments/amg2023/experiment.py @@ -143,4 +143,3 @@ def compute_package_section(self): self.name, [f"amg2023{self.determine_version()} {mixedint}"] ) self.add_package_spec("hypre", ["hypre+lapack"]) - #self.add_package_spec("hypre", ["hypre+lapack+mpi"]) From 198f8eb9efdfd6e12b5d233a139c5e343fd8c949 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Mon, 3 Nov 2025 13:08:13 -0800 Subject: [PATCH 26/37] add gtl and cuda variants, cuda software --- systems/lbnl-perlmutter/system.py | 94 ++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 0bb4b3d91..22e59fc02 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: Apache-2.0 from benchpark.directives import variant, maintainers +from benchpark.cudasystem import CudaSystem from benchpark.system import ( System, compiler_section_for, @@ -45,6 +46,13 @@ class LbnlPerlmutter(System): description="Which constraint to use" ) + variant( + "gtl", + default=False, + values=(True, False), + description="Use GTL-enabled MPI", + ) + variant( "queue", default="regular", @@ -53,13 +61,22 @@ class LbnlPerlmutter(System): description="Submit to queue", ) + variant( + "cuda", + default="12.4", + values=("12.2", "12.4"), + description="CUDA version", + ) + def __init__(self, spec): super().__init__(spec) - self.programming_models = [OpenMPCPUOnlySystem()] + self.programming_models = [CudaSystem(), OpenMPCPUOnlySystem()] self.gcc_version = Version("12.3.0") self.mpi_version = Version("8.1.30") self.cce_version = Version("16.0.0") + self.cuda_version = Version(self.spec.variants["cuda"][0]) + self.gtl_flag = self.spec.variants["gtl"][0] self.scheduler = "slurm" attrs = self.id_to_resources.get("perlmutter") @@ -174,6 +191,10 @@ def compute_packages_section(self): } } + selections["packages"] |= self.cuda_config(self.spec.variants["cuda"][0])[ + "packages" + ] + return selections def system_specific_variables(self): @@ -205,6 +226,77 @@ def system_specific_variables(self): return opts + def cuda_config(self, cuda_version): + return { + "packages": { + "blas": {"require": "intel-oneapi-mkl"}, + "lapack": {"require": "intel-oneapi-mkl"}, + "curand": { + "externals": [ + { + "spec": f"curand@{cuda_version}", + "prefix": f"/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/math_libs/12.4/lib64", + } + ], + "buildable": False, + }, + "cuda": { + "externals": [ + { + "spec": f"cuda@{cuda_version}+allow-unsupported-compilers", + "prefix": f"/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/cuda/{cuda_version}", + } + ], + "buildable": False, + }, + "cub": { + "externals": [ + { + "spec": f"cub@{cuda_version}", + "prefix": f"/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/cuda/{cuda_version}", + } + ], + "buildable": False, + }, +# "cusparse": { +# "externals": [ +# { +# "spec": f"cusparse@{cuda_version}", +# "prefix": f"/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/cuda/{cuda_version}", +# } +# ], +# "buildable": False, +# }, + "cublas": { + "externals": [ + { + "spec": f"cublas@{cuda_version}", + "prefix": f"/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/math_libs/12.4/lib64", + } + ], + "buildable": False, + }, + "cusolver": { + "externals": [ + { + "spec": f"cusolver@{cuda_version}", + "prefix": f"/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/math_libs/12.4/lib64", + } + ], + "buildable": False, + }, + "cufft": { + "externals": [ + { + "spec": f"cufft@{cuda_version}", + "prefix": f"/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/math_libs/12.4/lib64", + } + ], + "buildable": False, + }, + } + } + def compute_software_section(self): """This is somewhat vestigial: for the Tioga config that is committed to the repo, multiple instances of mpi/compilers are stored and From 505ca6643c1860b7dc0026b25a93c2a6b3b9201c Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Mon, 3 Nov 2025 22:32:41 -0800 Subject: [PATCH 27/37] checkpoint, adding cuda --- systems/lbnl-perlmutter/system.py | 43 +++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 22e59fc02..4bfac16d1 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -48,7 +48,7 @@ class LbnlPerlmutter(System): variant( "gtl", - default=False, + default=True, values=(True, False), description="Use GTL-enabled MPI", ) @@ -73,8 +73,11 @@ def __init__(self, spec): self.programming_models = [CudaSystem(), OpenMPCPUOnlySystem()] self.gcc_version = Version("12.3.0") + self.short_gcc_version = ( + f"{self.gcc_version.major}.{self.gcc_version.minor}" + ) self.mpi_version = Version("8.1.30") - self.cce_version = Version("16.0.0") + #self.cce_version = Version("16.0.0") self.cuda_version = Version(self.spec.variants["cuda"][0]) self.gtl_flag = self.spec.variants["gtl"][0] @@ -83,6 +86,33 @@ def __init__(self, spec): for k, v in attrs.items(): setattr(self, k, v) + def mpi_config(self): + gtl = self.spec.variants["gtl"][0] + + if self.spec.satisfies("compiler=gcc"): + if gtl: + gtl_spec = "+gtl" + else: + gtl_spec = "~gtl" + + return { + "packages": { + "cray-mpich": { + "externals": [ + { + "spec": f"cray-mpich@{self.mpi_version}{gtl_spec}+wrappers %gcc@{self.gcc_version}", + "prefix": f"/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/{self.short_gcc_version}", + "extra_attributes": { + "gtl_lib_path": f"/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", + "ldflags": f"-L/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/{self.short_gcc_version}/lib -lmpi -L/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", + "gtl_libs": "libmpi_gtl_hsa", + }, + } + ] + } + } + } + def compute_compilers_section(self): # nvhpc_cfg = compiler_section_for( # "nvhpc", @@ -166,6 +196,8 @@ def compute_packages_section(self): } } + selections["packages"] |= self.mpi_config()["packages"] + if self.spec.satisfies("compiler=gcc"): selections["packages"] |= { "cray-libsci": { @@ -229,8 +261,8 @@ def system_specific_variables(self): def cuda_config(self, cuda_version): return { "packages": { - "blas": {"require": "intel-oneapi-mkl"}, - "lapack": {"require": "intel-oneapi-mkl"}, + "blas": {"require": "cray-libsci"}, + "lapack": {"require": "cray-libsci"}, "curand": { "externals": [ { @@ -310,8 +342,9 @@ def compute_software_section(self): "software": { "packages": { "default-compiler": {"pkg_spec": "gcc@12.3.0"}, - "default-mpi": {"pkg_spec": "cray-mpich@8.1.28"}, + "default-mpi": {"pkg_spec": "cray-mpich@8.1.30"}, "compiler-gcc": {"pkg_spec": "gcc@12.3.0"}, + "mpi-gcc": {"pkg_spec": "cray-mpich+gtl"}, } } } From 0b8a64a8d43838b66e0f942d54d78b38c784d75a Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Tue, 4 Nov 2025 14:04:41 -0800 Subject: [PATCH 28/37] add +cuda cuda_arch to system.py --- systems/lbnl-perlmutter/system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 4bfac16d1..a6086bc43 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -100,7 +100,7 @@ def mpi_config(self): "cray-mpich": { "externals": [ { - "spec": f"cray-mpich@{self.mpi_version}{gtl_spec}+wrappers %gcc@{self.gcc_version}", + "spec": f"cray-mpich@{self.mpi_version}{gtl_spec}+wrappers +cuda cuda_arch=80 %gcc@{self.gcc_version}", "prefix": f"/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/{self.short_gcc_version}", "extra_attributes": { "gtl_lib_path": f"/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", From b2bef95a1234620e53e29582eb009cfdbe7a278b Mon Sep 17 00:00:00 2001 From: Nathan Hanford Date: Tue, 4 Nov 2025 14:04:54 -0800 Subject: [PATCH 29/37] add cray-mpich package to repo --- repo/cray-mpich/package.py | 246 +++++++++++++++++++++++++++++++++---- 1 file changed, 220 insertions(+), 26 deletions(-) diff --git a/repo/cray-mpich/package.py b/repo/cray-mpich/package.py index cff4bf129..1f1cca55e 100644 --- a/repo/cray-mpich/package.py +++ b/repo/cray-mpich/package.py @@ -3,45 +3,239 @@ # # SPDX-License-Identifier: Apache-2.0 +import os + +from spack_repo.builtin.build_systems.cuda import CudaPackage +from spack_repo.builtin.build_systems.rocm import ROCmPackage +from spack_repo.builtin.packages.mpich.package import MpichEnvironmentModifications + from spack.package import * -from spack_repo.builtin.packages.cray_mpich.package import CrayMpich as BuiltinCM -class CrayMpich(BuiltinCM): +class CrayMpich(MpichEnvironmentModifications, Package, CudaPackage, ROCmPackage): + """Cray's MPICH is a high performance and widely portable implementation of + the Message Passing Interface (MPI) standard.""" + + homepage = "https://docs.nersc.gov/development/compilers/wrappers/" + + maintainers("etiennemlb", "haampie") + + version("9.0.1") + version("8.1.33") + version("8.1.32") + version("8.1.31") + version("8.1.30") + version("8.1.28") + version("8.1.25") + version("8.1.24") + version("8.1.21") + version("8.1.14") + version("8.1.7") + version("8.1.0") + version("8.0.16") + version("8.0.14") + version("8.0.11") + version("8.0.9") + version("7.7.16") + version("7.7.15") + version("7.7.14") + version("7.7.13") + + depends_on("cray-pmi") + depends_on("libfabric") + + # cray-mpich 8.1.7: features MPI compiler wrappers + variant("wrappers", default=True, when="@8.1.7:", description="enable MPI wrappers") + variant("gtl", default=True, when="+rocm", description="Enable GPU Accelerated Transports") + variant("gtl", default=True, when="+cuda", description="Enable GPU Accelerated Transports") + + provides("mpi@3") + + canonical_names = { + "gcc": "GNU", + "cce": "CRAY", + "intel": "INTEL", + "clang": "ALLINEA", + "aocc": "AOCC", + } + + has_code = False # Skip attempts to fetch a source that is not available + + # Allows attaching compilers to externals in packages.yaml + depends_on("c", type="build") + + requires("platform=linux", msg="Cray software is only available on linux") - variant("gtl", default=True, description="enable GPU-aware mode") + def install(self, spec, prefix): + raise InstallError( + self.spec.format( + "{name} is not installable, you need to specify " + "it as an external package in packages.yaml" + ) + ) @property - def libs(self): - libs = super().libs + def modname(self): + return f"cray-mpich/{self.version}" - if self.spec.satisfies("+gtl"): - gtl_lib_prefix = self.spec.extra_attributes["gtl_lib_path"] - # gtl_libs, if set, must be a single string. You can pass multiple - # libs by adding a space between each - gtl_libs = self.spec.extra_attributes["gtl_libs"].split() - libs += find_libraries(gtl_libs, root=gtl_lib_prefix, recursive=True) + @property + def external_prefix(self): + mpich_module = module_command("show", self.modname).splitlines() - return libs + for line in mpich_module: + if "CRAY_MPICH_DIR" in line: + return get_path_args_from_module_line(line)[0] - def setup_run_environment(self, env): + # Fixes an issue on Archer2 cray-mpich/8.0.16 where there is + # no CRAY_MPICH_DIR variable in the module file. + for line in mpich_module: + if "CRAY_LD_LIBRARY_PATH" in line: + libdir = get_path_args_from_module_line(line)[0] + return os.path.dirname(os.path.normpath(libdir)) - super().setup_run_environment(env) + def setup_run_environment(self, env: EnvironmentModifications) -> None: + if self.spec.satisfies("+wrappers"): + self.setup_mpi_wrapper_variables(env) + return - if self.spec.satisfies("+gtl"): - env.set("MPICH_GPU_SUPPORT_ENABLED", "1") - env.prepend_path("LD_LIBRARY_PATH", self.spec.extra_attributes["gtl_lib_path"]) + if self.spec.has_virtual_dependency("c"): + env.set("MPICC", self["c"].cc) + + if self.spec.has_virtual_dependency("cxx"): + env.set("MPICXX", self["cxx"].cxx) + + if self.spec.has_virtual_dependency("fortran"): + env.set("MPIFC", self["fortran"].fortran) + env.set("MPIF77", self["fortran"].fortran) + + def setup_dependent_package(self, module, dependent_spec): + spec = self.spec + if spec.satisfies("+wrappers"): + MpichEnvironmentModifications.setup_dependent_package(self, module, dependent_spec) else: - env.set("MPICH_GPU_SUPPORT_ENABLED", "0") - gtl_path = self.spec.extra_attributes.get("gtl_lib_path", "") - if gtl_path: - env.prepend_path("LD_LIBRARY_PATH", gtl_path) + if "c" in dependent_spec: + spec.mpicc = dependent_spec["c"].package.cc + if "cxx" in dependent_spec: + spec.mpicxx = dependent_spec["cxx"].package.cxx + if "fortran" in dependent_spec: + spec.mpifc = dependent_spec["fortran"].package.fortran + spec.mpif77 = dependent_spec["fortran"].package.fortran - def cmake_args(self): - args = super().cmake_args(self) + def setup_dependent_build_environment(self, env, dependent_spec): + if self.spec.satisfies("+gtl"): + for flag in self.gtl_lib["ldflags"]: + env.prepend_path("LDFLAGS", flag) + env.prepend_path("LIBS", self.gtl_lib["ldlibs"][0]) + def setup_dependent_run_environment(self, env, dependent_spec): if self.spec.satisfies("+gtl"): - # Link GTL for MPICH GPU-aware - args.append(self.define("CMAKE_EXE_LINKER_FLAGS", self.spec['mpi'].libs.ld_flags)) + env.set("MPICH_GPU_SUPPORT_ENABLED", "1") + + @property + def headers(self): + hdrs = find_headers("mpi", self.prefix.include, recursive=True) + hdrs.directories = os.path.dirname(hdrs[0]) + return hdrs + + @property + def libs(self): + query_parameters = self.spec.last_query.extra_parameters + + libraries = ["libmpich"] + + if "cxx" in query_parameters: + libraries.extend(["libmpicxx", "libmpichcxx"]) + + if "f77" in query_parameters: + libraries.extend(["libmpifort", "libmpichfort", "libfmpi", "libfmpich"]) + + if "f90" in query_parameters: + libraries.extend(["libmpif90", "libmpichf90"]) + + libs = find_libraries(libraries, root=self.prefix.lib, recursive=True) + libs += find_libraries(libraries, root=self.prefix.lib64, recursive=True) + + return libs + + @property + def gtl_lib(self): + # GPU transport Layer (GTL) handling background: + # - The cray-mpich module defines an environment variable per supported + # GPU (say, PE_MPICH_GTL_LIBS_amd_gfx942). So we should read the + # appropriate variable. + # In practice loading a module and checking its content is a PITA. We + # simplify by assuming that the GTL for a given vendor (say, AMD), is + # one and the same for all the targets of this vendor (one GTL for all + # Nvidia or one GTL for all AMD devices). + # - Second, except if you have a very weird mpich layout, the GTL are + # located in /opt/cray/pe/mpich//gtl/lib when the + # MPI libraries are in + # /opt/cray/pe/mpich//ofi//. + # Example: + # /opt/cray/pe/mpich/8.1.28/gtl/lib + # /opt/cray/pe/mpich/8.1.28/ofi// + # /opt/cray/pe/mpich/8.1.28/ofi///../../../gtl/lib + + gtl_kinds = { + "cuda": { + "lib": "libmpi_gtl_cuda", + "variant": "cuda_arch", + "values": {"70", "80", "90"}, + }, + "rocm": { + "lib": "libmpi_gtl_hsa", + "variant": "amdgpu_target", + "values": {"gfx906", "gfx908", "gfx90a", "gfx940", "gfx942"}, + }, + } + + for variant, gtl_kind in gtl_kinds.items(): + arch_variant = gtl_kind["variant"] + arch_values = gtl_kind["values"] + gtl_lib = gtl_kind["lib"] + + if self.spec.satisfies(f"+{variant} {arch_variant}=*"): + accelerator_architecture_set = set(self.spec.variants[arch_variant].value) + + if len( + accelerator_architecture_set + ) >= 1 and not accelerator_architecture_set.issubset(arch_values): + raise InstallError( + f"cray-mpich variant '+{variant} {arch_variant}'" + " was specified but no GTL support could be found for it." + ) + + mpi_root = os.path.abspath( + os.path.join(self.prefix, os.pardir, os.pardir, os.pardir) + ) + + gtl_root = os.path.join(mpi_root, "gtl", "lib") + + gtl_shared_libraries = find_libraries( + [gtl_lib], root=gtl_root, shared=True, recursive=False + ) + + if len(gtl_shared_libraries) != 1: + raise InstallError( + f"cray-mpich variant '+{variant} {arch_variant}'" + " was specified and GTL support was found for it but" + f" the '{gtl_lib}' could not be correctly found on disk." + ) + + gtl_library_fullpath = list(gtl_shared_libraries)[0] + tty.debug(f"Selected GTL: {gtl_library_fullpath}") + + gtl_library_directory = os.path.dirname(gtl_library_fullpath) + gtl_library_name = os.path.splitext( + os.path.basename(gtl_library_fullpath).split("lib")[1] + )[0] - return args + # Early break. Only one GTL can be active at a given time. + return { + "ldflags": [ + f"-L{gtl_library_directory}", + f"-Wl,-rpath,{gtl_library_directory}", + ], + "ldlibs": [f"-l{gtl_library_name}"], + } + return {} From 658c429398c05c463833ac933c7a197125056088 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Tue, 4 Nov 2025 14:19:06 -0800 Subject: [PATCH 30/37] code cleanup --- systems/lbnl-perlmutter/system.py | 77 ++++++++----------------------- 1 file changed, 18 insertions(+), 59 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index a6086bc43..c6556c4ed 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -43,7 +43,7 @@ class LbnlPerlmutter(System): "constraint", default="cpu", values=("cpu", "gpu", "gpu&hbmg40", "gpu&hbmg80"), - description="Which constraint to use" + description="Which constraint to use", ) variant( @@ -73,11 +73,8 @@ def __init__(self, spec): self.programming_models = [CudaSystem(), OpenMPCPUOnlySystem()] self.gcc_version = Version("12.3.0") - self.short_gcc_version = ( - f"{self.gcc_version.major}.{self.gcc_version.minor}" - ) + self.short_gcc_version = (f"{self.gcc_version.major}.{self.gcc_version.minor}") self.mpi_version = Version("8.1.30") - #self.cce_version = Version("16.0.0") self.cuda_version = Version(self.spec.variants["cuda"][0]) self.gtl_flag = self.spec.variants["gtl"][0] @@ -114,40 +111,6 @@ def mpi_config(self): } def compute_compilers_section(self): -# nvhpc_cfg = compiler_section_for( -# "nvhpc", -# [ -# compiler_def( -# "cray-mpich@8.1.30", -# "/opt/cray/pe/craype/2.7.32/", -# {"c": "cc", "cxx": "CC", "fortran": "ftn"}, -# modules=["PrgEnv-cray/8.5.0", "craype/2.7.32"], -# ) -# ], -# ) -# if self.spec.satisfies("compiler=gcc"): -# gcc_cfg = compiler_section_for( -# "gcc", -# [ -# compiler_def( -# "gcc@12.3.0 languages:=c,c++,fortran", -# f"/opt/cray/pe/gcc-native/12/", -# {"c": "gcc", "cxx": "g++", "fortran": "gfortran"}, -# modules=["PrgEnv-gnu", "gcc/12.3.0"], -# compilers_use_relative_paths=True, -# env={ -# "append_path": { -# "LD_LIBRARY_PATH": "/opt/cray/libfabric/1.22.0/lib64/:/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/cuda/12.4/lib64" -# } -# }, -# ) -# ], -# ) -# cfg = merge_dicts(nvhpc_cfg, gcc_cfg) -# else: -# cfg = nvhpc_cfg -# -# return cfg return compiler_section_for( "gcc", [ @@ -157,11 +120,11 @@ def compute_compilers_section(self): {"c": "gcc", "cxx": "g++", "fortran": "gfortran"}, modules=["PrgEnv-gnu", "gcc/12.3.0"], compilers_use_relative_paths=True, - env={ - "append_path": { - "LD_LIBRARY_PATH": "/opt/cray/libfabric/1.22.0/lib64/:/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/cuda/12.4/lib64" - } - }, + env={ + "append_path": { + "LD_LIBRARY_PATH": "/opt/cray/libfabric/1.22.0/lib64/:/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/cuda/12.4/lib64" + } + }, ) ], ) @@ -189,7 +152,12 @@ def compute_packages_section(self): "buildable": False, }, "cmake": { - "externals": [{"spec": "cmake@3.30.2", "prefix": "/global/common/software/nersc9/cmake/3.30.2"}], + "externals": [ + { + "spec": "cmake@3.30.2", + "prefix": "/global/common/software/nersc9/cmake/3.30.2", + } + ], "buildable": False, }, "mpi": {"buildable": False}, @@ -203,7 +171,7 @@ def compute_packages_section(self): "cray-libsci": { "externals": [ { - "spec": "cray-libsci@24.07.0", + "spec": "cray-libsci@24.07.0", "prefix": "/opt/cray/pe/libsci/24.07.0/GNU/12.3/x86_64/", } ], @@ -234,25 +202,25 @@ def system_specific_variables(self): if self.spec.satisfies("constraint=cpu"): opts.update( { - "extra_batch_opts":f"--constraint={self.spec.variants['constraint'][0]}", + "extra_batch_opts": f"--constraint={self.spec.variants['constraint'][0]}", } ) elif self.spec.satisfies("constraint=gpu"): opts.update( { - "extra_batch_opts":f"--constraint={self.spec.variants['constraint'][0]}", + "extra_batch_opts": f"--constraint={self.spec.variants['constraint'][0]}", } ) elif self.spec.satisfies("constraint=gpu&hbmg40"): opts.update( { - "extra_batch_opts":f"--constraint={self.spec.variants['constraint'][0]}", + "extra_batch_opts": f"--constraint={self.spec.variants['constraint'][0]}", } ) elif self.spec.satisfies("constraint=gpu&hbmg80"): opts.update( { - "extra_batch_opts":f"--constraint={self.spec.variants['constraint'][0]}", + "extra_batch_opts": f"--constraint={self.spec.variants['constraint'][0]}", } ) @@ -290,15 +258,6 @@ def cuda_config(self, cuda_version): ], "buildable": False, }, -# "cusparse": { -# "externals": [ -# { -# "spec": f"cusparse@{cuda_version}", -# "prefix": f"/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/cuda/{cuda_version}", -# } -# ], -# "buildable": False, -# }, "cublas": { "externals": [ { From 0551a1a1989408e4d26690b7fe3247932a299d38 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Tue, 4 Nov 2025 14:30:53 -0800 Subject: [PATCH 31/37] style --- systems/lbnl-perlmutter/system.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index c6556c4ed..510b9a7ad 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -73,7 +73,7 @@ def __init__(self, spec): self.programming_models = [CudaSystem(), OpenMPCPUOnlySystem()] self.gcc_version = Version("12.3.0") - self.short_gcc_version = (f"{self.gcc_version.major}.{self.gcc_version.minor}") + self.short_gcc_version = f"{self.gcc_version.major}.{self.gcc_version.minor}" self.mpi_version = Version("8.1.30") self.cuda_version = Version(self.spec.variants["cuda"][0]) self.gtl_flag = self.spec.variants["gtl"][0] @@ -223,7 +223,7 @@ def system_specific_variables(self): "extra_batch_opts": f"--constraint={self.spec.variants['constraint'][0]}", } ) - + return opts def cuda_config(self, cuda_version): From 06c68ec04fc87e169baa3173b7a9c52a8ebf302c Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Tue, 4 Nov 2025 14:38:42 -0800 Subject: [PATCH 32/37] isort fixes --- repo/cray-mpich/package.py | 3 +-- systems/lbnl-perlmutter/system.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/repo/cray-mpich/package.py b/repo/cray-mpich/package.py index 1f1cca55e..fb12c4282 100644 --- a/repo/cray-mpich/package.py +++ b/repo/cray-mpich/package.py @@ -5,12 +5,11 @@ import os +from spack.package import * from spack_repo.builtin.build_systems.cuda import CudaPackage from spack_repo.builtin.build_systems.rocm import ROCmPackage from spack_repo.builtin.packages.mpich.package import MpichEnvironmentModifications -from spack.package import * - class CrayMpich(MpichEnvironmentModifications, Package, CudaPackage, ROCmPackage): """Cray's MPICH is a high performance and widely portable implementation of diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 510b9a7ad..767f5685f 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -3,18 +3,20 @@ # # SPDX-License-Identifier: Apache-2.0 -from benchpark.directives import variant, maintainers + +from packaging.version import Version + from benchpark.cudasystem import CudaSystem +from benchpark.directives import maintainers, variant +from benchpark.openmpsystem import OpenMPCPUOnlySystem +from benchpark.paths import hardware_descriptions from benchpark.system import ( + JobQueue, System, - compiler_section_for, compiler_def, - JobQueue, + compiler_section_for, merge_dicts, ) -from benchpark.openmpsystem import OpenMPCPUOnlySystem -from packaging.version import Version -from benchpark.paths import hardware_descriptions class LbnlPerlmutter(System): From 309ba0c277aeb433e4e0f8ab85c6ac9357e8af01 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Tue, 4 Nov 2025 14:53:47 -0800 Subject: [PATCH 33/37] flake8 --- systems/lbnl-perlmutter/system.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 767f5685f..7d648ce03 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -15,7 +15,6 @@ System, compiler_def, compiler_section_for, - merge_dicts, ) @@ -118,7 +117,7 @@ def compute_compilers_section(self): [ compiler_def( "gcc@12.3.0 languages:=c,c++,fortran", - f"/opt/cray/pe/gcc-native/12/", + "/opt/cray/pe/gcc-native/12/", {"c": "gcc", "cxx": "g++", "fortran": "gfortran"}, modules=["PrgEnv-gnu", "gcc/12.3.0"], compilers_use_relative_paths=True, @@ -140,7 +139,7 @@ def compute_packages_section(self): "externals": [ { "spec": "cray-mpich@8.1.30", - "prefix": f"/opt/cray/pe/mpich/8.1.30/ofi/gnu/12.3/", + "prefix": "/opt/cray/pe/mpich/8.1.30/ofi/gnu/12.3/", } ], "buildable": False, @@ -237,7 +236,7 @@ def cuda_config(self, cuda_version): "externals": [ { "spec": f"curand@{cuda_version}", - "prefix": f"/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/math_libs/12.4/lib64", + "prefix": "/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/math_libs/12.4/lib64", } ], "buildable": False, @@ -264,7 +263,7 @@ def cuda_config(self, cuda_version): "externals": [ { "spec": f"cublas@{cuda_version}", - "prefix": f"/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/math_libs/12.4/lib64", + "prefix": "/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/math_libs/12.4/lib64", } ], "buildable": False, @@ -273,7 +272,7 @@ def cuda_config(self, cuda_version): "externals": [ { "spec": f"cusolver@{cuda_version}", - "prefix": f"/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/math_libs/12.4/lib64", + "prefix": "/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/math_libs/12.4/lib64", } ], "buildable": False, @@ -282,7 +281,7 @@ def cuda_config(self, cuda_version): "externals": [ { "spec": f"cufft@{cuda_version}", - "prefix": f"/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/math_libs/12.4/lib64", + "prefix": "/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/math_libs/12.4/lib64", } ], "buildable": False, From 713dfc9f229ce9bd3ba0eb05bcc25ca9d7d72d93 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Tue, 4 Nov 2025 15:56:32 -0800 Subject: [PATCH 34/37] yamlfix --- .../HPECray-zen3-A100-Slingshot/hardware_description.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/systems/all_hardware_descriptions/HPECray-zen3-A100-Slingshot/hardware_description.yaml b/systems/all_hardware_descriptions/HPECray-zen3-A100-Slingshot/hardware_description.yaml index af09b4687..bcb3eb049 100644 --- a/systems/all_hardware_descriptions/HPECray-zen3-A100-Slingshot/hardware_description.yaml +++ b/systems/all_hardware_descriptions/HPECray-zen3-A100-Slingshot/hardware_description.yaml @@ -2,7 +2,6 @@ # Benchpark Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: Apache-2.0 - system_definition: name: HPECray-zen3-A100-Slingshot integrator: From 3a66b2ab5c00fc1a7706b70c5910b513ba823671 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Mon, 10 Nov 2025 12:54:58 -0800 Subject: [PATCH 35/37] switch to libmpi_gtl_cuda --- systems/lbnl-perlmutter/system.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 7d648ce03..bad44c511 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -102,8 +102,8 @@ def mpi_config(self): "prefix": f"/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/{self.short_gcc_version}", "extra_attributes": { "gtl_lib_path": f"/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", - "ldflags": f"-L/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/{self.short_gcc_version}/lib -lmpi -L/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", - "gtl_libs": "libmpi_gtl_hsa", + "ldflags": f"-L/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/{self.short_gcc_version}/lib -lmpi -L/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -lmpi_gtl_cuda", + "gtl_libs": "libmpi_gtl_cuda", }, } ] From de6f6220ed539956b32780b22eb6a504728e21c5 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Mon, 10 Nov 2025 13:02:17 -0800 Subject: [PATCH 36/37] restore to earlier cray-mpich --- repo/cray-mpich/package.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/repo/cray-mpich/package.py b/repo/cray-mpich/package.py index fb12c4282..e598724e7 100644 --- a/repo/cray-mpich/package.py +++ b/repo/cray-mpich/package.py @@ -1,15 +1,16 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. + +# Copyright Spack Project Developers. See COPYRIGHT file for details. # -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: (Apache-2.0 OR MIT) import os -from spack.package import * from spack_repo.builtin.build_systems.cuda import CudaPackage from spack_repo.builtin.build_systems.rocm import ROCmPackage from spack_repo.builtin.packages.mpich.package import MpichEnvironmentModifications +from spack.package import * + class CrayMpich(MpichEnvironmentModifications, Package, CudaPackage, ROCmPackage): """Cray's MPICH is a high performance and widely portable implementation of @@ -238,3 +239,4 @@ def gtl_lib(self): "ldlibs": [f"-l{gtl_library_name}"], } return {} + From 6988462abf52c4348247069f22c5bf5b01179313 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Mon, 10 Nov 2025 14:05:43 -0800 Subject: [PATCH 37/37] restore cray-mpich and set ld_flags explicitly --- repo/cray-mpich/package.py | 254 +++++-------------------------------- 1 file changed, 30 insertions(+), 224 deletions(-) diff --git a/repo/cray-mpich/package.py b/repo/cray-mpich/package.py index e598724e7..206e54a71 100644 --- a/repo/cray-mpich/package.py +++ b/repo/cray-mpich/package.py @@ -1,242 +1,48 @@ - -# Copyright Spack Project Developers. See COPYRIGHT file for details. +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. # -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import os - -from spack_repo.builtin.build_systems.cuda import CudaPackage -from spack_repo.builtin.build_systems.rocm import ROCmPackage -from spack_repo.builtin.packages.mpich.package import MpichEnvironmentModifications +# SPDX-License-Identifier: Apache-2.0 from spack.package import * +from spack_repo.builtin.packages.cray_mpich.package import CrayMpich as BuiltinCM -class CrayMpich(MpichEnvironmentModifications, Package, CudaPackage, ROCmPackage): - """Cray's MPICH is a high performance and widely portable implementation of - the Message Passing Interface (MPI) standard.""" - - homepage = "https://docs.nersc.gov/development/compilers/wrappers/" - - maintainers("etiennemlb", "haampie") - - version("9.0.1") - version("8.1.33") - version("8.1.32") - version("8.1.31") - version("8.1.30") - version("8.1.28") - version("8.1.25") - version("8.1.24") - version("8.1.21") - version("8.1.14") - version("8.1.7") - version("8.1.0") - version("8.0.16") - version("8.0.14") - version("8.0.11") - version("8.0.9") - version("7.7.16") - version("7.7.15") - version("7.7.14") - version("7.7.13") - - depends_on("cray-pmi") - depends_on("libfabric") - - # cray-mpich 8.1.7: features MPI compiler wrappers - variant("wrappers", default=True, when="@8.1.7:", description="enable MPI wrappers") - variant("gtl", default=True, when="+rocm", description="Enable GPU Accelerated Transports") - variant("gtl", default=True, when="+cuda", description="Enable GPU Accelerated Transports") - - provides("mpi@3") - - canonical_names = { - "gcc": "GNU", - "cce": "CRAY", - "intel": "INTEL", - "clang": "ALLINEA", - "aocc": "AOCC", - } - - has_code = False # Skip attempts to fetch a source that is not available - - # Allows attaching compilers to externals in packages.yaml - depends_on("c", type="build") - - requires("platform=linux", msg="Cray software is only available on linux") +class CrayMpich(BuiltinCM): - def install(self, spec, prefix): - raise InstallError( - self.spec.format( - "{name} is not installable, you need to specify " - "it as an external package in packages.yaml" - ) - ) - - @property - def modname(self): - return f"cray-mpich/{self.version}" - - @property - def external_prefix(self): - mpich_module = module_command("show", self.modname).splitlines() - - for line in mpich_module: - if "CRAY_MPICH_DIR" in line: - return get_path_args_from_module_line(line)[0] - - # Fixes an issue on Archer2 cray-mpich/8.0.16 where there is - # no CRAY_MPICH_DIR variable in the module file. - for line in mpich_module: - if "CRAY_LD_LIBRARY_PATH" in line: - libdir = get_path_args_from_module_line(line)[0] - return os.path.dirname(os.path.normpath(libdir)) - - def setup_run_environment(self, env: EnvironmentModifications) -> None: - if self.spec.satisfies("+wrappers"): - self.setup_mpi_wrapper_variables(env) - return - - if self.spec.has_virtual_dependency("c"): - env.set("MPICC", self["c"].cc) - - if self.spec.has_virtual_dependency("cxx"): - env.set("MPICXX", self["cxx"].cxx) - - if self.spec.has_virtual_dependency("fortran"): - env.set("MPIFC", self["fortran"].fortran) - env.set("MPIF77", self["fortran"].fortran) - - def setup_dependent_package(self, module, dependent_spec): - spec = self.spec - if spec.satisfies("+wrappers"): - MpichEnvironmentModifications.setup_dependent_package(self, module, dependent_spec) - else: - if "c" in dependent_spec: - spec.mpicc = dependent_spec["c"].package.cc - if "cxx" in dependent_spec: - spec.mpicxx = dependent_spec["cxx"].package.cxx - if "fortran" in dependent_spec: - spec.mpifc = dependent_spec["fortran"].package.fortran - spec.mpif77 = dependent_spec["fortran"].package.fortran - - def setup_dependent_build_environment(self, env, dependent_spec): - if self.spec.satisfies("+gtl"): - for flag in self.gtl_lib["ldflags"]: - env.prepend_path("LDFLAGS", flag) - env.prepend_path("LIBS", self.gtl_lib["ldlibs"][0]) - - def setup_dependent_run_environment(self, env, dependent_spec): - if self.spec.satisfies("+gtl"): - env.set("MPICH_GPU_SUPPORT_ENABLED", "1") - - @property - def headers(self): - hdrs = find_headers("mpi", self.prefix.include, recursive=True) - hdrs.directories = os.path.dirname(hdrs[0]) - return hdrs + variant("gtl", default=True, description="enable GPU-aware mode") @property def libs(self): - query_parameters = self.spec.last_query.extra_parameters - - libraries = ["libmpich"] - - if "cxx" in query_parameters: - libraries.extend(["libmpicxx", "libmpichcxx"]) + libs = super().libs - if "f77" in query_parameters: - libraries.extend(["libmpifort", "libmpichfort", "libfmpi", "libfmpich"]) - - if "f90" in query_parameters: - libraries.extend(["libmpif90", "libmpichf90"]) - - libs = find_libraries(libraries, root=self.prefix.lib, recursive=True) - libs += find_libraries(libraries, root=self.prefix.lib64, recursive=True) + if self.spec.satisfies("+gtl"): + ld_flags = self.spec.extra_attributes["ldflags"] + gtl_lib_prefix = self.spec.extra_attributes["gtl_lib_path"] + # gtl_libs, if set, must be a single string. You can pass multiple + # libs by adding a space between each + gtl_libs = self.spec.extra_attributes["gtl_libs"].split() + libs += find_libraries(gtl_libs, root=gtl_lib_prefix, recursive=True) return libs - @property - def gtl_lib(self): - # GPU transport Layer (GTL) handling background: - # - The cray-mpich module defines an environment variable per supported - # GPU (say, PE_MPICH_GTL_LIBS_amd_gfx942). So we should read the - # appropriate variable. - # In practice loading a module and checking its content is a PITA. We - # simplify by assuming that the GTL for a given vendor (say, AMD), is - # one and the same for all the targets of this vendor (one GTL for all - # Nvidia or one GTL for all AMD devices). - # - Second, except if you have a very weird mpich layout, the GTL are - # located in /opt/cray/pe/mpich//gtl/lib when the - # MPI libraries are in - # /opt/cray/pe/mpich//ofi//. - # Example: - # /opt/cray/pe/mpich/8.1.28/gtl/lib - # /opt/cray/pe/mpich/8.1.28/ofi// - # /opt/cray/pe/mpich/8.1.28/ofi///../../../gtl/lib - - gtl_kinds = { - "cuda": { - "lib": "libmpi_gtl_cuda", - "variant": "cuda_arch", - "values": {"70", "80", "90"}, - }, - "rocm": { - "lib": "libmpi_gtl_hsa", - "variant": "amdgpu_target", - "values": {"gfx906", "gfx908", "gfx90a", "gfx940", "gfx942"}, - }, - } - - for variant, gtl_kind in gtl_kinds.items(): - arch_variant = gtl_kind["variant"] - arch_values = gtl_kind["values"] - gtl_lib = gtl_kind["lib"] - - if self.spec.satisfies(f"+{variant} {arch_variant}=*"): - accelerator_architecture_set = set(self.spec.variants[arch_variant].value) - - if len( - accelerator_architecture_set - ) >= 1 and not accelerator_architecture_set.issubset(arch_values): - raise InstallError( - f"cray-mpich variant '+{variant} {arch_variant}'" - " was specified but no GTL support could be found for it." - ) - - mpi_root = os.path.abspath( - os.path.join(self.prefix, os.pardir, os.pardir, os.pardir) - ) + def setup_run_environment(self, env): - gtl_root = os.path.join(mpi_root, "gtl", "lib") + super().setup_run_environment(env) - gtl_shared_libraries = find_libraries( - [gtl_lib], root=gtl_root, shared=True, recursive=False - ) - - if len(gtl_shared_libraries) != 1: - raise InstallError( - f"cray-mpich variant '+{variant} {arch_variant}'" - " was specified and GTL support was found for it but" - f" the '{gtl_lib}' could not be correctly found on disk." - ) - - gtl_library_fullpath = list(gtl_shared_libraries)[0] - tty.debug(f"Selected GTL: {gtl_library_fullpath}") + if self.spec.satisfies("+gtl"): + env.set("MPICH_GPU_SUPPORT_ENABLED", "1") + env.prepend_path("LD_LIBRARY_PATH", self.spec.extra_attributes["gtl_lib_path"]) + else: + env.set("MPICH_GPU_SUPPORT_ENABLED", "0") + gtl_path = self.spec.extra_attributes.get("gtl_lib_path", "") + if gtl_path: + env.prepend_path("LD_LIBRARY_PATH", gtl_path) - gtl_library_directory = os.path.dirname(gtl_library_fullpath) - gtl_library_name = os.path.splitext( - os.path.basename(gtl_library_fullpath).split("lib")[1] - )[0] + def cmake_args(self): + args = super().cmake_args(self) - # Early break. Only one GTL can be active at a given time. - return { - "ldflags": [ - f"-L{gtl_library_directory}", - f"-Wl,-rpath,{gtl_library_directory}", - ], - "ldlibs": [f"-l{gtl_library_name}"], - } - return {} + if self.spec.satisfies("+gtl"): + # Link GTL for MPICH GPU-aware + args.append(self.define("CMAKE_EXE_LINKER_FLAGS", self.spec['mpi'].libs.ld_flags)) + return args