From e15efa0e2a691ad98709d35bffdb4fe157a0941c Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Wed, 9 Apr 2025 22:42:47 -0700 Subject: [PATCH 1/6] add new system: perlmutter --- .../hardware_description.yaml | 47 +++ systems/lbnl-perlmutter/system.py | 307 ++++++++++++++++++ 2 files changed, 354 insertions(+) create mode 100644 systems/all_hardware_descriptions/HPECray-zen3-A100-Slingshot/hardware_description.yaml create mode 100644 systems/lbnl-perlmutter/system.py diff --git a/systems/all_hardware_descriptions/HPECray-zen3-A100-Slingshot/hardware_description.yaml b/systems/all_hardware_descriptions/HPECray-zen3-A100-Slingshot/hardware_description.yaml new file mode 100644 index 000000000..af09b4687 --- /dev/null +++ b/systems/all_hardware_descriptions/HPECray-zen3-A100-Slingshot/hardware_description.yaml @@ -0,0 +1,47 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +system_definition: + name: HPECray-zen3-A100-Slingshot + integrator: + vendor: HPECray + name: EX235a + processor: + vendor: AMD + name: EPYC-Zen3 + ISA: x86_64 + uArch: zen3 + accelerator: + vendor: NVIDIA + name: A100 + ISA: PTX + uArch: sm_80 + interconnect: + vendor: HPECray + name: Slingshot11 + systems-tested: + csc-lumi: + os: HPECrayOS + scheduler: slurm + compiler: cce + runtime: rocm + mpi: cray-mpich + installation-year: 2023 + llnl-elcapitan: + os: TOSS + scheduler: flux + compiler: cce + runtime: rocm + mpi: cray-mpich + top500-system-instances: + Frontier: + benchpark_system: + top500: + Lumi: + benchpark_system: csc-lumi + top500: + Tioga: + benchpark_system: llnl-elcapitan + top500: diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py new file mode 100644 index 000000000..01c7552ce --- /dev/null +++ b/systems/lbnl-perlmutter/system.py @@ -0,0 +1,307 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +from packaging.version import Version + +from benchpark.directives import variant, maintainers +from benchpark.system import System +from benchpark.paths import hardware_descriptions + + +class LbnlPerlmutter(System): + + maintainers("slabasan") + + id_to_resources = { + "perlmutter": { + "cuda_arch": "80", + "sys_cores_per_node": 64, + "sys_gpus_per_node": 4, + "system_site": "lbnl", + "hardware_key": str(hardware_descriptions) + + "/HPECray-zen3-A100-Slingshot/hardware_description.yaml", + }, + } + + variant( + "compiler", + default="cce", + values=("cce", "gcc"), + description="Which compiler to use", + ) + + def __init__(self, spec): + super().__init__(spec) + + if self.spec.satisfies("compiler=gcc"): + self.gcc_version = Version("12.2.0") + self.mpi_version = Version("8.1.26") + + self.scheduler = "slurm" + attrs = self.id_to_resources.get("perlmutter") + for k, v in attrs.items(): + setattr(self, k, v) + + def compute_packages_section(self): + selections = { + "packages": { + "all": {"require": "target=x86_64:"}, + "tar": {"externals": [{"spec": "tar@1.30", "prefix": "/usr"}]}, + "coreutils": { + "externals": [{"spec": "coreutils@8.30", "prefix": "/usr"}] + }, + "libtool": {"externals": [{"spec": "libtool@2.4.6", "prefix": "/usr"}]}, + "flex": {"externals": [{"spec": "flex@2.6.1+lex", "prefix": "/usr"}]}, + "openssl": { + "externals": [{"spec": "openssl@1.1.1k", "prefix": "/usr"}] + }, + "m4": {"externals": [{"spec": "m4@1.4.18", "prefix": "/usr"}]}, + "groff": {"externals": [{"spec": "groff@1.22.3", "prefix": "/usr"}]}, + "cmake": { + "externals": [ + {"spec": "cmake@3.20.2", "prefix": "/usr"}, + {"spec": "cmake@3.23.1", "prefix": "/usr/tce"}, + {"spec": "cmake@3.24.2", "prefix": "/usr/tce"}, + ], + "buildable": False, + }, + "elfutils": { + "externals": [{"spec": "elfutils@0.190", "prefix": "/usr"}], + "buildable": False, + }, + "papi": { + "externals": [{"spec": "papi@5.6.0.0", "prefix": "/usr"}], + "buildable": False, + }, + "unwind": { + "externals": [{"spec": "unwind@8.0.1", "prefix": "/usr"}], + "buildable": False, + }, + "pkgconf": {"externals": [{"spec": "pkgconf@1.4.2", "prefix": "/usr"}]}, + "curl": { + "externals": [ + {"spec": "curl@7.61.1+gssapi+ldap+nghttp2", "prefix": "/usr"} + ] + }, + "gmake": {"externals": [{"spec": "gmake@4.2.1", "prefix": "/usr"}]}, + "subversion": { + "externals": [{"spec": "subversion@1.10.2", "prefix": "/usr"}] + }, + "diffutils": { + "externals": [{"spec": "diffutils@3.6", "prefix": "/usr"}] + }, + "swig": {"externals": [{"spec": "swig@3.0.12", "prefix": "/usr"}]}, + "gawk": {"externals": [{"spec": "gawk@4.2.1", "prefix": "/usr"}]}, + "binutils": { + "externals": [{"spec": "binutils@2.30.113", "prefix": "/usr"}] + }, + "findutils": { + "externals": [{"spec": "findutils@4.6.0", "prefix": "/usr"}] + }, + "git-lfs": { + "externals": [{"spec": "git-lfs@2.11.0", "prefix": "/usr/tce"}] + }, + "ccache": {"externals": [{"spec": "ccache@3.7.7", "prefix": "/usr"}]}, + "automake": { + "externals": [{"spec": "automake@1.16.1", "prefix": "/usr"}] + }, + "cvs": {"externals": [{"spec": "cvs@1.11.23", "prefix": "/usr"}]}, + "git": { + "externals": [ + {"spec": "git@2.31.1+tcltk", "prefix": "/usr"}, + {"spec": "git@2.29.1+tcltk", "prefix": "/usr/tce"}, + ] + }, + "openssh": {"externals": [{"spec": "openssh@8.0p1", "prefix": "/usr"}]}, + "autoconf": { + "externals": [{"spec": "autoconf@2.69", "prefix": "/usr"}] + }, + "texinfo": {"externals": [{"spec": "texinfo@6.5", "prefix": "/usr"}]}, + "bison": {"externals": [{"spec": "bison@3.0.4", "prefix": "/usr"}]}, + "python": { + "externals": [ + { + "spec": "python@3.9.12", + "prefix": "/usr/tce/packages/python/python-3.9.12", + "buildable": False, + } + ] + }, + "unzip": { + "buildable": False, + "externals": [{"spec": "unzip@6.0", "prefix": "/usr"}], + }, + "hypre": {"variants": "amdgpu_target=gfx90a"}, + "hwloc": { + "externals": [ + {"spec": "hwloc@2.9.1", "prefix": "/usr", "buildable": False} + ] + }, + "fftw": {"buildable": False}, + "intel-oneapi-mkl": { + "externals": [ + { + "spec": "intel-oneapi-mkl@2023.2.0", + "prefix": "/opt/intel/oneapi", + } + ], + "buildable": False, + }, + "mpi": {"buildable": False}, + "libfabric": { + "externals": [ + {"spec": "libfabric@2.1", "prefix": "/opt/cray/libfabric/2.1"} + ], + "buildable": False, + }, + } + } + + selections["packages"] |= self.mpi_config()["packages"] + + if self.spec.satisfies("compiler=cce"): + selections["packages"] |= { + "cray-libsci": { + "externals": [ + { + "spec": "cray-libsci@23.05.1.4%cce", + "prefix": "/opt/cray/pe/libsci/23.05.1.4/cray/12.0/x86_64/", + } + ] + } + } + elif self.spec.satisfies("compiler=gcc"): + selections["packages"] |= { + "cray-libsci": { + "externals": [ + { + "spec": "cray-libsci@23.05.1.4%gcc", + "prefix": "/opt/cray/pe/libsci/23.05.1.4/gnu/10.3/x86_64/", + } + ] + } + } + + selections["packages"] |= self.compiler_weighting_cfg()["packages"] + + return selections + + def compiler_weighting_cfg(self): + compiler = self.spec.variants["compiler"][0] + + if compiler == "cce": + return {"packages": {"all": {"require": [{"one_of": ["%cce", "%gcc"]}]}}} + elif compiler == "gcc": + return {"packages": {}} + else: + raise ValueError(f"Unexpected value for compiler: {compiler}") + + def compute_compilers_section(self): + selections = { + "compilers": [ + { + "compiler": { + "spec": "gcc@12.2.0", + "paths": { + "cc": "/opt/cray/pe/gcc/12.2.0/bin/gcc", + "cxx": "/opt/cray/pe/gcc/12.2.0/bin/g++", + "f77": "/opt/cray/pe/gcc/12.2.0/bin/gfortran", + "fc": "/opt/cray/pe/gcc/12.2.0/bin/gfortran", + }, + "flags": {}, + "operating_system": "rhel8", + "target": "x86_64", + "modules": [], + "environment": {}, + "extra_rpaths": [], + } + } + ] + } + + return selections + + def mpi_config(self): + gtl = self.spec.variants["gtl"][0] + + if self.spec.satisfies("compiler=cce"): + dont_use_gtl = { + "gtl_lib_path": f"/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", + "ldflags": f"-L/opt/cray/pe/mpich/{self.mpi_version}/ofi/crayclang/{self.short_cce_version}/lib -lmpi -L/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", + } + + use_gtl = { + "gtl_flags": "$MV2_COMM_WORLD_LOCAL_RANK", + "gtl_cutoff_size": 4096, + "fi_cxi_ats": 0, + "gtl_lib_path": f"/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", + "gtl_libs": ["libmpi_gtl_hsa"], + "ldflags": f"-L/opt/cray/pe/mpich/{self.mpi_version}/ofi/crayclang/{self.short_cce_version}/lib -lmpi -L/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -lmpi_gtl_hsa", + } + + if gtl: + gtl_spec = "+gtl" + gtl_cfg = use_gtl + else: + gtl_spec = "~gtl" + gtl_cfg = dont_use_gtl + + return { + "packages": { + "cray-mpich": { + "externals": [ + { + "spec": f"cray-mpich@{self.mpi_version}%cce@{self.cce_version} {gtl_spec} +wrappers", + "prefix": f"/opt/cray/pe/mpich/{self.mpi_version}/ofi/crayclang/{self.short_cce_version}", + "extra_attributes": gtl_cfg, # Assuming `gtl_cfg` is already defined elsewhere + } + ] + } + } + } + + elif self.spec.satisfies("compiler=gcc"): + return { + "packages": { + "cray-mpich": { + "externals": [ + { + "spec": f"cray-mpich@{self.mpi_version}%gcc@{self.gcc_version} ~gtl +wrappers", + "prefix": f"/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/10.3", + "extra_attributes": { + "gtl_lib_path": f"/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", + "ldflags": f"-L/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/10.3/lib -lmpi -L/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", + }, + } + ] + } + } + } + + def compute_software_section(self): + """This is somewhat vestigial: for the Tioga config that is committed + to the repo, multiple instances of mpi/compilers are stored and + and these variables were used to choose consistent dependencies. + The configs generated by this class should only ever have one + instance of MPI etc., so there is no need for that. The experiments + will fail if these variables are not defined though, so for now + they are still generated (but with more-generic values). + """ + return { + "software": { + "packages": { + "default-compiler": { + "pkg_spec": f"{self.spec.variants['compiler'][0]}" + }, + "default-mpi": {"pkg_spec": "cray-mpich"}, + "compiler-amdclang": {"pkg_spec": "clang"}, + "compiler-gcc": {"pkg_spec": "gcc"}, + "mpi-gcc": {"pkg_spec": "cray-mpich~gtl"}, + "blas": {"pkg_spec": f"{self.spec.variants['blas'][0]}"}, + "lapack": {"pkg_spec": f"{self.spec.variants['lapack'][0]}"}, + "lapack-oneapi": {"pkg_spec": "intel-oneapi-mkl"}, + } + } + } From 5c7da00887f0c1feb4fbf808b1d24eff3ec70aba Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Thu, 24 Apr 2025 12:00:05 -0700 Subject: [PATCH 2/6] update externals with benchpark system external --- systems/lbnl-perlmutter/system.py | 141 +++++++++++++++--------------- 1 file changed, 71 insertions(+), 70 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 01c7552ce..d7ff09375 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -36,8 +36,8 @@ def __init__(self, spec): super().__init__(spec) if self.spec.satisfies("compiler=gcc"): - self.gcc_version = Version("12.2.0") - self.mpi_version = Version("8.1.26") + self.gcc_version = Version("13.2.1") + self.mpi_version = Version("8.1.30") self.scheduler = "slurm" attrs = self.id_to_resources.get("perlmutter") @@ -48,116 +48,117 @@ def compute_packages_section(self): selections = { "packages": { "all": {"require": "target=x86_64:"}, - "tar": {"externals": [{"spec": "tar@1.30", "prefix": "/usr"}]}, + "tar": {"externals": [{"spec": "tar@1.34", "prefix": "/usr"}]}, "coreutils": { - "externals": [{"spec": "coreutils@8.30", "prefix": "/usr"}] + "externals": [{"spec": "coreutils@8.32", "prefix": "/usr"}] }, "libtool": {"externals": [{"spec": "libtool@2.4.6", "prefix": "/usr"}]}, - "flex": {"externals": [{"spec": "flex@2.6.1+lex", "prefix": "/usr"}]}, + "flex": {"externals": [{"spec": "flex@2.6.4+lex", "prefix": "/usr"}]}, "openssl": { - "externals": [{"spec": "openssl@1.1.1k", "prefix": "/usr"}] + "externals": [{"spec": "openssl@1.1.1l-fips", "prefix": "/usr"}] }, "m4": {"externals": [{"spec": "m4@1.4.18", "prefix": "/usr"}]}, - "groff": {"externals": [{"spec": "groff@1.22.3", "prefix": "/usr"}]}, + "groff": {"externals": [{"spec": "groff@1.22.4", "prefix": "/usr"}]}, "cmake": { - "externals": [ - {"spec": "cmake@3.20.2", "prefix": "/usr"}, - {"spec": "cmake@3.23.1", "prefix": "/usr/tce"}, - {"spec": "cmake@3.24.2", "prefix": "/usr/tce"}, - ], - "buildable": False, - }, - "elfutils": { - "externals": [{"spec": "elfutils@0.190", "prefix": "/usr"}], + "externals": [{"spec": "cmake@3.20.4", "prefix": "/usr"}], "buildable": False, }, - "papi": { - "externals": [{"spec": "papi@5.6.0.0", "prefix": "/usr"}], - "buildable": False, - }, - "unwind": { - "externals": [{"spec": "unwind@8.0.1", "prefix": "/usr"}], - "buildable": False, - }, - "pkgconf": {"externals": [{"spec": "pkgconf@1.4.2", "prefix": "/usr"}]}, "curl": { "externals": [ - {"spec": "curl@7.61.1+gssapi+ldap+nghttp2", "prefix": "/usr"} + {"spec": "curl@8.0.1+gssapi+ldap+nghttp2", "prefix": "/usr"} ] }, "gmake": {"externals": [{"spec": "gmake@4.2.1", "prefix": "/usr"}]}, "subversion": { - "externals": [{"spec": "subversion@1.10.2", "prefix": "/usr"}] + "externals": [{"spec": "subversion@1.14.1", "prefix": "/usr"}], + "buildable": False, }, "diffutils": { - "externals": [{"spec": "diffutils@3.6", "prefix": "/usr"}] + "externals": [{"spec": "diffutils@3.6", "prefix": "/usr"}], + "buildable": False, }, - "swig": {"externals": [{"spec": "swig@3.0.12", "prefix": "/usr"}]}, "gawk": {"externals": [{"spec": "gawk@4.2.1", "prefix": "/usr"}]}, "binutils": { - "externals": [{"spec": "binutils@2.30.113", "prefix": "/usr"}] + "externals": [ + {"spec": "binutils@2.40~gold~headers", "prefix": "/opt/cray/pe/cce/18.0.0/binutils/x86_64/x86_64-pc-linux-gnu"} + {"spec": "binutils@2.43.1~gold~headers", "prefix": "/usr"} + ], + "buildable": False, }, "findutils": { - "externals": [{"spec": "findutils@4.6.0", "prefix": "/usr"}] + "externals": [{"spec": "findutils@4.8.0", "prefix": "/usr"}], + "buildable": False, }, - "git-lfs": { - "externals": [{"spec": "git-lfs@2.11.0", "prefix": "/usr/tce"}] + "ccache": { + "externals": [{"spec": "ccache@3.4.7", "prefix": "/usr"}]}, + "buildable": False, }, - "ccache": {"externals": [{"spec": "ccache@3.7.7", "prefix": "/usr"}]}, "automake": { - "externals": [{"spec": "automake@1.16.1", "prefix": "/usr"}] + "externals": [{"spec": "automake@1.15.1", "prefix": "/usr"}], + "buildable": False, }, - "cvs": {"externals": [{"spec": "cvs@1.11.23", "prefix": "/usr"}]}, "git": { - "externals": [ - {"spec": "git@2.31.1+tcltk", "prefix": "/usr"}, - {"spec": "git@2.29.1+tcltk", "prefix": "/usr/tce"}, - ] + "externals": [{"spec": "git@2.35.3~tcltk", "prefix": "/usr"}], + "buildable": False, + }, + "openssh": { + "externals": [{"spec": "openssh@8.4p1", "prefix": "/usr"}], + "buildable": False, }, - "openssh": {"externals": [{"spec": "openssh@8.0p1", "prefix": "/usr"}]}, "autoconf": { - "externals": [{"spec": "autoconf@2.69", "prefix": "/usr"}] + "externals": [{"spec": "autoconf@2.69", "prefix": "/usr"}], + "buildable": False, + }, + "bison": { + "externals": [{"spec": "bison@3.0.4", "prefix": "/usr"}], + "buildable": False, }, - "texinfo": {"externals": [{"spec": "texinfo@6.5", "prefix": "/usr"}]}, - "bison": {"externals": [{"spec": "bison@3.0.4", "prefix": "/usr"}]}, "python": { "externals": [ { - "spec": "python@3.9.12", - "prefix": "/usr/tce/packages/python/python-3.9.12", - "buildable": False, - } - ] + "prefix": "/usr", + "spec": "python@2.7.18+bz2+crypt+ctypes~dbm~lzma+nis~pyexpat+pythoncmd~readline~sqlite3~ssl~tkinter+uuid+zlib", + }, + { + "prefix": "/usr", + "spec": "python@3.6.15+bz2+crypt+ctypes~dbm+lzma+nis+pyexpat~pythoncmd+readline+sqlite3+ssl~tkinter+uuid+zlib", + }, + ], + "buildable": False, }, - "unzip": { + "doxygen": { + "externals": [{"spec": "doxygen@1.8.14~graphviz~mscgen", "prefix": "/usr"}], "buildable": False, - "externals": [{"spec": "unzip@6.0", "prefix": "/usr"}], }, - "hypre": {"variants": "amdgpu_target=gfx90a"}, - "hwloc": { - "externals": [ - {"spec": "hwloc@2.9.1", "prefix": "/usr", "buildable": False} - ] + "gettext": { + "externals": [{"spec": "gettext@0.20.2", "prefix": "/usr"}], + "buildable": False, }, - "fftw": {"buildable": False}, - "intel-oneapi-mkl": { - "externals": [ - { - "spec": "intel-oneapi-mkl@2023.2.0", - "prefix": "/opt/intel/oneapi", - } - ], + "ninja": { + "externals": [{"spec": "ninja@1.10.0", "prefix": "/usr"}], "buildable": False, }, - "mpi": {"buildable": False}, - "libfabric": { - "externals": [ - {"spec": "libfabric@2.1", "prefix": "/opt/cray/libfabric/2.1"} - ], + "perl": { + "externals": [{"spec": "perl@5.26.1~cpanm+opcode+open+shared+threads", "prefix": "/usr"}], + "buildable": False, + }, + "pkg-config": { + "externals": [{"spec": "pkg-config@0.29.2", "prefix": "/usr"}], + "buildable": False, + }, + "sed": { + "externals": [{"spec": "sed@4.4", "prefix": "/usr"}], + "buildable": False, + }, + "tar": { + "externals": [{"spec": "tar@1.34", "prefix": "/usr"}], + "buildable": False, + }, + "zlib": { + "externals": [{"spec": "zlib@1.2.13", "prefix": "/usr"}], "buildable": False, }, } - } selections["packages"] |= self.mpi_config()["packages"] From 16610312f876ace1bbb05ef36daa414d83cd4a89 Mon Sep 17 00:00:00 2001 From: Stephanie Labasan Date: Thu, 8 May 2025 11:11:53 -0700 Subject: [PATCH 3/6] remove cce compiler for now --- systems/lbnl-perlmutter/system.py | 59 +++---------------------------- 1 file changed, 5 insertions(+), 54 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index d7ff09375..797d4f23f 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -27,8 +27,8 @@ class LbnlPerlmutter(System): variant( "compiler", - default="cce", - values=("cce", "gcc"), + default="gcc", + values=("gcc"), description="Which compiler to use", ) @@ -162,18 +162,7 @@ def compute_packages_section(self): selections["packages"] |= self.mpi_config()["packages"] - if self.spec.satisfies("compiler=cce"): - selections["packages"] |= { - "cray-libsci": { - "externals": [ - { - "spec": "cray-libsci@23.05.1.4%cce", - "prefix": "/opt/cray/pe/libsci/23.05.1.4/cray/12.0/x86_64/", - } - ] - } - } - elif self.spec.satisfies("compiler=gcc"): + if self.spec.satisfies("compiler=gcc"): selections["packages"] |= { "cray-libsci": { "externals": [ @@ -192,9 +181,7 @@ def compute_packages_section(self): def compiler_weighting_cfg(self): compiler = self.spec.variants["compiler"][0] - if compiler == "cce": - return {"packages": {"all": {"require": [{"one_of": ["%cce", "%gcc"]}]}}} - elif compiler == "gcc": + if compiler == "gcc": return {"packages": {}} else: raise ValueError(f"Unexpected value for compiler: {compiler}") @@ -227,43 +214,7 @@ def compute_compilers_section(self): def mpi_config(self): gtl = self.spec.variants["gtl"][0] - if self.spec.satisfies("compiler=cce"): - dont_use_gtl = { - "gtl_lib_path": f"/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", - "ldflags": f"-L/opt/cray/pe/mpich/{self.mpi_version}/ofi/crayclang/{self.short_cce_version}/lib -lmpi -L/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", - } - - use_gtl = { - "gtl_flags": "$MV2_COMM_WORLD_LOCAL_RANK", - "gtl_cutoff_size": 4096, - "fi_cxi_ats": 0, - "gtl_lib_path": f"/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", - "gtl_libs": ["libmpi_gtl_hsa"], - "ldflags": f"-L/opt/cray/pe/mpich/{self.mpi_version}/ofi/crayclang/{self.short_cce_version}/lib -lmpi -L/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -lmpi_gtl_hsa", - } - - if gtl: - gtl_spec = "+gtl" - gtl_cfg = use_gtl - else: - gtl_spec = "~gtl" - gtl_cfg = dont_use_gtl - - return { - "packages": { - "cray-mpich": { - "externals": [ - { - "spec": f"cray-mpich@{self.mpi_version}%cce@{self.cce_version} {gtl_spec} +wrappers", - "prefix": f"/opt/cray/pe/mpich/{self.mpi_version}/ofi/crayclang/{self.short_cce_version}", - "extra_attributes": gtl_cfg, # Assuming `gtl_cfg` is already defined elsewhere - } - ] - } - } - } - - elif self.spec.satisfies("compiler=gcc"): + if self.spec.satisfies("compiler=gcc"): return { "packages": { "cray-mpich": { From 6dee0d43fa6198ab67251409d4ba3511e4b31ddf Mon Sep 17 00:00:00 2001 From: Stephanie Labasan Date: Thu, 8 May 2025 12:02:37 -0700 Subject: [PATCH 4/6] checkpoint --- systems/lbnl-perlmutter/system.py | 81 ++++++++++++++----------------- 1 file changed, 36 insertions(+), 45 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 797d4f23f..4c3986217 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -28,7 +28,6 @@ class LbnlPerlmutter(System): variant( "compiler", default="gcc", - values=("gcc"), description="Which compiler to use", ) @@ -68,6 +67,18 @@ def compute_packages_section(self): {"spec": "curl@8.0.1+gssapi+ldap+nghttp2", "prefix": "/usr"} ] }, + "cusolver": { + "externals": [{"spec": "cusolver@12.4", "prefix": "/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/cuda/12.4"}], + "buildable": False, + }, + "cublas": { + "externals": [{"spec": "cublas@12.4", "prefix": "/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/cuda/12.4"}], + "buildable": False, + }, + "automake": { + "externals": [{"spec": "automake@1.15.1", "prefix": "/usr"}], + "buildable": False, + }, "gmake": {"externals": [{"spec": "gmake@4.2.1", "prefix": "/usr"}]}, "subversion": { "externals": [{"spec": "subversion@1.14.1", "prefix": "/usr"}], @@ -80,7 +91,7 @@ def compute_packages_section(self): "gawk": {"externals": [{"spec": "gawk@4.2.1", "prefix": "/usr"}]}, "binutils": { "externals": [ - {"spec": "binutils@2.40~gold~headers", "prefix": "/opt/cray/pe/cce/18.0.0/binutils/x86_64/x86_64-pc-linux-gnu"} + {"spec": "binutils@2.40~gold~headers", "prefix": "/opt/cray/pe/cce/18.0.0/binutils/x86_64/x86_64-pc-linux-gnu"}, {"spec": "binutils@2.43.1~gold~headers", "prefix": "/usr"} ], "buildable": False, @@ -93,10 +104,6 @@ def compute_packages_section(self): "externals": [{"spec": "ccache@3.4.7", "prefix": "/usr"}]}, "buildable": False, }, - "automake": { - "externals": [{"spec": "automake@1.15.1", "prefix": "/usr"}], - "buildable": False, - }, "git": { "externals": [{"spec": "git@2.35.3~tcltk", "prefix": "/usr"}], "buildable": False, @@ -160,32 +167,22 @@ def compute_packages_section(self): }, } - selections["packages"] |= self.mpi_config()["packages"] + #selections["packages"] |= self.mpi_config() - if self.spec.satisfies("compiler=gcc"): - selections["packages"] |= { - "cray-libsci": { - "externals": [ - { - "spec": "cray-libsci@23.05.1.4%gcc", - "prefix": "/opt/cray/pe/libsci/23.05.1.4/gnu/10.3/x86_64/", - } - ] - } - } - - selections["packages"] |= self.compiler_weighting_cfg()["packages"] +# if self.spec.satisfies("compiler=gcc"): +# selections["packages"] |= { +# "cray-libsci": { +# "externals": [ +# { +# "spec": "cray-libsci@23.05.1.4%gcc", +# "prefix": "/opt/cray/pe/libsci/23.05.1.4/gnu/10.3/x86_64/", +# } +# ] +# } +# } return selections - def compiler_weighting_cfg(self): - compiler = self.spec.variants["compiler"][0] - - if compiler == "gcc": - return {"packages": {}} - else: - raise ValueError(f"Unexpected value for compiler: {compiler}") - def compute_compilers_section(self): selections = { "compilers": [ @@ -212,23 +209,19 @@ def compute_compilers_section(self): return selections def mpi_config(self): - gtl = self.spec.variants["gtl"][0] - if self.spec.satisfies("compiler=gcc"): return { - "packages": { - "cray-mpich": { - "externals": [ - { - "spec": f"cray-mpich@{self.mpi_version}%gcc@{self.gcc_version} ~gtl +wrappers", - "prefix": f"/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/10.3", - "extra_attributes": { - "gtl_lib_path": f"/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", - "ldflags": f"-L/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/10.3/lib -lmpi -L/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", - }, - } - ] - } + "cray-mpich": { + "externals": [ + { + "spec": f"cray-mpich@{self.mpi_version}%gcc@{self.gcc_version} ~gtl +wrappers", + "prefix": f"/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/10.3", + "extra_attributes": { + "gtl_lib_path": f"/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", + "ldflags": f"-L/opt/cray/pe/mpich/{self.mpi_version}/ofi/gnu/10.3/lib -lmpi -L/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{self.mpi_version}/gtl/lib", + }, + } + ] } } @@ -251,8 +244,6 @@ def compute_software_section(self): "compiler-amdclang": {"pkg_spec": "clang"}, "compiler-gcc": {"pkg_spec": "gcc"}, "mpi-gcc": {"pkg_spec": "cray-mpich~gtl"}, - "blas": {"pkg_spec": f"{self.spec.variants['blas'][0]}"}, - "lapack": {"pkg_spec": f"{self.spec.variants['lapack'][0]}"}, "lapack-oneapi": {"pkg_spec": "intel-oneapi-mkl"}, } } From 6a2a302a9eedf0dc41f46d9c3a106f9bd154458a Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Thu, 8 May 2025 13:05:39 -0700 Subject: [PATCH 5/6] fix system.py --- systems/lbnl-perlmutter/system.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 4c3986217..8faede6b3 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -101,7 +101,7 @@ def compute_packages_section(self): "buildable": False, }, "ccache": { - "externals": [{"spec": "ccache@3.4.7", "prefix": "/usr"}]}, + "externals": [{"spec": "ccache@3.4.7", "prefix": "/usr"}], "buildable": False, }, "git": { @@ -166,6 +166,7 @@ def compute_packages_section(self): "buildable": False, }, } + } #selections["packages"] |= self.mpi_config() From 870536c5c87f668013bd12ac58044cdfdff13220 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Thu, 8 May 2025 14:39:05 -0700 Subject: [PATCH 6/6] fix path to cublas --- systems/lbnl-perlmutter/system.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/systems/lbnl-perlmutter/system.py b/systems/lbnl-perlmutter/system.py index 8faede6b3..a5cc3cca5 100644 --- a/systems/lbnl-perlmutter/system.py +++ b/systems/lbnl-perlmutter/system.py @@ -28,6 +28,7 @@ class LbnlPerlmutter(System): variant( "compiler", default="gcc", + values=("gcc",), description="Which compiler to use", ) @@ -68,11 +69,11 @@ def compute_packages_section(self): ] }, "cusolver": { - "externals": [{"spec": "cusolver@12.4", "prefix": "/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/cuda/12.4"}], + "externals": [{"spec": "cusolver@12.4", "prefix": "/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/math_libs/12.4"}], "buildable": False, }, "cublas": { - "externals": [{"spec": "cublas@12.4", "prefix": "/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/cuda/12.4"}], + "externals": [{"spec": "cublas@12.4", "prefix": "/opt/nvidia/hpc_sdk/Linux_x86_64/24.5/math_libs/12.4"}], "buildable": False, }, "automake": {