From 22e6424c60c7dd80706b234d9517b159e2acd53b Mon Sep 17 00:00:00 2001 From: eap Date: Tue, 17 Dec 2024 15:44:09 -0700 Subject: [PATCH 01/31] add changes to intel container --- .../containers/docker-ubuntu-intel-impi.yaml | 192 ++++++++++++++---- 1 file changed, 151 insertions(+), 41 deletions(-) diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 6dd5ad15d..672004eb3 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -9,21 +9,47 @@ spack: compilers: - compiler: - spec: intel@2021.6.0 + spec: intel@2021.10.0 paths: cc: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/icc cxx: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/icpc f77: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/ifort fc: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/ifort flags: {} - operating_system: ubuntu22.04 + operating_system: ubuntu24.04 modules: [] environment: prepend_path: PATH: /usr/bin - LD_LIBRARY_PATH: /usr/lib64:/opt/intel/oneapi/compiler/2022.1.0/linux/compiler/lib/intel64_lin + LD_LIBRARY_PATH: /opt/intel/oneapi/compiler/2023.2.3/linux/compiler/lib/intel64_lin CPATH: /usr/include extra_rpaths: [] + - compiler: + spec: gcc@=11.4.0 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: {} + operating_system: ubuntu24.04 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: oneapi@=2023.2.3 + paths: + cc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/icx + cxx: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/icpx + f77: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/ifx + fc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/ifx + flags: {} + operating_system: ubuntu24.04 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] # Basic package config from configs/common/packages.yaml # Additional package config for container @@ -32,29 +58,50 @@ spack: require: '%intel' target: [core2] providers: - mpi: [intel-oneapi-mpi@2021.6.0] - compiler: [intel@2021.6.0] - intel: - buildable: false + mpi: [intel-oneapi-mpi@2021.10.0] + compiler: [intel@2021.10.0, gcc@11.4.0] + autoconf: externals: - - spec: intel@2021.6.0 - prefix: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64 - intel-oneapi-mpi: - buildable: false + - spec: autoconf@2.71 + prefix: /usr + automake: externals: - - spec: intel-oneapi-mpi@2021.6.0 - prefix: /opt/intel/oneapi - # Comment out for now so that fftw-api uses fftw and not mkl - #intel-oneapi-mkl: - # buildable: false - # externals: - # - spec: intel-oneapi-mkl@2022.1.0 - # prefix: /opt/intel/oneapi + - spec: automake@1.16.5 + prefix: /usr + binutils: + externals: + - spec: binutils@2.38 + prefix: /usr + coreutils: + externals: + - spec: coreutils@8.32 + prefix: /usr diffutils: buildable: false externals: - spec: diffutils@3.8 prefix: /usr + ectrans: + require:: + - '@1.2.0 ~mkl +fftw' + findutils: + externals: + - spec: findutils@4.8.0 + prefix: /usr + # DO Not submit - figure out what is going on - MLK is not installed so it must be built but might cause errors + #intel-oneapi-mkl: + # buildable: false + #externals: + #- spec: intel-oneapi-mkl@2022.1.0 + # prefix: /opt/intel/oneapi + flex: + externals: + - spec: flex@2.6.4+lex + prefix: /usr + gawk: + externals: + - spec: gawk@5.1.0 + prefix: /usr git: buildable: false externals: @@ -65,36 +112,89 @@ spack: externals: - spec: git-lfs@3.0.2 prefix: /usr - mysql: - buildable: false + gmake: externals: - - spec: mysql@8.0.39 + - spec: gmake@4.3 prefix: /usr - qt: + groff: + externals: + - spec: groff@1.22.4 + prefix: /usr + gsibec: + require:: + - '@1.2.1 ~mkl' + intel: buildable: false externals: - - spec: qt@5.15.3 + - spec: intel@2021.10.0 + prefix: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64 + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.10.0 + prefix: /opt/intel/oneapi + libtool: + externals: + - spec: libtool@2.4.6 prefix: /usr - version: [5.15.3] - wget: + mpi: + buildable: False + m4: + externals: + - spec: m4@1.4.18 + prefix: /usr + lcms: + externals: + - spec: lcms@2.13.1 + prefix: /usr + mysql: buildable: false externals: - - spec: wget@1.21.2 + - spec: mysql@8.0.37 prefix: /usr - version: [1.21.2] - gettext: + perl: externals: - - spec: gettext@0.21 + - spec: perl@5.34.0~cpanm+opcode+open+shared+threads + prefix: /usr + pkg-config: + externals: + - spec: pkg-config@0.29.2 prefix: /usr py-pynacl: buildable: false externals: - spec: py-pynacl@1.5.0 prefix: /usr + py-numpy: + require:: + - '@:1.23 ^openblas' # Turn off crypt, because libxcrypt doesn't # build with Intel. python: variants: ~crypt + qt: + buildable: false + externals: + - spec: qt@5.15.3 + prefix: /usr + version: [5.15.3] + sed: + externals: + - spec: sed@4.8 + prefix: /usr + tar: + externals: + - spec: tar@1.34 + prefix: /usr + wget: + buildable: false + externals: + - spec: wget@1.21.2 + prefix: /usr + gettext: + externals: + - spec: gettext@0.21 + prefix: /usr specs: [] @@ -116,7 +216,7 @@ spack: # Sets the base images for the stages where Spack builds the # software or where the software gets installed after being built.. images: - os: ubuntu:22.04 + os: ubuntu:24.04 spack: url: https://github.com/jcsda/spack ref: spack-stack-dev @@ -130,39 +230,43 @@ spack: build: - autopoint - bc - - cpp - - g++ - - gcc + - cpp-11 + - g++-11 + - gcc-11 - gettext - - gfortran + - gfortran-11 - git - git-lfs - gpg + - libgomp1 - make - mysql-server - qtbase5-dev - qt5-qmake - libqt5svg5-dev - qt5dxcb-plugin + - libcurl4-openssl-dev - wget final: - autopoint - bc - - cpp - - g++ - - gcc + - cpp-11 + - g++-11 + - gcc-11 - gettext - - gfortran + - gfortran-11 - git - git-lfs - gpg + - libgomp1 - make - mysql-server - qtbase5-dev - qt5-qmake - libqt5svg5-dev - qt5dxcb-plugin + - libcurl4-openssl-dev - wget # Implicitly included in build step - build-essential @@ -192,7 +296,10 @@ spack: wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \ echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \ apt update && \ - apt install intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2022.1.0 intel-oneapi-compiler-fortran-2022.1.0 intel-oneapi-mkl-devel-2022.1.0 intel-oneapi-mpi-devel-2021.6.0 -y && \ + apt install intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2023.2.3 intel-oneapi-compiler-fortran-2023.2.3 intel-oneapi-mkl-devel-2023.2.0 intel-oneapi-mpi-devel-2021.10.0 -y && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ + update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-11 100 && \ rm -rf /var/lib/apt/lists/* pre_final: | # Set environment variables for installing tzdata @@ -212,7 +319,10 @@ spack: wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \ echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \ apt update && \ - apt install intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2022.1.0 intel-oneapi-compiler-fortran-2022.1.0 intel-oneapi-mkl-devel-2022.1.0 intel-oneapi-mpi-devel-2021.6.0 -y && \ + apt install intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2023.2.3 intel-oneapi-compiler-fortran-2023.2.3 intel-oneapi-mkl-devel-2023.2.0 intel-oneapi-mpi-devel-2021.10.0 -y && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ + update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-11 100 && \ rm -rf /var/lib/apt/lists/* && \ /opt/views/view/bin/pip3 install pynacl # Copy spack find output from builder From 89665b69c1fee4fcd01590eecc014389bf7d9c8c Mon Sep 17 00:00:00 2001 From: eap Date: Tue, 17 Dec 2024 23:02:12 -0700 Subject: [PATCH 02/31] Several jedi-ci spec changes --- configs/containers/specs/jedi-ci.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/containers/specs/jedi-ci.yaml b/configs/containers/specs/jedi-ci.yaml index 6f93c5432..d5d4c423d 100644 --- a/configs/containers/specs/jedi-ci.yaml +++ b/configs/containers/specs/jedi-ci.yaml @@ -37,11 +37,11 @@ py-eccodes@1.5.0, py-f90nml@1.4.3, py-gitpython@3.1.40, - py-h5py@3.11.0, + py-h5py@3.12.1, py-numpy@1.23.5, py-pandas@1.5.3, py-pip, - py-pyyaml@6.0, + py-pyyaml@6.0.2, py-scipy@1.12.0, py-shapely@1.8.0, py-xarray@2023.7.0, @@ -51,11 +51,11 @@ nco@5.1.6, esmf@8.6.1, mapl@2.46.3, - zlib-ng@2.1.5, + zlib-ng@2.2.1, zstd@1.5.2, odc@1.5.2, shumlib@macos_clang_linux_intel_port, - awscli-v2@2.13.22, + awscli-v2@2.15.53, py-globus-cli@3.16.0, py-fortranformat@2.0.0, # Added for new CI system 2024/04/30 From e2b9ffdd69ed76480e877fd965f8133a00e16491 Mon Sep 17 00:00:00 2001 From: Evan Parker Date: Wed, 18 Dec 2024 22:56:16 +0000 Subject: [PATCH 03/31] several updates --- configs/containers/docker-ubuntu-intel-impi.yaml | 16 ++++------------ configs/containers/specs/jedi-ci.yaml | 6 +++++- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 672004eb3..eff6e99a0 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -11,10 +11,10 @@ spack: - compiler: spec: intel@2021.10.0 paths: - cc: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/icc - cxx: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/icpc - f77: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/ifort - fc: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/ifort + cc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/icc + cxx: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/icpc + f77: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/ifort + fc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/ifort flags: {} operating_system: ubuntu24.04 modules: [] @@ -81,9 +81,6 @@ spack: externals: - spec: diffutils@3.8 prefix: /usr - ectrans: - require:: - - '@1.2.0 ~mkl +fftw' findutils: externals: - spec: findutils@4.8.0 @@ -123,11 +120,6 @@ spack: gsibec: require:: - '@1.2.1 ~mkl' - intel: - buildable: false - externals: - - spec: intel@2021.10.0 - prefix: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64 intel-oneapi-mpi: buildable: false externals: diff --git a/configs/containers/specs/jedi-ci.yaml b/configs/containers/specs/jedi-ci.yaml index d5d4c423d..4fb1600e9 100644 --- a/configs/containers/specs/jedi-ci.yaml +++ b/configs/containers/specs/jedi-ci.yaml @@ -15,7 +15,7 @@ eckit@1.28.3, ecmwf-atlas@0.40.0 +fckit +trans +tesselation +fftw, fiat@1.4.1, - ectrans@1.5.0 +fftw, + ectrans@1.5.0, eigen@3.4.0, fckit@0.13.2, fms@2024.02, @@ -32,8 +32,12 @@ netcdf-c@4.9.2, netcdf-cxx4@4.3.1, netcdf-fortran@4.6.1, + openssl@3.4.0, parallelio@2.6.2 +pnetcdf, parallel-netcdf@1.12.3, + py-boto3@1.34.44, + py-cartopy@0.24.1, + py-cryptography@42.0.8, py-eccodes@1.5.0, py-f90nml@1.4.3, py-gitpython@3.1.40, From 90eb8629f271cec4b1ec451b66acd05ad20af2a0 Mon Sep 17 00:00:00 2001 From: eap Date: Wed, 18 Dec 2024 19:48:04 -0700 Subject: [PATCH 04/31] add gcc require for cross compiler packages --- configs/containers/docker-ubuntu-intel-impi.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index eff6e99a0..766101d40 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -72,6 +72,9 @@ spack: externals: - spec: binutils@2.38 prefix: /usr + boost: + require: + - '%gcc' coreutils: externals: - spec: coreutils@8.32 @@ -81,6 +84,9 @@ spack: externals: - spec: diffutils@3.8 prefix: /usr + ecflow: + require: + - '%gcc' findutils: externals: - spec: findutils@4.8.0 @@ -144,6 +150,9 @@ spack: externals: - spec: mysql@8.0.37 prefix: /usr + pixman: + require: + - '%gcc' perl: externals: - spec: perl@5.34.0~cpanm+opcode+open+shared+threads From 1e0906e27a6f3e5e9aa7d79ff63a0dd8fe3cf156 Mon Sep 17 00:00:00 2001 From: eap Date: Wed, 18 Dec 2024 20:27:43 -0700 Subject: [PATCH 05/31] unpin fixes from #1429 --- configs/containers/specs/jedi-ci.yaml | 32 +++++++++++++-------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/configs/containers/specs/jedi-ci.yaml b/configs/containers/specs/jedi-ci.yaml index 4fb1600e9..7fb01480b 100644 --- a/configs/containers/specs/jedi-ci.yaml +++ b/configs/containers/specs/jedi-ci.yaml @@ -38,25 +38,25 @@ py-boto3@1.34.44, py-cartopy@0.24.1, py-cryptography@42.0.8, - py-eccodes@1.5.0, - py-f90nml@1.4.3, - py-gitpython@3.1.40, - py-h5py@3.12.1, - py-numpy@1.23.5, - py-pandas@1.5.3, + py-eccodes, + py-f90nml, + py-gitpython, + py-h5py, + py-numpy, + py-pandas, py-pip, - py-pyyaml@6.0.2, - py-scipy@1.12.0, - py-shapely@1.8.0, - py-xarray@2023.7.0, + py-pyyaml, + py-scipy, + py-shapely, + py-xarray, sp@2.5.0, udunits@2.2.28, w3emc@2.10.0, - nco@5.1.6, + nco, esmf@8.6.1, mapl@2.46.3, - zlib-ng@2.2.1, - zstd@1.5.2, + zlib-ng, + zstd, odc@1.5.2, shumlib@macos_clang_linux_intel_port, awscli-v2@2.15.53, @@ -68,6 +68,6 @@ jq@1.6 ] # Notes: # 1. Don't build CRTM by default so that it gets built in the JEDI bundles - # 2. Comment out for now until build problems are solved - # https://github.com/jcsda/spack-stack/issues/522 - # py-mysql-connector-python@8.0.32 + # 2. Avoid pinning packages to specific versions/variants unless absolutely + # needed, because config/common/packages.yaml does that already (removal + # of pinned versions in the above list is work in progress, DH 2024/12/18) \ No newline at end of file From a51c344c654364e11bd3dce44258b52402af3dd2 Mon Sep 17 00:00:00 2001 From: eap Date: Wed, 18 Dec 2024 23:31:13 -0700 Subject: [PATCH 06/31] add build essentials --- configs/containers/docker-ubuntu-intel-impi.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 766101d40..e08c8ebaf 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -229,9 +229,13 @@ spack: ## Additional system packages that are needed at runtime os_packages: build: + - autoconf + - automake - autopoint - bc + - build-essential - cpp-11 + - cmake - g++-11 - gcc-11 - gettext From 9081611177c042719c0c8d7b8764fc77488e5bac Mon Sep 17 00:00:00 2001 From: eap Date: Thu, 19 Dec 2024 15:14:10 -0700 Subject: [PATCH 07/31] Many fixes --- .../containers/docker-ubuntu-intel-impi.yaml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index e08c8ebaf..118e650d4 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -4,7 +4,7 @@ spack: config: checksum: false - build_jobs: 2 + build_jobs: 5 connect_timeout: 60 compilers: @@ -233,9 +233,9 @@ spack: - automake - autopoint - bc - - build-essential - cpp-11 - cmake + - flex - g++-11 - gcc-11 - gettext @@ -244,8 +244,8 @@ spack: - git-lfs - gpg - libgomp1 - - make - mysql-server + - pkg-config - qtbase5-dev - qt5-qmake - libqt5svg5-dev @@ -267,6 +267,7 @@ spack: - libgomp1 - make - mysql-server + - pkg-config - qtbase5-dev - qt5-qmake - libqt5svg5-dev @@ -305,6 +306,7 @@ spack: update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 && \ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-11 100 && \ + rm /usr/bin/cpp && ln -s /usr/bin/cpp-11 /usr/bin/cpp && \ rm -rf /var/lib/apt/lists/* pre_final: | # Set environment variables for installing tzdata @@ -315,7 +317,8 @@ spack: ENV FC=ifort build: | # Put output of spack find into a file - RUN cd /opt/spack-environment && \ + RUN source /opt/intel/oneapi/setvars.sh && \ + cd /opt/spack-environment && \ spack env activate -d . && \ spack find 2>&1 | tee /root/spack_find.out final: | @@ -328,6 +331,7 @@ spack: update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 && \ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-11 100 && \ + rm /usr/bin/cpp && ln -s /usr/bin/cpp-11 /usr/bin/cpp && \ rm -rf /var/lib/apt/lists/* && \ /opt/views/view/bin/pip3 install pynacl # Copy spack find output from builder @@ -339,8 +343,7 @@ spack: echo "export CC=icc" >> /home/nonroot/.bashrc && \ echo "export CXX=icpc" >> /home/nonroot/.bashrc && \ echo "export FC=ifort" >> /home/nonroot/.bashrc && \ - echo "source /opt/intel/oneapi/compiler/latest/env/vars.sh" >> /home/nonroot/.bashrc && \ - echo "source /opt/intel/oneapi/mpi/latest/env/vars.sh" >> /home/nonroot/.bashrc && \ + echo "source /opt/intel/oneapi/setvars.sh" > /etc/profile.d/02-intel-vars.sh && \ printf "[credential]\n helper = cache --timeout=7200\n" >> /home/nonroot/.gitconfig && \ chown -R nonroot:nonroot /home/nonroot/.gitconfig # Replicate settings for root user @@ -350,8 +353,6 @@ spack: echo "export CXX=icpc" >> /root/.bashrc && \ echo "export FC=ifort" >> /root/.bashrc && \ echo "export jedi_cmake_ROOT=$(ls -d /opt/software/linux-ubuntu*/intel-*/jedi-cmake-*)" >> /root/.bashrc && \ - echo "source /opt/intel/oneapi/compiler/latest/env/vars.sh" >> /root/.bashrc && \ - echo "source /opt/intel/oneapi/mpi/latest/env/vars.sh" >> /root/.bashrc && \ printf "[credential]\n helper = cache --timeout=7200\n" >> /root/.gitconfig # Labels for the image From 578375bc9b4602cee66f4ef44ed03012be1022d3 Mon Sep 17 00:00:00 2001 From: eap Date: Fri, 20 Dec 2024 12:54:20 -0700 Subject: [PATCH 08/31] llvm fixes --- .../containers/docker-ubuntu-intel-impi.yaml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 118e650d4..5ed76d910 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -129,7 +129,8 @@ spack: intel-oneapi-mpi: buildable: false externals: - - spec: intel-oneapi-mpi@2021.10.0 + # This spec is required in order set paths allowing execution of mpiicx. + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 +classic-names prefix: /opt/intel/oneapi libtool: externals: @@ -145,6 +146,11 @@ spack: externals: - spec: lcms@2.13.1 prefix: /usr + llvm: + buildable: false + externals: + - spec: llvm@14.0.6 + prefix: /usr mysql: buildable: false externals: @@ -244,6 +250,7 @@ spack: - git-lfs - gpg - libgomp1 + - llvm-14 - mysql-server - pkg-config - qtbase5-dev @@ -265,6 +272,7 @@ spack: - git-lfs - gpg - libgomp1 + - llvm-14 - make - mysql-server - pkg-config @@ -307,6 +315,9 @@ spack: update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-11 100 && \ rm /usr/bin/cpp && ln -s /usr/bin/cpp-11 /usr/bin/cpp && \ + # Needed by py-llvmlite + update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \ + echo "source /opt/intel/oneapi/setvars.sh" > /etc/profile.d/02-intel-vars.sh && \ rm -rf /var/lib/apt/lists/* pre_final: | # Set environment variables for installing tzdata @@ -317,8 +328,7 @@ spack: ENV FC=ifort build: | # Put output of spack find into a file - RUN source /opt/intel/oneapi/setvars.sh && \ - cd /opt/spack-environment && \ + RUN cd /opt/spack-environment && \ spack env activate -d . && \ spack find 2>&1 | tee /root/spack_find.out final: | @@ -332,6 +342,8 @@ spack: update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-11 100 && \ rm /usr/bin/cpp && ln -s /usr/bin/cpp-11 /usr/bin/cpp && \ + # Needed by py-llvmlite + update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \ rm -rf /var/lib/apt/lists/* && \ /opt/views/view/bin/pip3 install pynacl # Copy spack find output from builder From 90a4c263f125ab56142e1116822d55249dc3dcb4 Mon Sep 17 00:00:00 2001 From: eap Date: Fri, 20 Dec 2024 22:30:56 -0700 Subject: [PATCH 09/31] add tbb and mkl --- configs/containers/docker-ubuntu-intel-impi.yaml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 5ed76d910..856e3a585 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -126,12 +126,22 @@ spack: gsibec: require:: - '@1.2.1 ~mkl' + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mpi@2023.2.0 + prefix: /opt/intel/oneapi intel-oneapi-mpi: buildable: false externals: # This spec is required in order set paths allowing execution of mpiicx. - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 +classic-names prefix: /opt/intel/oneapi + intel-oneapi-tbb: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.10.0 + prefix: /opt/intel/oneapi libtool: externals: - spec: libtool@2.4.6 @@ -317,7 +327,6 @@ spack: rm /usr/bin/cpp && ln -s /usr/bin/cpp-11 /usr/bin/cpp && \ # Needed by py-llvmlite update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \ - echo "source /opt/intel/oneapi/setvars.sh" > /etc/profile.d/02-intel-vars.sh && \ rm -rf /var/lib/apt/lists/* pre_final: | # Set environment variables for installing tzdata @@ -355,7 +364,7 @@ spack: echo "export CC=icc" >> /home/nonroot/.bashrc && \ echo "export CXX=icpc" >> /home/nonroot/.bashrc && \ echo "export FC=ifort" >> /home/nonroot/.bashrc && \ - echo "source /opt/intel/oneapi/setvars.sh" > /etc/profile.d/02-intel-vars.sh && \ + echo "source /opt/intel/oneapi/setvars.sh" >> /etc/bash.bashrc && \ printf "[credential]\n helper = cache --timeout=7200\n" >> /home/nonroot/.gitconfig && \ chown -R nonroot:nonroot /home/nonroot/.gitconfig # Replicate settings for root user From 2d3dea5f15775f4485b95127808d04d61bd08c3a Mon Sep 17 00:00:00 2001 From: eap Date: Fri, 20 Dec 2024 22:38:41 -0700 Subject: [PATCH 10/31] gcc updates --- .../containers/docker-ubuntu-gcc-openmpi.yaml | 43 ++++++++----------- .../containers/docker-ubuntu-intel-impi.yaml | 2 +- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index 43b5b2fb8..197583b63 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -4,19 +4,19 @@ spack: config: checksum: false - build_jobs: 2 + build_jobs: 5 connect_timeout: 60 compilers: - compiler: - spec: gcc@12.3.0 + spec: gcc@13.3.0 paths: - cc: /usr/bin/gcc-12 - cxx: /usr/bin/g++-12 - f77: /usr/bin/gfortran-12 - fc: /usr/bin/gfortran-12 + cc: /usr/bin/gcc-13 + cxx: /usr/bin/g++-13 + f77: /usr/bin/gfortran-13 + fc: /usr/bin/gfortran-13 flags: {} - operating_system: ubuntu22.04 + operating_system: ubuntu24.04 modules: [] environment: {} extra_rpaths: [] @@ -29,16 +29,16 @@ spack: target: [x86_64] providers: mpi: [openmpi@5.0.5] - compiler: [gcc@12.3.0] + compiler: [gcc@13.3.0] gcc: buildable: false externals: - - spec: gcc@12.3.0 + - spec: gcc@13.3.0 prefix: /usr gcc-runtime: buildable: false externals: - - spec: gcc-runtime@12.3.0 + - spec: gcc-runtime@13.3.0 prefix: /usr diffutils: buildable: false @@ -58,7 +58,7 @@ spack: mysql: buildable: false externals: - - spec: mysql@8.0.39 + - spec: mysql@8.0.40 prefix: /usr qt: buildable: false @@ -71,7 +71,6 @@ spack: externals: - spec: wget@1.21.2 prefix: /usr - version: [1.21.2] specs: [] @@ -106,10 +105,10 @@ spack: os_packages: build: - bc - - cpp-12 - - g++-12 - - gcc-12 - - gfortran-12 + - cpp-13 + - g++-13 + - gcc-13 + - gfortran-13 - git - git-lfs - make @@ -122,10 +121,10 @@ spack: final: - bc - - cpp-12 - - g++-12 - - gcc-12 - - gfortran-12 + - cpp-13 + - g++-13 + - gcc-13 + - gfortran-13 - git - git-lfs - make @@ -154,10 +153,6 @@ spack: # Register spack-stack extension repo RUN spack repo add $SPACK_ROOT/var/spack/repos/spack-stack --scope defaults # Set the GNU 12.x toolchain as the default compilers. - RUN ln -svf /usr/bin/gfortran-12 /usr/bin/gfortran && \ - ln -svf /usr/bin/cpp-12 /usr/bin/cpp && \ - ln -svf /usr/bin/gcc-12 /usr/bin/gcc && \ - ln -svf /usr/bin/g++-12 /usr/bin/g++ pre_final: | # Set environment variables for installing tzdata ENV DEBIAN_FRONTEND=noninteractive diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 856e3a585..60e7bfe66 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -164,7 +164,7 @@ spack: mysql: buildable: false externals: - - spec: mysql@8.0.37 + - spec: mysql@8.0.40 prefix: /usr pixman: require: From bdff3ac3b6c33db25a1df0c8de36852b2851b795 Mon Sep 17 00:00:00 2001 From: eap Date: Fri, 20 Dec 2024 22:47:49 -0700 Subject: [PATCH 11/31] Fix ubuntu version for gcc --- configs/containers/docker-ubuntu-gcc-openmpi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index 197583b63..af25f97e8 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -92,7 +92,7 @@ spack: # Sets the base images for the stages where Spack builds the # software or where the software gets installed after being built.. images: - os: ubuntu:22.04 + os: ubuntu:24.04 spack: url: https://github.com/jcsda/spack ref: spack-stack-dev From 21e8d58552fb0763f6d1b581c010a4aa0bafe5dc Mon Sep 17 00:00:00 2001 From: eap Date: Sat, 21 Dec 2024 15:48:30 -0700 Subject: [PATCH 12/31] fixes for intel gcc --- .../containers/docker-ubuntu-gcc-openmpi.yaml | 17 +++++++++++------ .../containers/docker-ubuntu-intel-impi.yaml | 4 ++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index af25f97e8..c2a635431 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -55,6 +55,11 @@ spack: externals: - spec: git-lfs@3.0.2 prefix: /usr + llvm: + buildable: false + externals: + - spec: llvm@14.0.6 + prefix: /usr mysql: buildable: false externals: @@ -111,6 +116,7 @@ spack: - gfortran-13 - git - git-lfs + - llvm-14 - make - mysql-server - qtbase5-dev @@ -127,6 +133,7 @@ spack: - gfortran-13 - git - git-lfs + - llvm-14 - make - mysql-server - qtbase5-dev @@ -152,7 +159,8 @@ spack: COPY spack-ext-SPACK_STACK_HASH/repos/spack-stack $SPACK_ROOT/var/spack/repos/spack-stack # Register spack-stack extension repo RUN spack repo add $SPACK_ROOT/var/spack/repos/spack-stack --scope defaults - # Set the GNU 12.x toolchain as the default compilers. + # Needed by py-llvmlite + update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 pre_final: | # Set environment variables for installing tzdata ENV DEBIAN_FRONTEND=noninteractive @@ -168,11 +176,6 @@ spack: final: | # Copy spack find output from builder COPY --from=builder /root/spack_find.out /root/spack_find.out - # Set the GNU 12.x toolchain as the default compilers. - RUN ln -svf /usr/bin/gfortran-12 /usr/bin/gfortran && \ - ln -svf /usr/bin/cpp-12 /usr/bin/cpp && \ - ln -svf /usr/bin/gcc-12 /usr/bin/gcc && \ - ln -svf /usr/bin/g++-12 /usr/bin/g++ # Make a non-root user:nonroot / group:nonroot for running MPI RUN useradd -U -k /etc/skel -s /bin/bash -d /home/nonroot -m nonroot --uid 43891 && \ echo "ulimit -s unlimited" >> /home/nonroot/.bashrc && \ @@ -180,6 +183,8 @@ spack: echo "export CC=gcc" >> /home/nonroot/.bashrc && \ echo "export CXX=g++" >> /home/nonroot/.bashrc && \ echo "export FC=gfortran" >> /home/nonroot/.bashrc && \ + # Needed by py-llvmlite + update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \ printf "[credential]\n helper = cache --timeout=7200\n" >> /home/nonroot/.gitconfig && \ mkdir /home/nonroot/.pmix && \ echo "rmaps_default_mapping_policy=:oversubscribe" >> /home/nonroot/.pmix/mca-params.conf && \ diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 60e7bfe66..f552987e0 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -129,7 +129,7 @@ spack: intel-oneapi-mkl: buildable: false externals: - - spec: intel-oneapi-mpi@2023.2.0 + - spec: intel-oneapi-mkl@2023.2.0 prefix: /opt/intel/oneapi intel-oneapi-mpi: buildable: false @@ -140,7 +140,7 @@ spack: intel-oneapi-tbb: buildable: false externals: - - spec: intel-oneapi-mpi@2021.10.0 + - spec: intel-oneapi-tbb@2021.10.0 prefix: /opt/intel/oneapi libtool: externals: From 14495cb51c5d38943f2cc71718042cba26dda88f Mon Sep 17 00:00:00 2001 From: eap Date: Sat, 21 Dec 2024 15:51:34 -0700 Subject: [PATCH 13/31] fix llvm alternative --- configs/containers/docker-ubuntu-gcc-openmpi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index c2a635431..7469d904d 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -158,7 +158,7 @@ spack: # Copy the spack-stack extension repo into the spack code COPY spack-ext-SPACK_STACK_HASH/repos/spack-stack $SPACK_ROOT/var/spack/repos/spack-stack # Register spack-stack extension repo - RUN spack repo add $SPACK_ROOT/var/spack/repos/spack-stack --scope defaults + RUN spack repo add $SPACK_ROOT/var/spack/repos/spack-stack --scope defaults && \ # Needed by py-llvmlite update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 pre_final: | From f982b7f1b6c411fd2e902c268f2c93154e5b517c Mon Sep 17 00:00:00 2001 From: eap Date: Sat, 21 Dec 2024 21:11:30 -0700 Subject: [PATCH 14/31] remove unnecessary requirements --- configs/containers/docker-ubuntu-intel-impi.yaml | 4 ++-- configs/containers/specs/jedi-ci.yaml | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index f2843e266..15fa3703d 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -187,8 +187,8 @@ spack: - spec: py-pynacl@1.5.0 prefix: /usr py-numpy: - require:: - - '@:1.23 ^openblas' + require: + - '^intel-oneapi-mkl' # Turn off crypt, because libxcrypt doesn't # build with Intel. python: diff --git a/configs/containers/specs/jedi-ci.yaml b/configs/containers/specs/jedi-ci.yaml index fdf5b8708..dcf3ce744 100644 --- a/configs/containers/specs/jedi-ci.yaml +++ b/configs/containers/specs/jedi-ci.yaml @@ -32,17 +32,12 @@ netcdf-c@4.9.2, netcdf-cxx4@4.3.1, netcdf-fortran@4.6.1, - openssl@3.4.0, parallelio@2.6.2 +pnetcdf, parallel-netcdf@1.12.3, - py-boto3@1.34.44, - py-cartopy@0.24.1, - py-cryptography@42.0.8, py-eccodes, py-f90nml, py-gitpython, py-h5py, - py-numpy, py-pandas, py-pip, py-pyyaml, @@ -58,7 +53,7 @@ zstd, odc@1.5.2, shumlib@macos_clang_linux_intel_port, - awscli-v2@2.15.53, + awscli-v2, py-globus-cli@3.16.0, py-fortranformat@2.0.0, # Added for new CI system 2024/04/30 From 941bd4ae24a254f0e374dd0259f5f40eda319faf Mon Sep 17 00:00:00 2001 From: eap Date: Sat, 21 Dec 2024 21:13:26 -0700 Subject: [PATCH 15/31] remove old oneapi-mkl with 'DO NOT SUBMIT' --- configs/containers/docker-ubuntu-intel-impi.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 15fa3703d..a6206e5bc 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -91,12 +91,6 @@ spack: externals: - spec: findutils@4.8.0 prefix: /usr - # DO Not submit - figure out what is going on - MLK is not installed so it must be built but might cause errors - #intel-oneapi-mkl: - # buildable: false - #externals: - #- spec: intel-oneapi-mkl@2022.1.0 - # prefix: /opt/intel/oneapi flex: externals: - spec: flex@2.6.4+lex From b24b879ad6bcfcd74567e52a01a87d877704a98c Mon Sep 17 00:00:00 2001 From: eap Date: Sat, 21 Dec 2024 21:17:08 -0700 Subject: [PATCH 16/31] re-order 'all' packages --- configs/containers/docker-ubuntu-intel-impi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index a6206e5bc..8f2bd3c8a 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -57,9 +57,9 @@ spack: all: require: '%intel' target: [core2] + compiler: [intel@2021.10.0, gcc@11.4.0] providers: mpi: [intel-oneapi-mpi@2021.10.0] - compiler: [intel@2021.10.0, gcc@11.4.0] autoconf: externals: - spec: autoconf@2.71 From 8d83be8234bfb8356ca8753af7927dda23a0f9c7 Mon Sep 17 00:00:00 2001 From: eap Date: Sun, 22 Dec 2024 15:39:49 -0700 Subject: [PATCH 17/31] Update ubuntu-clang image --- .../containers/docker-ubuntu-clang-mpich.yaml | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/configs/containers/docker-ubuntu-clang-mpich.yaml b/configs/containers/docker-ubuntu-clang-mpich.yaml index 9bcc1c866..cf83dc500 100644 --- a/configs/containers/docker-ubuntu-clang-mpich.yaml +++ b/configs/containers/docker-ubuntu-clang-mpich.yaml @@ -16,7 +16,7 @@ spack: f77: /usr/bin/gfortran fc: /usr/bin/gfortran flags: {} - operating_system: ubuntu22.04 + operating_system: ubuntu24.04 modules: [] environment: prepend_path: @@ -30,7 +30,7 @@ spack: require: '%clang' target: [x86_64] providers: - mpi: [mpich@4.2.1] + mpi: [mpich@4.2.3] compiler: [clang@14.0.0] gcc: buildable: false @@ -56,9 +56,9 @@ spack: mpich: buildable: false externals: - - spec: mpich@4.2.1 - prefix: /opt/mpich-4.2.1 - version: [4.2.1] + - spec: mpich@4.2.3 + prefix: /opt/mpich-4.2.3 + version: [4.2.3] diffutils: buildable: false externals: @@ -82,7 +82,7 @@ spack: mysql: buildable: false externals: - - spec: mysql@8.0.39 + - spec: mysql@8.0.40 prefix: /usr qt: buildable: false @@ -93,9 +93,9 @@ spack: wget: buildable: false externals: - - spec: wget@1.21.2 + - spec: wget@1.21.4 prefix: /usr - version: [1.21.2] + version: [1.21.4] specs: [] @@ -117,7 +117,7 @@ spack: # Sets the base images for the stages where Spack builds the # software or where the software gets installed after being built.. images: - os: ubuntu:22.04 + os: ubuntu:24.04 spack: url: https://github.com/jcsda/spack ref: spack-stack-dev @@ -197,15 +197,15 @@ spack: # Register spack-stack extension repo RUN spack repo add $SPACK_ROOT/var/spack/repos/spack-stack --scope defaults # Set default binaries for clang-14 and GNU-12. - RUN ln -svf /usr/bin/gfortran-12 /usr/bin/gfortran && \ - ln -svf /usr/bin/cpp-12 /usr/bin/cpp && \ - ln -svf /usr/bin/gcc-12 /usr/bin/gcc && \ - ln -svf /usr/bin/g++-12 /usr/bin/g++ && \ - ln -svf /usr/bin/clang-14 /usr/bin/clang && \ - ln -svf /usr/bin/clang++-14 /usr/bin/clang++ && \ - ln -svf /usr/bin/clang-cpp-14 /usr/bin/clang-cpp + RUN rm /usr/bin/cpp && ln -s /usr/bin/cpp-12 /usr/bin/cpp && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100 && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100 && \ + update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-12 100 && \ + update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 100 && \ + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-14 100 && \ + update-alternatives --install /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-14 100 # Build mpich outside of spack-stack - ENV MPICH_VERSION=4.2.1 + ENV MPICH_VERSION=4.2.3 ENV CC=clang ENV CXX=clang++ ENV FC=gfortran @@ -231,23 +231,23 @@ spack: # Set environment variables for installing tzdata ENV DEBIAN_FRONTEND=noninteractive ENV TZ=Etc/UTC - ENV MPICH_VERSION=4.2.1 + ENV MPICH_VERSION=4.2.3 ENV PATH=/opt/mpich-${MPICH_VERSION}/bin:${PATH} ENV CPATH=/opt/mpich-${MPICH_VERSION}/include:${CPATH} ENV LD_LIBRARY_PATH=/opt/mpich-${MPICH_VERSION}/lib:${LD_LIBRARY_PATH} final: | # Copy spack find output from builder COPY --from=builder /root/spack_find.out /root/spack_find.out - ENV MPICH_VERSION=4.2.1 + ENV MPICH_VERSION=4.2.3 # Set the GNU 12.x toolchain as the default compilers. # Set default binaries for clang-14 and GNU-12. - RUN ln -svf /usr/bin/gfortran-12 /usr/bin/gfortran && \ - ln -svf /usr/bin/cpp-12 /usr/bin/cpp && \ - ln -svf /usr/bin/gcc-12 /usr/bin/gcc && \ - ln -svf /usr/bin/g++-12 /usr/bin/g++ && \ - ln -svf /usr/bin/clang-14 /usr/bin/clang && \ - ln -svf /usr/bin/clang++-14 /usr/bin/clang++ && \ - ln -svf /usr/bin/clang-cpp-14 /usr/bin/clang-cpp + RUN rm /usr/bin/cpp && ln -s /usr/bin/cpp-12 /usr/bin/cpp && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100 && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100 && \ + update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-12 100 && \ + update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 100 && \ + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-14 100 && \ + update-alternatives --install /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-14 100 # Copy mpich-${MPICH_VERSION} installation from builder COPY --from=builder /opt/mpich-${MPICH_VERSION} /opt/mpich-${MPICH_VERSION} # Make a non-root user:nonroot / group:nonroot for running MPI From fe3da1d5371fc917d5a25ba12d5a5122ee0c4df5 Mon Sep 17 00:00:00 2001 From: eap Date: Sun, 22 Dec 2024 21:49:13 -0700 Subject: [PATCH 18/31] add intel oneapi --- .../specs/docker-ubuntu-oneapi-impi.yaml | 368 ++++++++++++++++++ 1 file changed, 368 insertions(+) create mode 100644 configs/containers/specs/docker-ubuntu-oneapi-impi.yaml diff --git a/configs/containers/specs/docker-ubuntu-oneapi-impi.yaml b/configs/containers/specs/docker-ubuntu-oneapi-impi.yaml new file mode 100644 index 000000000..fabb219fa --- /dev/null +++ b/configs/containers/specs/docker-ubuntu-oneapi-impi.yaml @@ -0,0 +1,368 @@ +spack: + concretizer: + unify: true + + config: + checksum: false + build_jobs: 2 + connect_timeout: 60 + + compilers: + - compiler: + spec: gcc@=13.4.0 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: {} + operating_system: ubuntu24.04 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: oneapi@=2025.0.0 + paths: + cc: /opt/intel/oneapi/compiler/2025.0/bin/icx + cxx: /opt/intel/oneapi/compiler/2025.0/bin/icpx + f77: /opt/intel/oneapi/compiler/2025.0/bin/ifx + fc: /opt/intel/oneapi/compiler/2025.0/bin/ifx + flags: {} + operating_system: ubuntu24.04 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + + # Basic package config from configs/common/packages.yaml + # Additional package config for container + packages: + all: + require: '%oneapi' + target: [core2] + compiler: [oneapi@2025.0.0, gcc@13.4.0] + providers: + mpi: [intel-oneapi-mpi@2021.14.0] + autoconf: + externals: + - spec: autoconf@2.71 + prefix: /usr + automake: + externals: + - spec: automake@1.16.5 + prefix: /usr + binutils: + externals: + - spec: binutils@2.38 + prefix: /usr + bison: + require: + - '%gcc' + #boost: + # require: + # - '%gcc' + coreutils: + externals: + - spec: coreutils@8.32 + prefix: /usr + diffutils: + buildable: false + externals: + - spec: diffutils@3.8 + prefix: /usr + ectrans: + require: + - '+mkl ~fftw' + #ecflow: + # require: + # - '%gcc' + #findutils: + # externals: + # - spec: findutils@4.8.0 + # prefix: /usr + #flex: + # externals: + # - spec: flex@2.6.4+lex + # prefix: /usr + #gawk: + # externals: + # - spec: gawk@5.1.0 + # prefix: /usr + git: + buildable: false + externals: + - spec: git@2.34.1~tcltk + prefix: /usr + git-lfs: + buildable: false + externals: + - spec: git-lfs@3.0.2 + prefix: /usr + gmake: + externals: + - spec: gmake@4.3 + prefix: /usr + gsibec: + require: + - '+mkl' + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2025.0.0 + prefix: /opt/intel/oneapi + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.14.0%oneapi@2025.0.0 + prefix: /opt/intel/oneapi + intel-oneapi-tbb: + buildable: false + externals: + - spec: intel-oneapi-tbb@2022.0.0 + prefix: /opt/intel/oneapi + #libtool: + # externals: + # - spec: libtool@2.4.6 + # prefix: /usr + mpi: + buildable: False + #m4: + # externals: + # - spec: m4@1.4.18 + # prefix: /usr + #lcms: + # externals: + # - spec: lcms@2.13.1 + # prefix: /usr + libmd: + require: + - '%gcc' + libbsd: + require: + - '%gcc' + llvm: + buildable: false + externals: + - spec: llvm@14.0.6 + #grep: + # buildable: false + # externals: + # - spec: grep@3.7 + # prefix: /usr + mysql: + buildable: false + externals: + - spec: mysql@8.0.40 + prefix: /usr + #perl: + # externals: + # - spec: perl@5.34.0~cpanm+opcode+open+shared+threads + # prefix: /usr + pkg-config: + externals: + - spec: pkg-config@0.29.2 + prefix: /usr + py-pynacl: + buildable: false + externals: + - spec: py-pynacl@1.5.0 + prefix: /usr + py-numpy: + require: + - '^intel-oneapi-mkl' + # Turn off crypt, because libxcrypt doesn't + # build with Intel. + #python: + # variants: ~crypt + #qt: + # buildable: false + # externals: + # - spec: qt@5.15.3 + # prefix: /usr + # version: [5.15.3] + #sed: + # externals: + # - spec: sed@4.8 + # prefix: /usr + tar: + externals: + - spec: tar@1.34 + prefix: /usr + wget: + buildable: false + externals: + - spec: wget@1.21.2 + prefix: /usr + gettext: + externals: + - spec: gettext@0.21 + prefix: /usr + + specs: [] + + container: + + # Select the format of the recipe e.g. docker, + # singularity or anything else that is currently supported + format: docker + # How to use: + #$ spack containerize > Dockerfile + #$ sudo docker build -t myimage . + #$ sudo docker run -it myimage + + #format: singularity + # How to use: + #$ spack containerize > singularity.def + #$ sudo singularity build singularity.sif singularity.def + + # Sets the base images for the stages where Spack builds the + # software or where the software gets installed after being built.. + images: + os: ubuntu:24.04 + spack: + url: https://github.com/jcsda/spack + ref: spack-stack-dev + resolve_sha: true + + # Whether or not to strip binaries + strip: false + + ## Additional system packages that are needed at runtime + os_packages: + build: + - autoconf + - automake + - autopoint + - bc + - cpp-13 + - cmake + - flex + - g++-13 + - gcc-13 + - gettext + - gfortran-13 + - git + - git-lfs + - gpg + - libgomp1 + - llvm-14 + - mysql-server + - pkg-config + - qtbase5-dev + - qt5-qmake + - libqt5svg5-dev + - qt5dxcb-plugin + - libcurl4-openssl-dev + - wget + + final: + - autopoint + - bc + - cpp-13 + - g++-13 + - gcc-13 + - gettext + - gfortran-13 + - git + - git-lfs + - gpg + - libgomp1 + - llvm-14 + - make + - mysql-server + - pkg-config + - qtbase5-dev + - qt5-qmake + - libqt5svg5-dev + - qt5dxcb-plugin + - libcurl4-openssl-dev + - wget + # Implicitly included in build step + - build-essential + - ca-certificates + - curl + - file + - gnupg2 + - iproute2 + - locales + - python3 + - python3-pip + - python3-setuptools + - unzip + - vim + - nano + - less + + # Extra instructions + extra_instructions: + pre_build: | + # Copy the spack-stack extension repo into the spack code + COPY spack-ext-SPACK_STACK_HASH/repos/spack-stack $SPACK_ROOT/var/spack/repos/spack-stack + # Register spack-stack extension repo + RUN spack repo add $SPACK_ROOT/var/spack/repos/spack-stack --scope defaults + # Install Intel compilers and MPI library + RUN apt update && apt install apt-utils && \ + wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \ + echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \ + apt update && \ + apt install intel-oneapi-compiler-dpcpp-cpp-2025.0 intel-oneapi-compiler-fortran-2025.0 intel-oneapi-mkl-devel-2025.0 intel-oneapi-mpi-devel-2021.14 intel-oneapi-tbb-devel-2022.0 -y && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100 && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 && \ + update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-13 100 && \ + rm /usr/bin/cpp && ln -s /usr/bin/cpp-13 /usr/bin/cpp && \ + # Needed by py-llvmlite + update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \ + rm -rf /var/lib/apt/lists/* + pre_final: | + # Set environment variables for installing tzdata + ENV DEBIAN_FRONTEND=noninteractive + ENV TZ=Etc/UTC + ENV CC=icc + ENV CXX=icpc + ENV FC=ifort + build: | + # Put output of spack find into a file + RUN cd /opt/spack-environment && \ + spack env activate -d . && \ + spack find 2>&1 | tee /root/spack_find.out + final: | + # Install Intel compilers and MPI library and pynacl + RUN apt update && apt install apt-utils && \ + wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \ + echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \ + apt update && \ + apt install intel-oneapi-compiler-dpcpp-cpp-2025.0 intel-oneapi-compiler-fortran-2025.0 intel-oneapi-mkl-devel-2025.0 intel-oneapi-mpi-devel-2021.14 intel-oneapi-tbb-devel-2022.0 -y && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100 && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 && \ + update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-13 100 && \ + rm /usr/bin/cpp && ln -s /usr/bin/cpp-13 /usr/bin/cpp && \ + # Needed by py-llvmlite + update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \ + rm -rf /var/lib/apt/lists/* && \ + /opt/views/view/bin/pip3 install pynacl + # Copy spack find output from builder + COPY --from=builder /root/spack_find.out /root/spack_find.out + # Make a non-root user:nonroot / group:nonroot for running MPI + RUN useradd -U -k /etc/skel -s /bin/bash -d /home/nonroot -m nonroot --uid 43891 && \ + echo "ulimit -s unlimited" >> /home/nonroot/.bashrc && \ + echo "ulimit -v unlimited" >> /home/nonroot/.bashrc && \ + echo "export CC=icc" >> /home/nonroot/.bashrc && \ + echo "export CXX=icpc" >> /home/nonroot/.bashrc && \ + echo "export FC=ifort" >> /home/nonroot/.bashrc && \ + echo "source /opt/intel/oneapi/setvars.sh" >> /etc/bash.bashrc && \ + printf "[credential]\n helper = cache --timeout=7200\n" >> /home/nonroot/.gitconfig && \ + chown -R nonroot:nonroot /home/nonroot/.gitconfig + # Replicate settings for root user + RUN echo "ulimit -s unlimited" >> /root/.bashrc && \ + echo "ulimit -v unlimited" >> /root/.bashrc && \ + echo "export CC=icc" >> /root/.bashrc && \ + echo "export CXX=icpc" >> /root/.bashrc && \ + echo "export FC=ifort" >> /root/.bashrc && \ + echo "export jedi_cmake_ROOT=$(ls -d /opt/software/linux-ubuntu*/intel-*/jedi-cmake-*)" >> /root/.bashrc && \ + printf "[credential]\n helper = cache --timeout=7200\n" >> /root/.gitconfig + + # Labels for the image + labels: + app: "" + mpi: "intel-oneapi-mpi" From 4c5520be5a3314a2d7faa2ec8ad918aab2c80fbc Mon Sep 17 00:00:00 2001 From: eap Date: Sun, 22 Dec 2024 21:53:43 -0700 Subject: [PATCH 19/31] move file --- configs/containers/{specs => }/docker-ubuntu-oneapi-impi.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename configs/containers/{specs => }/docker-ubuntu-oneapi-impi.yaml (100%) diff --git a/configs/containers/specs/docker-ubuntu-oneapi-impi.yaml b/configs/containers/docker-ubuntu-oneapi-impi.yaml similarity index 100% rename from configs/containers/specs/docker-ubuntu-oneapi-impi.yaml rename to configs/containers/docker-ubuntu-oneapi-impi.yaml From 517ae53081d0a7ebac1e1757eecaddd751dd75bc Mon Sep 17 00:00:00 2001 From: eap Date: Mon, 23 Dec 2024 00:17:39 -0700 Subject: [PATCH 20/31] several fixes --- .../containers/docker-ubuntu-oneapi-impi.yaml | 33 ++++++++++--------- configs/containers/specs/jedi-ci.yaml | 2 +- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/configs/containers/docker-ubuntu-oneapi-impi.yaml b/configs/containers/docker-ubuntu-oneapi-impi.yaml index fabb219fa..d17b78007 100644 --- a/configs/containers/docker-ubuntu-oneapi-impi.yaml +++ b/configs/containers/docker-ubuntu-oneapi-impi.yaml @@ -4,7 +4,7 @@ spack: config: checksum: false - build_jobs: 2 + build_jobs: 6 connect_timeout: 60 compilers: @@ -43,7 +43,7 @@ spack: target: [core2] compiler: [oneapi@2025.0.0, gcc@13.4.0] providers: - mpi: [intel-oneapi-mpi@2021.14.0] + mpi: [intel-oneapi-mpi@2021.14] autoconf: externals: - spec: autoconf@2.71 @@ -109,17 +109,17 @@ spack: intel-oneapi-mkl: buildable: false externals: - - spec: intel-oneapi-mkl@2025.0.0 + - spec: intel-oneapi-mkl@2025.0 prefix: /opt/intel/oneapi intel-oneapi-mpi: buildable: false externals: - - spec: intel-oneapi-mpi@2021.14.0%oneapi@2025.0.0 + - spec: intel-oneapi-mpi@2021.14%oneapi@2025.0.0 prefix: /opt/intel/oneapi intel-oneapi-tbb: buildable: false externals: - - spec: intel-oneapi-tbb@2022.0.0 + - spec: intel-oneapi-tbb@2022.0 prefix: /opt/intel/oneapi #libtool: # externals: @@ -307,6 +307,9 @@ spack: echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \ apt update && \ apt install intel-oneapi-compiler-dpcpp-cpp-2025.0 intel-oneapi-compiler-fortran-2025.0 intel-oneapi-mkl-devel-2025.0 intel-oneapi-mpi-devel-2021.14 intel-oneapi-tbb-devel-2022.0 -y && \ + ln -s /opt/intel/oneapi/mpi/2021.14 /opt/intel/oneapi/mpi/2021.14.0 && \ + ln -s /opt/intel/oneapi/mkl/2025.0 /opt/intel/oneapi/mkl/2025.0.0 && \ + ln -s /opt/intel/oneapi/compiler/2025.0 /opt/intel/oneapi/compiler/2025.0.0 && \ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100 && \ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 && \ update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-13 100 && \ @@ -318,9 +321,9 @@ spack: # Set environment variables for installing tzdata ENV DEBIAN_FRONTEND=noninteractive ENV TZ=Etc/UTC - ENV CC=icc - ENV CXX=icpc - ENV FC=ifort + ENV CC=icx + ENV CXX=icpx + ENV FC=ifx build: | # Put output of spack find into a file RUN cd /opt/spack-environment && \ @@ -347,19 +350,19 @@ spack: RUN useradd -U -k /etc/skel -s /bin/bash -d /home/nonroot -m nonroot --uid 43891 && \ echo "ulimit -s unlimited" >> /home/nonroot/.bashrc && \ echo "ulimit -v unlimited" >> /home/nonroot/.bashrc && \ - echo "export CC=icc" >> /home/nonroot/.bashrc && \ - echo "export CXX=icpc" >> /home/nonroot/.bashrc && \ - echo "export FC=ifort" >> /home/nonroot/.bashrc && \ + echo "export CC=icx" >> /home/nonroot/.bashrc && \ + echo "export CXX=icpx" >> /home/nonroot/.bashrc && \ + echo "export FC=ifx" >> /home/nonroot/.bashrc && \ echo "source /opt/intel/oneapi/setvars.sh" >> /etc/bash.bashrc && \ printf "[credential]\n helper = cache --timeout=7200\n" >> /home/nonroot/.gitconfig && \ chown -R nonroot:nonroot /home/nonroot/.gitconfig # Replicate settings for root user RUN echo "ulimit -s unlimited" >> /root/.bashrc && \ echo "ulimit -v unlimited" >> /root/.bashrc && \ - echo "export CC=icc" >> /root/.bashrc && \ - echo "export CXX=icpc" >> /root/.bashrc && \ - echo "export FC=ifort" >> /root/.bashrc && \ - echo "export jedi_cmake_ROOT=$(ls -d /opt/software/linux-ubuntu*/intel-*/jedi-cmake-*)" >> /root/.bashrc && \ + echo "export CC=icx" >> /root/.bashrc && \ + echo "export CXX=icpx" >> /root/.bashrc && \ + echo "export FC=ifx" >> /root/.bashrc && \ + echo "export jedi_cmake_ROOT=$(ls -d /opt/software/linux-ubuntu*/oneapi-*/jedi-cmake-*)" >> /root/.bashrc && \ printf "[credential]\n helper = cache --timeout=7200\n" >> /root/.gitconfig # Labels for the image diff --git a/configs/containers/specs/jedi-ci.yaml b/configs/containers/specs/jedi-ci.yaml index dcf3ce744..1f4061e0d 100644 --- a/configs/containers/specs/jedi-ci.yaml +++ b/configs/containers/specs/jedi-ci.yaml @@ -15,7 +15,7 @@ eckit@1.28.3, ecmwf-atlas@0.40.0 +fckit +trans +tesselation +fftw, fiat@1.4.1, - ectrans@1.2.0 +fftw, + ectrans@1.2.0, eigen@3.4.0, fckit@0.13.2, fms@2024.02, From 0d1dfcde609534ef2b32a5c3a992a5c6074e23af Mon Sep 17 00:00:00 2001 From: eap Date: Mon, 23 Dec 2024 10:12:48 -0700 Subject: [PATCH 21/31] fix build jobs for intel and gcc --- configs/containers/docker-ubuntu-gcc-openmpi.yaml | 2 +- configs/containers/docker-ubuntu-intel-impi.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index 7469d904d..9a1ed0eb9 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -4,7 +4,7 @@ spack: config: checksum: false - build_jobs: 5 + build_jobs: 2 connect_timeout: 60 compilers: diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 8f2bd3c8a..c2a418083 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -4,7 +4,7 @@ spack: config: checksum: false - build_jobs: 5 + build_jobs: 2 connect_timeout: 60 compilers: From 60b1cc1b79ab61de4658367e0b9b9f5f11c438d5 Mon Sep 17 00:00:00 2001 From: eap Date: Mon, 23 Dec 2024 11:00:40 -0700 Subject: [PATCH 22/31] fix intel compiler version --- .../containers/docker-ubuntu-oneapi-impi.yaml | 52 +++++++++++++------ 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/configs/containers/docker-ubuntu-oneapi-impi.yaml b/configs/containers/docker-ubuntu-oneapi-impi.yaml index d17b78007..53aa57c03 100644 --- a/configs/containers/docker-ubuntu-oneapi-impi.yaml +++ b/configs/containers/docker-ubuntu-oneapi-impi.yaml @@ -1,3 +1,20 @@ +###### DO NOT SUBMIT THIS BLOCK ##### +## Intel packages +# intel-oneapi-compiler-dpcpp-cpp-2024.2 \ +# intel-oneapi-compiler-fortran-2024.2 \ +# intel-oneapi-mpi-devel-2021.13 \ +# intel-oneapi-tbb-devel-2021.13 \ +# intel-oneapi-mkl-devel-2024.2 +# +# +## Versions reported by apt cache policy +# compiler - 2024.2.1 +# fortran 2024.2.1 +# mpi - 2021.13.1 +# tbb - 2021.13.1 +# mkl - 2024.2.2 +#### /end DNS ######################## + spack: concretizer: unify: true @@ -22,17 +39,20 @@ spack: environment: {} extra_rpaths: [] - compiler: - spec: oneapi@=2025.0.0 + spec: oneapi@=2024.2.1 paths: - cc: /opt/intel/oneapi/compiler/2025.0/bin/icx - cxx: /opt/intel/oneapi/compiler/2025.0/bin/icpx - f77: /opt/intel/oneapi/compiler/2025.0/bin/ifx - fc: /opt/intel/oneapi/compiler/2025.0/bin/ifx + cc: /opt/intel/oneapi/compiler/2024.2/bin/icx + cxx: /opt/intel/oneapi/compiler/2024.2/bin/icpx + f77: /opt/intel/oneapi/compiler/2024.2/bin/ifx + fc: /opt/intel/oneapi/compiler/2024.2/bin/ifx flags: {} operating_system: ubuntu24.04 target: x86_64 modules: [] - environment: {} + environment: + # change to append_path once relevant PR is merged + append_path: + CPATH: /opt/intel/oneapi/compiler/2024.2/opt/compiler/include/intel64 extra_rpaths: [] # Basic package config from configs/common/packages.yaml @@ -41,9 +61,9 @@ spack: all: require: '%oneapi' target: [core2] - compiler: [oneapi@2025.0.0, gcc@13.4.0] + compiler: [oneapi@2024.2.1, gcc@13.4.0] providers: - mpi: [intel-oneapi-mpi@2021.14] + mpi: [intel-oneapi-mpi@2021.13] autoconf: externals: - spec: autoconf@2.71 @@ -109,17 +129,17 @@ spack: intel-oneapi-mkl: buildable: false externals: - - spec: intel-oneapi-mkl@2025.0 + - spec: intel-oneapi-mkl@2024.2 prefix: /opt/intel/oneapi intel-oneapi-mpi: buildable: false externals: - - spec: intel-oneapi-mpi@2021.14%oneapi@2025.0.0 + - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.1 prefix: /opt/intel/oneapi intel-oneapi-tbb: buildable: false externals: - - spec: intel-oneapi-tbb@2022.0 + - spec: intel-oneapi-tbb@2021.13 prefix: /opt/intel/oneapi #libtool: # externals: @@ -306,10 +326,10 @@ spack: wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \ echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \ apt update && \ - apt install intel-oneapi-compiler-dpcpp-cpp-2025.0 intel-oneapi-compiler-fortran-2025.0 intel-oneapi-mkl-devel-2025.0 intel-oneapi-mpi-devel-2021.14 intel-oneapi-tbb-devel-2022.0 -y && \ - ln -s /opt/intel/oneapi/mpi/2021.14 /opt/intel/oneapi/mpi/2021.14.0 && \ - ln -s /opt/intel/oneapi/mkl/2025.0 /opt/intel/oneapi/mkl/2025.0.0 && \ - ln -s /opt/intel/oneapi/compiler/2025.0 /opt/intel/oneapi/compiler/2025.0.0 && \ + apt install intel-oneapi-compiler-dpcpp-cpp-2024.2 intel-oneapi-compiler-fortran-2024.2 intel-oneapi-mpi-devel-2021.13 intel-oneapi-tbb-devel-2021.13 intel-oneapi-mkl-devel-2024.2 -y && \ + #ln -s /opt/intel/oneapi/mpi/2021.13 /opt/intel/oneapi/mpi/2021.13.1 && \ + #ln -s /opt/intel/oneapi/mkl/2024.2 /opt/intel/oneapi/mkl/2024.2.2 && \ + #ln -s /opt/intel/oneapi/compiler/2024.2 /opt/intel/oneapi/compiler/2024.2.1 && \ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100 && \ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 && \ update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-13 100 && \ @@ -335,7 +355,7 @@ spack: wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \ echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \ apt update && \ - apt install intel-oneapi-compiler-dpcpp-cpp-2025.0 intel-oneapi-compiler-fortran-2025.0 intel-oneapi-mkl-devel-2025.0 intel-oneapi-mpi-devel-2021.14 intel-oneapi-tbb-devel-2022.0 -y && \ + apt install intel-oneapi-compiler-dpcpp-cpp-2024.2 intel-oneapi-compiler-fortran-2024.2 intel-oneapi-mpi-devel-2021.13 intel-oneapi-tbb-devel-2021.13 intel-oneapi-mkl-devel-2024.2 -y && \ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100 && \ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 && \ update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-13 100 && \ From 6c59f6ad58b0c649740abb480d3c5fec98c13d25 Mon Sep 17 00:00:00 2001 From: eap Date: Mon, 23 Dec 2024 14:12:35 -0700 Subject: [PATCH 23/31] ifort correct path --- configs/containers/docker-ubuntu-oneapi-impi.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/configs/containers/docker-ubuntu-oneapi-impi.yaml b/configs/containers/docker-ubuntu-oneapi-impi.yaml index 53aa57c03..d6b85e63f 100644 --- a/configs/containers/docker-ubuntu-oneapi-impi.yaml +++ b/configs/containers/docker-ubuntu-oneapi-impi.yaml @@ -43,8 +43,10 @@ spack: paths: cc: /opt/intel/oneapi/compiler/2024.2/bin/icx cxx: /opt/intel/oneapi/compiler/2024.2/bin/icpx - f77: /opt/intel/oneapi/compiler/2024.2/bin/ifx - fc: /opt/intel/oneapi/compiler/2024.2/bin/ifx + #f77: /opt/intel/oneapi/compiler/2024.2/bin/ifx + #fc: /opt/intel/oneapi/compiler/2024.2/bin/ifx + f77: /opt/intel/oneapi/compiler/2024.2/bin/ifort + fc: /opt/intel/oneapi/compiler/2024.2/bin/ifort flags: {} operating_system: ubuntu24.04 target: x86_64 @@ -343,7 +345,7 @@ spack: ENV TZ=Etc/UTC ENV CC=icx ENV CXX=icpx - ENV FC=ifx + ENV FC=ifort build: | # Put output of spack find into a file RUN cd /opt/spack-environment && \ @@ -372,7 +374,7 @@ spack: echo "ulimit -v unlimited" >> /home/nonroot/.bashrc && \ echo "export CC=icx" >> /home/nonroot/.bashrc && \ echo "export CXX=icpx" >> /home/nonroot/.bashrc && \ - echo "export FC=ifx" >> /home/nonroot/.bashrc && \ + echo "export FC=ifort" >> /home/nonroot/.bashrc && \ echo "source /opt/intel/oneapi/setvars.sh" >> /etc/bash.bashrc && \ printf "[credential]\n helper = cache --timeout=7200\n" >> /home/nonroot/.gitconfig && \ chown -R nonroot:nonroot /home/nonroot/.gitconfig @@ -381,7 +383,7 @@ spack: echo "ulimit -v unlimited" >> /root/.bashrc && \ echo "export CC=icx" >> /root/.bashrc && \ echo "export CXX=icpx" >> /root/.bashrc && \ - echo "export FC=ifx" >> /root/.bashrc && \ + echo "export FC=ifort" >> /root/.bashrc && \ echo "export jedi_cmake_ROOT=$(ls -d /opt/software/linux-ubuntu*/oneapi-*/jedi-cmake-*)" >> /root/.bashrc && \ printf "[credential]\n helper = cache --timeout=7200\n" >> /root/.gitconfig From fe41b1d7922315ef20aaa11be0832cbd526b23af Mon Sep 17 00:00:00 2001 From: eap Date: Mon, 23 Dec 2024 17:19:39 -0700 Subject: [PATCH 24/31] Cleanup comments and definitions --- .../containers/docker-ubuntu-oneapi-impi.yaml | 85 +++---------------- 1 file changed, 13 insertions(+), 72 deletions(-) diff --git a/configs/containers/docker-ubuntu-oneapi-impi.yaml b/configs/containers/docker-ubuntu-oneapi-impi.yaml index d6b85e63f..618e19c3c 100644 --- a/configs/containers/docker-ubuntu-oneapi-impi.yaml +++ b/configs/containers/docker-ubuntu-oneapi-impi.yaml @@ -43,10 +43,10 @@ spack: paths: cc: /opt/intel/oneapi/compiler/2024.2/bin/icx cxx: /opt/intel/oneapi/compiler/2024.2/bin/icpx - #f77: /opt/intel/oneapi/compiler/2024.2/bin/ifx - #fc: /opt/intel/oneapi/compiler/2024.2/bin/ifx - f77: /opt/intel/oneapi/compiler/2024.2/bin/ifort - fc: /opt/intel/oneapi/compiler/2024.2/bin/ifort + f77: /opt/intel/oneapi/compiler/2024.2/bin/ifx + fc: /opt/intel/oneapi/compiler/2024.2/bin/ifx + #f77: /opt/intel/oneapi/compiler/2024.2/bin/ifort + #fc: /opt/intel/oneapi/compiler/2024.2/bin/ifort flags: {} operating_system: ubuntu24.04 target: x86_64 @@ -81,9 +81,6 @@ spack: bison: require: - '%gcc' - #boost: - # require: - # - '%gcc' coreutils: externals: - spec: coreutils@8.32 @@ -96,21 +93,6 @@ spack: ectrans: require: - '+mkl ~fftw' - #ecflow: - # require: - # - '%gcc' - #findutils: - # externals: - # - spec: findutils@4.8.0 - # prefix: /usr - #flex: - # externals: - # - spec: flex@2.6.4+lex - # prefix: /usr - #gawk: - # externals: - # - spec: gawk@5.1.0 - # prefix: /usr git: buildable: false externals: @@ -128,6 +110,10 @@ spack: gsibec: require: - '+mkl' + # Building icu4c with oneapi intel causes segfault. + icu4c: + require: + - '%gcc' intel-oneapi-mkl: buildable: false externals: @@ -143,20 +129,8 @@ spack: externals: - spec: intel-oneapi-tbb@2021.13 prefix: /opt/intel/oneapi - #libtool: - # externals: - # - spec: libtool@2.4.6 - # prefix: /usr mpi: buildable: False - #m4: - # externals: - # - spec: m4@1.4.18 - # prefix: /usr - #lcms: - # externals: - # - spec: lcms@2.13.1 - # prefix: /usr libmd: require: - '%gcc' @@ -167,20 +141,11 @@ spack: buildable: false externals: - spec: llvm@14.0.6 - #grep: - # buildable: false - # externals: - # - spec: grep@3.7 - # prefix: /usr mysql: buildable: false externals: - spec: mysql@8.0.40 prefix: /usr - #perl: - # externals: - # - spec: perl@5.34.0~cpanm+opcode+open+shared+threads - # prefix: /usr pkg-config: externals: - spec: pkg-config@0.29.2 @@ -193,20 +158,6 @@ spack: py-numpy: require: - '^intel-oneapi-mkl' - # Turn off crypt, because libxcrypt doesn't - # build with Intel. - #python: - # variants: ~crypt - #qt: - # buildable: false - # externals: - # - spec: qt@5.15.3 - # prefix: /usr - # version: [5.15.3] - #sed: - # externals: - # - spec: sed@4.8 - # prefix: /usr tar: externals: - spec: tar@1.34 @@ -228,18 +179,8 @@ spack: # Select the format of the recipe e.g. docker, # singularity or anything else that is currently supported format: docker - # How to use: - #$ spack containerize > Dockerfile - #$ sudo docker build -t myimage . - #$ sudo docker run -it myimage - - #format: singularity - # How to use: - #$ spack containerize > singularity.def - #$ sudo singularity build singularity.sif singularity.def - # Sets the base images for the stages where Spack builds the - # software or where the software gets installed after being built.. + # Sets the base images for the stages where Spack builds and installs. images: os: ubuntu:24.04 spack: @@ -345,7 +286,7 @@ spack: ENV TZ=Etc/UTC ENV CC=icx ENV CXX=icpx - ENV FC=ifort + ENV FC=ifx build: | # Put output of spack find into a file RUN cd /opt/spack-environment && \ @@ -374,7 +315,7 @@ spack: echo "ulimit -v unlimited" >> /home/nonroot/.bashrc && \ echo "export CC=icx" >> /home/nonroot/.bashrc && \ echo "export CXX=icpx" >> /home/nonroot/.bashrc && \ - echo "export FC=ifort" >> /home/nonroot/.bashrc && \ + echo "export FC=ifx" >> /home/nonroot/.bashrc && \ echo "source /opt/intel/oneapi/setvars.sh" >> /etc/bash.bashrc && \ printf "[credential]\n helper = cache --timeout=7200\n" >> /home/nonroot/.gitconfig && \ chown -R nonroot:nonroot /home/nonroot/.gitconfig @@ -383,11 +324,11 @@ spack: echo "ulimit -v unlimited" >> /root/.bashrc && \ echo "export CC=icx" >> /root/.bashrc && \ echo "export CXX=icpx" >> /root/.bashrc && \ - echo "export FC=ifort" >> /root/.bashrc && \ + echo "export FC=ifx" >> /root/.bashrc && \ echo "export jedi_cmake_ROOT=$(ls -d /opt/software/linux-ubuntu*/oneapi-*/jedi-cmake-*)" >> /root/.bashrc && \ printf "[credential]\n helper = cache --timeout=7200\n" >> /root/.gitconfig # Labels for the image labels: app: "" - mpi: "intel-oneapi-mpi" + mpi: "intel-llvm-oneapi-mpi" From a9ded31cc2ad91a7170ad587984720e1d3caa843 Mon Sep 17 00:00:00 2001 From: eap Date: Mon, 23 Dec 2024 17:49:02 -0700 Subject: [PATCH 25/31] reduce parallelism --- configs/containers/docker-ubuntu-oneapi-impi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/containers/docker-ubuntu-oneapi-impi.yaml b/configs/containers/docker-ubuntu-oneapi-impi.yaml index 618e19c3c..468c77957 100644 --- a/configs/containers/docker-ubuntu-oneapi-impi.yaml +++ b/configs/containers/docker-ubuntu-oneapi-impi.yaml @@ -21,7 +21,7 @@ spack: config: checksum: false - build_jobs: 6 + build_jobs: 2 connect_timeout: 60 compilers: From 752ae44c817a4fd0f69f8eec1d538a5f81291173 Mon Sep 17 00:00:00 2001 From: eap Date: Mon, 23 Dec 2024 17:51:53 -0700 Subject: [PATCH 26/31] Fix DNS comments --- .../containers/docker-ubuntu-oneapi-impi.yaml | 28 +++++-------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/configs/containers/docker-ubuntu-oneapi-impi.yaml b/configs/containers/docker-ubuntu-oneapi-impi.yaml index 468c77957..640acd62d 100644 --- a/configs/containers/docker-ubuntu-oneapi-impi.yaml +++ b/configs/containers/docker-ubuntu-oneapi-impi.yaml @@ -1,20 +1,3 @@ -###### DO NOT SUBMIT THIS BLOCK ##### -## Intel packages -# intel-oneapi-compiler-dpcpp-cpp-2024.2 \ -# intel-oneapi-compiler-fortran-2024.2 \ -# intel-oneapi-mpi-devel-2021.13 \ -# intel-oneapi-tbb-devel-2021.13 \ -# intel-oneapi-mkl-devel-2024.2 -# -# -## Versions reported by apt cache policy -# compiler - 2024.2.1 -# fortran 2024.2.1 -# mpi - 2021.13.1 -# tbb - 2021.13.1 -# mkl - 2024.2.2 -#### /end DNS ######################## - spack: concretizer: unify: true @@ -270,9 +253,12 @@ spack: echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \ apt update && \ apt install intel-oneapi-compiler-dpcpp-cpp-2024.2 intel-oneapi-compiler-fortran-2024.2 intel-oneapi-mpi-devel-2021.13 intel-oneapi-tbb-devel-2021.13 intel-oneapi-mkl-devel-2024.2 -y && \ - #ln -s /opt/intel/oneapi/mpi/2021.13 /opt/intel/oneapi/mpi/2021.13.1 && \ - #ln -s /opt/intel/oneapi/mkl/2024.2 /opt/intel/oneapi/mkl/2024.2.2 && \ - #ln -s /opt/intel/oneapi/compiler/2024.2 /opt/intel/oneapi/compiler/2024.2.1 && \ + # Intel package versions reported by apt cache policy + # compiler - 2024.2.1 + # fortran - 2024.2.1 + # mpi - 2021.13.1 + # tbb - 2021.13.1 + # mkl - 2024.2.2 update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100 && \ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 && \ update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-13 100 && \ @@ -331,4 +317,4 @@ spack: # Labels for the image labels: app: "" - mpi: "intel-llvm-oneapi-mpi" + mpi: "intel-oneapi-mpi" From 08f7518202c1980ac8df130cc88237de3320b8b0 Mon Sep 17 00:00:00 2001 From: Evan Parker Date: Tue, 24 Dec 2024 05:51:05 +0000 Subject: [PATCH 27/31] Switch from ifx to ifort --- .../containers/docker-ubuntu-oneapi-impi.yaml | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/configs/containers/docker-ubuntu-oneapi-impi.yaml b/configs/containers/docker-ubuntu-oneapi-impi.yaml index 640acd62d..5aec25af1 100644 --- a/configs/containers/docker-ubuntu-oneapi-impi.yaml +++ b/configs/containers/docker-ubuntu-oneapi-impi.yaml @@ -26,10 +26,10 @@ spack: paths: cc: /opt/intel/oneapi/compiler/2024.2/bin/icx cxx: /opt/intel/oneapi/compiler/2024.2/bin/icpx - f77: /opt/intel/oneapi/compiler/2024.2/bin/ifx - fc: /opt/intel/oneapi/compiler/2024.2/bin/ifx - #f77: /opt/intel/oneapi/compiler/2024.2/bin/ifort - #fc: /opt/intel/oneapi/compiler/2024.2/bin/ifort + #f77: /opt/intel/oneapi/compiler/2024.2/bin/ifx + #fc: /opt/intel/oneapi/compiler/2024.2/bin/ifx + f77: /opt/intel/oneapi/compiler/2024.2/bin/ifort + fc: /opt/intel/oneapi/compiler/2024.2/bin/ifort flags: {} operating_system: ubuntu24.04 target: x86_64 @@ -272,7 +272,7 @@ spack: ENV TZ=Etc/UTC ENV CC=icx ENV CXX=icpx - ENV FC=ifx + ENV FC=ifort build: | # Put output of spack find into a file RUN cd /opt/spack-environment && \ @@ -297,22 +297,17 @@ spack: COPY --from=builder /root/spack_find.out /root/spack_find.out # Make a non-root user:nonroot / group:nonroot for running MPI RUN useradd -U -k /etc/skel -s /bin/bash -d /home/nonroot -m nonroot --uid 43891 && \ - echo "ulimit -s unlimited" >> /home/nonroot/.bashrc && \ - echo "ulimit -v unlimited" >> /home/nonroot/.bashrc && \ - echo "export CC=icx" >> /home/nonroot/.bashrc && \ - echo "export CXX=icpx" >> /home/nonroot/.bashrc && \ - echo "export FC=ifx" >> /home/nonroot/.bashrc && \ echo "source /opt/intel/oneapi/setvars.sh" >> /etc/bash.bashrc && \ + echo "ulimit -s unlimited" >> /etc/bash.bashrc && \ + echo "ulimit -v unlimited" >> /etc/bash.bashrc && \ + echo "export CC=icx" >> /etc/bash.bashrc && \ + echo "export CXX=icpx" >> /etc/bash.bashrc && \ + echo "export FC=ifort" >> /etc/bash.bashrc && \ + echo "export jedi_cmake_ROOT=$(ls -d /opt/software/linux-ubuntu*/oneapi-*/jedi-cmake-*)" >> /etc/bash.bashrc && \ + printf "[credential]\n helper = cache --timeout=7200\n" >> /root/.gitconfig && \ + # Replicate gitconfig for nonroot user. printf "[credential]\n helper = cache --timeout=7200\n" >> /home/nonroot/.gitconfig && \ chown -R nonroot:nonroot /home/nonroot/.gitconfig - # Replicate settings for root user - RUN echo "ulimit -s unlimited" >> /root/.bashrc && \ - echo "ulimit -v unlimited" >> /root/.bashrc && \ - echo "export CC=icx" >> /root/.bashrc && \ - echo "export CXX=icpx" >> /root/.bashrc && \ - echo "export FC=ifx" >> /root/.bashrc && \ - echo "export jedi_cmake_ROOT=$(ls -d /opt/software/linux-ubuntu*/oneapi-*/jedi-cmake-*)" >> /root/.bashrc && \ - printf "[credential]\n helper = cache --timeout=7200\n" >> /root/.gitconfig # Labels for the image labels: From 64c70377ae7685b834fe1b135124e35973e1f83f Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 27 Dec 2024 09:09:19 -0700 Subject: [PATCH 28/31] Update KnownIssues.rst: python on Cray is non-functional: Could not find platform dependent libraries (#1427) --- doc/source/KnownIssues.rst | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/doc/source/KnownIssues.rst b/doc/source/KnownIssues.rst index 6d411af8a..6bb20905c 100644 --- a/doc/source/KnownIssues.rst +++ b/doc/source/KnownIssues.rst @@ -7,9 +7,9 @@ Known Issues General ============================== -1. ``gcc@13`` (``gcc``, ``g++``, ``gfortran``) and ``apple-clang@15`` (``clang``, ``clang++``) not yet supported +1. ``python`` on Cray is non-functional: Could not find platform dependent libraries - Our software stack doesn't build with ``gcc@13`` yet. This is also true when combining the LLVM or Apple ``clang`` compiler with ``gfortran@13``. We also don't support the latest release of ``apple-clang@15`` with Xcode 15.3 yet, and with Xcode 15.0 a workaround is described in :numref:`Section %s `. + This problem and the solution to unset the environment variable ``CONFIG_SITE`` before building spack-stack environments is described in https://github.com/spack/spack/issues/40110 and https://github.com/spack/spack/issues/42132. 2. Build errors for ``mapl@2.35.2`` with ``mpich@4.1.1`` @@ -19,23 +19,15 @@ General On some systems, ``ecflow_server`` gets confused by multiple hostnames, e.g. ``localhost`` and ``MYORG-L-12345``. The ``ecflow_start.sh`` script reports the hostname it wants to use. This name (or both) must be in ``/etc/hosts`` in the correct address line, often the loopback address (``127.0.0.1``). -4. Installation of duplicate packages ``ecbuild``, ``hdf5`` - - One reason for this is an external ``cmake@3.20`` installation, which confuses the concretizer when building a complex environment such as the ``skylab-dev`` or ```unified-dev`` environment. For certain packages (and thus their dependencies), a newer version than ``cmake@3.20`` is required, for others ``cmake@3.20`` works, and spack then thinks that it needs to build two identical versions of the same package with different versions of ``cmake``. The solution is to remove any external ``cmake@3.20`` package (and best also earlier versions) in the site config and run the concretization step again. Another reason on Ubuntu 20 is the presence of external ``openssl`` packages, which should be removed before re-running the concretization step. - -5. Installation of duplicate package ``nco`` - - We tracked this down to multiple versions of ``bison`` being used. The best solution is to remove external ``bison`` versions earlier than 3.8 from the site config (``packages.yaml``). - -6. Installing/using graphical applications after switching user using ``sudo su`` +4. Installing/using graphical applications after switching user using ``sudo su`` When using a role account to install spack-stack, it is sometimes necessary to run graphical applications such as the ``qt`` online installer. The following website describes in detail how this can be done: https://www.thegeekdiary.com/how-to-set-x11-forwarding-export-remote-display-for-users-who-switch-accounts-using-sudo/ -7. ``==> Error: the key "core_compilers" must be set in modules.yaml`` during ``spack module [lmod|tcl] refresh`` +5. ``==> Error: the key "core_compilers" must be set in modules.yaml`` during ``spack module [lmod|tcl] refresh`` This error usually indicates that the wrong module type is used in the ``spack module ... refresh`` command. For example, the system is configured for ``lmod``, but the command used is ``spack module tcl refresh``. -8. Runtime segmentation faults for applications with ``intel@2021.10.0``: ``Relink `/opt/intel/oneapi/compiler/2024.0/lib/libirc.so' with `/lib/x86_64-linux-gnu/libc.so.6' for IFUNC symbol `memmove' - Segmentation fault (core dumped)``. +6. Runtime segmentation faults for applications with ``intel@2021.10.0``: ``Relink `/opt/intel/oneapi/compiler/2024.0/lib/libirc.so' with `/lib/x86_64-linux-gnu/libc.so.6' for IFUNC symbol `memmove' - Segmentation fault (core dumped)``. This problem is caused by a bad library in the Intel oneAPI installation. The solution is to fix the library using patchelf, which requires write access to the oneAPI installation: First, verify that ``ldd /opt/intel/oneapi/compiler/2024.0/lib/libirc.so`` says it is statically linked (it isn't). Then, run: ``patchelf --add-needed libc.so.6 /opt/intel/oneapi/compiler/2024.0/lib/libirc.so`` and your application should run rightaway (no need to recompile). From ba23d4cc67243ccb06a50c6105cea496e913366f Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Sun, 29 Dec 2024 19:43:21 -0700 Subject: [PATCH 29/31] Add configs/template/ccpp-scm (#1400) --- configs/templates/ccpp-scm/spack.yaml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 configs/templates/ccpp-scm/spack.yaml diff --git a/configs/templates/ccpp-scm/spack.yaml b/configs/templates/ccpp-scm/spack.yaml new file mode 100644 index 000000000..07ee4ff2d --- /dev/null +++ b/configs/templates/ccpp-scm/spack.yaml @@ -0,0 +1,23 @@ +spack: + concretizer: + unify: when_possible + + config: + install_tree: + root: $env/install + modules: + default: + roots: + lmod: $env/install/modulefiles + tcl: $env/install/modulefiles + + view: false + include: [] + + specs: + - netcdf-fortran + - bacio + - sp + - w3emc + - py-f90nml + From fdaf01ca8e940bdfe60285fa69b072a36bf31c6e Mon Sep 17 00:00:00 2001 From: Evan Parker Date: Mon, 30 Dec 2024 23:07:57 +0000 Subject: [PATCH 30/31] add ectrans variant to old intel --- configs/containers/docker-ubuntu-intel-impi.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index c2a418083..1cbdd9620 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -87,6 +87,9 @@ spack: ecflow: require: - '%gcc' + ectrans: + require: + - '+mkl ~fftw' findutils: externals: - spec: findutils@4.8.0 From b21e3fb7d79f75aa778d7c36fab689566b5b5fc6 Mon Sep 17 00:00:00 2001 From: Steve G Date: Thu, 2 Jan 2025 12:17:14 -0700 Subject: [PATCH 31/31] Add a tier2 site config for AWS AMIs (#1393) This is to add AWS as a tier2 supported platform. The two images are Rock8 and Ubuntu24.04. --- configs/sites/tier2/aws-rocky8/README.md | 452 ++++++++++++++++++ configs/sites/tier2/aws-rocky8/compilers.yaml | 32 ++ configs/sites/tier2/aws-rocky8/config.yaml | 9 + configs/sites/tier2/aws-rocky8/modules.yaml | 7 + configs/sites/tier2/aws-rocky8/packages.yaml | 94 ++++ .../sites/tier2/aws-rocky8/packages_gcc.yaml | 5 + .../tier2/aws-rocky8/packages_intel.yaml | 29 ++ configs/sites/tier2/aws-ubuntu2404/README.md | 384 +++++++++++++++ .../sites/tier2/aws-ubuntu2404/compilers.yaml | 30 ++ .../sites/tier2/aws-ubuntu2404/config.yaml | 9 + .../sites/tier2/aws-ubuntu2404/modules.yaml | 7 + .../sites/tier2/aws-ubuntu2404/packages.yaml | 94 ++++ .../tier2/aws-ubuntu2404/packages_gcc.yaml | 5 + .../tier2/aws-ubuntu2404/packages_intel.yaml | 30 ++ doc/source/PreConfiguredSites.rst | 22 + 15 files changed, 1209 insertions(+) create mode 100644 configs/sites/tier2/aws-rocky8/README.md create mode 100644 configs/sites/tier2/aws-rocky8/compilers.yaml create mode 100644 configs/sites/tier2/aws-rocky8/config.yaml create mode 100644 configs/sites/tier2/aws-rocky8/modules.yaml create mode 100644 configs/sites/tier2/aws-rocky8/packages.yaml create mode 100644 configs/sites/tier2/aws-rocky8/packages_gcc.yaml create mode 100644 configs/sites/tier2/aws-rocky8/packages_intel.yaml create mode 100644 configs/sites/tier2/aws-ubuntu2404/README.md create mode 100644 configs/sites/tier2/aws-ubuntu2404/compilers.yaml create mode 100644 configs/sites/tier2/aws-ubuntu2404/config.yaml create mode 100644 configs/sites/tier2/aws-ubuntu2404/modules.yaml create mode 100644 configs/sites/tier2/aws-ubuntu2404/packages.yaml create mode 100644 configs/sites/tier2/aws-ubuntu2404/packages_gcc.yaml create mode 100644 configs/sites/tier2/aws-ubuntu2404/packages_intel.yaml diff --git a/configs/sites/tier2/aws-rocky8/README.md b/configs/sites/tier2/aws-rocky8/README.md new file mode 100644 index 000000000..9705277ea --- /dev/null +++ b/configs/sites/tier2/aws-rocky8/README.md @@ -0,0 +1,452 @@ +## spack-stack AMI (Rocky8) + +This document is to go over the the running and usage of this specific AMI for Rocky8. This image has two environments: gnu (gcc-11.2), and intel (intel@2021.10.0). You can use either one of these environments for development purposes. + +### Using the AMI + +Here is an example `aws cli` command-line to run this AMI with your designated key-pair. + +```bash +aws ec2 run-instances \ + --image-id "ami-02391db2758465a87" --instance-type "m6i.4xlarge" \ + --key-name YOUR-KEYPAIR \ + --block-device-mappings '{"DeviceName":"/dev/sda1","Ebs":{"Encrypted":false,"DeleteOnTermination":true,"Iops":3000,"SnapshotId":"snap-05fb00e35af5550e7","VolumeSize":150,"VolumeType":"gp3","Throughput":125}}' \ + --network-interfaces '{"SubnetId":"subnet-00575f6b8ccc2005d","AssociatePublicIpAddress":true,"DeviceIndex":0,"Groups":["sg-0436b207bc220df08"]}' \ + --tag-specifications '{"ResourceType":"instance","Tags":[{"Key":"Name","Value":"rocky8-spack-stack-gcc-intel"},{"Key":"User","Value":$(whoami)}]}' \ + --metadata-options '{"HttpEndpoint":"enabled","HttpPutResponseHopLimit":2,"HttpTokens":"required"}' \ + --private-dns-name-options '{"HostnameType":"ip-name","EnableResourceNameDnsARecord":false,"EnableResourceNameDnsAAAARecord":false}' \ + --count "1" +``` + +## Using this Site Config + +JCSDA publishes a fully configured and built installation of spack stack derived +from this config as an Amazon Machine Images (AMI). The easiest way to use this +configuration of spack stack is to launch a VM using that AMI. The instructions +below are included for maintainance of the history of this site config and +their possible relevance to debugging issues should they arise. + +### Base Instance + +This AMI was built on an instance with these properties: + +* AMI Name: Rocky-8-EC2-Base-8.9-20231119.0.x86_64-d6577ceb-8ea8-4e0e-84c6-f098fc302e82 +* AMI ID: ami-02391db2758465a87 +* Instance m6i.4xlarge (uses Intel Xeon processor) +* 300GB of gp3 storage as / + +## Pre-requisites (For All Compilers) + +
+Tmux Usage + +### Tmux + +Rocky 8 doesn't have screen so here's a guide to tmux. + +```bash +# Start a session +tmux new -s my-session-name + +# Detach from a session +ctr-b d + +# list sessions +tmux ls + +# Attach to existing session +tmux attach -t my-session-name +``` + +More information on how to use `tmux`: + +
+ +### Installing Packages + +```bash +# Update system software and start a tmux session. +sudo dnf -y update +sudo dnf -y install tmux +tmux new -s setup +sudo su - + +# Compilers +dnf install gcc-toolset-11-gcc-c++ \ + gcc-toolset-11-gcc-gfortran \ + gcc-toolset-11-gdb + +#Install other requirements. +dnf install binutils-devel \ + m4 \ + wget \ + git \ + git-lfs \ + bash-completion \ + bzip2 bzip2-devel \ + libcurl libcurl-devel \ + unzip \ + patch \ + automake \ + xorg-x11-xauth \ + xterm \ + perl-IPC-Cmd \ + pearl-core \ + gettext-devel \ + texlive \ + tcl-devel \ + nano \ + vim \ + bison \ + qt5-qtbase \ + qt5-qtbase-devel \ + qt5-qtsvg \ + qt5-qtsvg-devel \ + autoconf2.7x \ + libxml2 libxml2-devel + +# Python develop. +dnf install python3-devel + +# Enable gcc toolset. This is needed for later builds. Once lmod is installed +# this will be configured as a pass-through module. +scl enable gcc-toolset-11 bash + +# Configure git credential caching and git lfs for the rocky user and root. +git config --global credential.helper 'cache --timeout=3600' +git lfs install + +# Configure x11 forwarding. +echo "X11Forwarding yes" >> /etc/ssh/sshd_config +service sshd restart +``` + +### Install MySQL Community Server + +```bash +dnf config-manager --set-enabled crb + +dnf install mysql-server mysql-devel +sudo systemctl start mysqld.service +sudo systemctl enable mysqld + +# Use the mysql server. +mysql -u root +exit +``` + +### Install Lmod + +```bash +# Enable gcc using system module +scl enable gcc-toolset-11 bash + +# Install lua/lmod manually because apt only has older versions +# that are not compatible with the modern lua modules spack produces +# https://lmod.readthedocs.io/en/latest/030_installing.html#install-lua-x-y-z-tar-gz +sudo su - +mkdir -p /opt/lua/5.1.4.9/src && cd $_ +wget https://sourceforge.net/projects/lmod/files/lua-5.1.4.9.tar.bz2 +tar -xvf lua-5.1.4.9.tar.bz2 +cd lua-5.1.4.9 +./configure --prefix=/opt/lua/5.1.4.9 2>&1 | tee log.config +make VERBOSE=1 2>&1 | tee log.make +make install 2>&1 | tee log.install + +cat << 'EOF' >> /etc/profile.d/02-lua.sh +# Set environment variables for lua +export PATH="/opt/lua/5.1.4.9/bin:$PATH" +export LD_LIBRARY_PATH="/opt/lua/5.1.4.9/lib:$LD_LIBRARY_PATH" +export CPATH="/opt/lua/5.1.4.9/include:$CPATH" +export MANPATH="/opt/lua/5.1.4.9/man:$MANPATH" +EOF + +source /etc/profile.d/02-lua.sh +mkdir -p /opt/lmod/8.7/src +cd /opt/lmod/8.7/src +wget https://sourceforge.net/projects/lmod/files/Lmod-8.7.tar.bz2 +tar -xvf Lmod-8.7.tar.bz2 +cd Lmod-8.7 +# Note the weird prefix, lmod installs in PREFIX/lmod/X.Y automatically +./configure --prefix=/opt/ \ + --with-lmodConfigDir=/opt/lmod/8.7/config \ + 2>&1 | tee log.config +make install 2>&1 | tee log.install +ln -sf /opt/lmod/lmod/init/profile /etc/profile.d/z00_lmod.sh +ln -sf /opt/lmod/lmod/init/cshrc /etc/profile.d/z00_lmod.csh +ln -sf /opt/lmod/lmod/init/profile.fish /etc/profile.d/z00_lmod.fish + +# Add custom module for system gcc +cat << 'EOF' >> /opt/rh/gcc-toolset-11/gcc-toolset.lua +--%Module +family("compiler") +help([[Wrapper module for gcc-toolset-11 using Software Collections]]) +whatis("Description: GCC Toolset 11") + +-- Execute 'scl enable' to load gcc-toolset-11 environment +execute {cmd="scl enable gcc-toolset-11 bash", modeA={"load"}} +EOF + +# Add a number of default module locations to the lmod startup script. +cat << 'EOF' >> /etc/profile.d/z01_lmod.sh +module use /opt/rh/gcc-toolset-11 +EOF + +# Log out completely, ssh back into the instance and check if lua modules work +exit +exit +``` + +### Clone `spack-stack` + +```bash +# if in sudo su - +cd /opt +git clone -b develop --recursive https://github.com/jcsda/spack-stack.git +``` + +## Install Steps by Compiler + +
+GCC Installation + +```bash +sudo su - + +cd /opt/spack-stack +source setup.sh + +scl enable gcc-toolset-11 bash + +# Swap default module type for default linux. +sed -i 's/tcl/lmod/g' configs/sites/tier2/linux.default/modules.yaml +spack stack create env --site linux.default --template unified-dev --name unified-env-gcc --compiler gcc +cd envs/unified-env-gcc +spack env activate -p . +export SPACK_SYSTEM_CONFIG_PATH="$PWD/site" +spack external find --scope system \ + --exclude cmake \ + --exclude curl --exclude openssl \ + --exclude openssh --exclude python +spack external find --scope system wget +spack external find --scope system mysql +spack external find --scope system grep +spack compiler find --scope system +unset SPACK_SYSTEM_CONFIG_PATH + +# ACTION: Edit the site/packages.yaml and add these packages +# If not present. +cat << 'EOF' >> $PWD/site/packages.yaml + gcc: + buildable: false + externals: + - spec: gcc@11.2.1 + prefix: /usr + gcc-runtime: + buildable: false + externals: + - spec: gcc-runtime@11.2.1 + prefix: /usr + qt: + buildable: false + externals: + - spec: qt@5.15.3 + prefix: /usr + version: [5.15.3] +EOF + +# Continue configuration. +spack config add "packages:all:compiler:[gcc@11.2.1]" +spack config add "packages:all:providers:mpi:[openmpi@5.0.5]" +spack config add "packages:fontconfig:variants:+pic" +spack config add "packages:pixman:variants:+pic" +spack config add "packages:cairo:variants:+pic" +spack config add "packages:ewok-env:variants:+mysql" + +# Concretize and install +spack concretize 2>&1 | tee log.concretize +${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d -c log.concretize +spack install --fail-fast -j 16 2>&1 | tee log.install + +# Install modules +spack module lmod refresh +spack stack setup-meta-modules + +cat << 'EOF' >> /etc/profile.d/z01_lmod.sh +module use /opt/spack-stack/envs/unified-env-gcc/install/modulefiles/Core +EOF +``` + +
+ +
+Intel Installation + +#### Install Intel Compiler + +```bash +rm -rf /opt/intel +rm -rf /var/intel + +mkdir -p /opt/intel/src +pushd /opt/intel/src + +# Download Intel install assets. +wget -O cpp-compiler.sh https://registrationcenter-download.intel.com/akdlm/IRC_NAS/d85fbeee-44ec-480a-ba2f-13831bac75f7/l_dpcpp-cpp-compiler_p_2023.2.3.12_offline.sh +wget -O fortran-compiler.sh https://registrationcenter-download.intel.com/akdlm/IRC_NAS/0ceccee5-353c-4fd2-a0cc-0aecb7492f87/l_fortran-compiler_p_2023.2.3.13_offline.sh +wget -O tbb.sh https://registrationcenter-download.intel.com/akdlm/IRC_NAS/c95cd995-586b-4688-b7e8-2d4485a1b5bf/l_tbb_oneapi_p_2021.10.0.49543_offline.sh +wget -O mpi.sh https://registrationcenter-download.intel.com/akdlm/IRC_NAS/4f5871da-0533-4f62-b563-905edfb2e9b7/l_mpi_oneapi_p_2021.10.0.49374_offline.sh +wget -O math.sh https://registrationcenter-download.intel.com/akdlm/IRC_NAS/adb8a02c-4ee7-4882-97d6-a524150da358/l_onemkl_p_2023.2.0.49497_offline.sh + +# Install the Intel assets. +sh cpp-compiler.sh -a --silent --eula accept 2>&1 | tee install.cpp-compiler.log +sh fortran-compiler.sh -a --silent --eula accept | tee install.fortran-compiler.log +sh tbb.sh -a --silent --eula accept | tee install.tbb.log +sh mpi.sh -a --silent --eula accept | tee install.mpi.log +sh math.sh -a --silent --eula accept | tee install.math.log + +popd +``` + +#### Create Intel Environment +```bash +tmux -s intel +sudo su - +module load gcc-toolset + +source /opt/intel/oneapi/compiler/2023.2.3/env/vars.sh +source /opt/intel/oneapi/mpi/2021.10.0/env/vars.sh +source /opt/intel/oneapi/setvars.sh + +cd /opt/spack-stack +source ./setup.sh + +spack stack create env --site linux.default --template unified-dev --name unified-env-intel --compiler intel +cd envs/unified-env-intel +spack env activate -p . + +export SPACK_SYSTEM_CONFIG_PATH="${PWD}/site" + +spack external find --scope system --exclude python --exclude curl +spack external find --scope system perl +spack external find --scope system wget +spack external find --scope system texlive +spack external find --scope system mysql +spack external find --scope system grep + +# No external find for pre-installed intel-oneapi-mpi (from pcluster AMI), +# and no way to add object entry to list using "spack config add". +cat << 'EOF' >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 + prefix: /opt/intel/oneapi +EOF + +# Can't find qt5 because qtpluginfo is broken, +# and no way to add object entry to list using "spack config add". +cat << 'EOF' >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + qt: + buildable: false + externals: + - spec: qt@5.15.3 + prefix: /usr +EOF + +spack compiler find --scope system + +export -n SPACK_SYSTEM_CONFIG_PATH + +# spack config add "packages:mpi:buildable:False" +spack config add "packages:all:providers:mpi:[intel-oneapi-mpi@2021.10.0]" +spack config add "packages:all:compiler:[intel@2021.10.0, gcc@11.2.1]" + +# Edit envs/unified-intel/spack.yaml. +# 1) Find this line: +# compilers: ['%aocc', '%apple-clang', '%gcc', '%intel'] +# 2) Delete all compilers except for your target compiler. In the case of intel +# the line should look like this: +# compilers: ['%intel'] + +# edit envs/unified-env/site/compilers.yaml and replace the following line in the **Intel** compiler section: +# environment: {} +# --> +# environment: +# prepend_path: +# PATH: /opt/rh/gcc-toolset-11/root/usr/bin +# LD_LIBRARY_PATH: '/opt/intel/oneapi/compiler/2023.2.3/linux/compiler/lib/intel64_lin:/usr/lib:/usr/lib64' +# CPATH: /opt/rh/gcc-toolset-11/root/usr/include + +spack concretize 2>&1 | tee log.concretize +${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize +spack install -j 12 --verbose 2>&1 | tee log.install +spack module lmod refresh +spack stack setup-meta-modules + +cat << 'EOF' >> /etc/profile.d/z01_lmod.sh +module use /opt/spack-stack/envs/unified-env-gcc/install/modulefiles/Core +EOF +``` + +
+ +## Test Installation + +
+GCC + +```bash +# Example given for building jedi-bundle +module use /opt/spack-stack/envs/unified-dev-gcc/install/modulefiles/Core +module load stack-gcc/11.2.1 +module load stack-openmpi/5.0.5 +module load base-env +module load jedi-mpas-env +module load jedi-fv3-env +module load ewok-env +module load sp + +mkdir /opt/jedi +cd /opt/jedi +git clone https://github.com/JCSDA-internal/jedi-bundle.git +cd jedi-bundle +mkdir build && cd build +ecbuild ../ +make update +make -j10 +ctest +``` + +
+ +
+Intel + +```bash +# Example given for building jedi-bundle +module use /opt/spack-stack/envs/unified-dev-intel/install/modulefiles/Core +module load stack-intel/2021.10.0 +module load stack-intel-oneapi-mpi/2021.10.0 +module load base-env +module load jedi-mpas-env +module load jedi-fv3-env +module load ewok-env +module load sp + +mkdir /opt/jedi +cd /opt/jedi +git clone https://github.com/JCSDA-internal/jedi-bundle.git +cd jedi-bundle +mkdir build && cd build +ecbuild ../ +make update +make -j10 +ctest +``` + +**Note**: If the `make -j10` shows any errors, re-run it and it should be successful. + +
+ +The installation and configuration is now complete for the instance. diff --git a/configs/sites/tier2/aws-rocky8/compilers.yaml b/configs/sites/tier2/aws-rocky8/compilers.yaml new file mode 100644 index 000000000..f52755104 --- /dev/null +++ b/configs/sites/tier2/aws-rocky8/compilers.yaml @@ -0,0 +1,32 @@ +compilers: +- compiler: + spec: intel@=2021.10.0 + paths: + cc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/icc + cxx: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/icpc + f77: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/ifort + fc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/ifort + flags: {} + operating_system: rocky8 + target: x86_64 + modules: [] + environment: + prepend_path: + PATH: /opt/rh/gcc-toolset-11/root/user/bin + LD_LIBRARY_PATH: '/opt/intel/oneapi/compiler/2023.2.3/linux/compiler/lib/intel64_lin:/usr/lib64:/usr/lib' + CPATH: /opt/rh/gcc-toolset-11/root/user/include + extra_rpaths: [] +- compiler: + spec: gcc@11.2.1 + paths: + cc: /opt/rh/gcc-toolset-11/root/user/bin/gcc + cxx: /opt/rh/gcc-toolset-11/root/user/bin/g++ + f77: /opt/rh/gcc-toolset-11/root/user/bin/gfortran + fc: /opt/rh/gcc-toolset-11/root/user/bin/gfortran + flags: {} + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + diff --git a/configs/sites/tier2/aws-rocky8/config.yaml b/configs/sites/tier2/aws-rocky8/config.yaml new file mode 100644 index 000000000..cf54e6360 --- /dev/null +++ b/configs/sites/tier2/aws-rocky8/config.yaml @@ -0,0 +1,9 @@ +config: + build_jobs: 8 + + # Overrides for spack build and staging areas to speed up builds + # by using a local directory instead of the EFS shared filesystem + build_stage: /tmp/spack-stack/cache/build_stage + test_stage: /tmp/spack-stack/cache/test_stage + source_cache: /tmp/spack-stack/cache/source_cache + misc_cache: /tmp/spack-stack/cache/misc_cache diff --git a/configs/sites/tier2/aws-rocky8/modules.yaml b/configs/sites/tier2/aws-rocky8/modules.yaml new file mode 100644 index 000000000..65801706f --- /dev/null +++ b/configs/sites/tier2/aws-rocky8/modules.yaml @@ -0,0 +1,7 @@ +modules: + default: + enable:: + - lmod + lmod: + include: + - python diff --git a/configs/sites/tier2/aws-rocky8/packages.yaml b/configs/sites/tier2/aws-rocky8/packages.yaml new file mode 100644 index 000000000..6158d0119 --- /dev/null +++ b/configs/sites/tier2/aws-rocky8/packages.yaml @@ -0,0 +1,94 @@ +packages: + mysql: + externals: + - spec: mysql@8.0.37 + prefix: /usr + wget: + externals: + - spec: wget@1.21.2 + prefix: /usr + binutils: + externals: + - spec: binutils@2.38 + prefix: /usr + coreutils: + externals: + - spec: coreutils@8.32 + prefix: /usr + diffutils: + externals: + - spec: diffutils@3.8 + prefix: /usr + findutils: + externals: + - spec: findutils@4.8.0 + prefix: /usr + gawk: + externals: + - spec: gawk@5.1.0 + prefix: /usr + tar: + externals: + - spec: tar@1.34 + prefix: /usr + autoconf: + externals: + - spec: autoconf@2.71 + prefix: /usr + automake: + externals: + - spec: automake@1.16.5 + prefix: /usr + flex: + externals: + - spec: flex@2.6.4+lex + prefix: /usr + gettext: + externals: + - spec: gettext@0.21 + prefix: /usr + git: + externals: + - spec: git@2.34.1~tcltk + prefix: /usr + git-lfs: + externals: + - spec: git-lfs@3.0.2 + prefix: /usr + gmake: + externals: + - spec: gmake@4.3 + prefix: /usr + groff: + externals: + - spec: groff@1.22.4 + prefix: /usr + libtool: + externals: + - spec: libtool@2.4.6 + prefix: /usr + m4: + externals: + - spec: m4@1.4.18 + prefix: /usr + perl: + externals: + - spec: perl@5.34.0~cpanm+opcode+open+shared+threads + prefix: /usr + pkg-config: + externals: + - spec: pkg-config@0.29.2 + prefix: /usr + qt: + buildable: false + externals: + - spec: qt@5.15.3 + prefix: /usr + sed: + externals: + - spec: sed@4.8 + prefix: /usr + lcms: + externals: + - spec: lcms@2.13.1 + prefix: /usr diff --git a/configs/sites/tier2/aws-rocky8/packages_gcc.yaml b/configs/sites/tier2/aws-rocky8/packages_gcc.yaml new file mode 100644 index 000000000..8ee1dc22b --- /dev/null +++ b/configs/sites/tier2/aws-rocky8/packages_gcc.yaml @@ -0,0 +1,5 @@ +packages: + all: + compiler:: [gcc@11.2.1] + providers: + mpi:: [openmpi@4.1.6] diff --git a/configs/sites/tier2/aws-rocky8/packages_intel.yaml b/configs/sites/tier2/aws-rocky8/packages_intel.yaml new file mode 100644 index 000000000..97c09dcb0 --- /dev/null +++ b/configs/sites/tier2/aws-rocky8/packages_intel.yaml @@ -0,0 +1,29 @@ +packages: + all: + compiler:: [intel@2021.10.0,gcc@11.2.1] + providers: + mpi:: [intel-oneapi-mpi@2021.10.0] + # Remove the next three lines to switch to intel-oneapi-mkl + blas:: [openblas] + fftw-api:: [fftw] + lapack:: [openblas] + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 + prefix: /opt/intel/oneapi + intel-oneapi-mkl: + # Remove buildable: False and configure version+uncomment externals section below to use intel-oneapi-mkl + buildable: False + #externals: + #- spec: intel-oneapi-mkl@2022.0.2%intel@2021.10.0 + # prefix: /opt/intel/oneapi + ectrans: + require:: + - '@1.2.0 ~mkl +fftw' + gsibec: + require:: + - '@1.2.1 ~mkl' + py-numpy: + require:: + - '@:1.23 ^openblas' diff --git a/configs/sites/tier2/aws-ubuntu2404/README.md b/configs/sites/tier2/aws-ubuntu2404/README.md new file mode 100644 index 000000000..2fddc5d2f --- /dev/null +++ b/configs/sites/tier2/aws-ubuntu2404/README.md @@ -0,0 +1,384 @@ +## spack-stack AMI (Ubuntu 24.04) + +This document is to go over the the running and usage of this specific AMI for Ubuntu 24.04 LTS. This image has two environments: gnu (gcc-12.3), and intel (intel@2021.10.0). You can use either one of these environments for development purposes. + +### Using the Snapshot + +Here is an example `aws cli` command-line to run a pre-built snapshot with your designated key-pair. + +```bash +aws ec2 run-instances \ + --image-id "ami-0ea3c35c5c3284d82" --instance-type "m6i.4xlarge" \ + --key-name YOUR-KEYPAIR \ + --block-device-mappings '{"DeviceName":"/dev/sda1","Ebs":{"Encrypted":false,"DeleteOnTermination":true,"Iops":3000,"SnapshotId":"snap-05fb00e35af5550e7","VolumeSize":150,"VolumeType":"gp3","Throughput":125}}' \ + --network-interfaces '{"SubnetId":"subnet-072fb62ff85b32a7a","AssociatePublicIpAddress":true,"DeviceIndex":0,"Groups":["sg-0091fa8e748fbe355"]}' \ + --tag-specifications '{"ResourceType":"instance","Tags":[{"Key":"Name","Value":"ubuntu2404-spack-stack-gcc-intel"},{"Key":"User","Value":$(whoami)}]}' \ + --metadata-options '{"HttpEndpoint":"enabled","HttpPutResponseHopLimit":2,"HttpTokens":"required"}' \ + --private-dns-name-options '{"HostnameType":"ip-name","EnableResourceNameDnsARecord":false,"EnableResourceNameDnsAAAARecord":false}' \ + --count "1" +``` + +## Using this Site Config + +JCSDA publishes a fully configured and built installation of spack stack derived +from this config as an AWS Snapshot. The easiest way to use this +configuration of spack stack is to launch a VM using that AMI. The instructions +below are included for maintainance of the history of this site config and +their possible relevance to debugging issues should they arise. + +### Base Instance + +This AMI was built on an instance with these properties: + +* AMI Name: ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-20240927 +* AMI ID: ami-0ea3c35c5c3284d82 +* Instance m6i.4xlarge (uses Intel Xeon processor) +* 300GB of gp3 storage as / + +## Pre-requisites (For All Compilers) + +### Installing Packages + +```bash +# Update system software and start a tmux session. +sudo su - +apt update +apt upgrade + +# Build tools +apt install -y build-essential g++-11 gcc-11 gfortran-11 make cmake automake autoconf + +#Install other requirements. +apt install -y cpp-11 libgomp1 git git-lfs autopoint mysql-server libmysqlclient-dev qtbase5-dev qt5-qmake libqt5svg5-dev qt5dxcb-plugin wget curl file tcl-dev gnupg2 iproute2 locales unzip less bzip2 gettext libtree pkg-config libcurl4-openssl-dev + +# Editors +apt install -y vim nano + +# Python develop. +apt install -y python3 python3-pip python3-setuptools + +# Configure git credential caching and git lfs for the rocky user and root. +git config --global credential.helper 'cache --timeout=3600' +git lfs install + +# Change the gcc, g++, and gfortran version to 11 and give it the highest priority +update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 +update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 +update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-11 100 + +exit # Exit root access +``` + +**Important**: Running the `update-alternatives` changes defaults, and if you need to use a different version of gcc/g++/gfortran you can run: `update-alternatives --config [gcc|g++|gfortran]` and select the version you want that is installed. + +### Install Lmod + +```bash +# Install lua/lmod manually because apt only has older versions +# that are not compatible with the modern lua modules spack produces +# https://lmod.readthedocs.io/en/latest/030_installing.html#install-lua-x-y-z-tar-gz +sudo su - +mkdir -p /opt/lua/5.1.4.9/src && cd $_ +wget https://sourceforge.net/projects/lmod/files/lua-5.1.4.9.tar.bz2 +tar -xvf lua-5.1.4.9.tar.bz2 +cd lua-5.1.4.9 +./configure --prefix=/opt/lua/5.1.4.9 2>&1 | tee log.config +make VERBOSE=1 2>&1 | tee log.make +make install 2>&1 | tee log.install + +cat << 'EOF' >> /etc/profile.d/02-lua.sh +# Set environment variables for lua +export PATH="/opt/lua/5.1.4.9/bin:$PATH" +export LD_LIBRARY_PATH="/opt/lua/5.1.4.9/lib:$LD_LIBRARY_PATH" +export CPATH="/opt/lua/5.1.4.9/include:$CPATH" +export MANPATH="/opt/lua/5.1.4.9/man:$MANPATH" +EOF + +source /etc/profile.d/02-lua.sh +mkdir -p /opt/lmod/8.7/src +cd /opt/lmod/8.7/src +wget https://sourceforge.net/projects/lmod/files/Lmod-8.7.tar.bz2 +tar -xvf Lmod-8.7.tar.bz2 +cd Lmod-8.7 +# Note the weird prefix, lmod installs in PREFIX/lmod/X.Y automatically +./configure --prefix=/opt/ \ + --with-lmodConfigDir=/opt/lmod/8.7/config \ + 2>&1 | tee log.config +make install 2>&1 | tee log.install +ln -sf /opt/lmod/lmod/init/profile /etc/profile.d/z00_lmod.sh +ln -sf /opt/lmod/lmod/init/cshrc /etc/profile.d/z00_lmod.csh +ln -sf /opt/lmod/lmod/init/profile.fish /etc/profile.d/z00_lmod.fish + +# Log out completely, ssh back into the instance and check if lua modules work +exit +exit +``` + +### Setup MySQL Sercer + +```bash +sudo systemctl start mysqld.service +sudo systemctl enable mysqld + +# Use the mysql server. +mysql -u root +exit +``` + +### Clone `spack-stack` + +```bash +cd /opt +sudo git clone -b develop --depth 1 --recursive https://github.com/jcsda/spack-stack.git +``` + +## Install Spack-Stack Steps by Compiler + +
+GCC Installation + +```bash +sudo su - + +cd /opt/spack-stack +source setup.sh +# Swap default module type for default linux. +sed -i 's/tcl/lmod/g' configs/sites/tier2/linux.default/modules.yaml +spack stack create env --site linux.default --template unified-dev --name unified-env-gcc --compiler gcc +cd envs/unified-env-gcc +spack env activate -p . +export SPACK_SYSTEM_CONFIG_PATH="$PWD/site" +spack external find --scope system \ + --exclude cmake \ + --exclude curl --exclude openssl \ + --exclude openssh --exclude python +spack external find --scope system wget +spack external find --scope system mysql +spack external find --scope system grep +spack compiler find --scope system +unset SPACK_SYSTEM_CONFIG_PATH +# ACTION: Edit the site/compilers.yaml with the following. +# 1) Delete or comment gcc-13 refs and preserve only gcc-12 +# 2) Delete or comment clang refs. +# ACTION: Edit the site/packages.yaml and add these packages +# If not present. +cat << 'EOF' >> $PWD/site/packages.yaml + gcc: + buildable: false + externals: + - spec: gcc@11.4.0 + prefix: /usr + gcc-runtime: + buildable: false + externals: + - spec: gcc-runtime@11.4.0 + prefix: /usr + qt: + buildable: false + externals: + - spec: qt@5.15.3 + prefix: /usr + version: [5.15.3] +EOF + +# Continue configuration. +spack config add "packages:all:compiler:[gcc@11.4.0]" +spack config add "packages:all:providers:mpi:[openmpi@5.0.5]" +spack config add "packages:fontconfig:variants:+pic" +spack config add "packages:pixman:variants:+pic" +spack config add "packages:cairo:variants:+pic" +spack config add "packages:ewok-env:variants:+mysql" + +# Concretize and install +spack concretize 2>&1 | tee log.concretize +${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d -c log.concretize +spack install --verbose --fail-fast 2>&1 | tee log.install + +# Install modules +spack module lmod refresh +spack stack setup-meta-modules + +# Add a number of default module locations to the lmod startup script. +cat << 'EOF' >> /etc/profile.d/z01_lmod.sh +module use /opt/spack-stack/envs/unified-env-gcc/install/modulefiles/Core +EOF +``` + +
+ +
+Intel Installation + +#### Install Intel Compiler + +```bash +sudo su - + +mkdir -p /opt/intel/src +pushd /opt/intel/src + +# Download Intel install assets. +wget -O cpp-compiler.sh https://registrationcenter-download.intel.com/akdlm/IRC_NAS/d85fbeee-44ec-480a-ba2f-13831bac75f7/l_dpcpp-cpp-compiler_p_2023.2.3.12_offline.sh +wget -O fortran-compiler.sh https://registrationcenter-download.intel.com/akdlm/IRC_NAS/0ceccee5-353c-4fd2-a0cc-0aecb7492f87/l_fortran-compiler_p_2023.2.3.13_offline.sh +wget -O tbb.sh https://registrationcenter-download.intel.com/akdlm/IRC_NAS/c95cd995-586b-4688-b7e8-2d4485a1b5bf/l_tbb_oneapi_p_2021.10.0.49543_offline.sh +wget -O mpi.sh https://registrationcenter-download.intel.com/akdlm/IRC_NAS/4f5871da-0533-4f62-b563-905edfb2e9b7/l_mpi_oneapi_p_2021.10.0.49374_offline.sh +wget -O math.sh https://registrationcenter-download.intel.com/akdlm/IRC_NAS/adb8a02c-4ee7-4882-97d6-a524150da358/l_onemkl_p_2023.2.0.49497_offline.sh + +# Install the Intel assets. +sh cpp-compiler.sh -a --silent --eula accept 2>&1 | tee install.cpp-compiler.log +sh fortran-compiler.sh -a --silent --eula accept | tee install.fortran-compiler.log +sh tbb.sh -a --silent --eula accept | tee install.tbb.log +sh mpi.sh -a --silent --eula accept | tee install.mpi.log +sh math.sh -a --silent --eula accept | tee install.math.log + +popd +exit +``` + +#### Install Intel Spack-Stack Environment + +```bash +sudo su - + +source /opt/intel/oneapi/compiler/2023.2.3/env/vars.sh +source /opt/intel/oneapi/mpi/2021.10.0/env/vars.sh +source /opt/intel/oneapi/setvars.sh + +cd /opt/spack-stack +source ./setup.sh + +spack stack create env --site linux.default --template unified-dev --name unified-env-intel --compiler intel +cd envs/unified-env-intel +spack env activate -p . + +export SPACK_SYSTEM_CONFIG_PATH="${PWD}/site" + +spack external find --scope system --exclude bison --exclude openssl --exclude python --exclude gettext --exclude m4 --exclude cmake --exclude curl +spack external find --scope system perl +spack external find --scope system wget +spack external find --scope system texlive +spack external find --scope system mysql +spack external find --scope system grep + +# No external find for pre-installed intel-oneapi-mpi, +# and no way to add object entry to list using "spack config add". +cat << 'EOF' >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + gcc: + buildable: false + externals: + - spec: gcc@11.4.0 + prefix: /usr + gcc-runtime: + buildable: false + externals: + - spec: gcc-runtime@11.4.0 + prefix: /usr + grep: + buildable: False + externals: + - spec: grep@3.11 + prefix: /usr + intel-oneapi-mpi: + externals: + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 +classic-names + prefix: /opt/intel/oneapi +EOF + +# Can't find qt5 because qtpluginfo is broken, +# and no way to add object entry to list using "spack config add". +cat << 'EOF' >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + qt: + buildable: false + externals: + - spec: qt@5.15.3 + prefix: /usr +EOF + +spack compiler find --scope system + +unset SPACK_SYSTEM_CONFIG_PATH + +spack config add "packages:mpi:buildable:False" +spack config add "packages:all:providers:mpi:[intel-oneapi-mpi@2021.10.0]" +spack config add "packages:all:compiler:[intel@2021.10.0, gcc@11.4.0]" + +# edit envs/unified-env/site/compilers.yaml and replace the following line in the **Intel** compiler section: +# environment: {} +# --> +# environment: +# prepend_path: +# LD_LIBRARY_PATH: '/opt/intel/oneapi/compiler/2023.2.3/linux/compiler/lib/intel64_lin' + +spack concretize 2>&1 | tee log.concretize +${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize +spack install --fail-fast -j 12 2>&1 | tee log.install +spack module lmod refresh +spack stack setup-meta-modules + +cat << 'EOF' >> /etc/profile.d/z01_lmod.sh +module use /opt/spack-stack/envs/unified-env-gcc/install/modulefiles/Core +EOF +``` + +
+ +## Test Installation + +
+GCC + +```bash +# Example given for building jedi-bundle +module use /opt/spack-stack/envs/unified-dev-gcc/install/modulefiles/Core +module load stack-gcc/12.3.0 +module load stack-openmpi/5.0.5 +module load base-env +module load jedi-mpas-env +module load jedi-fv3-env +module load ewok-env +module load sp + +mkdir /opt/jedi +cd /opt/jedi +git clone https://github.com/JCSDA-internal/jedi-bundle.git +cd jedi-bundle +mkdir build && cd build +ecbuild ../ +make update +make -j10 +ctest +``` + +
+ +
+Intel + +```bash +# Example given for building jedi-bundle +module use /opt/spack-stack/envs/unified-env-intel/install/modulefiles/Core +module load stack-intel/2021.10.0 +module load stack-intel-oneapi-mpi/2021.10.0 +module load base-env +module load jedi-mpas-env +module load jedi-fv3-env +module load ewok-env +module load sp + +# (Optional) Re-source the intel environment +source /opt/intel/oneapi/setvars.sh + +mkdir /opt/jedi +cd /opt/jedi +git clone https://github.com/JCSDA-internal/jedi-bundle.git +cd jedi-bundle +mkdir build && cd build +ecbuild ../ +make update +make -j10 +ctest +``` + +
+ +The installation and configuration is now complete for the instance. diff --git a/configs/sites/tier2/aws-ubuntu2404/compilers.yaml b/configs/sites/tier2/aws-ubuntu2404/compilers.yaml new file mode 100644 index 000000000..7cfcf40d8 --- /dev/null +++ b/configs/sites/tier2/aws-ubuntu2404/compilers.yaml @@ -0,0 +1,30 @@ +compilers: +- compiler: + spec: intel@=2021.10.0 + paths: + cc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/icc + cxx: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/icpc + f77: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/ifort + fc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/ifort + flags: {} + operating_system: ubuntu24.04 + target: x86_64 + modules: [] + environment: + prepend_path: + LD_LIBRARY_PATH: '/opt/intel/oneapi/compiler/2023.2.3/linux/compiler/lib/intel64_lin' + extra_rpaths: [] +- compiler: + spec: gcc@11.4.0 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: {} + operating_system: ubuntu24.04 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + diff --git a/configs/sites/tier2/aws-ubuntu2404/config.yaml b/configs/sites/tier2/aws-ubuntu2404/config.yaml new file mode 100644 index 000000000..cf54e6360 --- /dev/null +++ b/configs/sites/tier2/aws-ubuntu2404/config.yaml @@ -0,0 +1,9 @@ +config: + build_jobs: 8 + + # Overrides for spack build and staging areas to speed up builds + # by using a local directory instead of the EFS shared filesystem + build_stage: /tmp/spack-stack/cache/build_stage + test_stage: /tmp/spack-stack/cache/test_stage + source_cache: /tmp/spack-stack/cache/source_cache + misc_cache: /tmp/spack-stack/cache/misc_cache diff --git a/configs/sites/tier2/aws-ubuntu2404/modules.yaml b/configs/sites/tier2/aws-ubuntu2404/modules.yaml new file mode 100644 index 000000000..65801706f --- /dev/null +++ b/configs/sites/tier2/aws-ubuntu2404/modules.yaml @@ -0,0 +1,7 @@ +modules: + default: + enable:: + - lmod + lmod: + include: + - python diff --git a/configs/sites/tier2/aws-ubuntu2404/packages.yaml b/configs/sites/tier2/aws-ubuntu2404/packages.yaml new file mode 100644 index 000000000..6158d0119 --- /dev/null +++ b/configs/sites/tier2/aws-ubuntu2404/packages.yaml @@ -0,0 +1,94 @@ +packages: + mysql: + externals: + - spec: mysql@8.0.37 + prefix: /usr + wget: + externals: + - spec: wget@1.21.2 + prefix: /usr + binutils: + externals: + - spec: binutils@2.38 + prefix: /usr + coreutils: + externals: + - spec: coreutils@8.32 + prefix: /usr + diffutils: + externals: + - spec: diffutils@3.8 + prefix: /usr + findutils: + externals: + - spec: findutils@4.8.0 + prefix: /usr + gawk: + externals: + - spec: gawk@5.1.0 + prefix: /usr + tar: + externals: + - spec: tar@1.34 + prefix: /usr + autoconf: + externals: + - spec: autoconf@2.71 + prefix: /usr + automake: + externals: + - spec: automake@1.16.5 + prefix: /usr + flex: + externals: + - spec: flex@2.6.4+lex + prefix: /usr + gettext: + externals: + - spec: gettext@0.21 + prefix: /usr + git: + externals: + - spec: git@2.34.1~tcltk + prefix: /usr + git-lfs: + externals: + - spec: git-lfs@3.0.2 + prefix: /usr + gmake: + externals: + - spec: gmake@4.3 + prefix: /usr + groff: + externals: + - spec: groff@1.22.4 + prefix: /usr + libtool: + externals: + - spec: libtool@2.4.6 + prefix: /usr + m4: + externals: + - spec: m4@1.4.18 + prefix: /usr + perl: + externals: + - spec: perl@5.34.0~cpanm+opcode+open+shared+threads + prefix: /usr + pkg-config: + externals: + - spec: pkg-config@0.29.2 + prefix: /usr + qt: + buildable: false + externals: + - spec: qt@5.15.3 + prefix: /usr + sed: + externals: + - spec: sed@4.8 + prefix: /usr + lcms: + externals: + - spec: lcms@2.13.1 + prefix: /usr diff --git a/configs/sites/tier2/aws-ubuntu2404/packages_gcc.yaml b/configs/sites/tier2/aws-ubuntu2404/packages_gcc.yaml new file mode 100644 index 000000000..4f9287676 --- /dev/null +++ b/configs/sites/tier2/aws-ubuntu2404/packages_gcc.yaml @@ -0,0 +1,5 @@ +packages: + all: + compiler:: [gcc@11.4.0] + providers: + mpi:: [openmpi@5.0.5] diff --git a/configs/sites/tier2/aws-ubuntu2404/packages_intel.yaml b/configs/sites/tier2/aws-ubuntu2404/packages_intel.yaml new file mode 100644 index 000000000..66a9c4263 --- /dev/null +++ b/configs/sites/tier2/aws-ubuntu2404/packages_intel.yaml @@ -0,0 +1,30 @@ +packages: + all: + compiler:: [intel@2021.10.0,gcc@11.4.0] + providers: + mpi:: [intel-oneapi-mpi@2021.10.0] + # Remove the next three lines to switch to intel-oneapi-mkl + blas:: [openblas] + fftw-api:: [fftw] + lapack:: [openblas] + mpi: + buildable: False + intel-oneapi-mpi: + externals: + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 + prefix: /opt/intel/oneapi + intel-oneapi-mkl: + # Remove buildable: False and configure version+uncomment externals section below to use intel-oneapi-mkl + buildable: False + #externals: + #- spec: intel-oneapi-mkl@2023.2.0%intel@2021.10.0 + # prefix: /opt/intel/oneapi + ectrans: + require:: + - '@1.2.0 ~mkl +fftw' + gsibec: + require:: + - '@1.2.1 ~mkl' + py-numpy: + require:: + - '@:1.23 ^openblas' diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index e5d900829..c11a2d782 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -532,6 +532,28 @@ Linux/macOS default configs The Linux and macOS configurations are **not** meant to be used as is, as user setups and package versions vary considerably. Instructions for adding this information can be found in :numref:`Section %s `. +.. _Preconfigured_Sites_AWS_Ubuntu2404: + +---------------- +AWS Ubuntu 24.04 +---------------- + +To build consult the `README.md` in the `sites/tier2/aws-ubuntu2404`. + +This image can contain GCC and Intel compilers. It is strongly suggested that if you are to use either environment, it is suggested to separate environments into their own terminal. + + +.. _Preconfigured_Sites_AWS_Rocky8: + +---------- +AWS Rocky8 +---------- + +To build consult the `README.md` in the `sites/tier2/aws-rocky8`. + +This image can contain GCC and Intel compilers. It is strongly suggested that if you are to use either environment, it is suggested to separate environments into their own terminal. + + .. _Configurable_Sites_CreateEnv: ========================