From 572d0f5eaacd90b1443afc51c4fc078c8a32589c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 20 Dec 2022 16:11:02 -0700 Subject: [PATCH] release/1.2.0: final updates (site configs, documentation, misc) (#419) * Add skylab-3.0.0 template * Add mirror to aws-pcluster site config * Add S4 mirror, update Cheyenne and Discover packages * Update container configs * Add gaea mirror * Update submodule pointer for spack * Add S4 GNU site config * Update Narwhal site config * Update casper site config * Make crtm@v2.4-jedi.2 default for JEDI * Documentation updates --- .gitmodules | 2 +- configs/containers/README.md | 4 +- .../containers/docker-ubuntu-clang-mpich.yaml | 3 +- .../containers/docker-ubuntu-gcc-openmpi.yaml | 3 +- .../containers/docker-ubuntu-intel-impi.yaml | 7 +- configs/sites/aws-pcluster/mirrors.yaml | 18 +++ configs/sites/casper/compilers.yaml | 2 +- configs/sites/casper/mirrors.yaml | 18 +++ configs/sites/casper/packages.yaml | 14 +- configs/sites/cheyenne/packages.yaml | 11 +- configs/sites/discover/packages.yaml | 11 +- configs/sites/gaea/mirrors.yaml | 18 +++ configs/sites/gaea/modules.yaml | 3 + configs/sites/narwhal/compilers.yaml | 2 +- configs/sites/narwhal/mirrors.yaml | 2 +- configs/sites/narwhal/modules.yaml | 3 + configs/sites/narwhal/packages.yaml | 4 - configs/sites/s4/compilers.yaml | 2 - configs/sites/s4/mirrors.yaml | 18 +++ configs/sites/s4/modules.yaml | 2 + configs/sites/s4/packages.yaml | 5 + configs/templates/skylab-3.0.0/spack.yaml | 95 ++++++++++++ configs/templates/skylab-dev/spack.yaml | 5 +- .../templates/skylab-no-python-dev/spack.yaml | 5 +- doc/source/MaintainersSection.rst | 14 +- doc/source/Platforms.rst | 141 ++++++++---------- doc/source/Quickstart.rst | 31 +++- spack | 2 +- 28 files changed, 325 insertions(+), 120 deletions(-) create mode 100644 configs/sites/aws-pcluster/mirrors.yaml create mode 100644 configs/sites/casper/mirrors.yaml create mode 100644 configs/sites/gaea/mirrors.yaml create mode 100644 configs/sites/s4/mirrors.yaml create mode 100644 configs/templates/skylab-3.0.0/spack.yaml diff --git a/.gitmodules b/.gitmodules index 06dda9b36..cf6751e3d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,7 +3,7 @@ #url = https://github.com/spack/spack #branch = develop url = https://github.com/NOAA-EMC/spack - branch = release/1.2.0 + branch = spack-stack-1.2.0 [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/configs/containers/README.md b/configs/containers/README.md index 74c2a6e54..bf0e8100b 100644 --- a/configs/containers/README.md +++ b/configs/containers/README.md @@ -6,7 +6,7 @@ To avoid hardcoding specs in the generic container recipes, we keep the specs li ``` specs: [base-env@1.0.0, jedi-base-env@1.0.0 ~fftw, jedi-ewok-env@1.0.0, jedi-fv3-env@1.0.0, jedi-mpas-env@1.0.0, bacio@2.4.1, bison@3.8.2, bufr@11.7.1, ecbuild@3.6.5, eccodes@2.27.0, ecflow@5, - eckit@1.19.0, ecmwf-atlas@0.30.0 +trans ~fftw, ectrans@1.0.0 ~fftw, eigen@3.4.0, + eckit@1.20.2, ecmwf-atlas@0.31.1 +trans ~fftw, ectrans@1.1.0 ~fftw, eigen@3.4.0, fckit@0.9.5, flex@2.6.4, fms@release-jcsda, g2@3.4.5, g2tmpl@1.10.0, gftl-shared@1.5.0, gsibec@1.0.6, hdf5@1.12.1, hdf@4.2.15, ip@3.3.3, jasper@2.0.32, jedi-cmake@1.4.0, libpng@1.6.37, nccmp@1.9.0.1, netcdf-c@4.8.1, netcdf-cxx4@4.3.1, @@ -14,7 +14,7 @@ To avoid hardcoding specs in the generic container recipes, we keep the specs li parallel-netcdf@1.12.2, parallelio@2.5.7, py-eccodes@1.4.2, py-f90nml@1.4.3, py-numpy@1.22.3, py-pandas@1.4.0, py-pyyaml@6.0, py-scipy@1.8.0, py-shapely@1.8.0, py-xarray@2022.3.0, sp@2.3.3, udunits@2.2.28, w3nco@2.4.1, nco@5.0.6, - yafyaml@0.5.1, zlib@1.2.13, odc@1.4.5, crtm@v2.4-jedi.1, shumlib@macos_clang_linux_intel_port] + yafyaml@0.5.1, zlib@1.2.13, odc@1.4.5, crtm@v2.4-jedi.2, shumlib@macos_clang_linux_intel_port] # Don't build ESMF and MAPL for now: # https://github.com/JCSDA-internal/MPAS-Model/issues/38 # https://github.com/NOAA-EMC/spack-stack/issues/326 diff --git a/configs/containers/docker-ubuntu-clang-mpich.yaml b/configs/containers/docker-ubuntu-clang-mpich.yaml index ec9e7bdb5..0e7c8d322 100644 --- a/configs/containers/docker-ubuntu-clang-mpich.yaml +++ b/configs/containers/docker-ubuntu-clang-mpich.yaml @@ -29,6 +29,7 @@ spack: # Additional package config for container packages: all: + require: '%clang' target: [x86_64] providers: mpi: [mpich@4.0.2] @@ -95,7 +96,7 @@ spack: os: ubuntu:20.04 spack: url: https://github.com/noaa-emc/spack - ref: jcsda_emc_spack_stack + ref: spack-stack-1.2.0 resolve_sha: false # Whether or not to strip binaries diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index 0e35eb257..f672a3570 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -27,6 +27,7 @@ spack: # Additional package config for container packages: all: + require: '%gcc' target: [x86_64] providers: mpi: [openmpi@4.1.4] @@ -82,7 +83,7 @@ spack: os: ubuntu:20.04 spack: url: https://github.com/noaa-emc/spack - ref: jcsda_emc_spack_stack + ref: spack-stack-1.2.0 resolve_sha: false # Whether or not to strip binaries diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index c64985753..128ee8be8 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -31,6 +31,7 @@ spack: # Additional package config for container packages: all: + require: '%intel' target: [core2] providers: mpi: [intel-oneapi-mpi@2021.6.0] @@ -60,6 +61,10 @@ spack: externals: - spec: git-lfs@2.9.2 prefix: /usr + # Turn off crypt, because libxcrypt doesn't + # build with Intel. + python: + variants: ~crypt qt: buildable: false externals: @@ -96,7 +101,7 @@ spack: os: ubuntu:20.04 spack: url: https://github.com/noaa-emc/spack - ref: jcsda_emc_spack_stack + ref: spack-stack-1.2.0 resolve_sha: false # Whether or not to strip binaries diff --git a/configs/sites/aws-pcluster/mirrors.yaml b/configs/sites/aws-pcluster/mirrors.yaml new file mode 100644 index 000000000..903867028 --- /dev/null +++ b/configs/sites/aws-pcluster/mirrors.yaml @@ -0,0 +1,18 @@ +mirrors: + local-source: + fetch: + url: file:///mnt/experiments-efs/spack-stack/source-cache + access_pair: + - null + - null + access_token: null + profile: null + endpoint_url: null + push: + url: file:///mnt/experiments-efs/spack-stack/source-cache + access_pair: + - null + - null + access_token: null + profile: null + endpoint_url: null diff --git a/configs/sites/casper/compilers.yaml b/configs/sites/casper/compilers.yaml index ae59673cd..a3724a0bf 100644 --- a/configs/sites/casper/compilers.yaml +++ b/configs/sites/casper/compilers.yaml @@ -15,7 +15,7 @@ compilers:: prepend_path: PATH: '/glade/u/apps/dav/opt/gnu/9.1.0/bin' CPATH: '/glade/u/apps/dav/opt/gnu/9.1.0/include' - LD_LIBRARY_PATH: '/glade/u/apps/opt/intel/2020u1/compilers_and_libraries/linux/lib/intel64_lin:/glade/u/apps/dav/opt/gnu/9.1.0/lib64' + LD_LIBRARY_PATH: '/glade/u/apps/opt/intel/2020u1/compilers_and_libraries/linux/lib/intel64_lin:/glade/u/apps/opt/intel/2020u1/compilers_and_libraries_2020.1.217/linux/mpi/intel64/libfabric/lib:/glade/u/apps/dav/opt/gnu/9.1.0/lib64' set: INTEL_LICENSE_FILE: '28518@128.117.177.41' LM_LICENSE_FILE: '28518@128.117.177.41' diff --git a/configs/sites/casper/mirrors.yaml b/configs/sites/casper/mirrors.yaml new file mode 100644 index 000000000..8325126cc --- /dev/null +++ b/configs/sites/casper/mirrors.yaml @@ -0,0 +1,18 @@ +mirrors: + local-source: + fetch: + url: file:///glade/work/jedipara/cheyenne/spack-stack/source-cache + access_pair: + - null + - null + access_token: null + profile: null + endpoint_url: null + push: + url: file:///glade/work/jedipara/cheyenne/spack-stack/source-cache + access_pair: + - null + - null + access_token: null + profile: null + endpoint_url: null diff --git a/configs/sites/casper/packages.yaml b/configs/sites/casper/packages.yaml index 803a428ed..bc636d3b9 100644 --- a/configs/sites/casper/packages.yaml +++ b/configs/sites/casper/packages.yaml @@ -80,10 +80,11 @@ packages: prefix: /glade/work/jedipara/cheyenne/spack-stack/ecflow-5.8.4 modules: - ecflow/5.8.4 - expat: - externals: - - spec: expat@1.6.0 - prefix: /usr + # Don't use, leads to duplicate packages being built + #expat: + # externals: + # - spec: expat@1.6.0 + # prefix: /usr file: externals: - spec: file@5.11 @@ -93,6 +94,8 @@ packages: - spec: findutils@4.5.11 prefix: /usr flex: + # Must set buildable: false to avoid duplicate packages + buildable: false externals: - spec: flex@2.5.37+lex prefix: /usr @@ -164,6 +167,9 @@ packages: externals: - spec: openssl@1.0.2k-fips prefix: /usr + # Pin patchelf to 0.15.0 for Intel compiler (no C++-17 features) + patchelf: + version:: [0.15.0] perl: externals: - spec: perl@5.16.3~cpanm+shared+threads diff --git a/configs/sites/cheyenne/packages.yaml b/configs/sites/cheyenne/packages.yaml index 03a908ae1..c0e748075 100644 --- a/configs/sites/cheyenne/packages.yaml +++ b/configs/sites/cheyenne/packages.yaml @@ -115,11 +115,12 @@ packages: externals: - spec: findutils@4.5.12 prefix: /usr - # Don't use, leads to duplicate packages being built - #flex: - # externals: - # - spec: flex@2.5.37+lex - # prefix: /usr + flex: + # Must set buildable: false to avoid duplicate packages + buildable: false + externals: + - spec: flex@2.5.37+lex + prefix: /usr gawk: externals: - spec: gawk@4.1.0 diff --git a/configs/sites/discover/packages.yaml b/configs/sites/discover/packages.yaml index 7201f18cd..5a0dfad00 100644 --- a/configs/sites/discover/packages.yaml +++ b/configs/sites/discover/packages.yaml @@ -102,11 +102,12 @@ packages: externals: - spec: findutils@4.5.12 prefix: /usr - # Don't use, leads to duplicate packages being built - #flex: - # externals: - # - spec: flex@2.5.37+lex - # prefix: /usr + flex: + # Must set buildable: false to avoid duplicate packages + buildable: false + externals: + - spec: flex@2.5.37+lex + prefix: /usr gawk: externals: - spec: gawk@4.1.0 diff --git a/configs/sites/gaea/mirrors.yaml b/configs/sites/gaea/mirrors.yaml new file mode 100644 index 000000000..372fe7bd3 --- /dev/null +++ b/configs/sites/gaea/mirrors.yaml @@ -0,0 +1,18 @@ +mirrors: + local-source: + fetch: + url: file:///lustre/f2/pdata/esrl/gsd/spack-stack/source-cache + access_pair: + - null + - null + access_token: null + profile: null + endpoint_url: null + push: + url: file:///lustre/f2/pdata/esrl/gsd/spack-stack/source-cache + access_pair: + - null + - null + access_token: null + profile: null + endpoint_url: null diff --git a/configs/sites/gaea/modules.yaml b/configs/sites/gaea/modules.yaml index e4c2e5300..07cc3c509 100644 --- a/configs/sites/gaea/modules.yaml +++ b/configs/sites/gaea/modules.yaml @@ -2,3 +2,6 @@ modules: default: enable:: - tcl + tcl: + blacklist: + - ecflow diff --git a/configs/sites/narwhal/compilers.yaml b/configs/sites/narwhal/compilers.yaml index 5af3dbd07..3c0485d0f 100644 --- a/configs/sites/narwhal/compilers.yaml +++ b/configs/sites/narwhal/compilers.yaml @@ -10,7 +10,7 @@ compilers:: operating_system: sles15 target: any modules: - - PrgEnv-intel + - PrgEnv-intel/8.3.2 - intel/2021.3.0 environment: prepend_path: diff --git a/configs/sites/narwhal/mirrors.yaml b/configs/sites/narwhal/mirrors.yaml index 24638e049..8f845ff68 100644 --- a/configs/sites/narwhal/mirrors.yaml +++ b/configs/sites/narwhal/mirrors.yaml @@ -1,5 +1,5 @@ mirrors: - local: + local-source: fetch: url: file:///p/app/projects/NEPTUNE/spack-stack/spack-stack-v1/source-cache access_pair: diff --git a/configs/sites/narwhal/modules.yaml b/configs/sites/narwhal/modules.yaml index e4c2e5300..07cc3c509 100644 --- a/configs/sites/narwhal/modules.yaml +++ b/configs/sites/narwhal/modules.yaml @@ -2,3 +2,6 @@ modules: default: enable:: - tcl + tcl: + blacklist: + - ecflow diff --git a/configs/sites/narwhal/packages.yaml b/configs/sites/narwhal/packages.yaml index abbd75070..c722fba5b 100644 --- a/configs/sites/narwhal/packages.yaml +++ b/configs/sites/narwhal/packages.yaml @@ -49,10 +49,6 @@ packages: externals: - spec: bison@3.0.4 prefix: /usr - cmake: - externals: - - spec: cmake@3.17.0 - prefix: /usr curl: externals: - spec: curl@7.66.0+gssapi+ldap+nghttp2 diff --git a/configs/sites/s4/compilers.yaml b/configs/sites/s4/compilers.yaml index 98fd7d6c8..5bdbb957a 100644 --- a/configs/sites/s4/compilers.yaml +++ b/configs/sites/s4/compilers.yaml @@ -18,7 +18,6 @@ compilers: LD_LIBRARY_PATH: '/home/opt/intel/oneapi/2022.1/compiler/2022.0.1/linux/compiler/lib/intel64_lin:/data/prod/hpc-stack/gnu/9.3.0/lib64' CPATH: '/data/prod/hpc-stack/gnu/9.3.0/include' extra_rpaths: [] -# Not yet tested: - compiler: spec: gcc@9.3.0 paths: @@ -31,7 +30,6 @@ compilers: target: x86_64 modules: - gnu/9.3.0 - prefix: /data/prod/hpc-stack/gnu/9.3.0 modules: [] environment: {} extra_rpaths: [] diff --git a/configs/sites/s4/mirrors.yaml b/configs/sites/s4/mirrors.yaml new file mode 100644 index 000000000..152750f81 --- /dev/null +++ b/configs/sites/s4/mirrors.yaml @@ -0,0 +1,18 @@ +mirrors: + local-source: + fetch: + url: file:///data/prod/jedi/spack-stack/source-cache + access_pair: + - null + - null + access_token: null + profile: null + endpoint_url: null + push: + url: file:///data/prod/jedi/spack-stack/source-cache + access_pair: + - null + - null + access_token: null + profile: null + endpoint_url: null diff --git a/configs/sites/s4/modules.yaml b/configs/sites/s4/modules.yaml index 7728eff92..e6d6fac4b 100644 --- a/configs/sites/s4/modules.yaml +++ b/configs/sites/s4/modules.yaml @@ -5,3 +5,5 @@ modules: lmod: blacklist: - ecflow + whitelist: + - mpich diff --git a/configs/sites/s4/packages.yaml b/configs/sites/s4/packages.yaml index 539c4d900..558b8db33 100644 --- a/configs/sites/s4/packages.yaml +++ b/configs/sites/s4/packages.yaml @@ -1,10 +1,13 @@ packages: all: compiler:: [intel@2021.5.0] + #compiler:: [gcc@9.3.0] providers: mpi:: [intel-oneapi-mpi@2021.5.0] + #mpi:: [mpich@4.0.2%gcc@9.3.0] ### MPI, Python, MKL + # Comment out next two lines for gnu@9.3.0 with mpich@4.0.2 built by spack mpi: buildable: False intel-oneapi-mpi: @@ -21,6 +24,8 @@ packages: # externals: # - spec: intel-oneapi-mkl@2022.0.1%intel@2021.5.0 # prefix: /opt/intel/oneapi/2022.1 + mpich: + version: [4.0.2] python: buildable: False externals: diff --git a/configs/templates/skylab-3.0.0/spack.yaml b/configs/templates/skylab-3.0.0/spack.yaml new file mode 100644 index 000000000..de2bec4dd --- /dev/null +++ b/configs/templates/skylab-3.0.0/spack.yaml @@ -0,0 +1,95 @@ +spack: + concretizer: + unify: when_possible + + view: false + + packages: + fiat: + version: [1.0.0] + ectrans: + version: [1.1.0] + + view: false + include: [] + + specs: + # Virtual environment packages + - jedi-ewok-env + - jedi-fv3-env + - jedi-mpas-env + - jedi-ufs-env + - jedi-um-env + - soca-env + + # Individual packages + - bacio@2.4.1 + # Do not request specific version of bison - this leads to duplicate packages being built + #- bison@3.8.2 + - bufr@11.7.1 + - crtm@2.3.0 + - crtm@2.4.0 + - crtm@v2.3-jedi.4 + - crtm@v2.4-jedi.2 + - ecbuild@3.6.5 + - eccodes@2.27.0 + - ecflow@5 + - eckit@1.20.2 + - ecmwf-atlas@0.31.1 + - ectrans@1.1.0 + - eigen@3.4.0 + - esmf@8.3.0b09+pio + # DH* fake version number + #- ewok@0.0.1 + - fckit@0.9.5 + # DH* fake version number + - fiat@1.0.0 + # Do not request specific version of flex - this leads to duplicate packages being built + #- flex@2.6.4 + - fms@2022.02 + # DH* fake version number + - fms@release-jcsda + - g2@3.4.5 + - g2tmpl@1.10.0 + #- gdal@3.4.3 + #- geos@3.9.1 + - gftl-shared@1.5.0 + - gsibec@1.0.6 + - hdf5@1.12.1 + - hdf@4.2.15 + - ip@3.3.3 + - jasper@2.0.32 + - jedi-cmake@1.4.0 + - libpng@1.6.37 + - mapl@2.22.0 + - nccmp@1.9.0.1 + - ncview@2.1.8 + - nemsio@2.5.2 + - netcdf-c@4.8.1 + - netcdf-cxx4@4.3.1 + - netcdf-fortran@4.5.4 + - nlohmann-json-schema-validator@2.1.0 + - nlohmann-json@3.10.5 + - odc@1.4.5 + - parallel-netcdf@1.12.2 + - parallelio@2.5.7 + - py-eccodes@1.4.2 + - py-f90nml@1.4.3 + - py-numpy@1.22.3 + - py-pandas@1.4.0 + - py-pyyaml@6.0 + - py-scipy@1.8.0 + - py-shapely@1.8.0 + - py-xarray@2022.3.0 + # DH* fake version number + #- r2d2@0.0.1 + # DH* fake version number + - shumlib@macos_clang_linux_intel_port + - sigio@2.3.2 + #- solo@1.0.0 + - sp@2.3.3 + - udunits@2.2.28 + - w3emc@2.9.2 + - w3nco@2.4.1 + - yafyaml@0.5.1 + - zlib@1.2.13 diff --git a/configs/templates/skylab-dev/spack.yaml b/configs/templates/skylab-dev/spack.yaml index a42bcadd2..de2bec4dd 100644 --- a/configs/templates/skylab-dev/spack.yaml +++ b/configs/templates/skylab-dev/spack.yaml @@ -8,7 +8,7 @@ spack: fiat: version: [1.0.0] ectrans: - version: [1.0.0] + version: [1.1.0] view: false include: [] @@ -30,8 +30,7 @@ spack: - crtm@2.3.0 - crtm@2.4.0 - crtm@v2.3-jedi.4 - - crtm@v2.4-jedi - - crtm@v2.4-jedi.1 + - crtm@v2.4-jedi.2 - ecbuild@3.6.5 - eccodes@2.27.0 - ecflow@5 diff --git a/configs/templates/skylab-no-python-dev/spack.yaml b/configs/templates/skylab-no-python-dev/spack.yaml index 03b0072a8..23e6fe009 100644 --- a/configs/templates/skylab-no-python-dev/spack.yaml +++ b/configs/templates/skylab-no-python-dev/spack.yaml @@ -8,7 +8,7 @@ spack: fiat: version: [1.0.0] ectrans: - version: [1.0.0] + version: [1.1.0] view: false include: [] @@ -44,8 +44,7 @@ spack: - crtm@2.3.0 - crtm@2.4.0 - crtm@v2.3-jedi.4 - - crtm@v2.4-jedi - - crtm@v2.4-jedi.1 + - crtm@v2.4-jedi.2 - ecbuild@3.6.5 - eccodes@2.27.0 # ecflow depends on Python diff --git a/doc/source/MaintainersSection.rst b/doc/source/MaintainersSection.rst index 15cf913b5..5c43b1a64 100644 --- a/doc/source/MaintainersSection.rst +++ b/doc/source/MaintainersSection.rst @@ -7,6 +7,16 @@ Maintainers/Developers Section Pre-configuring sites ============================== +.. _MaintainersSection_Preface: + +------------------------------ +Preface/general instructions +------------------------------ + +Preconfigured sites are defined through spack configuration files in the spack-stack directory ``configs/sites``, for example ``configs/sites/orion``. All files in the site-specific subdirectory will be copied into the environment into ``envs/env-name/site``. Site-specific configurations consist of general definitions (``config.yaml``), packages (``packages.yaml``), compilers (``compilers.yaml``), modules (``modules.yaml``), mirrors (``mirrors.yaml``) etc. These configurations overwrite the common configurations that are copied from ``configs/common`` into ``envs/env-name/common``. + +The instructions below are platform-specific tasks that only need to be done once and can be reused for new spack environments. To build new environments on preconfigured platforms, follow the instructions in :numref:`Section %s `. + .. _MaintainersSection_Orion: ------------------------------ @@ -373,7 +383,7 @@ The procedure is similar to using spack mirrors for local reuse, but a few addit 3. On the machine with full internet access: Load the basic external modules, if using a machine that is preconfigured for spack-stack (see :numref:`Section %s `) and make sure that ``git`` supports ``lfs`` (if necessary, load the external modules that spack-stack also uses). -4. On the machine with full internet access: check out the same version of ``spack-stack``, run ``setup.sh``, and then the following sequence of commands: +4. On the machine with full internet access: check out the same version of ``spack-stack``, run ``setup.sh``, and then the following sequence of commands. The mirror will be created in directory ``./spack/var/spack/environments/air_gapped_mirror_env``. .. code-block:: console @@ -381,8 +391,6 @@ The procedure is similar to using spack mirrors for local reuse, but a few addit spack env activate air_gapped_mirror_env spack mirror create -a - The mirror will be created in directory ``./spack/var/spack/environments/air_gapped_mirror_env`` - 5. On the air-gapped system: Copy the directory from the system with internet access to the local destination for the spack mirror. It is recommended to use ``rsync`` to avoid deleting existing packages, if updating an existing mirror on the air-gapped system. 6.. On the air-gapped system: Add the mirror to the spack environment's mirror list, unless already included in the site config. diff --git a/doc/source/Platforms.rst b/doc/source/Platforms.rst index 274204f73..cf93e9447 100644 --- a/doc/source/Platforms.rst +++ b/doc/source/Platforms.rst @@ -18,34 +18,34 @@ spack-stack-v1 -------------- .. note:: - This version supports the JEDI Skylab release end of June/beginning of July 2022, and can be used for testing spack-stack with other applications (e.g. the UFS Weather Model). Amazon Web Services AMI are available in the US East 1 region. + This version supports the JEDI Skylab release 3 of December 2022, and can be used for testing spack-stack with other applications (e.g. the UFS Weather Model). Amazon Web Services AMI are available in the US East 1 or 2 regions. +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ | System | Maintainers | Location | +==========================================================+===========================+====================================================================================================================+ -| MSU Orion Intel | Dom Heinzeller | ``/work/noaa/da/role-da/spack-stack/spack-stack-v1/envs/skylab-2.0.0-intel-2022.0.2/install`` | +| MSU Orion Intel | Dom Heinzeller | ``/work/noaa/da/role-da/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2022.0.2/install`` | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ -| MSU Orion GNU | Dom Heinzeller | ``/work/noaa/da/role-da/spack-stack/spack-stack-v1/envs/skylab-2.0.0-gnu-10.2.0/install`` | +| MSU Orion GNU | Dom Heinzeller | ``/work/noaa/da/role-da/spack-stack/spack-stack-v1/envs/skylab-3.0.0-gnu-10.2.0/install`` | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ -| NASA Discover Intel | Dom Heinzeller | ``/discover/swdev/jcsda/spack-stack/spack-stack-v1/envs/skylab-2.0.0-intel-2022.0.1/install`` | +| NASA Discover Intel | Dom Heinzeller | ``/discover/swdev/jcsda/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2022.0.1/install`` | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ -| NASA Discover GNU | Dom Heinzeller | ``/discover/swdev/jcsda/spack-stack/spack-stack-v1/envs/skylab-2.0.0-gnu-10.1.0/install`` | +| NASA Discover GNU | Dom Heinzeller | ``/discover/swdev/jcsda/spack-stack/spack-stack-v1/envs/skylab-3.0.0-gnu-10.1.0/install`` | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ -| NAVY HPCMP Narwhal | Dom Heinzeller | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-v1/envs/skylab-2.0.0-intel-2021.3.0/install`` | +| NAVY HPCMP Narwhal | Dom Heinzeller | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2021.3.0/install`` | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ -| NCAR-Wyoming Casper | Dom Heinzeller | ``/glade/work/jedipara/cheyenne/spack-stack/spack-stack-v1/envs/skylab-2.0.0-intel-19.1.1.217-casper/install`` | +| NCAR-Wyoming Casper | Dom Heinzeller | ``/glade/work/jedipara/cheyenne/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-19.1.1.217-casper/install`` | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ -| NCAR-Wyoming Cheyenne Intel | Dom Heinzeller | ``/glade/work/jedipara/cheyenne/spack-stack/spack-stack-v1/envs/skylab-2.0.0-intel-19.1.1.217/install`` | +| NCAR-Wyoming Cheyenne Intel | Dom Heinzeller | ``/glade/work/jedipara/cheyenne/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-19.1.1.217/install`` | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ -| NCAR-Wyoming Cheyenne GNU | Dom Heinzeller | ``/glade/work/jedipara/cheyenne/spack-stack/spack-stack-v1/envs/skylab-2.0.0-gnu-10.1.0/install`` | +| NCAR-Wyoming Cheyenne GNU | Dom Heinzeller | ``/glade/work/jedipara/cheyenne/spack-stack/spack-stack-v1/envs/skylab-3.0.0-gnu-10.1.0/install`` | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ | NOAA Parallel Works (AWS, Azure, Gcloud) | | not yet supported - coming soon | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ -| NOAA RDHPCS Gaea | Dom Heinzeller | ``/lustre/f2/pdata/esrl/gsd/spack-stack/spack-stack-v1/envs/skylab-2.0.0-intel-2021.3.0/install`` | +| NOAA RDHPCS Gaea | Dom Heinzeller | ``/lustre/f2/pdata/esrl/gsd/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2021.3.0/install`` | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ -| NOAA RDHPCS Hera Intel | Hang Lei / Dom Heinzeller | ``/scratch1/NCEPDEV/global/spack-stack/spack-stack-v1/envs/skylab-2.0.0-intel-2021.5.0/install`` | +| NOAA RDHPCS Hera Intel | Hang Lei / Dom Heinzeller | ``/scratch1/NCEPDEV/global/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2021.5.0/install`` | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ -| NOAA RDHPCS Hera GNU | Hang Lei / Dom Heinzeller | ``/scratch1/NCEPDEV/global/spack-stack/spack-stack-v1/envs/skylab-2.0.0-gnu-9.2.0/install`` | +| NOAA RDHPCS Hera GNU | Hang Lei / Dom Heinzeller | ``/scratch1/NCEPDEV/global/spack-stack/spack-stack-v1/envs/skylab-3.0.0-gnu-9.2.0/install`` | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ | NOAA RDHPCS Jet Intel | | not yet supported - coming soon | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ @@ -55,15 +55,13 @@ spack-stack-v1 +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ | TACC Frontera GNU | | not yet supported - coming soon | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ -| UW (Univ. of Wisc.) S4 | Dom Heinzeller | ``/data/prod/jedi/spack-stack/spack-stack-v1/envs/skylab-2.0.0-intel-2021.5.0/install`` | +| UW (Univ. of Wisc.) S4 Intel | Dom Heinzeller | ``/data/prod/jedi/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2021.5.0/install`` | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ -| Amazon Web Services AMI Parallelcluster Ubuntu 20.04 GNU | | not yet supported - coming soon | +| UW (Univ. of Wisc.) S4 GNU | Dom Heinzeller | ``/data/prod/jedi/spack-stack/spack-stack-v1/envs/skylab-3.0.0-gnu-9.3.0/install`` | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ -| Amazon Web Services AMI Ubuntu 20.04 GNU | Dom Heinzeller | ``/home/ubuntu/spack-stack-v1/envs/skylab-2.0.0-gcc-10.3.0/install`` | +| Amazon Web Services AMI Parallelcluster Ubuntu 20.04 GNU | Dom Heinzeller | not yet supported - coming soon | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ -| Amazon Web Services AMI Ubuntu 22.04 GNU | Dom Heinzeller | ``/home/ubuntu/spack-stack-v1/envs/skylab-2.0.0-gnu-11.2.0/install`` | -+----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ -| Amazon Web Services AMI Red Hat 8 GNU | Dom Heinzeller | ``/home/ec2-user/spack-stack-v1/envs/skylab-2.0.0-gcc-11.2.1/install`` | +| Amazon Web Services AMI Red Hat 8 GNU | Dom Heinzeller | ``/home/ec2-user/spack-stack-v1/envs/skylab-3.0.0-gcc-11.2.1/install`` | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ For questions or problems, please consult the known issues in :numref:`Section %s `, the currently open GitHub `issues `_ and `discussions `_ first. @@ -83,21 +81,21 @@ The following is required for building new spack environments and for using spac module load miniconda/3.9.7 module load ecflow/5.8.4 -For ``spack-stack-2.0.0`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.2.0``/``skylab-3.0.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /work/noaa/da/role-da/spack-stack/spack-stack-v1/envs/skylab-2.0.0-intel-2022.0.2/install/modulefiles/Core + module use /work/noaa/da/role-da/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2022.0.2/install/modulefiles/Core module load stack-intel/2022.0.2 module load stack-intel-oneapi-mpi/2021.5.1 module load stack-python/3.9.7 module available -For ``spack-stack-2.0.0`` with GNU, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.2.0``/``skylab-3.0.0`` with GNU, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /work/noaa/da/role-da/spack-stack/spack-stack-v1/envs/skylab-2.0.0-gnu-10.2.0/install/modulefiles/Core + module use /work/noaa/da/role-da/spack-stack/spack-stack-v1/envs/skylab-3.0.0-gnu-10.2.0/install/modulefiles/Core module load stack-gcc/10.2.0 module load stack-openmpi/4.0.4 module load stack-python/3.9.7 @@ -118,21 +116,21 @@ The following is required for building new spack environments and for using spac module load miniconda/3.9.7 module load ecflow/5.8.4 -For ``spack-stack-2.0.0`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.2.0``/``skylab-3.0.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /discover/swdev/jcsda/spack-stack/spack-stack-v1/envs/skylab-2.0.0-intel-2022.0.1/install/modulefiles/Core + module use /discover/swdev/jcsda/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2022.0.1/install/modulefiles/Core module load stack-intel/2022.0.1 module load stack-intel-oneapi-mpi/2021.5.0 module load stack-python/3.9.7 module available -For ``spack-stack-2.0.0`` with GNU, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.2.0``/``skylab-3.0.0`` with GNU, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /discover/swdev/jcsda/spack-stack/spack-stack-v1/envs/skylab-2.0.0-gnu-10.1.0/install/modulefiles/Core + module use /discover/swdev/jcsda/spack-stack/spack-stack-v1/envs/skylab-3.0.0-gnu-10.1.0/install/modulefiles/Core module load stack-gcc/10.1.0 module load stack-openmpi/4.1.3 module load stack-python/3.9.7 @@ -149,8 +147,11 @@ The following is required for building new spack environments and for using spac .. code-block:: console module unload PrgEnv-cray - module load PrgEnv-intel/8.1.0 + module load PrgEnv-intel/8.3.2 module unload intel + module load intel/2021.3.0 + module unload cray-mpich + module load cray-mpich/8.1.14 module unload cray-python module load cray-python/3.9.7.1 module unload cray-libsci @@ -159,11 +160,11 @@ The following is required for building new spack environments and for using spac module use /p/app/projects/NEPTUNE/spack-stack/modulefiles module load ecflow/5.8.4 -For ``spack-stack-2.0.0`` with Intel, load the following modules after loading the above modules. +For ``spack-stack-1.2.0``/``skylab-3.0.0`` with Intel, load the following modules after loading the above modules. .. code-block:: console - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-v1/envs/skylab-2.0.0-intel-2021.3.0/install/modulefiles/Core + module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2021.3.0/install/modulefiles/Core module load stack-intel/2021.3.0 module load stack-cray-mpich/8.1.14 module load stack-python/3.9.7 @@ -184,11 +185,11 @@ The following is required for building new spack environments and for using spac module load miniconda/3.9.12 module load ecflow/5.8.4 -For ``spack-stack-2.0.0`` with Intel, load the following modules after loading miniconda and ecflow. +For ``spack-stack-1.2.0``/``skylab-3.0.0`` with Intel, load the following modules after loading miniconda and ecflow. .. code-block:: console - module use /glade/work/jedipara/cheyenne/spack-stack/spack-stack-v1/envs/skylab-2.0.0-intel-19.1.1.217-casper/install/modulefiles/Core + module use /glade/work/jedipara/cheyenne/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-19.1.1.217-casper/install/modulefiles/Core module load stack-intel/19.1.1.217 module load stack-intel-mpi/2019.7.217 module load stack-python/3.9.12 @@ -210,21 +211,21 @@ The following is required for building new spack environments and for using spac module load miniconda/3.9.12 module load ecflow/5.8.4 -For ``spack-stack-2.0.0`` with Intel, load the following modules after loading miniconda and ecflow. Note that there are problems with newer versions of the Intel compiler/MPI library when trying to run MPI jobs with just one task (``mpiexec -np 1``) - for JEDI, job hangs forever in a particular MPI communication call in oops. +For ``spack-stack-1.2.0``/``skylab-3.0.0`` with Intel, load the following modules after loading miniconda and ecflow. Note that there are problems with newer versions of the Intel compiler/MPI library when trying to run MPI jobs with just one task (``mpiexec -np 1``) - for JEDI, job hangs forever in a particular MPI communication call in oops. .. code-block:: console - module use /glade/work/jedipara/cheyenne/spack-stack/spack-stack-v1/envs/skylab-2.0.0-intel-19.1.1.217/install/modulefiles/Core + module use /glade/work/jedipara/cheyenne/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-19.1.1.217/install/modulefiles/Core module load stack-intel/19.1.1.217 module load stack-intel-mpi/2019.7.217 module load stack-python/3.9.12 module available -For ``spack-stack-2.0.0`` with GNU, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.2.0``/``skylab-3.0.0`` with GNU, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /glade/work/jedipara/cheyenne/spack-stack/spack-stack-v1/envs/skylab-2.0.0-gnu-10.1.0/install/modulefiles/Core + module use /glade/work/jedipara/cheyenne/spack-stack/spack-stack-v1/envs/skylab-3.0.0-gnu-10.1.0/install/modulefiles/Core module load stack-gcc/10.1.0 module load stack-openmpi/4.1.1 module load stack-python/3.9.12 @@ -237,7 +238,7 @@ NOAA Acorn (WCOSS2 test system) ------------------------------- .. note:: - ``spack-stack-2.0.0`` is currently not supported on this platform and will be added in the near future. + ``spack-stack-1.2.0``/``skylab-3.0.0`` is currently not supported on this platform and will be added in the near future. On WCOSS2 OpenSUSE sets `CONFIG_SITE` which causes libraries to be installed in `lib64`, breaking the `lib` assumption made by some packages. @@ -253,7 +254,7 @@ NOAA Parallel Works (AWS, Azure, Gcloud) ---------------------------------------- .. note:: - ``spack-stack-2.0.0`` is currently not supported on this platform and will be added in the near future. + ``spack-stack-1.2.0``/``skylab-3.0.0`` is currently not supported on this platform and will be added in the near future. The following is required for building new spack environments and for using spack to build and run software. The default module path needs to be removed, otherwise spack detect the system as Cray. It is also necessary to add ``git-lfs`` and some other utilities to the search path. @@ -283,11 +284,11 @@ The following is required for building new spack environments and for using spac module load miniconda/3.9.12 module load ecflow/5.8.4 -For ``spack-stack-2.0.0`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.2.0``/``skylab-3.0.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /lustre/f2/pdata/esrl/gsd/spack-stack/spack-stack-v1/envs/skylab-2.0.0-intel-2021.3.0/install/modulefiles/Core + module use /lustre/f2/pdata/esrl/gsd/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2021.3.0/install/modulefiles/Core module load stack-intel/2021.3.0 module load stack-cray-mpich/7.7.11 module load stack-python/3.9.12 @@ -315,21 +316,21 @@ The following is required for building new spack environments and for using spac module load miniconda/3.9.12 module load ecflow/5.5.3 -For ``spack-stack-2.0.0`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.2.0``/``skylab-3.0.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /scratch1/NCEPDEV/global/spack-stack/spack-stack-v1/envs/skylab-2.0.0-intel-2021.5.0/install/modulefiles/Core + module use /scratch1/NCEPDEV/global/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2021.5.0/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-intel-oneapi-mpi/2021.5.1 module load stack-python/3.9.12 module available -For ``spack-stack-2.0.0`` with GNU, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.2.0``/``skylab-3.0.0`` with GNU, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /scratch1/NCEPDEV/global/spack-stack/spack-stack-v1/envs/skylab-2.0.0-gnu-9.2.0/install/modulefiles/Core + module use /scratch1/NCEPDEV/global/spack-stack/spack-stack-v1/envs/skylab-3.0.0-gnu-9.2.0/install/modulefiles/Core module load stack-gcc/9.2.0 module load stack-openmpi/3.1.4 module load stack-python/3.9.12 @@ -344,7 +345,7 @@ NOAA RDHPCS Jet ------------------------------ .. note:: - ``spack-stack-2.0.0`` is currently not supported on this platform and will be added in the near future. + ``spack-stack-1.2.0``/``skylab-3.0.0`` is currently not supported on this platform and will be added in the near future. **WORK IN PROGRESS** @@ -353,7 +354,7 @@ TACC Frontera ------------------------------ .. note:: - ``spack-stack-2.0.0`` is currently not supported on this platform and will be added in the near future. + ``spack-stack-1.2.0``/``skylab-3.0.0`` is currently not supported on this platform and will be added in the near future. The following is required for building new spack environments and for using spack to build and run software. @@ -377,11 +378,11 @@ The following is required for building new spack environments and for using spac module load miniconda/3.9.12 module load ecflow/5.8.4 -For ``spack-stack-2.0.0`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.2.0``/``skylab-3.0.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /data/prod/jedi/spack-stack/spack-stack-v1/envs/skylab-2.0.0-intel-2021.5.0/install/modulefiles/Core + module use /data/prod/jedi/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2021.5.0/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-intel-oneapi-mpi/2021.5.0 module load stack-python/3.9.12 @@ -389,55 +390,41 @@ For ``spack-stack-2.0.0`` with Intel, load the following modules after loading m module unuse /opt/apps/modulefiles/Compiler/intel/22 module available -Note the two `module unuse` statements, that need to be run after the stack metamodules are loaded. Loading the Intel compiler meta module loads the Intel compiler module provided by the sysadmins, which adds those two directories to the module path. These contain duplicate libraries that are not compatible with our stack, such as ``hdf4``. - ------------------------------------------------- -Amazon Web Services Parallelcluster Ubuntu 20.04 ------------------------------------------------- - -.. note:: - ``spack-stack-2.0.0`` is currently not supported on this platform and will be added in the near future. - -**COMING SOON** +Note the two `module unuse` commands, that need to be run after the stack metamodules are loaded. Loading the Intel compiler meta module loads the Intel compiler module provided by the sysadmins, which adds those two directories to the module path. These contain duplicate libraries that are not compatible with our stack, such as ``hdf4``. --------------------------------- -Amazon Web Services Ubuntu 20.04 --------------------------------- -For ``spack-stack-2.0.0``, use a t2.2xlarge instance or similar with AMI "skylab-2.0.0-ubuntu20" (ami-06fce89fba374ea67), available on request in us-east-1. After logging in, run: +For ``spack-stack-1.2.0``/``skylab-3.0.0`` with GNU, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /home/ubuntu/spack-stack-v1/envs/skylab-2.0.0-gcc-10.3.0/install/modulefiles/Core - module load stack-gcc/10.3.0 + module use /data/prod/jedi/spack-stack/spack-stack-v1/envs/skylab-3.0.0-gnu-9.3.0/install/modulefiles/Core + module load stack-gcc/9.3.0 module load stack-mpich/4.0.2 - module load stack-python/3.8.10 + module load stack-python/3.9.12 + module unuse /data/prod/hpc-stack/modulefiles/compiler/gnu/9.3.0 module available --------------------------------- -Amazon Web Services Ubuntu 22.04 --------------------------------- +Note the additional `module unuse` command, that needs to be run after the stack metamodules are loaded. Loading the GNU compiler meta module loads the GNU compiler module provided by the sysadmins, which adds this directory to the module path. This directory contains duplicate libraries that are not compatible with our stack, such as ``sp`` or ``bufr``. -For ``spack-stack-2.0.0``, use a t2.2xlarge instance or similar with AMI "skylab-2.0.0-ubuntu22" (ami-09e3055e325abe91e), available on request in us-east-2. After logging in, run: +------------------------------------------------ +Amazon Web Services Parallelcluster Ubuntu 20.04 +------------------------------------------------ -.. code-block:: console +.. note:: + ``spack-stack-1.2.0``/``skylab-3.0.0`` is currently not supported on this platform and will be added in the near future. - module use /home/ubuntu/spack-stack-v1/envs/skylab-2.0.0-gnu-11.2.0/install/modulefiles/Core - module load stack-gcc/11.2.0 - module load stack-mpich/4.0.2 - module load stack-python/3.10.6 - module available +**COMING SOON** ----------------------------- Amazon Web Services Red hat 8 ----------------------------- -For ``spack-stack-2.0.0``, use a t2.2xlarge instance or similar with AMI "skylab-2.0.0-redhat8" (ami-0f6b5f8a07d2f4350), available on request in us-east-1. After logging in, run: +For ``spack-stack-1.2.0``/``skylab-3.0.0``, use a c6i.2xlarge instance or similar with AMI "skylab-3.0.0-redhat8" (ami-0e5d6871c77b1c1b3), available on request in us-east-1. After logging in, run: .. code-block:: console scl enable gcc-toolset-11 bash - module use /home/ec2-user/spack-stack-v1/envs/skylab-2.0.0-gcc-11.2.1/install/modulefiles/Core + module use /home/ec2-user/spack-stack-v1/envs/skylab-3.0.0-gcc-11.2.1/install/modulefiles/Core module load stack-gcc/11.2.1 module load stack-openmpi/4.1.4 module load stack-python/3.9.7 @@ -927,7 +914,7 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi 7. Edit site config files and common config files, for example to remove duplicate versions of external packages that are unwanted, add specs in ``envs/jedi-ufs.mylinux/spack.yaml``, etc. .. warning:: - **Important:** Remove any external ``cmake@3.20`` package from ``envs/jedi-ufs.mylinux/site/packages.yaml``. It is in fact recommended to remove all versions of ``cmake`` up to ``3.20``. + **Important:** Remove any external ``cmake@3.20`` package from ``envs/jedi-ufs.mylinux/site/packages.yaml``. It is in fact recommended to remove all versions of ``cmake`` up to ``3.20``. Further, on Red Hat/CentOS, remove any external curl that might have been found. .. code-block:: console diff --git a/doc/source/Quickstart.rst b/doc/source/Quickstart.rst index 4a3b0c1cf..559e14451 100644 --- a/doc/source/Quickstart.rst +++ b/doc/source/Quickstart.rst @@ -24,6 +24,8 @@ Quickstart Using spack to create environments and containers ================================================= +.. _Quickstart_CreateEnv: + ------------------------ Create local environment ------------------------ @@ -51,10 +53,14 @@ The following instructions install a new spack environment on a pre-configured s emacs envs/jedi-fv3.hera/common/*.yaml emacs envs/jedi-fv3.hera/site/*.yaml - # Process the specs and install - # Note: both steps will take some time! + # Process/concretize the specs spack concretize - spack install [--verbose] [--fail-fast] + + # Optional step for systems with a pre-configured spack mirror, see below. + + # Install the environment, recommended to always use --source + # to install the source code with the compiled binary package + spack install --source [--verbose] [--fail-fast] # Create lua module files spack module lmod refresh @@ -67,13 +73,30 @@ The following instructions install a new spack environment on a pre-configured s For example: .. code-block:: bash - + spack concretize 2>&1 | tee log.concretize spack install [--verbose] [--fail-fast] 2>&1 | tee log.install .. note:: For platforms with multiple compilers in the site config, make sure that the correct compiler and corresponding MPI library are set correctly in ``envs/jedi-fv3.hera/site/packages.yaml`` before running ``spack concretize``. Also, check the output of ``spack concretize`` to make sure that the correct compiler is used (e.g. ``%intel-2022.0.1``). If not, edit ``envs/jedi-fv3.hera/site/compilers.yaml`` and remove the offending compiler. Then, remove ``envs/jedi-fv3.hera/spack.lock`` and rerun ``spack concretize``. +Optional step for sites with a preconfigured spack mirror +--------------------------------------------------------- + +To check if a mirror is configured, look for ``local-source`` in the output of + +.. code-block:: bash + + spack mirror list + +If a mirror exists, add new packages to the mirror. Here, ``/path/to/mirror`` is the location from the above list command without the leading ``file://`` + +.. code-block:: bash + + spack mirror create -a -d /path/to/mirror + +If this fails with ``git lfs`` errors, check the site config for which module to load for ``git lfs`` support. Load the module, then run the ``spack mirror add`` command, then unload the module and proceed with the installation. + ---------------- Create container ---------------- diff --git a/spack b/spack index 7f0be04af..c4c11ce49 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 7f0be04afb0b45475ea480fdf8f2c7279fe1f4ba +Subproject commit c4c11ce496cc60e3333336706c9eb4bd22756758