release/1.2.0: final updates (site configs, documentation, misc) (#419)

* Add skylab-3.0.0 template * Add mirror to aws-pcluster site config * Add S4 mirror, update Cheyenne and Discover packages * Update container configs * Add gaea mirror * Update submodule pointer for spack * Add S4 GNU site config * Update Narwhal site config * Update casper site config * Make crtm@v2.4-jedi.2 default for JEDI * Documentation updates
JCSDA · Dec 20, 2022 · 572d0f5 · 572d0f5
1 parent 41f249f
commit 572d0f5
Show file tree

Hide file tree

Showing 28 changed files with 325 additions and 120 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -3,7 +3,7 @@
   #url = https://github.com/spack/spack
   #branch = develop
   url = https://github.com/NOAA-EMC/spack
-  branch = release/1.2.0
+  branch = spack-stack-1.2.0
 [submodule "doc/CMakeModules"]
   path = doc/CMakeModules
   url = https://github.com/noaa-emc/cmakemodules

diff --git a/configs/containers/README.md b/configs/containers/README.md
@@ -6,15 +6,15 @@ To avoid hardcoding specs in the generic container recipes, we keep the specs li
 ```
   specs: [base-env@1.0.0, jedi-base-env@1.0.0 ~fftw, jedi-ewok-env@1.0.0, jedi-fv3-env@1.0.0,
     jedi-mpas-env@1.0.0, bacio@2.4.1, bison@3.8.2, bufr@11.7.1, ecbuild@3.6.5, eccodes@2.27.0, ecflow@5,
-    eckit@1.19.0, ecmwf-atlas@0.30.0 +trans ~fftw, ectrans@1.0.0 ~fftw, eigen@3.4.0,
+    eckit@1.20.2, ecmwf-atlas@0.31.1 +trans ~fftw, ectrans@1.1.0 ~fftw, eigen@3.4.0,
     fckit@0.9.5, flex@2.6.4, fms@release-jcsda, g2@3.4.5, g2tmpl@1.10.0, gftl-shared@1.5.0,
     gsibec@1.0.6, hdf5@1.12.1, hdf@4.2.15, ip@3.3.3, jasper@2.0.32, jedi-cmake@1.4.0,
     libpng@1.6.37, nccmp@1.9.0.1, netcdf-c@4.8.1, netcdf-cxx4@4.3.1,
     netcdf-fortran@4.5.4, nlohmann-json-schema-validator@2.1.0, nlohmann-json@3.10.5,
     parallel-netcdf@1.12.2, parallelio@2.5.7, py-eccodes@1.4.2, py-f90nml@1.4.3, py-numpy@1.22.3,
     py-pandas@1.4.0, py-pyyaml@6.0, py-scipy@1.8.0, py-shapely@1.8.0, py-xarray@2022.3.0,
     sp@2.3.3, udunits@2.2.28, w3nco@2.4.1, nco@5.0.6,
-    yafyaml@0.5.1, zlib@1.2.13, odc@1.4.5, crtm@v2.4-jedi.1, shumlib@macos_clang_linux_intel_port]
+    yafyaml@0.5.1, zlib@1.2.13, odc@1.4.5, crtm@v2.4-jedi.2, shumlib@macos_clang_linux_intel_port]
     # Don't build ESMF and MAPL for now:
     # https://github.com/JCSDA-internal/MPAS-Model/issues/38
     # https://github.com/NOAA-EMC/spack-stack/issues/326

diff --git a/configs/containers/docker-ubuntu-clang-mpich.yaml b/configs/containers/docker-ubuntu-clang-mpich.yaml
@@ -29,6 +29,7 @@ spack:
   # Additional package config for container
   packages:
     all:
+      require: '%clang'
       target: [x86_64]
       providers:
         mpi: [mpich@4.0.2]
@@ -95,7 +96,7 @@ spack:
       os: ubuntu:20.04
       spack:
         url: https://github.com/noaa-emc/spack
-        ref: jcsda_emc_spack_stack
+        ref: spack-stack-1.2.0
         resolve_sha: false
 
     # Whether or not to strip binaries

diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml
@@ -27,6 +27,7 @@ spack:
   # Additional package config for container
   packages:
     all:
+      require: '%gcc'
       target: [x86_64]
       providers:
         mpi: [openmpi@4.1.4]
@@ -82,7 +83,7 @@ spack:
       os: ubuntu:20.04
       spack:
         url: https://github.com/noaa-emc/spack
-        ref: jcsda_emc_spack_stack
+        ref: spack-stack-1.2.0
         resolve_sha: false
 
     # Whether or not to strip binaries

diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml
@@ -31,6 +31,7 @@ spack:
   # Additional package config for container
   packages:
     all:
+      require: '%intel'
       target: [core2]
       providers:
         mpi: [intel-oneapi-mpi@2021.6.0]
@@ -60,6 +61,10 @@ spack:
       externals:
       - spec: git-lfs@2.9.2
         prefix: /usr
+    # Turn off crypt, because libxcrypt doesn't
+    # build with Intel.
+    python:
+      variants: ~crypt
     qt:
       buildable: false
       externals:
@@ -96,7 +101,7 @@ spack:
       os: ubuntu:20.04
       spack:
         url: https://github.com/noaa-emc/spack
-        ref: jcsda_emc_spack_stack
+        ref: spack-stack-1.2.0
         resolve_sha: false
 
     # Whether or not to strip binaries

diff --git a/configs/sites/aws-pcluster/mirrors.yaml b/configs/sites/aws-pcluster/mirrors.yaml
@@ -0,0 +1,18 @@
+mirrors:
+  local-source:
+    fetch:
+      url: file:///mnt/experiments-efs/spack-stack/source-cache
+      access_pair:
+      - null
+      - null
+      access_token: null
+      profile: null
+      endpoint_url: null
+    push:
+      url: file:///mnt/experiments-efs/spack-stack/source-cache
+      access_pair:
+      - null
+      - null
+      access_token: null
+      profile: null
+      endpoint_url: null
diff --git a/configs/sites/casper/compilers.yaml b/configs/sites/casper/compilers.yaml
@@ -15,7 +15,7 @@ compilers::
         prepend_path:
           PATH: '/glade/u/apps/dav/opt/gnu/9.1.0/bin'
           CPATH: '/glade/u/apps/dav/opt/gnu/9.1.0/include'
-          LD_LIBRARY_PATH: '/glade/u/apps/opt/intel/2020u1/compilers_and_libraries/linux/lib/intel64_lin:/glade/u/apps/dav/opt/gnu/9.1.0/lib64'
+          LD_LIBRARY_PATH: '/glade/u/apps/opt/intel/2020u1/compilers_and_libraries/linux/lib/intel64_lin:/glade/u/apps/opt/intel/2020u1/compilers_and_libraries_2020.1.217/linux/mpi/intel64/libfabric/lib:/glade/u/apps/dav/opt/gnu/9.1.0/lib64'
         set:
           INTEL_LICENSE_FILE: '28518@128.117.177.41'
           LM_LICENSE_FILE: '28518@128.117.177.41'

diff --git a/configs/sites/casper/mirrors.yaml b/configs/sites/casper/mirrors.yaml
@@ -0,0 +1,18 @@
+mirrors:
+  local-source:
+    fetch:
+      url: file:///glade/work/jedipara/cheyenne/spack-stack/source-cache
+      access_pair:
+      - null
+      - null
+      access_token: null
+      profile: null
+      endpoint_url: null
+    push:
+      url: file:///glade/work/jedipara/cheyenne/spack-stack/source-cache
+      access_pair:
+      - null
+      - null
+      access_token: null
+      profile: null
+      endpoint_url: null
diff --git a/configs/sites/casper/packages.yaml b/configs/sites/casper/packages.yaml
@@ -80,10 +80,11 @@ packages:
       prefix: /glade/work/jedipara/cheyenne/spack-stack/ecflow-5.8.4
       modules:
       - ecflow/5.8.4
-  expat:
-    externals:
-    - spec: expat@1.6.0
-      prefix: /usr
+  # Don't use, leads to duplicate packages being built
+  #expat:
+  #  externals:
+  #  - spec: expat@1.6.0
+  #    prefix: /usr
   file:
     externals:
     - spec: file@5.11
@@ -93,6 +94,8 @@ packages:
     - spec: findutils@4.5.11
       prefix: /usr
   flex:
+    # Must set buildable: false to avoid duplicate packages
+    buildable: false
     externals:
     - spec: flex@2.5.37+lex
       prefix: /usr
@@ -164,6 +167,9 @@ packages:
     externals:
     - spec: openssl@1.0.2k-fips
       prefix: /usr
+  # Pin patchelf to 0.15.0 for Intel compiler (no C++-17 features)
+  patchelf:
+    version:: [0.15.0]
   perl:
     externals:
     - spec: perl@5.16.3~cpanm+shared+threads

diff --git a/configs/sites/cheyenne/packages.yaml b/configs/sites/cheyenne/packages.yaml
@@ -115,11 +115,12 @@ packages:
     externals:
     - spec: findutils@4.5.12
       prefix: /usr
-  # Don't use, leads to duplicate packages being built
-  #flex:
-  #  externals:
-  #  - spec: flex@2.5.37+lex
-  #    prefix: /usr
+  flex:
+    # Must set buildable: false to avoid duplicate packages
+    buildable: false
+    externals:
+    - spec: flex@2.5.37+lex
+      prefix: /usr
   gawk:
     externals:
     - spec: gawk@4.1.0

diff --git a/configs/sites/discover/packages.yaml b/configs/sites/discover/packages.yaml
@@ -102,11 +102,12 @@ packages:
     externals:
     - spec: findutils@4.5.12
       prefix: /usr
-  # Don't use, leads to duplicate packages being built
-  #flex:
-  #  externals:
-  #  - spec: flex@2.5.37+lex
-  #    prefix: /usr
+  flex:
+    # Must set buildable: false to avoid duplicate packages
+    buildable: false
+    externals:
+    - spec: flex@2.5.37+lex
+      prefix: /usr
   gawk:
     externals:
     - spec: gawk@4.1.0

diff --git a/configs/sites/gaea/mirrors.yaml b/configs/sites/gaea/mirrors.yaml
@@ -0,0 +1,18 @@
+mirrors:
+  local-source:
+    fetch:
+      url: file:///lustre/f2/pdata/esrl/gsd/spack-stack/source-cache
+      access_pair:
+      - null
+      - null
+      access_token: null
+      profile: null
+      endpoint_url: null
+    push:
+      url: file:///lustre/f2/pdata/esrl/gsd/spack-stack/source-cache
+      access_pair:
+      - null
+      - null
+      access_token: null
+      profile: null
+      endpoint_url: null
diff --git a/configs/sites/gaea/modules.yaml b/configs/sites/gaea/modules.yaml
@@ -2,3 +2,6 @@ modules:
   default:
     enable::
     - tcl
+    tcl:
+      blacklist:
+      - ecflow
diff --git a/configs/sites/narwhal/compilers.yaml b/configs/sites/narwhal/compilers.yaml
@@ -10,7 +10,7 @@ compilers::
       operating_system: sles15
       target: any
       modules:
-      - PrgEnv-intel
+      - PrgEnv-intel/8.3.2
       - intel/2021.3.0
       environment:
         prepend_path:

diff --git a/configs/sites/narwhal/mirrors.yaml b/configs/sites/narwhal/mirrors.yaml
@@ -1,5 +1,5 @@
 mirrors:
-  local:
+  local-source:
     fetch:
       url: file:///p/app/projects/NEPTUNE/spack-stack/spack-stack-v1/source-cache
       access_pair:

diff --git a/configs/sites/narwhal/modules.yaml b/configs/sites/narwhal/modules.yaml
@@ -2,3 +2,6 @@ modules:
   default:
     enable::
     - tcl
+    tcl:
+      blacklist:
+      - ecflow
diff --git a/configs/sites/narwhal/packages.yaml b/configs/sites/narwhal/packages.yaml
@@ -49,10 +49,6 @@ packages:
     externals:
     - spec: bison@3.0.4
       prefix: /usr
-  cmake:
-    externals:
-    - spec: cmake@3.17.0
-      prefix: /usr
   curl:
     externals:
     - spec: curl@7.66.0+gssapi+ldap+nghttp2

diff --git a/configs/sites/s4/compilers.yaml b/configs/sites/s4/compilers.yaml
@@ -18,7 +18,6 @@ compilers:
         LD_LIBRARY_PATH: '/home/opt/intel/oneapi/2022.1/compiler/2022.0.1/linux/compiler/lib/intel64_lin:/data/prod/hpc-stack/gnu/9.3.0/lib64'
         CPATH: '/data/prod/hpc-stack/gnu/9.3.0/include'
     extra_rpaths: []
-# Not yet tested:
 - compiler:
     spec: gcc@9.3.0
     paths:
@@ -31,7 +30,6 @@ compilers:
     target: x86_64
     modules:
     - gnu/9.3.0
-    prefix: /data/prod/hpc-stack/gnu/9.3.0
     modules: []
     environment: {}
     extra_rpaths: []
diff --git a/configs/sites/s4/mirrors.yaml b/configs/sites/s4/mirrors.yaml
@@ -0,0 +1,18 @@
+mirrors:
+  local-source:
+    fetch:
+      url: file:///data/prod/jedi/spack-stack/source-cache
+      access_pair:
+      - null
+      - null
+      access_token: null
+      profile: null
+      endpoint_url: null
+    push:
+      url: file:///data/prod/jedi/spack-stack/source-cache
+      access_pair:
+      - null
+      - null
+      access_token: null
+      profile: null
+      endpoint_url: null
diff --git a/configs/sites/s4/modules.yaml b/configs/sites/s4/modules.yaml
@@ -5,3 +5,5 @@ modules:
     lmod:
       blacklist:
       - ecflow
+      whitelist:
+      - mpich
diff --git a/configs/sites/s4/packages.yaml b/configs/sites/s4/packages.yaml
@@ -1,10 +1,13 @@
 packages:
   all:
     compiler:: [intel@2021.5.0]
+    #compiler:: [gcc@9.3.0]
     providers:
       mpi:: [intel-oneapi-mpi@2021.5.0]
+      #mpi:: [mpich@4.0.2%gcc@9.3.0]
 
 ### MPI, Python, MKL
+  # Comment out next two lines for gnu@9.3.0 with mpich@4.0.2 built by spack
   mpi:
     buildable: False
   intel-oneapi-mpi:
@@ -21,6 +24,8 @@ packages:
   #  externals:
   #  - spec: intel-oneapi-mkl@2022.0.1%intel@2021.5.0
   #    prefix: /opt/intel/oneapi/2022.1
+  mpich:
+    version: [4.0.2]
   python:
     buildable: False
     externals: