release/1.2.0: add crtm@2.4-jedi.1 tag, update several site configs, …

…add documentation on spack mirrors (#418)

* Update JEDI crtm tag in configs
* Update hera mirror, add cheyenne, discover and orion mirrors
* Update documentation on creating and using spack mirrors (source caches)
* Update submodule pointer for spack

configs/containers/README.md

@@ -14,7 +14,7 @@ To avoid hardcoding specs in the generic container recipes, we keep the specs li
     parallel-netcdf@1.12.2, parallelio@2.5.7, py-eccodes@1.4.2, py-f90nml@1.4.3, py-numpy@1.22.3,
     py-pandas@1.4.0, py-pyyaml@6.0, py-scipy@1.8.0, py-shapely@1.8.0, py-xarray@2022.3.0,
     sp@2.3.3, udunits@2.2.28, w3nco@2.4.1, nco@5.0.6,
-    yafyaml@0.5.1, zlib@1.2.13, odc@1.4.5, crtm@v2.4_jedi, shumlib@macos_clang_linux_intel_port]
+    yafyaml@0.5.1, zlib@1.2.13, odc@1.4.5, crtm@v2.4-jedi.1, shumlib@macos_clang_linux_intel_port]
     # Don't build ESMF and MAPL for now:
     # https://github.com/JCSDA-internal/MPAS-Model/issues/38
     # https://github.com/NOAA-EMC/spack-stack/issues/326

configs/sites/cheyenne/mirrors.yaml

@@ -0,0 +1,18 @@
+mirrors:
+  local-source:
+    fetch:
+      url: file:///glade/work/jedipara/cheyenne/spack-stack/source-cache
+      access_pair:
+      - null
+      - null
+      access_token: null
+      profile: null
+      endpoint_url: null
+    push:
+      url: file:///glade/work/jedipara/cheyenne/spack-stack/source-cache
+      access_pair:
+      - null
+      - null
+      access_token: null
+      profile: null
+      endpoint_url: null

configs/sites/cheyenne/packages.yaml

@@ -63,10 +63,11 @@ packages:
     externals:
     - spec: berkeley-db@4.8.30
       prefix: /usr
-  bison:
-    externals:
-    - spec: bison@2.7
-      prefix: /usr
+  # Don't use, leads to duplicate packages being built
+  #bison:
+  # externals:
+  # - spec: bison@2.7
+  #   prefix: /usr
   bzip2:
     externals:
     - spec: bzip2@1.0.6
@@ -101,10 +102,11 @@ packages:
       prefix: /glade/work/jedipara/cheyenne/spack-stack/ecflow-5.8.4
       modules:
       - ecflow/5.8.4
-  expat:
-    externals:
-    - spec: expat@1.6.0
-      prefix: /usr
+  # Don't use, leads to duplicate packages being built
+  #expat:
+  #  externals:
+  #  - spec: expat@1.6.0
+  #    prefix: /usr
   file:
     externals:
     - spec: file@5.22
@@ -113,10 +115,11 @@ packages:
     externals:
     - spec: findutils@4.5.12
       prefix: /usr
-  flex:
-    externals:
-    - spec: flex@2.5.37+lex
-      prefix: /usr
+  # Don't use, leads to duplicate packages being built
+  #flex:
+  #  externals:
+  #  - spec: flex@2.5.37+lex
+  #    prefix: /usr
   gawk:
     externals:
     - spec: gawk@4.1.0

configs/sites/discover/mirrors.yaml

@@ -0,0 +1,18 @@
+mirrors:
+  local-source:
+    fetch:
+      url: file:///discover/swdev/jcsda/spack-stack/source-cache
+      access_pair:
+      - null
+      - null
+      access_token: null
+      profile: null
+      endpoint_url: null
+    push:
+      url: file:///discover/swdev/jcsda/spack-stack/source-cache
+      access_pair:
+      - null
+      - null
+      access_token: null
+      profile: null
+      endpoint_url: null

configs/sites/discover/packages.yaml

@@ -56,10 +56,11 @@ packages:
     externals:
     - spec: berkeley-db@4.8.30
       prefix: /usr
-  bison:
-    externals:
-    - spec: bison@2.7
-      prefix: /usr
+  # Don't use, leads to duplicate packages being built
+  #bison:
+  #  externals:
+  #  - spec: bison@2.7
+  #    prefix: /usr
   ccache:
     externals:
     - spec: ccache@3.1.9
@@ -101,10 +102,11 @@ packages:
     externals:
     - spec: findutils@4.5.12
       prefix: /usr
-  flex:
-    externals:
-    - spec: flex@2.5.37+lex
-      prefix: /usr
+  # Don't use, leads to duplicate packages being built
+  #flex:
+  #  externals:
+  #  - spec: flex@2.5.37+lex
+  #    prefix: /usr
   gawk:
     externals:
     - spec: gawk@4.1.0

configs/sites/hera/mirrors.yaml

@@ -1,15 +1,15 @@
 mirrors:
-  local:
+  local-source:
     fetch:
-      url: file:///scratch1/NCEPDEV/global/spack-stack/mirror
+      url: file:///scratch1/NCEPDEV/global/spack-stack/source-cache
       access_pair:
       - null
       - null
       access_token: null
       profile: null
       endpoint_url: null
     push:
-      url: file:///scratch1/NCEPDEV/global/spack-stack/mirror
+      url: file:///scratch1/NCEPDEV/global/spack-stack/source-cache
       access_pair:
       - null
       - null

configs/sites/orion/mirrors.yaml

@@ -0,0 +1,18 @@
+mirrors:
+  local-source:
+    fetch:
+      url: file:///work/noaa/da/role-da/spack-stack/source-cache
+      access_pair:
+      - null
+      - null
+      access_token: null
+      profile: null
+      endpoint_url: null
+    push:
+      url: file:///work/noaa/da/role-da/spack-stack/source-cache
+      access_pair:
+      - null
+      - null
+      access_token: null
+      profile: null
+      endpoint_url: null

configs/templates/skylab-dev/spack.yaml

@@ -30,7 +30,8 @@ spack:
     - crtm@2.3.0
     - crtm@2.4.0
     - crtm@v2.3-jedi.4
-    - crtm@v2.4_jedi
+    - crtm@v2.4-jedi
+    - crtm@v2.4-jedi.1
     - ecbuild@3.6.5
     - eccodes@2.27.0
     - ecflow@5

configs/templates/skylab-no-python-dev/spack.yaml

@@ -44,7 +44,8 @@ spack:
     - crtm@2.3.0
     - crtm@2.4.0
     - crtm@v2.3-jedi.4
-    - crtm@v2.4_jedi
+    - crtm@v2.4-jedi
+    - crtm@v2.4-jedi.1
     - ecbuild@3.6.5
     - eccodes@2.27.0
     # ecflow depends on Python

doc/source/MaintainersSection.rst

@@ -237,36 +237,7 @@ On Hera, ``miniconda`` must be installed as a one-off before spack can be used.
 miniconda
    Follow the instructions in :numref:`Section %s <Prerequisites_Miniconda>` to create a basic ``miniconda`` installation and associated modulefile for working with spack. Don't forget to log off and back on to forget about the conda environment.
 
-Hera sits behind the NOAA firewall and doesn't have access to all packages on the web. It is therefore necessary to create a spack mirror on another platform (e.g. Cheyenne). This can be done as follows.
-
-1. (On Hera) Create an environment as usual and run the concretization step (``spack concretize``), but do not start the installation yet.
-
-2. (On Cheyenne) Load the basic external modules (see :numref:`Section %s <Platforms_Cheyenne>`) and load module ``git/2.33.1`` (for ``git lfs``). Check out a fresh clone of ``spack-stack`` and run ``source setup.sh``.
-
-3. (On Hera) Copy (e.g. using ``scp``) the environment's ``spack.lock`` file to Cheyenne into the ``spack-stack`` directory.
-
-4. (On Cheyenne) Run the following sequence of commands:
-
-.. code-block:: console
-
-   spack env create hera_mirror_env spack.lock
-   spack env activate hera_mirror_env
-   spack mirror create -a
-
-   The mirror will be created in directory ``./spack/var/spack/environments/hera_mirror_env``
-
-5. (On Hera) Copy the directory from Cheyenne to ``/scratch1/NCEPDEV/global/spack-stack/mirror``. It is recommended to use ``rsync`` to avoid deleting existing packages in the mirror directory on Hera.
-
-6. (On Hera) Add the mirror to the spack environment's mirror list. Note that this is already included in the Hera site config in ``spack-stack`` (``configs/sites/hera/mirrors.yaml``).
-
-.. code-block:: console
-
-   spack mirror add local file:///scratch1/NCEPDEV/global/spack-stack/mirror
-   spack mirror list
-
-   The newly created local mirror should be listed at the top, which means that spack will search this directory first.
-
-7. (On Hera) Proceed with the installation as usual.
+Hera sits behind the NOAA firewall and doesn't have access to all packages on the web. It is therefore necessary to create a spack mirror on another platform (e.g. Cheyenne). This can be done as described in section :numref:`Section %s <MaintainersSection_spack_mirrors>` for air-gapped systems.
 
 .. _MaintainersSection_Jet:
 
@@ -354,6 +325,77 @@ See ``configs/sites/aws-pcluster/README.md``.
 
 .. _MaintainersSection_Testing_New_Packages:
 
+.. _MaintainersSection_spack_mirrors:
+
+==================================
+Creating/maintaining spack mirrors
+==================================
+
+Spack mirrors allow downloading the source code required to build environments once to a local directory (in the following also referred to as source cache), and then use this directory for subsequent installations. If a package cannot be found in the mirror (e.g. because a newer version is required), it will automatically be pulled from the web. It won't be added to the source cache automatically, this is a step that needs to be done manually.
+
+Spack mirrors also make it possible to download the source code for an air-gapped machine on another system, then transferring the entire mirror to the system without internet access and using it during the installation.
+
+-----------------------------
+Spack mirrors for local reuse
+-----------------------------
+
+Since all spack-stack installations are based on environments, we only cover spack mirrors for environments here. For a more general discussion, users are referred to the `Spack Documentation <https://spack.readthedocs.io/en/latest>`_.
+
+1. Create an environment as usual, activate it and run the concretization step (``spack concretize``), but do not start the installation yet.
+
+2. Create the spack mirror in ``/path/to/spack-mirror``.
+
+.. code-block:: console
+
+   spack mirror create -a -d /path/to/spack-source
+
+3. If the spack mirror already exists, then existing packages will be ignored and only new packages will be added to the mirror.
+
+4. If not already included in the environment (e.g. from the spack-stack site config), add the mirror:
+
+.. code-block:: console
+   spack mirror list
+   spack mirror add local-source file:///path/to/spack-source
+
+   The newly created local mirror should be listed at the top, which means that spack will search this directory first.
+
+7. Proceed with the installation as usual.
+
+------------------------------------
+Spack mirrors for air-gapped systems
+------------------------------------
+
+The procedure is similar to using spack mirrors for local reuse, but a few additional steps are needed in between.
+
+1. On the air-gapped system: Create an environment as usual, activate it and run the concretization step (``spack concretize``), but do not start the installation yet.
+
+2. Copy the file ``spack.lock`` (in ``envs/env-name/``) to the machine with full internet access using ``scp``, for example.
+
+3. On the machine with full internet access: Load the basic external modules, if using a machine that is preconfigured for spack-stack (see :numref:`Section %s <Platforms>`) and make sure that ``git`` supports ``lfs`` (if necessary, load the external modules that spack-stack also uses).
+
+4. On the machine with full internet access: check out the same version of ``spack-stack``, run ``setup.sh``, and then the following sequence of commands:
+
+.. code-block:: console
+
+   spack env create air_gapped_mirror_env spack.lock
+   spack env activate air_gapped_mirror_env
+   spack mirror create -a
+
+   The mirror will be created in directory ``./spack/var/spack/environments/air_gapped_mirror_env``
+
+5. On the air-gapped system: Copy the directory from the system with internet access to the local destination for the spack mirror. It is recommended to use ``rsync`` to avoid deleting existing packages, if updating an existing mirror on the air-gapped system.
+
+6.. On the air-gapped system: Add the mirror to the spack environment's mirror list, unless already included in the site config.
+
+.. code-block:: console
+
+   spack mirror add locals-source  file:///path/to/spack-source
+   spack mirror list
+
+   The newly created local mirror should be listed at the top, which means that spack will search this directory first.
+
+7. On the air-gapped system: Proceed with the installation as usual.
+
 ==============================
 Testing new packages
 ==============================

doc/source/Platforms.rst

@@ -174,6 +174,8 @@ For ``spack-stack-2.0.0`` with Intel, load the following modules after loading t
 NCAR-Wyoming Casper
 -------------------
 
+The following is required for building new spack environments and for using spack to build and run software.
+
 .. code-block:: console
 
    module purge
@@ -203,9 +205,7 @@ The following is required for building new spack environments and for using spac
 .. code-block:: console
 
    module purge
-   module unuse /glade/u/apps/ch/modulefiles/default/compilers
-   export MODULEPATH_ROOT=/glade/work/jedipara/cheyenne/spack-stack/modulefiles
-   module use /glade/work/jedipara/cheyenne/spack-stack/modulefiles/compilers
+   export LMOD_TMOD_FIND_FIRST=yes
    module use /glade/work/jedipara/cheyenne/spack-stack/modulefiles/misc
    module load miniconda/3.9.12
    module load ecflow/5.8.4