Add new template cylc-dev (and test building in GitHub actions) (#1410)

This PR adds a new template cylc-dev to build a special environment for running cylc. Because of the way cylc works, this is best accomplished by creating an environment view and then a cylc wrapper that contains the minimum settings necessary to use cylc from the view's bin directory. See cylc/cylc-flow#6532 for more information. Note that the creation of the cylc wrapper is not part of this PR.

Associated changes:
- To compile cylc-dev on the Ubuntu self-hosted runner, exclude meson from the spack external find call. Spack found a deprecated version meson@0.61 and despite it being deprecated it tried to use it - with the result that the harfbuzz build failed (needed for cylc-dev). Letting spack build meson (it picks meson@1.5.1) solved the problem.
- Remove the +excel variant from py-pandas, which was something the jedi-tools-env used, but that is no longer needed and that caused build errors (and added several more dependencies).
- Fix a bug in the GitHub actions that would allow the tests to fail silently (replace set +e with set -e).
- To avoid duplicate packages, require py-cython@3 and py-numpy@1.26
- With Intel Classic/LLVM, build qt with gcc (has no effect on pre-configured sites, where qt is configured as an external package)

Author: climbfuji

.github/workflows/macos-ci-aarch64.yaml

@@ -111,7 +111,7 @@ jobs:
             set +e
             spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/
             spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary)
-            set +e
+            set -e
             echo "Packages in spack binary cache:"
             spack buildcache list
           fi

.github/workflows/ubuntu-ci-x86_64-gnu.yaml

@@ -51,12 +51,14 @@ jobs:
           # Set up spack-stack
           source ./setup.sh
 
-          declare -a TEMPLATES=("unified-dev" "skylab-dev")
+          declare -a TEMPLATES=("unified-dev" "skylab-dev" "cylc-dev")
           for TEMPLATE in "${TEMPLATES[@]}"; do
             if [[ "${TEMPLATE}" == *"unified-dev"* ]]; then
               export ENVNAME=ue-gcc-11.4.0-buildcache
             elif [[ "${TEMPLATE}" == *"skylab-dev"* ]]; then
               export ENVNAME=se-gcc-11.4.0-buildcache
+            elif [[ "${TEMPLATE}" == *"cylc-dev"* ]]; then
+              export ENVNAME=ce-gcc-11.4.0-buildcache
             fi
             echo "Creating environment ${ENVNAME} from template ${TEMPLATE}"
 
@@ -68,7 +70,8 @@ jobs:
             # Find external packages
             spack external find --scope system \
                 --exclude bison --exclude openssl \
-                --exclude curl --exclude python
+                --exclude curl --exclude python \
+                --exclude meson
             spack external find --scope system grep
             spack external find --scope system sed
             spack external find --scope system perl
@@ -104,7 +107,7 @@ jobs:
               set +e
               spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
               spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary)
-              set +e
+              set -e
               echo "Packages in spack binary cache:"
               spack buildcache list
             fi
@@ -113,20 +116,22 @@ jobs:
             # This allows us to "spin up" builds that altogether take longer than
             # six hours, and/or fail later in the build process.
 
-            # base-env
-            echo "base-env ..."
-            spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.base-env
-            spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env
-
-            # jedi-base-env
-            echo "jedi-base-env ..."
-            spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-base-env
-            spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
-
-            # jedi-ufs-env
-            echo "jedi-ufs-env ..."
-            spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-ufs-env
-            spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env
+            if [[ "${TEMPLATE}" == *"unified-dev"* || "${TEMPLATE}" == *"skylab-dev"* ]]; then
+              # base-env
+              echo "base-env ..."
+              spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.base-env
+              spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env
+
+              # jedi-base-env
+              echo "jedi-base-env ..."
+              spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-base-env
+              spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
+
+              # jedi-ufs-env
+              echo "jedi-ufs-env ..."
+              spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-ufs-env
+              spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env
+            fi
 
             # the rest
             echo "${TEMPLATE} ..."

.github/workflows/ubuntu-ci-x86_64-intel.yaml

@@ -59,7 +59,8 @@ jobs:
           # Find external packages
           spack external find --scope system \
               --exclude bison --exclude openssl \
-              --exclude curl --exclude python
+              --exclude curl --exclude python \
+              --exclude meson
           spack external find --scope system grep
           spack external find --scope system sed
           spack external find --scope system perl
@@ -144,7 +145,7 @@ jobs:
             set +e
             spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
             spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary)
-            set +e
+            set -e
             echo "Packages in spack binary cache:"
             spack buildcache list
           fi

.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml

@@ -60,7 +60,7 @@ jobs:
           spack external find --scope system \
               --exclude bison --exclude openssl \
               --exclude curl --exclude python \
-              --exclude gmake
+              --exclude meson --exclude gmake
           spack external find --scope system grep
           spack external find --scope system sed
           spack external find --scope system perl
@@ -150,7 +150,7 @@ jobs:
             set +e
             spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
             spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary)
-            set +e
+            set -e
             echo "Packages in spack binary cache:"
             spack buildcache list
           fi

.github/workflows/ubuntu-ci-x86_64-oneapi.yaml

@@ -60,7 +60,7 @@ jobs:
           spack external find --scope system \
               --exclude bison --exclude openssl \
               --exclude curl --exclude python \
-              --exclude gmake
+              --exclude meson --exclude gmake
           spack external find --scope system grep
           spack external find --scope system sed
           spack external find --scope system perl
@@ -150,7 +150,7 @@ jobs:
             set +e
             spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
             spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary)
-            set +e
+            set -e
             echo "Packages in spack binary cache:"
             spack buildcache list
           fi

configs/common/packages.yaml

@@ -215,10 +215,9 @@ packages:
   # Avoid full rust dependency: version 43+ requires py-maturin
   py-cryptography:
     require: '@:42 +rust_bootstrap'
-  # Introduced in https://github.com/JCSDA/spack-stack/pull/894, pin py-cython
-  # to avoid duplicate packages being built (cylc dependencies soft-want @3:)
+  # Pin py-cython to avoid duplicate packages
   py-cython:
-    require: '@0.29.36'
+    require: '@3'
   # As long as we need the "delayed" variant for py-dask,
   # we are restricted to version 2021.03.0 (last one that has it)
   # and newer than 2021.01.0 because of these issues:
@@ -233,13 +232,15 @@ packages:
     require: '~mpi'
   # https://github.com/JCSDA/spack-stack/issues/1276
   py-matplotlib:
-     require: '@3.7.4'
+    require: '@3.7.4'
   # Variant ~mpi not working for latest py-netcdf4
   # https://github.com/spack/spack/issues/47652
   py-netcdf4:
     require: '@1.7.1 +mpi'
-  py-pandas:
-    require: '+excel'
+  # To avoid duplicate packages
+  py-numpy:
+    require:
+    - '@1.26'
   # To avoid duplicate packages
   py-ruamel-yaml:
     require: '@0.17.16'

configs/common/packages_intel.yaml

@@ -30,6 +30,9 @@ packages:
   boost:
     require:
     - '%gcc'
+  qt:
+    require:
+    - '%gcc'
   ecflow:
     require:
     - '%gcc'

configs/common/packages_oneapi.yaml

@@ -21,3 +21,6 @@ packages:
     require: '%gcc'
   libbsd:
     require: '%gcc'
+  qt:
+    require:
+    - '%gcc'

configs/sites/tier1/narwhal/compilers.yaml

@@ -48,10 +48,16 @@ compilers::
   - compiler:
       spec: gcc@10.3.0
       paths:
+        # Still need Cray wrappers for most environments
         cc: cc
         cxx: CC
         f77: ftn
         fc: ftn
+        # For cylc-dev, can't use Cray wrappers
+        #cc: /opt/cray/pe/gcc/10.3.0/snos/bin/gcc
+        #cxx: /opt/cray/pe/gcc/10.3.0/snos/bin/g++
+        #f77: /opt/cray/pe/gcc/10.3.0/snos/bin/gfortran
+        #fc: /opt/cray/pe/gcc/10.3.0/snos/bin/gfortran
       flags: {}
       operating_system: sles15
       modules:

configs/templates/cylc-dev/spack.yaml

@@ -0,0 +1,33 @@
+# Template for CYLC (flow, rose, GUI). Supported with GCC only.
+# Creates a view that allows users to define (manually for now)
+# simple wrappers for cylc that can be aliased as "cylc":
+#
+##!/bin/bash
+##CYLC_INSTALL_DIR=/path/to/env/view/bin
+##PATH=${CYLC_INSTALL_DIR}:$PATH
+##unset PYTHONPATH
+##cylc "$@"
+#
+spack:
+  concretizer:
+    unify: true
+
+  view:
+    cylc:
+      root: view
+      select: [^python]
+      link: run
+      link_type: symlink
+  include: []
+
+  definitions:
+  - compilers: ['%gcc']
+  - packages:
+    - py-cylc-flow@8.3.6
+    - py-cylc-rose@1.4.2
+    - py-cylc-uiserver@1.5.1
+
+  specs:
+  - matrix:
+    - [$packages]
+    - [$compilers]

spack

@@ -25,7 +25,7 @@ class JediToolsEnv(BundlePackage):
     # depends_on("aws-parallelcluster", type="run")
     depends_on("py-click", type="run")
     depends_on("py-openpyxl", type="run")
-    depends_on("py-pandas +excel", type="run")
+    depends_on("py-pandas", type="run")
     depends_on("py-pygithub", type="run")
     depends_on("py-scipy", type="run")
     depends_on("py-sphinx", type="run")