Merge branch 'develop' into feature/fire_blending

Author: bbakernoaa

.github/workflows/pw_aws_centos.yaml renamed to .github/workflows/pw_aws_ci.yaml

@@ -1,7 +1,4 @@
-name: gw-ci-aws-centos
-
-on: [workflow_dispatch]
-
+name: gw-ci-aws
 # TEST_DIR contains 2 directories;
 # 1. HOMEgfs: clone of the global-workflow
 # 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments
@@ -10,33 +7,73 @@ on: [workflow_dispatch]
 # ├── HOMEgfs
 # └── RUNTESTS
 #     ├── COMROT
-#     │   └── ${pslot}
+#     │   └── ${pslot}
 #     └── EXPDIR
 #         └── ${pslot}
+
+on:
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        description: 'Pull Request Number (use 0 for non-PR)'
+        required: true
+        default: '0'
+      os:
+        description: 'Operating System'
+        required: true
+        type: choice
+        options:
+          - rocky
+          - centos
+
 env:
   TEST_DIR: ${{ github.workspace }}/${{ github.run_id }}
   MACHINE_ID: noaacloud
 
 jobs:
+  fetch-branch:
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ secrets.GITHUBTOKEN }}
+    outputs:
+      branch: ${{ steps.get-branch.outputs.branch }}
+    steps:
+    - name: Fetch branch name for PR
+      id: get-branch
+      run: |
+        pr_number=${{ github.event.inputs.pr_number }}
+        repo=${{ github.repository }}
+        if [ "$pr_number" -eq "0" ]; then
+          branch=${{ github.event.inputs.ref }}
+        else
+          branch=$(gh pr view $pr_number --repo $repo --json headRefName --jq '.headRefName')
+        fi
+        echo "::set-output name=branch::$branch"
+
   checkout:
-    runs-on: [self-hosted, aws, parallelworks, centos]
+    needs: fetch-branch
+    runs-on: 
+      - self-hosted
+      - aws
+      - parallelworks
+      - ${{ github.event.inputs.os }}
     timeout-minutes: 600
-
     steps:
-
     - name: Checkout global-workflow
       uses: actions/checkout@v4
       with:
         path: ${{ github.run_id }}/HOMEgfs
         submodules: 'recursive'
-        ref: ${{ github.event.pull_request.head.ref }}
+        ref: ${{ needs.fetch-branch.outputs.branch }}
 
   build-link:      
-    runs-on: [self-hosted, aws, parallelworks, centos]
     needs: checkout
-
+    runs-on: 
+      - self-hosted
+      - aws
+      - parallelworks
+      - ${{ github.event.inputs.os }}
     steps:
-
     - name: Build components
       run: |
         cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
@@ -48,12 +85,15 @@ jobs:
         ./link_workflow.sh
 
   create-experiments:
-    needs: checkout
-    runs-on: [self-hosted, aws, parallelworks, centos]
+    needs: build-link
+    runs-on: 
+      - self-hosted
+      - aws
+      - parallelworks
+      - ${{ github.event.inputs.os }}
     strategy:
       matrix:
         case: ["C48_ATM"]
-
     steps:
       - name: Create Experiments ${{ matrix.case }}
         env:
@@ -68,9 +108,12 @@ jobs:
 
   run-experiments:
     needs: create-experiments
-    runs-on: [self-hosted, aws, parallelworks, centos]
+    runs-on: 
+      - self-hosted
+      - aws
+      - parallelworks
+      - ${{ github.event.inputs.os }}
     strategy:
-      max-parallel: 2
       matrix:
         case: ["C48_ATM"]
     steps:
@@ -81,9 +124,13 @@ jobs:
 
   clean-up:
     needs: run-experiments
-    runs-on: [self-hosted, aws, parallelworks, centos]
+    runs-on: 
+      - self-hosted
+      - aws
+      - parallelworks
+      - ${{ github.event.inputs.os }}
     steps:
-      - name: Clean-up
+      - name: Clean up workspace
         run: |
-          cd ${{ github.workspace }}
-          rm -rf ${{ github.run_id }}
+          echo "Cleaning up workspace"
+          rm -rf ${{ env.TEST_DIR }}

.gitignore

@@ -171,11 +171,6 @@ ush/bufr2ioda_insitu*
 versions/build.ver
 versions/run.ver
 
-# wxflow checkout and symlinks
-ush/python/wxflow
-workflow/wxflow
-ci/scripts/wxflow
-
 # jcb checkout and symlinks
 ush/python/jcb
 workflow/jcb

ci/Jenkinsfile

@@ -5,7 +5,7 @@ def cases = ''
 def GH = 'none'
 // Location of the custom workspaces for each machine in the CI system.  They are persitent for each iteration of the PR.
 def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea']
-def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI']
+def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI']
 def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git'
 def STATUS = 'Passed'
 
@@ -101,9 +101,10 @@ pipeline {
                 stages {
                     stage('Building') {
                         steps {
-                           catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
+                          catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
                             script {
                                 def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to build the system on per system basis under the custome workspace for each buile system
+                                env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
                                 sh(script: "mkdir -p ${HOMEgfs}")
                                 ws(HOMEgfs) {
                                     if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs
@@ -172,9 +173,10 @@ pipeline {
                                     }
                                     if (system == 'gfs') {
                                         cases = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
+                                        echo "Cases to run: ${cases}"
                                     }
                                }
-                           }
+                            }
                           }
                         }
                     }
@@ -192,32 +194,34 @@ pipeline {
                     def parallelStages = cases.collectEntries { caseName ->
                         ["${caseName}": {
                             stage("Create ${caseName}") {
-                                    catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
-                                        script {
-                                                sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp")
-                                                def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
-                                                system = yaml_case.experiment.system
-                                                def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}"   // local HOMEgfs is used to populate the XML on per system basis
-                                                env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS"
-                                                try {
-                                                    error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim()
-                                                } catch (Exception error_create) {
-                                                    sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """)
-                                                    error("Case ${caseName} failed to create experment directory")
-                                                }
+                                script {
+                                        sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp")
+                                        def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
+                                        def build_system = yaml_case.experiment.system
+                                        def HOMEgfs = "${CUSTOM_WORKSPACE}/${build_system}"   // local HOMEgfs is used to populate the XML on per system basis
+                                        env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
+                                        env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS"
+                                        try {
+                                              error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim()
+                                        } catch (Exception error_create) {
+                                            sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """)
+                                            error("Case ${caseName} failed to create experment directory")
                                         }
-                                }
+                                     }
                             }
 
                             stage("Running ${caseName}") {
                                     catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
                                         script {
                                                 HOMEgfs = "${CUSTOM_WORKSPACE}/gfs"  // common HOMEgfs is used to launch the scripts that run the experiments
+                                                env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
                                                 def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim()
                                                 def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs"
                                                 sh(script: " rm -f ${error_file}")
+                                                def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
+                                                def build_system = yaml_case.experiment.system
                                                 try {
-                                                    sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${system}")
+                                                    sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${build_system}")
                                                 } catch (Exception error_experment) {
                                                     sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}")
                                                     ws(CUSTOM_WORKSPACE) {
@@ -268,11 +272,11 @@ pipeline {
             }
         }
 
-
         stage( '5. FINALIZE' ) {
             agent { label NodeName[machine].toLowerCase() }
             steps {
                 script {
+                    env.HOME_GFS = "${CUSTOM_WORKSPACE}/gfs" // setting path to HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
                     sh(script: """
                         labels=\$(${GH} pr view ${env.CHANGE_ID} --repo ${repo_url} --json labels --jq '.labels[].name')
                         for label in \$labels; do

ci/cases/pr/C48mx500_hybAOWCDA.yaml

@@ -0,0 +1,26 @@
+experiment:
+  system: gfs
+  mode: cycled
+
+arguments:
+  pslot: {{ 'pslot' | getenv }}
+  app: S2S
+  resdetatmos: 48
+  resdetocean: 5.0
+  resensatmos: 48
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20240610
+  idate: 2021032412
+  edate: 2021032418
+  nens: 3
+  interval: 0
+  start: warm
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml
+
+skip_ci_on_hosts:
+  - wcoss2
+  - orion
+  - hercules
+  - hera
+  - gaea

ci/cases/pr/C96C48_hybatmaerosnowDA.yaml

@@ -18,6 +18,7 @@ arguments:
   yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml
 
 skip_ci_on_hosts:
+  - wcoss2
   - orion
   - gaea
   - hercules

ci/scripts/utils/launch_java_agent.sh

@@ -106,7 +106,7 @@ export GH="${HOME}/bin/gh"
 [[ -f "${GH}" ]] || echo "gh is not installed in ${HOME}/bin"
 ${GH} --version
 
-check_mark=$(gh auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true
+check_mark=$("${GH}" auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true
 if [[ "${check_mark}" != "✓" ]]; then
   echo "gh not authenticating with emcbot token"
   exit 1

ci/scripts/wxflow

@@ -0,0 +1 @@
+../../sorc/wxflow/src/wxflow

docs/source/clone.rst

@@ -9,7 +9,7 @@ Quick Instructions
 Quick clone/build/link instructions (more detailed instructions below).
 
 .. note::
-   Here we are making the assumption that you are using the workflow to run an experiment and so are working from the authoritative repository. If you are using a development branch then follow the instructions in :doc:`development.rst`. Once you do that you can follow the instructions here with the only difference being the repository/fork you are cloning from.
+   Here we are making the assumption that you are using the workflow to run an experiment and so are working from the authoritative repository. If you are using a development branch then follow the instructions in :doc:`development`. Once you do that you can follow the instructions here with the only difference being the repository/fork you are cloning from.
 
 Clone the `global-workflow` and `cd` into the `sorc` directory:
 

docs/source/development.rst

@@ -12,6 +12,7 @@ Code managers
 
  * Kate Friedman - @KateFriedman-NOAA / kate.friedman@noaa.gov
  * Walter Kolczynski - @WalterKolczynski-NOAA / walter.kolczynski@noaa.gov
+ * David Huber - @DavidHuber-NOAA / david.huber@noaa.gov
 
 .. _development:
 
@@ -70,7 +71,29 @@ The following steps should be followed in order to make changes to the develop b
 Development Tools
 =================
 
-See the ``/test`` folder in global-workflow for available development and testing tools.
+Two sets of testing are available for use by developers.  The first is the capability to run continuous integration tests locally and the second are a set of comparison tools.
+
+---------------------------
+Continuous Integration (CI)
+---------------------------
+
+The global workflow comes fitted with a suite of system tests that run various types of workflow.  These tests are commonly run for pull requests before they may be merged into the develop branch.  At a minimum, developers are expected to run the CI test(s) that will be impacted by their changes on at least one platform.
+
+The commonly run tests are written in YAML format and can be found in the ``ci/cases/pr`` directory.  The ``workflow/generate_workflows.sh`` tool is available to aid running these cases.  See the help documentation by running ``./generate_workflows.sh -h``.  The script has the capability to prepare the EXPDIR and COMROOT directories for a specified or implied suite of CI tests (see :doc:`setup` for details on these directories).  The script also has options to automatically build and run all tests for a given system (i.e. GFS or GEFS and a placeholder for SFS).  For instance, to build the workflow and run all of the GFS tests, one would execute
+
+::
+
+    cd workflow
+    ./generate_workflows.sh -A "your_hpc_account" -b -G -c /path/to/root/directory
+
+where:
+
+    * ``-A`` is used to specify the HPC (slurm or PBS) account to use
+    * ``-b`` indicates that the workflow should be built fresh
+    * ``-G`` specifies that all of the GFS cases should be run (this also influences the build flags to use)
+    * ``-c`` tells the tool to append the rocotorun commands for each experiment to your crontab
+
+More details on how to use the tool are provided by running ``generate_workflows.sh -h``.
 
 ----------------
 Comparison Tools

docs/source/hpc.rst

@@ -90,9 +90,9 @@ Optimizing the global workflow on S4
 
 The S4 cluster is relatively small and so optimizations are recommended to improve cycled runtimes. Please contact Innocent Souopgui (innocent.souopgui@noaa.gov) if you are planning on running a cycled experiment on this system to obtain optimized configuration files.
 
-========================================
+==================================================
 Stacksize on R&Ds (Hera, Orion, Hercules, Jet, S4)
-========================================
+==================================================
 
 Some GFS components, like the UPP, need an unlimited stacksize. Add the following setting into your appropriate .*rc file to support these components:
 

docs/source/init.rst

@@ -301,7 +301,7 @@ Operations/production output location on HPSS: /NCEPPROD/hpssprod/runhistory/rh
 |                |                                 |                                                                             |                                |
 |                |   gfs.t. ``hh`` z.sfcanl.nc     |                                                                             |                                |
 +----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+
-| v16.2[3]+ ops  |   gfs.t. ``hh`` z.atmanl.nc     | com_gfs_ ``gfs_ver`` _gfs. ``yyyymmdd`` _ ``hh`` .gfs_nca.tar               | gfs. ``yyyymmdd`` /``hh``/atmos|
+| v16.2[3]+ ops  |   gfs.t. ``hh`` z.atmanl.nc     | com_gfs\_ ``gfs_ver`` _gfs. ``yyyymmdd`` _ ``hh`` .gfs_nca.tar              | gfs. ``yyyymmdd`` /``hh``/atmos|
 |                |                                 |                                                                             |                                |
 |                |   gfs.t. ``hh`` z.sfcanl.nc     |                                                                             |                                |
 +----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+

docs/source/setup.rst

@@ -1,3 +1,5 @@
+.. _experiment-setup:
+
 ================
 Experiment Setup
 ================
@@ -179,8 +181,6 @@ where:
    * ``$EXPDIR`` is the path to your experiment directory where your configs will be placed and where you will find your workflow monitoring files (i.e. rocoto database and xml file). DO NOT include PSLOT folder at end of path, it will be built for you. [default: $HOME]
    * ``$ICSDIR`` is the path to the ICs for your run if generated separately. [default: None]
 
-.. [#]  More Coupled configurations in cycled mode are currently under development and not yet available
-
 Example:
 
 ::