From e5d857b060361cea5e5692ae6bb33e4299f0a08e Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Fri, 3 Jan 2025 18:44:43 -0500 Subject: [PATCH 1/2] Set runtime shell requirements within Jenkins Pipeline (#3171) This PR updates the Jenkins Pipeline script to source `gw_setup.sh` from within the PR itself for when ever a shell command is issued that needs it. Currently the runtime bash shell environment requirements for running the **global-workflow** (set by sourcing `$HOMEgfs/workflow/gw_setup.sh`) is established by the user's shell that makes the connection from the Java remote agent to the Jenkins Controller. This update simply sources `gw_setup.sh` directly during run time when ever needed instead. **NOTE:** This does not apply for **GitHub CLI** and **pyGitHub** since they are not currently supported by **Spack Stack** and therefore not incorporated in to `gw_setup.sh`. These remaining dependances are the responsibility of the _role account_ holders that manage the Jenkins Controller to Node Agent connections. --------- Co-authored-by: Terry McGuinness Co-authored-by: Rahul Mahajan --- ci/Jenkinsfile | 45 +++++++++++++++++++++------ ci/scripts/utils/launch_java_agent.sh | 15 ++++----- 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index b7a29e15b0..67ac0fdb17 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -138,8 +138,14 @@ pipeline { } } try { - sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}") - gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim() + sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID} + """) + gist_url=sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID} + """, returnStdout: true).trim() sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** in Build# ${env.BUILD_NUMBER} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """) } catch (Exception error_comment) { echo "Failed to comment on PR: ${error_comment.getMessage()}" @@ -158,7 +164,10 @@ pipeline { } } // Get a list of CI cases to run - CI_CASES = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split() + CI_CASES = sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine} + """, returnStdout: true).trim().split() echo "Cases to run: ${CI_CASES}" } } @@ -179,7 +188,10 @@ pipeline { script { env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS" try { - error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim() + error_output = sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml + """, returnStdout: true).trim() } catch (Exception error_create) { sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experiment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """) error("Case ${caseName} failed to create experiment directory") @@ -194,10 +206,19 @@ pipeline { def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs" sh(script: " rm -f ${error_file}") try { - sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow'") - sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}") + sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow' + """) + sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot} + """) } catch (Exception error_experment) { - sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}") + sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot} + """) ws(CUSTOM_WORKSPACE) { def error_logs = "" def error_logs_message = "" @@ -217,9 +238,15 @@ pipeline { } } try { - gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim() + gist_url = sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID} + """, returnStdout: true).trim() sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${caseName} **FAILED** on ${Machine} in Build# ${env.BUILD_NUMBER} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """) - sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}") + sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID} + """) } catch (Exception error_comment) { echo "Failed to comment on PR: ${error_comment.getMessage()}" } diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index eb78d3b1ef..ad79a75cbd 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -65,14 +65,14 @@ controller_url="https://jenkins.epic.oarcloud.noaa.gov" controller_user=${controller_user:-"terry.mcguinness"} controller_user_auth_token="jenkins_token" -HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" +HOMEGFS_="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" host=$(hostname) ######################################################################### # Set up runtime environment varibles for accounts on supproted machines ######################################################################### -source "${HOMEgfs}/ush/detect_machine.sh" +source "${HOMEGFS_}/ush/detect_machine.sh" case ${MACHINE_ID} in hera | orion | hercules | wcoss2 | gaea) echo "Launch Jenkins Java Controler on ${MACHINE_ID}";; @@ -84,10 +84,10 @@ esac LOG=lanuched_agent-$(date +%Y%m%d%M).log rm -f "${LOG}" -source "${HOMEgfs}/ush/module-setup.sh" -module use "${HOMEgfs}/modulefiles" +source "${HOMEGFS_}/ush/module-setup.sh" +module use "${HOMEGFS_}/modulefiles" module load "module_gwsetup.${MACHINE_ID}" -source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" +source "${HOMEGFS_}/ci/platforms/config.${MACHINE_ID}" JAVA_HOME="${JENKINS_AGENT_LANUCH_DIR}/JAVA/jdk-17.0.10" if [[ ! -d "${JAVA_HOME}" ]]; then @@ -102,9 +102,10 @@ JAVA="${JAVA_HOME}/bin/java" echo "JAVA VERSION: " ${JAVA} -version -export GH="${HOME}/bin/gh" -[[ -f "${GH}" ]] || echo "gh is not installed in ${HOME}/bin" +GH=$(command -v gh || echo "${HOME}/bin/gh") +[[ -f "${GH}" ]] || ( echo "ERROR: GitHub CLI (gh) not found. (exiting with error)"; exit 1 ) ${GH} --version +export GH check_mark=$("${GH}" auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true if [[ "${check_mark}" != "✓" ]]; then From 29089be113dc2f6da0dbade2c328cfadec54a04d Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 3 Jan 2025 20:28:33 -0500 Subject: [PATCH 2/2] Ensure OCNRES and ICERES have 3 digits in the archive script (#3199) This PR: - ensures the `OCNRES` and `ICERES` variables in `task_config` are 3 digits Resolves #3198 --- scripts/exglobal_archive.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index 5ae57ca7e3..ae613fb39c 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -17,6 +17,13 @@ def main(): # Instantiate the Archive object archive = Archive(config) + # update these keys to be 3 digits if they are part of archive.task_config.keys + for key in ['OCNRES', 'ICERES']: + try: + archive.task_config[key] = f"{archive.task_config[key]:03d}" + except KeyError as ee: + logger.info(f"key ({key}) not found in archive.task_config!") + # Pull out all the configuration keys needed to run the rest of archive steps keys = ['ATARDIR', 'current_cycle', 'FHMIN', 'FHMAX', 'FHOUT', 'RUN', 'PDY', 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_PREP_OBS_AERO', @@ -37,16 +44,15 @@ def main(): archive_dict = AttrDict() for key in keys: - archive_dict[key] = archive.task_config.get(key) - if archive_dict[key] is None: - print(f"Warning: key ({key}) not found in task_config!") + try: + archive_dict[key] = archive.task_config[key] + except KeyError as ee: + logger.warning(f"WARNING: key ({key}) not found in archive.task_config!") # Also import all COMIN* and COMOUT* directory and template variables for key in archive.task_config.keys(): - if key.startswith("COM_") or key.startswith("COMIN_") or key.startswith("COMOUT_"): + if key.startswith(("COM_", "COMIN_", "COMOUT_")): archive_dict[key] = archive.task_config.get(key) - if archive_dict[key] is None: - print(f"Warning: key ({key}) not found in task_config!") with chdir(config.ROTDIR):