Skip to content

Commit 7620805

Browse files
JessicaMeixner-NOAAJessicaMeixner-NOAA
committed
Merge remote-tracking branch 'EMC/develop' into enablepio_cycle_uglo15km
2 parents fe1719a + 060aec3 commit 7620805

File tree

4 files changed

+59
-23
lines changed

4 files changed

+59
-23
lines changed

‎ci/Jenkinsfile

Lines changed: 36 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -138,8 +138,14 @@ pipeline {
138138
}
139139
}
140140
try {
141-
sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}")
142-
gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim()
141+
sh(script: """
142+
source ${HOMEgfs}/workflow/gw_setup.sh
143+
${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}
144+
""")
145+
gist_url=sh(script: """
146+
source ${HOMEgfs}/workflow/gw_setup.sh
147+
${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}
148+
""", returnStdout: true).trim()
143149
sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** in Build# ${env.BUILD_NUMBER} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
144150
} catch (Exception error_comment) {
145151
echo "Failed to comment on PR: ${error_comment.getMessage()}"
@@ -158,7 +164,10 @@ pipeline {
158164
}
159165
}
160166
// Get a list of CI cases to run
161-
CI_CASES = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
167+
CI_CASES = sh(script: """
168+
source ${HOMEgfs}/workflow/gw_setup.sh
169+
${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}
170+
""", returnStdout: true).trim().split()
162171
echo "Cases to run: ${CI_CASES}"
163172
}
164173
}
@@ -179,7 +188,10 @@ pipeline {
179188
script {
180189
env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS"
181190
try {
182-
error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim()
191+
error_output = sh(script: """
192+
source ${HOMEgfs}/workflow/gw_setup.sh
193+
${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml
194+
""", returnStdout: true).trim()
183195
} catch (Exception error_create) {
184196
sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experiment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """)
185197
error("Case ${caseName} failed to create experiment directory")
@@ -194,10 +206,19 @@ pipeline {
194206
def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs"
195207
sh(script: " rm -f ${error_file}")
196208
try {
197-
sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow'")
198-
sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}")
209+
sh(script: """
210+
source ${HOMEgfs}/workflow/gw_setup.sh
211+
${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow'
212+
""")
213+
sh(script: """
214+
source ${HOMEgfs}/workflow/gw_setup.sh
215+
${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}
216+
""")
199217
} catch (Exception error_experment) {
200-
sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}")
218+
sh(script: """
219+
source ${HOMEgfs}/workflow/gw_setup.sh
220+
${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}
221+
""")
201222
ws(CUSTOM_WORKSPACE) {
202223
def error_logs = ""
203224
def error_logs_message = ""
@@ -217,9 +238,15 @@ pipeline {
217238
}
218239
}
219240
try {
220-
gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
241+
gist_url = sh(script: """
242+
source ${HOMEgfs}/workflow/gw_setup.sh
243+
${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}
244+
""", returnStdout: true).trim()
221245
sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${caseName} **FAILED** on ${Machine} in Build# ${env.BUILD_NUMBER} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
222-
sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}")
246+
sh(script: """
247+
source ${HOMEgfs}/workflow/gw_setup.sh
248+
${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}
249+
""")
223250
} catch (Exception error_comment) {
224251
echo "Failed to comment on PR: ${error_comment.getMessage()}"
225252
}

‎ci/scripts/utils/launch_java_agent.sh

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -65,14 +65,14 @@ controller_url="https://jenkins.epic.oarcloud.noaa.gov"
6565
controller_user=${controller_user:-"terry.mcguinness"}
6666
controller_user_auth_token="jenkins_token"
6767

68-
HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )"
68+
HOMEGFS_="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )"
6969
host=$(hostname)
7070

7171
#########################################################################
7272
# Set up runtime environment varibles for accounts on supproted machines
7373
#########################################################################
7474

75-
source "${HOMEgfs}/ush/detect_machine.sh"
75+
source "${HOMEGFS_}/ush/detect_machine.sh"
7676
case ${MACHINE_ID} in
7777
hera | orion | hercules | wcoss2 | gaea)
7878
echo "Launch Jenkins Java Controler on ${MACHINE_ID}";;
@@ -84,10 +84,10 @@ esac
8484
LOG=lanuched_agent-$(date +%Y%m%d%M).log
8585
rm -f "${LOG}"
8686

87-
source "${HOMEgfs}/ush/module-setup.sh"
88-
module use "${HOMEgfs}/modulefiles"
87+
source "${HOMEGFS_}/ush/module-setup.sh"
88+
module use "${HOMEGFS_}/modulefiles"
8989
module load "module_gwsetup.${MACHINE_ID}"
90-
source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
90+
source "${HOMEGFS_}/ci/platforms/config.${MACHINE_ID}"
9191

9292
JAVA_HOME="${JENKINS_AGENT_LANUCH_DIR}/JAVA/jdk-17.0.10"
9393
if [[ ! -d "${JAVA_HOME}" ]]; then
@@ -102,9 +102,10 @@ JAVA="${JAVA_HOME}/bin/java"
102102
echo "JAVA VERSION: "
103103
${JAVA} -version
104104

105-
export GH="${HOME}/bin/gh"
106-
[[ -f "${GH}" ]] || echo "gh is not installed in ${HOME}/bin"
105+
GH=$(command -v gh || echo "${HOME}/bin/gh")
106+
[[ -f "${GH}" ]] || ( echo "ERROR: GitHub CLI (gh) not found. (exiting with error)"; exit 1 )
107107
${GH} --version
108+
export GH
108109

109110
check_mark=$("${GH}" auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true
110111
if [[ "${check_mark}" != "✓" ]]; then

‎scripts/exglobal_archive.py

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,13 @@ def main():
1717
# Instantiate the Archive object
1818
archive = Archive(config)
1919

20+
# update these keys to be 3 digits if they are part of archive.task_config.keys
21+
for key in ['OCNRES', 'ICERES']:
22+
try:
23+
archive.task_config[key] = f"{archive.task_config[key]:03d}"
24+
except KeyError as ee:
25+
logger.info(f"key ({key}) not found in archive.task_config!")
26+
2027
# Pull out all the configuration keys needed to run the rest of archive steps
2128
keys = ['ATARDIR', 'current_cycle', 'FHMIN', 'FHMAX', 'FHOUT', 'RUN', 'PDY',
2229
'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_PREP_OBS_AERO',
@@ -37,16 +44,15 @@ def main():
3744

3845
archive_dict = AttrDict()
3946
for key in keys:
40-
archive_dict[key] = archive.task_config.get(key)
41-
if archive_dict[key] is None:
42-
print(f"Warning: key ({key}) not found in task_config!")
47+
try:
48+
archive_dict[key] = archive.task_config[key]
49+
except KeyError as ee:
50+
logger.warning(f"WARNING: key ({key}) not found in archive.task_config!")
4351

4452
# Also import all COMIN* and COMOUT* directory and template variables
4553
for key in archive.task_config.keys():
46-
if key.startswith("COM_") or key.startswith("COMIN_") or key.startswith("COMOUT_"):
54+
if key.startswith(("COM_", "COMIN_", "COMOUT_")):
4755
archive_dict[key] = archive.task_config.get(key)
48-
if archive_dict[key] is None:
49-
print(f"Warning: key ({key}) not found in task_config!")
5056

5157
with chdir(config.ROTDIR):
5258

‎workflow/rocoto/gfs_tasks.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2896,7 +2896,9 @@ def earc(self):
28962896
else:
28972897
dep_dict = {'type': 'task', 'name': f'{self.run}_esfc'}
28982898
deps.append(rocoto.add_dependency(dep_dict))
2899-
dependencies = rocoto.create_dependency(dep=deps)
2899+
dep_dict = {'type': 'task', 'name': f'{self.run}_echgres'}
2900+
deps.append(rocoto.add_dependency(dep_dict))
2901+
dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
29002902

29012903
earcenvars = self.envars.copy()
29022904
earcenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#'))

0 commit comments

Comments
 (0)