From 5f88b720f3cba6a6792998d61306fbe8eb8ef1a0 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 1 Nov 2024 07:29:07 -0500 Subject: [PATCH 01/15] Move machine-specific options to host files --- parm/config/gefs/config.base | 34 ++++++++++++++----------- parm/config/gfs/config.base | 39 +++++++++++++---------------- parm/config/gfs/config.prepoceanobs | 7 +++++- parm/config/gfs/yaml/defaults.yaml | 3 ++- workflow/hosts/awspw.yaml | 4 +++ workflow/hosts/azurepw.yaml | 1 + workflow/hosts/googlepw.yaml | 1 + workflow/hosts/hercules.yaml | 2 ++ workflow/hosts/orion.yaml | 2 ++ 9 files changed, 54 insertions(+), 39 deletions(-) diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 05aabaa323..1f34d5481f 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -131,6 +131,7 @@ export APP=@APP@ export DO_ATM="YES" export DO_COUPLED="NO" export DO_WAVE="NO" +export DISABLE_WAVE="@DISABLE_WAVE@" export DO_OCN="NO" export DO_ICE="NO" export DO_AERO="NO" @@ -188,8 +189,13 @@ case "${APP}" in ;; ATMW) export DO_COUPLED="YES" - export DO_WAVE="YES" - export WAVE_RUN="both" + if [[ "${DISABLE_WAVE}" != "YES" ]]; then + export DO_WAVE="YES" + export WAVE_RUN="both" + else + echo "WARNING: WW3 is explicitly disabled but APP=ATMW! Disabling waves." + export APP=ATM + fi ;; NG-GODAS) export DO_ATM="NO" @@ -208,14 +214,18 @@ case "${APP}" in fi if [[ "${APP}" =~ ^S2SW ]]; then - export DO_WAVE="YES" - export WAVE_RUN="both" - export cplwav2atm=".true." + if [[ "${DISABLE_WAVE}" != "YES" ]]; then + export DO_WAVE="YES" + export WAVE_RUN="both" + else + echo "WARNING WW3 is explicitly disabled, but APP=${APP}! Disabling waves." + export APP="${APP//W/}" + fi fi ;; *) - echo "Unrecognized APP: ${1}" - exit 1 + echo "Unrecognized APP: '${APP}'" + exit 2 ;; esac @@ -319,7 +329,7 @@ export MAKE_NSSTBUFR="@MAKE_NSSTBUFR@" export MAKE_ACFTBUFR="@MAKE_ACFTBUFR@" # Verification options -export DO_METP="NO" # Run METPLUS jobs - set METPLUS settings in config.metp +export DO_METP="@DO_METP@" # Run METPLUS jobs - set METPLUS settings in config.metp export DO_FIT2OBS="NO" # Run fit to observations package # Archiving options @@ -327,7 +337,7 @@ export HPSSARCH="@HPSSARCH@" # save data to HPSS archive export LOCALARCH="@LOCALARCH@" # save data to local archive if [[ ${HPSSARCH} = "YES" ]] && [[ ${LOCALARCH} = "YES" ]]; then echo "Both HPSS and local archiving selected. Please choose one or the other." - exit 2 + exit 3 fi export ARCH_CYC=00 # Archive data at this cycle for warm_start capability export ARCH_WARMICFREQ=4 # Archive frequency in days for warm_start capability @@ -338,10 +348,4 @@ export DELETE_COM_IN_ARCHIVE_JOB="YES" # NO=retain ROTDIR. YES default in arc # Number of regional collectives to create soundings for export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9} -# The tracker, genesis, and METplus jobs are not supported on CSPs yet -# TODO: we should place these in workflow/hosts/[aws|azure|google]pw.yaml as part of CSP's setup, not for general. -if [[ "${machine}" =~ "PW" ]]; then - export DO_WAVE="NO" -fi - echo "END: config.base" diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index ccb05abe88..b62ae3e584 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -172,6 +172,7 @@ shopt -u extglob export DO_ATM="YES" export DO_COUPLED="NO" export DO_WAVE="NO" +export DISABLE_WAVE="@DISABLE_WAVE@" export DO_OCN="NO" export DO_ICE="NO" export DO_AERO="NO" @@ -234,8 +235,13 @@ case "${APP}" in ;; ATMW) export DO_COUPLED="YES" - export DO_WAVE="YES" - export WAVE_RUN="both" + if [[ "${DISABLE_WAVE}" != "YES" ]]; then + export DO_WAVE="YES" + export WAVE_RUN="both" + else + echo "WARNING: WW3 is explicitly disabled but APP=ATMW! Disabling waves." + export APP=ATM + fi ;; NG-GODAS) export DO_ATM="NO" @@ -254,13 +260,18 @@ case "${APP}" in fi if [[ "${APP}" =~ ^S2SW ]]; then - export DO_WAVE="YES" - export WAVE_RUN="both" + if [[ "${DISABLE_WAVE}" != "YES" ]]; then + export DO_WAVE="YES" + export WAVE_RUN="both" + else + echo "WARNING WW3 is explicitly disabled, but APP=${APP}! Disabling waves." + export APP="${APP//W/}" + fi fi ;; *) echo "Unrecognized APP: '${APP}'" - exit 1 + exit 2 ;; esac @@ -462,7 +473,7 @@ export HPSSARCH="@HPSSARCH@" # save data to HPSS archive export LOCALARCH="@LOCALARCH@" # save data to local archive if [[ ${HPSSARCH} = "YES" ]] && [[ ${LOCALARCH} = "YES" ]]; then echo "Both HPSS and local archiving selected. Please choose one or the other." - exit 2 + exit 3 fi export ARCH_CYC=00 # Archive data at this cycle for warm_start capability export ARCH_WARMICFREQ=4 # Archive frequency in days for warm_start capability @@ -483,20 +494,4 @@ export OFFSET_START_HOUR=0 # Number of regional collectives to create soundings for export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9} -# The tracker, genesis, and METplus jobs are not supported on CSPs yet -# TODO: we should place these in workflow/hosts/awspw.yaml as part of AWS/AZURE setup, not for general. -if [[ "${machine}" =~ "PW" ]]; then - export DO_TRACKER="NO" - export DO_GENESIS="NO" - export DO_METP="NO" - export DO_WAVE="NO" -fi - -# The tracker and genesis are not installed on Orion/Hercules yet; this requires spack-stack builds of the package. -# TODO: we should place these in workflow/hosts/[orion|hercules].yaml. -if [[ "${machine}" == "ORION" || "${machine}" == "HERCULES" ]]; then - export DO_TRACKER="NO" - export DO_GENESIS="NO" -fi - echo "END: config.base" diff --git a/parm/config/gfs/config.prepoceanobs b/parm/config/gfs/config.prepoceanobs index 0963a5c42d..c72a958fc7 100644 --- a/parm/config/gfs/config.prepoceanobs +++ b/parm/config/gfs/config.prepoceanobs @@ -14,7 +14,12 @@ export OBS_LIST=@SOCA_OBS_LIST@ export OBS_YAML=${OBS_LIST} # ocean analysis needs own dmpdir until standard dmpdir has full ocean obs -export DMPDIR=@DMPDIR@ +use_exp_obs="@use_exp_obs@" +if [[ "use_exp_obs" == "YES" ]]; then + dmpdir_exp="@dmpdir_exp@" +fi + +export DMPDIR="${dmpdir_exp:-${DMPDIR}}" # For BUFR2IODA json and python scripts export JSON_TMPL_DIR="${PARMgfs}/gdas/ioda/bufr2ioda" diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index dfc67d1237..4513a8109b 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -61,4 +61,5 @@ prepoceanobs: SOCA_INPUT_FIX_DIR: "${FIXgfs}/gdas/soca/72x35x25/soca" SOCA_OBS_LIST: "${PARMgfs}/gdas/soca/obs/obs_list.yaml" # TODO: This is also repeated in ocnanal OBSPREP_YAML: "${PARMgfs}/gdas/soca/obsprep/obsprep_config.yaml" - DMPDIR: "${BASE_DATA}/experimental_obs" + use_exp_obs: "YES" + dmpdir_exp: "${BASE_DATA}/experimental_obs" diff --git a/workflow/hosts/awspw.yaml b/workflow/hosts/awspw.yaml index ef17d8f2f4..29ac71e2f8 100644 --- a/workflow/hosts/awspw.yaml +++ b/workflow/hosts/awspw.yaml @@ -24,4 +24,8 @@ LOCALARCH: 'NO' ATARDIR: '' # TODO: This will not yet work from AWS. MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' +DO_TRACKER: 'NO' +DO_GENESIS: 'NO' +DO_METP: 'NO' +DISABLE_WAVE: 'YES' SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions. diff --git a/workflow/hosts/azurepw.yaml b/workflow/hosts/azurepw.yaml index 1769f9ee19..99ac2af845 100644 --- a/workflow/hosts/azurepw.yaml +++ b/workflow/hosts/azurepw.yaml @@ -24,4 +24,5 @@ LOCALARCH: 'NO' ATARDIR: '' # TODO: This will not yet work from AZURE. MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' +DISABLE_WAVE: 'YES' SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions. diff --git a/workflow/hosts/googlepw.yaml b/workflow/hosts/googlepw.yaml index daf6cd1eb2..7a83d50b35 100644 --- a/workflow/hosts/googlepw.yaml +++ b/workflow/hosts/googlepw.yaml @@ -24,4 +24,5 @@ LOCALARCH: 'NO' ATARDIR: '' # TODO: This will not yet work from GOOGLE. MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' +DISABLE_WAVE: 'YES' SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions. diff --git a/workflow/hosts/hercules.yaml b/workflow/hosts/hercules.yaml index 73fde6cde6..2a2caa6a06 100644 --- a/workflow/hosts/hercules.yaml +++ b/workflow/hosts/hercules.yaml @@ -24,6 +24,8 @@ LOCALARCH: 'NO' ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' +DO_TRACKER: 'NO' +DO_GENESIS: 'NO' SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] COMINecmwf: /work/noaa/global/glopara/data/external_gempak/ecmwf COMINnam: /work/noaa/global/glopara/data/external_gempak/nam diff --git a/workflow/hosts/orion.yaml b/workflow/hosts/orion.yaml index d47b2b2bab..683d11ed30 100644 --- a/workflow/hosts/orion.yaml +++ b/workflow/hosts/orion.yaml @@ -24,6 +24,8 @@ LOCALARCH: 'NO' ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' +DO_TRACKER: 'NO' +DO_GENESIS: 'NO' SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] COMINecmwf: /work/noaa/global/glopara/data/external_gempak/ecmwf COMINnam: /work/noaa/global/glopara/data/external_gempak/nam From 1a146fb9e7da9c8b4d11a75e15e272023dfa7d57 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 1 Nov 2024 07:30:10 -0500 Subject: [PATCH 02/15] Rearrange setup_expt.py config setup so machine opts are updated last. --- workflow/setup_expt.py | 63 +++++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index f32203e600..b5bc96bc07 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -50,18 +50,30 @@ def fill_EXPDIR(inputs): def update_configs(host, inputs): def _update_defaults(dict_in: dict) -> dict: + # Given an input dict_in of the form + # {defaults: {config_name: {var1: value1, ...}, }, config_name: {var1: value1, ...}} + # Replace values in ['defaults']['config_name']['var1'] with ['config_name']['var1'] + # and return the ['defaults'] subdictionary as its own new dictionary. defaults = dict_in.pop('defaults', AttrDict()) defaults.update(dict_in) return defaults - # Read in the YAML file to fill out templates and override host defaults + # Convert the inputs to an AttrDict + # data = AttrDict(**inputs.__dict__) data = AttrDict(host.info, **inputs.__dict__) + + # Read in the YAML file to fill out templates data.HOMEgfs = _top yaml_path = inputs.yaml if not os.path.exists(yaml_path): - raise IOError(f'YAML file does not exist, check path:' + yaml_path) - yaml_dict = _update_defaults(AttrDict(parse_j2yaml(yaml_path, data))) + raise FileNotFoundError(f'YAML file does not exist, check path: {yaml_path}') + yaml_dict = parse_j2yaml(yaml_path, data) + + # yaml_dict is in the form {defaults: {key1: val1, ...}, base: {key1: val1, ...}, ...} + # _update_defaults replaces any keys/values in defaults with matching keys in base + yaml_dict = _update_defaults(yaml_dict) + # Override the YAML defaults with the host-specific capabilities # First update config.base edit_baseconfig(host, inputs, yaml_dict) @@ -92,20 +104,19 @@ def edit_baseconfig(host, inputs, yaml_dict): to `EXPDIR/pslot/config.base` """ - tmpl_dict = { + # Create base_dict which holds templated variables to be written to config.base + base_dict = { "@HOMEgfs@": _top, "@MACHINE@": host.machine.upper()} - # Replace host related items - extend_dict = get_template_dict(host.info) - tmpl_dict = dict(tmpl_dict, **extend_dict) - if inputs.start in ["warm"]: is_warm_start = ".true." elif inputs.start in ["cold"]: is_warm_start = ".false." + else: + raise ValueError(f"Invalid start type: {inputs.start}") - extend_dict = dict() + # Construct a dictionary from user inputs extend_dict = { "@PSLOT@": inputs.pslot, "@SDATE@": datetime_to_YMDH(inputs.idate), @@ -121,35 +132,37 @@ def edit_baseconfig(host, inputs, yaml_dict): "@APP@": inputs.app, "@NMEM_ENS@": getattr(inputs, 'nens', 0) } - tmpl_dict = dict(tmpl_dict, **extend_dict) - extend_dict = dict() if getattr(inputs, 'nens', 0) > 0: - extend_dict = { - "@CASEENS@": f'C{inputs.resensatmos}', - } - tmpl_dict = dict(tmpl_dict, **extend_dict) + extend_dict['@CASEENS@'] = f'C{inputs.resensatmos}' - extend_dict = dict() if inputs.mode in ['cycled']: - extend_dict = { - "@DOHYBVAR@": "YES" if inputs.nens > 0 else "NO", - } - tmpl_dict = dict(tmpl_dict, **extend_dict) + extend_dict["@DOHYBVAR@"] = "YES" if inputs.nens > 0 else "NO" - try: - tmpl_dict = dict(tmpl_dict, **get_template_dict(yaml_dict['base'])) - except KeyError: - pass + # Further extend/redefine base_dict with extend_dict + base_dict = dict(base_dict, **extend_dict) + + # Add/override 'base'-specific declarations in base_dict + if 'base' in yaml_dict: + base_dict = dict(base_dict, **get_template_dict(yaml_dict['base'])) + + # Finally, override defaults with machine-specific capabilties + # e.g. some machines are not able to run metp jobs + host_dict = get_template_dict(host.info) + base_dict = dict(base_dict, **host_dict) base_input = f'{inputs.configdir}/config.base' base_output = f'{inputs.expdir}/{inputs.pslot}/config.base' - edit_config(base_input, base_output, tmpl_dict) + edit_config(base_input, base_output, base_dict) return def edit_config(input_config, output_config, config_dict): + """ + Given a templated input_config filename, parse it based on config_dict and + write it out to the output_config filename. + """ # Read input config with open(input_config, 'rt') as fi: From f279db7f341637efe5c929bb2368812f53cf67f9 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 1 Nov 2024 08:59:22 -0500 Subject: [PATCH 03/15] Move machine aerosol inputs directories to host files --- parm/config/gfs/config.aero | 30 +----------------------------- workflow/hosts/gaea.yaml | 1 + workflow/hosts/hera.yaml | 1 + workflow/hosts/hercules.yaml | 1 + workflow/hosts/jet.yaml | 1 + workflow/hosts/orion.yaml | 1 + workflow/hosts/s4.yaml | 1 + workflow/hosts/wcoss2.yaml | 1 + 8 files changed, 8 insertions(+), 29 deletions(-) diff --git a/parm/config/gfs/config.aero b/parm/config/gfs/config.aero index f49593a439..bbfb782636 100644 --- a/parm/config/gfs/config.aero +++ b/parm/config/gfs/config.aero @@ -2,36 +2,8 @@ # UFS-Aerosols settings -# Turn off warnings about unused variables -# shellcheck disable=SC2034 - - # Path to the input data tree -case ${machine} in - "HERA") - AERO_INPUTS_DIR="/scratch1/NCEPDEV/global/glopara/data/gocart_emissions" - ;; - "ORION" | "HERCULES") - AERO_INPUTS_DIR="/work2/noaa/global/wkolczyn/noscrub/global-workflow/gocart_emissions" - ;; - "S4") - AERO_INPUTS_DIR="/data/prod/glopara/gocart_emissions" - ;; - "WCOSS2") - AERO_INPUTS_DIR="/lfs/h2/emc/global/noscrub/emc.global/data/gocart_emissions" - ;; - "GAEA") - AERO_INPUTS_DIR="/gpfs/f5/epic/proj-shared/global/glopara/data/gocart_emissions" - ;; - "JET") - AERO_INPUTS_DIR="/lfs5/HFIP/hfv3gfs/glopara/data/gocart_emissions" - ;; - *) - echo "FATAL ERROR: Machine ${machine} unsupported for aerosols" - exit 2 - ;; -esac -export AERO_INPUTS_DIR +export AERO_INPUTS_DIR=@AERO_INPUTS_DIR@ export AERO_DIAG_TABLE="${PARMgfs}/ufs/fv3/diag_table.aero" export AERO_FIELD_TABLE="${PARMgfs}/ufs/fv3/field_table.aero" diff --git a/workflow/hosts/gaea.yaml b/workflow/hosts/gaea.yaml index 5a37b5dabf..23ac75ca03 100644 --- a/workflow/hosts/gaea.yaml +++ b/workflow/hosts/gaea.yaml @@ -26,3 +26,4 @@ ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] +AERO_INPUTS_DIR: /gpfs/f5/epic/proj-shared/global/glopara/data/gocart_emissions diff --git a/workflow/hosts/hera.yaml b/workflow/hosts/hera.yaml index fa2c351aa1..e9e749ad3c 100644 --- a/workflow/hosts/hera.yaml +++ b/workflow/hosts/hera.yaml @@ -28,3 +28,4 @@ SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] COMINecmwf: /scratch1/NCEPDEV/global/glopara/data/external_gempak/ecmwf COMINnam: /scratch1/NCEPDEV/global/glopara/data/external_gempak/nam COMINukmet: /scratch1/NCEPDEV/global/glopara/data/external_gempak/ukmet +AERO_INPUTS_DIR: /scratch1/NCEPDEV/global/glopara/data/gocart_emissions diff --git a/workflow/hosts/hercules.yaml b/workflow/hosts/hercules.yaml index 2a2caa6a06..f528761cf1 100644 --- a/workflow/hosts/hercules.yaml +++ b/workflow/hosts/hercules.yaml @@ -30,3 +30,4 @@ SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] COMINecmwf: /work/noaa/global/glopara/data/external_gempak/ecmwf COMINnam: /work/noaa/global/glopara/data/external_gempak/nam COMINukmet: /work/noaa/global/glopara/data/external_gempak/ukmet +AERO_INPUTS_DIR: /work2/noaa/global/wkolczyn/noscrub/global-workflow/gocart_emissions diff --git a/workflow/hosts/jet.yaml b/workflow/hosts/jet.yaml index a53224fe52..737e2e7f94 100644 --- a/workflow/hosts/jet.yaml +++ b/workflow/hosts/jet.yaml @@ -28,3 +28,4 @@ SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] COMINecmwf: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/external_gempak/ecmwf COMINnam: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/external_gempak/nam COMINukmet: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/external_gempak/ukmet +AERO_INPUTS_DIR: /lfs5/HFIP/hfv3gfs/glopara/data/gocart_emissions diff --git a/workflow/hosts/orion.yaml b/workflow/hosts/orion.yaml index 683d11ed30..985c24c6fb 100644 --- a/workflow/hosts/orion.yaml +++ b/workflow/hosts/orion.yaml @@ -30,3 +30,4 @@ SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] COMINecmwf: /work/noaa/global/glopara/data/external_gempak/ecmwf COMINnam: /work/noaa/global/glopara/data/external_gempak/nam COMINukmet: /work/noaa/global/glopara/data/external_gempak/ukmet +AERO_INPUTS_DIR: /work2/noaa/global/wkolczyn/noscrub/global-workflow/gocart_emissions diff --git a/workflow/hosts/s4.yaml b/workflow/hosts/s4.yaml index b93fefec39..2e77c112b1 100644 --- a/workflow/hosts/s4.yaml +++ b/workflow/hosts/s4.yaml @@ -25,3 +25,4 @@ ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' MAKE_NSSTBUFR: 'YES' MAKE_ACFTBUFR: 'YES' SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] +AERO_INPUTS_DIR: /data/prod/glopara/gocart_emissions diff --git a/workflow/hosts/wcoss2.yaml b/workflow/hosts/wcoss2.yaml index 15f0705f91..4fb4b1d64a 100644 --- a/workflow/hosts/wcoss2.yaml +++ b/workflow/hosts/wcoss2.yaml @@ -28,3 +28,4 @@ SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] COMINecmwf: /lfs/h2/emc/global/noscrub/emc.global/data/external_gempak/ecmwf COMINnam: /lfs/h2/emc/global/noscrub/emc.global/data/external_gempak/nam COMINukmet: /lfs/h2/emc/global/noscrub/emc.global/data/external_gempak/ukmet +AERO_INPUTS_DIR: /lfs/h2/emc/global/noscrub/emc.global/data/gocart_emissions From c98a3c9d7653e9270dcd414e4ef97de0e5997ad0 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 1 Nov 2024 09:00:23 -0500 Subject: [PATCH 04/15] Enable HPSS archiving on WCOSS2 --- workflow/hosts/wcoss2.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/hosts/wcoss2.yaml b/workflow/hosts/wcoss2.yaml index 4fb4b1d64a..19b186c63d 100644 --- a/workflow/hosts/wcoss2.yaml +++ b/workflow/hosts/wcoss2.yaml @@ -18,7 +18,7 @@ RESERVATION: '' CHGRP_RSTPROD: 'YES' CLUSTERS: '' CHGRP_CMD: 'chgrp rstprod' -HPSSARCH: 'NO' +HPSSARCH: 'YES' HPSS_PROJECT: emc-global LOCALARCH: 'NO' ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}' From 7c22d0dad2e1c7a0e8350e83d1fd68e277ac0c6d Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 1 Nov 2024 09:13:55 -0500 Subject: [PATCH 05/15] Added missing braces in config.prepoceanobs --- parm/config/gfs/config.prepoceanobs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gfs/config.prepoceanobs b/parm/config/gfs/config.prepoceanobs index c72a958fc7..3aeff8a3de 100644 --- a/parm/config/gfs/config.prepoceanobs +++ b/parm/config/gfs/config.prepoceanobs @@ -15,7 +15,7 @@ export OBS_YAML=${OBS_LIST} # ocean analysis needs own dmpdir until standard dmpdir has full ocean obs use_exp_obs="@use_exp_obs@" -if [[ "use_exp_obs" == "YES" ]]; then +if [[ "${use_exp_obs}" == "YES" ]]; then dmpdir_exp="@dmpdir_exp@" fi From 00977b9014df2ed0ddac6a2a9e1dc3e912b3ee61 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 1 Nov 2024 09:30:22 -0500 Subject: [PATCH 06/15] Disable HPSS archiving on WCOSS2 --- workflow/hosts/wcoss2.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/hosts/wcoss2.yaml b/workflow/hosts/wcoss2.yaml index 19b186c63d..4fb4b1d64a 100644 --- a/workflow/hosts/wcoss2.yaml +++ b/workflow/hosts/wcoss2.yaml @@ -18,7 +18,7 @@ RESERVATION: '' CHGRP_RSTPROD: 'YES' CLUSTERS: '' CHGRP_CMD: 'chgrp rstprod' -HPSSARCH: 'YES' +HPSSARCH: 'NO' HPSS_PROJECT: emc-global LOCALARCH: 'NO' ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}' From 098a77403a3c0a212a6a3e1b6a22e0e9dd1d10d9 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 1 Nov 2024 13:40:12 -0500 Subject: [PATCH 07/15] Fix downstream archive issue, reenable wcoss2 HPSS archiving --- parm/archive/gfs_downstream.yaml.j2 | 4 ++-- workflow/hosts/wcoss2.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/parm/archive/gfs_downstream.yaml.j2 b/parm/archive/gfs_downstream.yaml.j2 index ed5317b42c..94bdd1df56 100644 --- a/parm/archive/gfs_downstream.yaml.j2 +++ b/parm/archive/gfs_downstream.yaml.j2 @@ -6,7 +6,7 @@ gfs_downstream: - "{{ COMIN_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.sfc" - "{{ COMIN_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.snd" {% for i in range(1, NUM_SND_COLLECTIVES) %} - - "{{ COMIN_ATMOS_WMO | relpath(ROTDIR) }}/gfs_collective{{ i }}.postsnd_{{ cycle_HH }}" + - "{{ COMIN_ATMOS_BUFR | relpath(ROTDIR) }}/gfs_collective{{ i }}.fil" {% endfor %} - - "{{ COMIN_ATMOS_BUFR | relpath(ROTDIR) }}/bufr.t{{ cycle_HH }}z" + - "{{ COMIN_ATMOS_BUFR | relpath(ROTDIR) }}/bufr.??????.{{ cycle_YMDH }}" - "{{ COMIN_ATMOS_BUFR | relpath(ROTDIR) }}/gfs.t{{ cycle_HH }}z.bufrsnd.tar.gz" diff --git a/workflow/hosts/wcoss2.yaml b/workflow/hosts/wcoss2.yaml index 4fb4b1d64a..19b186c63d 100644 --- a/workflow/hosts/wcoss2.yaml +++ b/workflow/hosts/wcoss2.yaml @@ -18,7 +18,7 @@ RESERVATION: '' CHGRP_RSTPROD: 'YES' CLUSTERS: '' CHGRP_CMD: 'chgrp rstprod' -HPSSARCH: 'NO' +HPSSARCH: 'YES' HPSS_PROJECT: emc-global LOCALARCH: 'NO' ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}' From df8d105a4d38616914ebd4f77fa4fb91c7fb3093 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 1 Nov 2024 13:45:45 -0500 Subject: [PATCH 08/15] Exit generate_workflows if a bad input is given --- workflow/generate_workflows.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/workflow/generate_workflows.sh b/workflow/generate_workflows.sh index c2565140d3..6a4cb9910a 100755 --- a/workflow/generate_workflows.sh +++ b/workflow/generate_workflows.sh @@ -150,10 +150,12 @@ while [[ $# -gt 0 && "$1" != "--" ]]; do :) echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" _usage + exit 1 ;; *) echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}" _usage + exit 1 ;; esac done From 86c66f9772de25ca766f1370b26d623134dfff8b Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Wed, 6 Nov 2024 07:58:54 -0500 Subject: [PATCH 09/15] Apply suggestions from code review Co-authored-by: Walter Kolczynski - NOAA --- parm/config/gefs/config.base | 2 +- workflow/hosts/wcoss2.yaml | 2 +- workflow/setup_expt.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 1f34d5481f..e5a4315ff5 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -329,7 +329,7 @@ export MAKE_NSSTBUFR="@MAKE_NSSTBUFR@" export MAKE_ACFTBUFR="@MAKE_ACFTBUFR@" # Verification options -export DO_METP="@DO_METP@" # Run METPLUS jobs - set METPLUS settings in config.metp +export DO_METP="NO" # Run METPLUS jobs - set METPLUS settings in config.metp export DO_FIT2OBS="NO" # Run fit to observations package # Archiving options diff --git a/workflow/hosts/wcoss2.yaml b/workflow/hosts/wcoss2.yaml index 19b186c63d..4fb4b1d64a 100644 --- a/workflow/hosts/wcoss2.yaml +++ b/workflow/hosts/wcoss2.yaml @@ -18,7 +18,7 @@ RESERVATION: '' CHGRP_RSTPROD: 'YES' CLUSTERS: '' CHGRP_CMD: 'chgrp rstprod' -HPSSARCH: 'YES' +HPSSARCH: 'NO' HPSS_PROJECT: emc-global LOCALARCH: 'NO' ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}' diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index b5bc96bc07..a27cfa0c53 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -55,11 +55,12 @@ def _update_defaults(dict_in: dict) -> dict: # Replace values in ['defaults']['config_name']['var1'] with ['config_name']['var1'] # and return the ['defaults'] subdictionary as its own new dictionary. defaults = dict_in.pop('defaults', AttrDict()) + if 'defaults' in defaults: + _update_defaults(defaults) defaults.update(dict_in) return defaults # Convert the inputs to an AttrDict - # data = AttrDict(**inputs.__dict__) data = AttrDict(host.info, **inputs.__dict__) # Read in the YAML file to fill out templates From 88aaa12fc1278b3cdb6d8f6046df28a50eaa4dbe Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 6 Nov 2024 07:49:31 -0600 Subject: [PATCH 10/15] Check wave support at setup time rather than runtime --- parm/config/gefs/config.base | 19 ++++--------------- parm/config/gfs/config.base | 19 ++++--------------- workflow/hosts/awspw.yaml | 2 +- workflow/hosts/azurepw.yaml | 2 +- workflow/hosts/googlepw.yaml | 2 +- workflow/setup_expt.py | 4 ++++ 6 files changed, 15 insertions(+), 33 deletions(-) diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index e5a4315ff5..8492db3c7d 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -131,7 +131,6 @@ export APP=@APP@ export DO_ATM="YES" export DO_COUPLED="NO" export DO_WAVE="NO" -export DISABLE_WAVE="@DISABLE_WAVE@" export DO_OCN="NO" export DO_ICE="NO" export DO_AERO="NO" @@ -189,13 +188,8 @@ case "${APP}" in ;; ATMW) export DO_COUPLED="YES" - if [[ "${DISABLE_WAVE}" != "YES" ]]; then - export DO_WAVE="YES" - export WAVE_RUN="both" - else - echo "WARNING: WW3 is explicitly disabled but APP=ATMW! Disabling waves." - export APP=ATM - fi + export DO_WAVE="YES" + export WAVE_RUN="both" ;; NG-GODAS) export DO_ATM="NO" @@ -214,13 +208,8 @@ case "${APP}" in fi if [[ "${APP}" =~ ^S2SW ]]; then - if [[ "${DISABLE_WAVE}" != "YES" ]]; then - export DO_WAVE="YES" - export WAVE_RUN="both" - else - echo "WARNING WW3 is explicitly disabled, but APP=${APP}! Disabling waves." - export APP="${APP//W/}" - fi + export DO_WAVE="YES" + export WAVE_RUN="both" fi ;; *) diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index b62ae3e584..4f702f9668 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -172,7 +172,6 @@ shopt -u extglob export DO_ATM="YES" export DO_COUPLED="NO" export DO_WAVE="NO" -export DISABLE_WAVE="@DISABLE_WAVE@" export DO_OCN="NO" export DO_ICE="NO" export DO_AERO="NO" @@ -235,13 +234,8 @@ case "${APP}" in ;; ATMW) export DO_COUPLED="YES" - if [[ "${DISABLE_WAVE}" != "YES" ]]; then - export DO_WAVE="YES" - export WAVE_RUN="both" - else - echo "WARNING: WW3 is explicitly disabled but APP=ATMW! Disabling waves." - export APP=ATM - fi + export DO_WAVE="YES" + export WAVE_RUN="both" ;; NG-GODAS) export DO_ATM="NO" @@ -260,13 +254,8 @@ case "${APP}" in fi if [[ "${APP}" =~ ^S2SW ]]; then - if [[ "${DISABLE_WAVE}" != "YES" ]]; then - export DO_WAVE="YES" - export WAVE_RUN="both" - else - echo "WARNING WW3 is explicitly disabled, but APP=${APP}! Disabling waves." - export APP="${APP//W/}" - fi + export DO_WAVE="YES" + export WAVE_RUN="both" fi ;; *) diff --git a/workflow/hosts/awspw.yaml b/workflow/hosts/awspw.yaml index 29ac71e2f8..b98c838faa 100644 --- a/workflow/hosts/awspw.yaml +++ b/workflow/hosts/awspw.yaml @@ -27,5 +27,5 @@ MAKE_ACFTBUFR: 'NO' DO_TRACKER: 'NO' DO_GENESIS: 'NO' DO_METP: 'NO' -DISABLE_WAVE: 'YES' +SUPPORT_WAVES: 'NO' SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions. diff --git a/workflow/hosts/azurepw.yaml b/workflow/hosts/azurepw.yaml index 99ac2af845..4725e28962 100644 --- a/workflow/hosts/azurepw.yaml +++ b/workflow/hosts/azurepw.yaml @@ -24,5 +24,5 @@ LOCALARCH: 'NO' ATARDIR: '' # TODO: This will not yet work from AZURE. MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' -DISABLE_WAVE: 'YES' +SUPPORT_WAVES: 'NO' SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions. diff --git a/workflow/hosts/googlepw.yaml b/workflow/hosts/googlepw.yaml index 7a83d50b35..1b979b6bc9 100644 --- a/workflow/hosts/googlepw.yaml +++ b/workflow/hosts/googlepw.yaml @@ -24,5 +24,5 @@ LOCALARCH: 'NO' ATARDIR: '' # TODO: This will not yet work from GOOGLE. MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' -DISABLE_WAVE: 'YES' +SUPPORT_WAVES: 'NO' SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions. diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index a27cfa0c53..26a5186fd2 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -364,6 +364,7 @@ def query_and_clean(dirname, force_clean=False): def validate_user_request(host, inputs): supp_res = host.info['SUPPORTED_RESOLUTIONS'] + supp_waves = host.info.get('SUPPORT_WAVES', 'YES') machine = host.machine for attr in ['resdetatmos', 'resensatmos']: try: @@ -373,6 +374,9 @@ def validate_user_request(host, inputs): if expt_res not in supp_res: raise NotImplementedError(f"Supported resolutions on {machine} are:\n{', '.join(supp_res)}") + if "W" in inputs.app and supp_waves == "NO": + raise NotImplementedError(f"Waves are not supported on {machine}") + def get_ocean_resolution(resdetatmos): """ From 2a89ef854d38370dec2254729634a5ab93cf9951 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Wed, 6 Nov 2024 09:15:40 -0500 Subject: [PATCH 11/15] Update parm/config/gefs/config.base Co-authored-by: Rahul Mahajan --- parm/config/gefs/config.base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 8492db3c7d..e769015937 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -213,7 +213,7 @@ case "${APP}" in fi ;; *) - echo "Unrecognized APP: '${APP}'" + echo "FATAL ERROR: Unrecognized APP: '${APP}'" exit 2 ;; esac From 4ad195fa13eab54d7f16c1944afdced9e8891214 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 6 Nov 2024 19:45:00 +0000 Subject: [PATCH 12/15] Satisfy flake8 --- workflow/setup_expt.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 26a5186fd2..353e0f9abe 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -7,7 +7,6 @@ import os import glob import shutil -import warnings from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, SUPPRESS, ArgumentTypeError from hosts import Host @@ -29,7 +28,7 @@ def makedirs_if_missing(dirname): os.makedirs(dirname) -def fill_EXPDIR(inputs): +def fill_expdir(inputs): """ Method to copy config files from workflow to experiment directory INPUTS: @@ -415,7 +414,7 @@ def main(*argv): if create_expdir: makedirs_if_missing(expdir) - fill_EXPDIR(user_inputs) + fill_expdir(user_inputs) update_configs(host, user_inputs) print(f"*" * 100) From fc75a225be62663bab82d52a0a18380dbf3369b2 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 6 Nov 2024 20:04:50 +0000 Subject: [PATCH 13/15] Allow config.aero to be modified by the machine yamls --- ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml | 1 - parm/config/gfs/yaml/defaults.yaml | 4 +++ workflow/setup_expt.py | 30 +++++++++++++------- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml b/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml index c4fa54dcc8..b956066f6b 100644 --- a/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml +++ b/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml @@ -17,4 +17,3 @@ nsst: NST_MODEL: "1" sfcanl: DONST: "NO" - diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index 4513a8109b..78caf46f5d 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -63,3 +63,7 @@ prepoceanobs: OBSPREP_YAML: "${PARMgfs}/gdas/soca/obsprep/obsprep_config.yaml" use_exp_obs: "YES" dmpdir_exp: "${BASE_DATA}/experimental_obs" + +# config.aero has just a system-specific path to add. +# This is handled by the setup_expt.py, but it has to be told to write to it. +aero: {} diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 353e0f9abe..27da4943d3 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -85,7 +85,7 @@ def _update_defaults(dict_in: dict) -> dict: stage_dict = dict(stage_dict, **host_dict) stage_input = f'{inputs.configdir}/config.stage_ic' stage_output = f'{inputs.expdir}/{inputs.pslot}/config.stage_ic' - edit_config(stage_input, stage_output, stage_dict) + edit_config(stage_input, stage_output, host_dict, stage_dict) # Loop over other configs and update them with defaults for cfg in yaml_dict.keys(): @@ -93,7 +93,7 @@ def _update_defaults(dict_in: dict) -> dict: continue cfg_file = f'{inputs.expdir}/{inputs.pslot}/config.{cfg}' cfg_dict = get_template_dict(yaml_dict[cfg]) - edit_config(cfg_file, cfg_file, cfg_dict) + edit_config(cfg_file, cfg_file, host_dict, cfg_dict) return @@ -146,24 +146,24 @@ def edit_baseconfig(host, inputs, yaml_dict): if 'base' in yaml_dict: base_dict = dict(base_dict, **get_template_dict(yaml_dict['base'])) - # Finally, override defaults with machine-specific capabilties - # e.g. some machines are not able to run metp jobs - host_dict = get_template_dict(host.info) - base_dict = dict(base_dict, **host_dict) - base_input = f'{inputs.configdir}/config.base' base_output = f'{inputs.expdir}/{inputs.pslot}/config.base' - edit_config(base_input, base_output, base_dict) + edit_config(base_input, base_output, host.info, base_dict) return -def edit_config(input_config, output_config, config_dict): +def edit_config(input_config, output_config, host_info, config_dict): """ Given a templated input_config filename, parse it based on config_dict and - write it out to the output_config filename. + host_info and write it out to the output_config filename. """ + # Override defaults with machine-specific capabilties + # e.g. some machines are not able to run metp jobs + host_dict = get_template_dict(host_info) + config_dict = dict(config_dict, **host_dict) + # Read input config with open(input_config, 'rt') as fi: config_str = fi.read() @@ -186,9 +186,17 @@ def edit_config(input_config, output_config, config_dict): def get_template_dict(input_dict): + # Reads a templated input dictionary and updates the output + output_dict = dict() + for key, value in input_dict.items(): - output_dict[f'@{key}@'] = value + # In some cases, the same config may be templated twice + # Prevent adding additional "@"s + if "@" in key: + output_dict[f'{key}'] = value + else: + output_dict[f'@{key}@'] = value return output_dict From 5c52594e66608f61d25c2fee97f04a65c7ca083a Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 12 Nov 2024 16:44:12 +0000 Subject: [PATCH 14/15] module_gwci.wcoss2.lua Load Python module for WCOSS2 setup --- modulefiles/module_gwsetup.wcoss2.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modulefiles/module_gwsetup.wcoss2.lua b/modulefiles/module_gwsetup.wcoss2.lua index a2440569db..86bdad3c56 100644 --- a/modulefiles/module_gwsetup.wcoss2.lua +++ b/modulefiles/module_gwsetup.wcoss2.lua @@ -4,5 +4,8 @@ Load environment to run GFS workflow ci scripts on WCOSS2 prepend_path("MODULEPATH", "/apps/ops/test/nco/modulefiles/core") load(pathJoin("rocoto","1.3.5")) +load(pathJoin("PrgEnv-intel")) +load(pathJoin("intel","19.1.3.304")) +load(pathJoin("python", "3.8.6")) whatis("Description: GFS run setup environment") From 1748b7d34038dc22e7dd3db8ea26aa2d27d0d232 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 13 Nov 2024 07:47:20 -0600 Subject: [PATCH 15/15] Temporarilly disable UFS DA test on WCOSS2 --- ci/cases/pr/C96C48_ufs_hybatmDA.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml index 41a8baa725..031054079a 100644 --- a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml +++ b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml @@ -21,4 +21,4 @@ skip_ci_on_hosts: - gaea - orion - hercules - + - wcoss2