diff --git a/Config/config_postprocess.xml b/Config/config_postprocess.xml
index c2df49b0..524cd247 100644
--- a/Config/config_postprocess.xml
+++ b/Config/config_postprocess.xml
@@ -53,7 +53,7 @@
- 128
- 16
+ 16
+ 16
6
/glade/p/cesm/amwg/amwg_data
- 128
+ 16
4
/glade/p/cesm/pcwg/ice/data
- 128
+ 16
12
6
/glade/p/cesm/lmwg/diag/lnd_diag_data
- 128
+ 16
16
- /glade/p/cesm
+ /glade/p/cesm/
diff --git a/Templates/env_diags.tmpl b/Templates/env_diags.tmpl
index 75bb6bd4..90f10c7f 100644
--- a/Templates/env_diags.tmpl
+++ b/Templates/env_diags.tmpl
@@ -23,7 +23,8 @@
{% for xml in v %}
-
+
+
{% endfor %}
{% endif %}
diff --git a/Templates/env_postprocess.tmpl b/Templates/env_postprocess.tmpl
index 7291717a..6027400e 100644
--- a/Templates/env_postprocess.tmpl
+++ b/Templates/env_postprocess.tmpl
@@ -22,7 +22,7 @@
{% for xml in v %}
-
+
{% endfor %}
{% endif %}
diff --git a/cesm_utils/cesm_utils/create_postprocess b/cesm_utils/cesm_utils/create_postprocess
index 786bd9bd..3d2a5d5e 100755
--- a/cesm_utils/cesm_utils/create_postprocess
+++ b/cesm_utils/cesm_utils/create_postprocess
@@ -94,10 +94,11 @@ import jinja2
# define a class to be used for the xml entry, id, desc values
# -------------------------------------------------------------------------------
class XmlEntry(object):
- def __init__(self, id, value, desc):
+ def __init__(self, id, value, desc, valid_values):
self._id = id
self._value = value
self._desc = desc
+ self._valid_values = valid_values
def id(self):
return self._id
@@ -108,6 +109,9 @@ class XmlEntry(object):
def desc(self):
return self._desc
+ def valid_values(self):
+ return self._valid_values
+
# -------------------------------------------------------------------------------
# commandline_options - parse any command line options
# -------------------------------------------------------------------------------
@@ -204,9 +208,11 @@ def create_env_file(envDict, configFile, tmplFile, envFile, obs_root, comp, stan
for entry_tag in group_tag.findall('entry'):
# check if the value needs to be inherited from the envDict
if entry_tag.get('value') == 'inherit':
- xml_list.append(XmlEntry(entry_tag.get('id'), envDict[entry_tag.get('id')], entry_tag.get('desc')))
+ xml_list.append(XmlEntry(entry_tag.get('id'), envDict[entry_tag.get('id')],
+ entry_tag.get('desc'), entry_tag.get('valid_values')))
else:
- xml_list.append(XmlEntry(entry_tag.get('id'), entry_tag.get('value'), entry_tag.get('desc')))
+ xml_list.append(XmlEntry(entry_tag.get('id'), entry_tag.get('value'),
+ entry_tag.get('desc'), entry_tag.get('valid_values')))
group_dict = { 'order' : order, 'name' : name, 'comment' : comment, 'xml_list' : xml_list }
group_list.append(group_dict)
@@ -217,9 +223,8 @@ def create_env_file(envDict, configFile, tmplFile, envFile, obs_root, comp, stan
xml_list = list()
if obs_root:
if len(obs_root) > 0:
- xml_obs = XmlEntry('{0}DIAG_DIAGOBSROOT'.format(comp.upper()),
- obs_root,
- 'Machine dependent diagnostics observation files root path')
+ xml_obs = XmlEntry('{0}DIAG_DIAGOBSROOT'.format(comp.upper()), obs_root,
+ 'Machine dependent diagnostics observation files root path', '')
xml_list.append(xml_obs)
# the xml_list now contains a list of XmlEntry classes that can be written to the template
@@ -562,7 +567,8 @@ def main(options):
config_file = '{0}/diagnostics/diagnostics/{1}/Config/config_diags_{1}.xml'.format(envDict['POSTPROCESS_PATH'], comp)
tmpl_file = 'env_diags.tmpl'
env_file = '{0}/env_diags_{1}.xml'.format(envDict['PP_CASE_PATH'], comp)
- create_env_file(envDict, config_file, tmpl_file, env_file, machine['{0}_obs_root'.format(comp)], comp, standalone)
+ create_env_file(envDict, config_file, tmpl_file, env_file, machine['{0}_obs_root'.format(comp)],
+ comp, standalone)
# define the template files for the batch scripts
batch_tmpl = 'batch_{0}.tmpl'.format(envDict['MACH'])
@@ -575,8 +581,9 @@ def main(options):
create_batch(envDict['POSTPROCESS_PATH'], machine['timeseries_pes'], batch_tmpl, run_tmpl, postProcessCmd,
machine['mpi_command'], outFile, processName, machine['project'], machine['pythonpath'],
- envDict['PP_CASE_PATH'], machine['reset_modules'], machine['modules'], machine['timeseries_queue'],
- machine['timeseries_ppn'], machine['timeseries_wallclock'], options, standalone)
+ envDict['PP_CASE_PATH'], machine['reset_modules'], machine['modules'],
+ machine['timeseries_queue'], machine['timeseries_ppn'], machine['timeseries_wallclock'],
+ options, standalone)
# generate the diagnostics batch submit scripts from template files
for comp in compList:
@@ -584,19 +591,24 @@ def main(options):
postProcessCmd = '{0}_avg_generator.py'.format(comp)
processName = '{0}_averages'.format(comp)
outFile = '{0}/{1}'.format(envDict['PP_CASE_PATH'], processName)
- create_batch(envDict['POSTPROCESS_PATH'], machine['{0}_averages_pes'.format(comp)], batch_tmpl, run_tmpl, postProcessCmd,
- machine['mpi_command'], outFile, processName, machine['project'], machine['pythonpath'],
- envDict['PP_CASE_PATH'], machine['reset_modules'], machine['modules'], machine['{0}_averages_queue'.format(comp)],
- machine['{0}_averages_ppn'.format(comp)], machine['{0}_averages_wallclock'.format(comp)], options, standalone)
+ create_batch(envDict['POSTPROCESS_PATH'], machine['{0}_averages_pes'.format(comp)], batch_tmpl,
+ run_tmpl, postProcessCmd, machine['mpi_command'], outFile, processName,
+ machine['project'], machine['pythonpath'], envDict['PP_CASE_PATH'],
+ machine['reset_modules'], machine['modules'], machine['{0}_averages_queue'.format(comp)],
+ machine['{0}_averages_ppn'.format(comp)], machine['{0}_averages_wallclock'.format(comp)],
+ options, standalone)
# generate the diagnostics batch submit script
postProcessCmd = '{0}_diags_generator.py'.format(comp)
processName = '{0}_diagnostics'.format(comp)
outFile = '{0}/{1}'.format(envDict['PP_CASE_PATH'], processName)
- create_batch(envDict['POSTPROCESS_PATH'], machine['{0}_diagnostics_pes'.format(comp)], batch_tmpl, run_tmpl, postProcessCmd,
- machine['mpi_command'], outFile, processName, machine['project'], machine['pythonpath'],
- envDict['PP_CASE_PATH'], machine['reset_modules'], machine['modules'], machine['{0}_diagnostics_queue'.format(comp)],
- machine['{0}_diagnostics_ppn'.format(comp)], machine['{0}_diagnostics_wallclock'.format(comp)], options, standalone)
+ create_batch(envDict['POSTPROCESS_PATH'], machine['{0}_diagnostics_pes'.format(comp)], batch_tmpl,
+ run_tmpl, postProcessCmd, machine['mpi_command'], outFile, processName,
+ machine['project'], machine['pythonpath'], envDict['PP_CASE_PATH'],
+ machine['reset_modules'], machine['modules'],
+ machine['{0}_diagnostics_queue'.format(comp)],
+ machine['{0}_diagnostics_ppn'.format(comp)],
+ machine['{0}_diagnostics_wallclock'.format(comp)], options, standalone)
# generate the regrid batch submit scripts from template files
for comp in regridList:
@@ -604,10 +616,12 @@ def main(options):
postProcessCmd = '{0}_regrid_generator.py'.format(comp)
processName = '{0}_regrid'.format(comp)
outFile = '{0}/{1}'.format(envDict['PP_CASE_PATH'], processName)
- create_batch(envDict['POSTPROCESS_PATH'], machine['{0}_regrid_pes'.format(comp)], batch_tmpl, run_tmpl, postProcessCmd,
- machine['mpi_command'], outFile, processName, machine['project'], machine['pythonpath'],
- envDict['PP_CASE_PATH'], machine['reset_modules'], machine['modules'], machine['{0}_regrid_queue'.format(comp)],
- machine['{0}_regrid_ppn'.format(comp)], machine['{0}_regrid_wallclock'.format(comp)], options, standalone)
+ create_batch(envDict['POSTPROCESS_PATH'], machine['{0}_regrid_pes'.format(comp)], batch_tmpl,
+ run_tmpl, postProcessCmd, machine['mpi_command'], outFile, processName,
+ machine['project'], machine['pythonpath'], envDict['PP_CASE_PATH'],
+ machine['reset_modules'], machine['modules'], machine['{0}_regrid_queue'.format(comp)],
+ machine['{0}_regrid_ppn'.format(comp)], machine['{0}_regrid_wallclock'.format(comp)],
+ options, standalone)
# copy some tools to the PP_CASE_PATH
files_to_copy = ['copy_html', 'pp_config', 'create_env_script', 'env_file.xsd']
diff --git a/diagnostics/diagnostics/ice/Config/config_diags_ice.xml b/diagnostics/diagnostics/ice/Config/config_diags_ice.xml
index 7f7d6579..d8f79051 100644
--- a/diagnostics/diagnostics/ice/Config/config_diags_ice.xml
+++ b/diagnostics/diagnostics/ice/Config/config_diags_ice.xml
@@ -45,6 +45,16 @@
desc="Controls whether to generate model vs. model diagnostics"
>
+
+
0:
# call the pyAverager with the inVarList
- callPyAverager(avg_start_year, avg_stop_year, in_dir, htype, key_infile, out_dir, case_prefix, averageList, inVarList,
- envDict, stream, reg_file, year0, year1, split, split_size, main_comm, debugMsg)
+ callPyAverager(avg_start_year, avg_stop_year, in_dir, htype, key_infile, out_dir,
+ case_prefix, averageList, inVarList, envDict, stream, reg_file, year0,
+ year1, split, split_size, netcdf_format, main_comm, debugMsg)
#============================================
@@ -271,9 +282,6 @@ def initialize_envDict(envDict, caseroot, debugMsg, standalone):
envDict (dictionary) - environment dictionary
"""
# setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files
- # TODO put this file list into the config_postprocess definition
-## env_file_list = ['../env_case.xml', '../env_run.xml', '../env_build.xml', '../env_mach_pes.xml', './env_postprocess.xml', './env_diags_ice.xml']
-## if standalone:
env_file_list = ['./env_postprocess.xml', './env_diags_ice.xml']
envDict = cesmEnvLib.readXML(caseroot, env_file_list)
@@ -288,7 +296,7 @@ def initialize_envDict(envDict, caseroot, debugMsg, standalone):
# add the os.environ['PATH'] to the envDict['PATH']
envDict['ICEDIAG_PATH'] = os.pathsep + os.environ['PATH']
- # strip the ATMDIAG_ prefix from the envDict entries before setting the
+ # strip the ICEDIAG_ prefix from the envDict entries before setting the
# enviroment to allow for compatibility with all the diag routine calls
envDict = diagUtilsLib.strip_prefix(envDict, 'ICEDIAG_')
@@ -379,8 +387,11 @@ def main(options, main_comm, debugMsg):
avg_BEGYR = (int(envDict['ENDYR_CONT']) - int(envDict['YRS_TO_AVG'])) + 1
createClimFiles(avg_BEGYR, envDict['ENDYR_CONT'], h_path, split, split_size,
- envDict['cont_htype'], envDict['cont_key_infile'], envDict['PATH_CLIMO_CONT'], envDict['CASE_TO_CONT'],
- 'cice.h', varList, envDict, envDict['GRIDFILECONT'], envDict['BEGYR_CONT'], envDict['ENDYR_CONT'], main_comm, debugMsg)
+ envDict['cont_htype'], envDict['cont_key_infile'],
+ envDict['PATH_CLIMO_CONT'], envDict['CASE_TO_CONT'],
+ 'cice.h', varList, envDict, envDict['GRIDFILECONT'],
+ envDict['BEGYR_CONT'], envDict['ENDYR_CONT'],
+ envDict['netcdf_format'], main_comm, debugMsg)
except Exception as error:
print(str(error))
traceback.print_exc()
@@ -411,8 +422,11 @@ def main(options, main_comm, debugMsg):
avg_BEGYR_DIFF = (int(envDict['ENDYR_DIFF']) - int(envDict['YRS_TO_AVG'])) + 1
createClimFiles(avg_BEGYR_DIFF, envDict['ENDYR_DIFF'], h_path, split, split_size,
- envDict['diff_htype'], envDict['diff_key_infile'], envDict['PATH_CLIMO_DIFF'], envDict['CASE_TO_DIFF'],
- 'cice.h', varList, envDict, envDict['GRIDFILEDIFF'], envDict['BEGYR_DIFF'], envDict['ENDYR_DIFF'], main_comm, debugMsg)
+ envDict['diff_htype'], envDict['diff_key_infile'],
+ envDict['PATH_CLIMO_DIFF'], envDict['CASE_TO_DIFF'],
+ 'cice.h', varList, envDict, envDict['GRIDFILEDIFF'],
+ envDict['BEGYR_DIFF'], envDict['ENDYR_DIFF'],
+ envDict['netcdf_format'], main_comm, debugMsg)
except Exception as error:
print(str(error))
traceback.print_exc()
@@ -441,9 +455,6 @@ def main(options, main_comm, debugMsg):
print(' Successfully completed generating ice climatology averages')
print('*************************************************************')
sys.exit(status)
-
-## except RunTimeError as error:
-
except Exception as error:
print(str(error))
if options.backtrace:
diff --git a/diagnostics/diagnostics/ice/ice_diags_bc.py b/diagnostics/diagnostics/ice/ice_diags_bc.py
index 4fa77575..c13d8cd8 100755
--- a/diagnostics/diagnostics/ice/ice_diags_bc.py
+++ b/diagnostics/diagnostics/ice/ice_diags_bc.py
@@ -74,44 +74,50 @@ def setup_workdir(self, env, t, scomm):
if (scomm.is_manager()):
climo_files = glob.glob(old_workdir+'/*.nc')
for climo_file in climo_files:
- name_split = climo_file.split('.') # Split on '.'
- if ('-' in name_split[-3]):
- fn = str.join('.',name_split[:len(name_split)-3] + name_split[-2:]) #Piece together w/o the date, but still has old path
+ if ('ice_vol_' in climo_file):
+ new_fn = workdir + '/' + os.path.basename(climo_file)
## if (scomm.is_manager()):
-## print('0. fn = {0}'.format(fn))
- path_split = fn.split('/') # Remove the path
- if ('jfm_climo' in path_split[-1]):
- s = 'jfm'
- elif ('amj_climo' in path_split[-1]):
- s = 'amj'
- elif ('jas_climo' in path_split[-1]):
- s = 'jas'
- elif ('ond_climo' in path_split[-1]):
- s = 'ond'
- elif ('fm_climo' in path_split[-1]):
- s = 'fm'
- elif ('on_climo' in path_split[-1]):
- s = 'on'
- elif ('_ANN_climo' in path_split[-1]):
- s = 'ann'
- else:
- s = None
- if s is not None:
- new_fn = workdir + '/' + s + '_avg_' + str(avg_BEGYR).zfill(4) + '-' + env['ENDYR_'+t].zfill(4) + '.nc'
-## if (scomm.is_manager()):
-## print('1. ice_diags_bc.py s = {0}: new_fn = {1}'.format(s, new_fn))
+## print('1. ice_diags_bc.py: new_fn = {0}'.format(new_fn))
+ else:
+ name_split = climo_file.split('.') # Split on '.'
+ if ('-' in name_split[-3]):
+ fn = str.join('.',name_split[:len(name_split)-3] + name_split[-2:]) #Piece together w/o the date, but still has old path
+ if (scomm.is_manager()):
+ print('1. fn = {0}'.format(fn))
+ path_split = fn.split('/') # Remove the path
+ if ('jfm_climo' in path_split[-1]):
+ s = 'jfm'
+ elif ('amj_climo' in path_split[-1]):
+ s = 'amj'
+ elif ('jas_climo' in path_split[-1]):
+ s = 'jas'
+ elif ('ond_climo' in path_split[-1]):
+ s = 'ond'
+ elif ('fm_climo' in path_split[-1]):
+ s = 'fm'
+ elif ('on_climo' in path_split[-1]):
+ s = 'on'
+ elif ('_ANN_climo' in path_split[-1]):
+ s = 'ann'
+ else:
+ s = None
+ if s is not None:
+ new_fn = workdir + '/' + s + '_avg_' + str(avg_BEGYR).zfill(4) + '-' + env['ENDYR_'+t].zfill(4) + '.nc'
+## if (scomm.is_manager()):
+## print('2. ice_diags_bc.py s = {0}: new_fn = {1}'.format(s, new_fn))
+ else:
+ new_fn = workdir + '/' +path_split[-1] # Take file name and add it to new path
+## if (scomm.is_manager()):
+## print('3. ice_diags_bc.py: new_fn = {0}'.format(new_fn))
else:
- new_fn = workdir + '/' +path_split[-1] # Take file name and add it to new path
+ new_fn = workdir + '/' + os.path.basename(climo_file)
## if (scomm.is_manager()):
-## print('2. ice_diags_bc.py: new_fn = {0}'.format(new_fn))
- else:
- new_fn = workdir + '/' + os.path.basename(climo_file)
-## if (scomm.is_manager()):
-## print('3. ice_diags_bc.py: new_fn = {0}'.format(new_fn))
-
+## print('4. ice_diags_bc.py: new_fn = {0}'.format(new_fn))
rc1, err_msg1 = cesmEnvLib.checkFile(new_fn, 'read')
if not rc1:
os.symlink(climo_file,new_fn)
+ else:
+ print('ice_diags_bc.py: unable to create link to file {0}'.format(new_fn))
return env
def check_prerequisites(self, env, scomm):
diff --git a/diagnostics/diagnostics/lnd/Config/config_diags_lnd.xml b/diagnostics/diagnostics/lnd/Config/config_diags_lnd.xml
index 7aaaf658..d7386947 100644
--- a/diagnostics/diagnostics/lnd/Config/config_diags_lnd.xml
+++ b/diagnostics/diagnostics/lnd/Config/config_diags_lnd.xml
@@ -191,6 +191,16 @@
desc="Controls whether to generate model vs. model diagnostics"
>
+
+
0:
# call the pyAverager with the inVarList
- callPyAverager(avg_start_year, avg_stop_year, in_dir, htype, key_infile, out_dir, case_prefix, averageList, inVarList,
- envDict, stream, trend_year0, trend_year1, main_comm, debugMsg)
+ callPyAverager(avg_start_year, avg_stop_year, in_dir, htype, key_infile,
+ out_dir, case_prefix, averageList, inVarList,
+ envDict, stream, trend_year0, trend_year1,
+ netcdf_format, main_comm, debugMsg)
#============================================
@@ -394,9 +406,11 @@ def main(options, main_comm, debugMsg):
case1_climo_dir = envDict['PTMPDIR_1']+'/climo/'+ envDict['caseid_1']+'/'
createClimFiles(envDict['clim_first_yr_1'], envDict['clim_last_yr_1'], h_path,
- envDict['case1_htype'], envDict['case1_key_infile'], case1_climo_dir, envDict['caseid_1'],
- suffix, varList, envDict, envDict['trends_first_yr_1'], envDict['trends_last_yr_1'],
- envDict['climo_'+model+'_1'], envDict['trends_'+model+'_1'], model, main_comm, debugMsg)
+ envDict['case1_htype'], envDict['case1_key_infile'], case1_climo_dir,
+ envDict['caseid_1'], suffix, varList, envDict,
+ envDict['trends_first_yr_1'], envDict['trends_last_yr_1'],
+ envDict['climo_'+model+'_1'], envDict['trends_'+model+'_1'], model,
+ envDict['netcdf_large'], main_comm, debugMsg)
except Exception as error:
print(str(error))
traceback.print_exc()
@@ -446,9 +460,11 @@ def main(options, main_comm, debugMsg):
case2_climo_dir = envDict['PTMPDIR_2']+'/climo/'+ envDict['caseid_2']+'/'
createClimFiles(envDict['clim_first_yr_2'], envDict['clim_last_yr_2'], h_path,
- envDict['case2_htype'], envDict['case2_key_infile'], case2_climo_dir, envDict['caseid_2'],
- suffix, varList, envDict, envDict['trends_first_yr_2'], envDict['trends_last_yr_2'],
- envDict['climo_'+model+'_2'], envDict['trends_'+model+'_2'], model, main_comm, debugMsg)
+ envDict['case2_htype'], envDict['case2_key_infile'], case2_climo_dir,
+ envDict['caseid_2'], suffix, varList, envDict,
+ envDict['trends_first_yr_2'], envDict['trends_last_yr_2'],
+ envDict['climo_'+model+'_2'], envDict['trends_'+model+'_2'], model,
+ envDict['netcdf_format'], main_comm, debugMsg)
except Exception as error:
print(str(error))
traceback.print_exc()