Merge pull request #210 from bertinia/master

updates to support cheyenne upgrade

.gitignore

@@ -14,13 +14,15 @@ ilamb/ilamb
 reshaper/pyReshaper
 conformer/conformer
 mpi_utils/pyTools
+ocean_remap/ocean_remap
 
 # Ignore python build and dist dirs
 cesm_utils/dist/
 conform/dist/
 diag_utils/dist/
 diagnostics/dist/
 timeseries/dist/
+ocean_remap/dist/
 
 # Ignore cesm-env2
 cesm-env2/

Externals.cfg

@@ -33,6 +33,13 @@ repo_url = https://bitbucket.org/ncollier/ilamb
 local_path = ilamb/ilamb
 required = True
 
+[ocean_remap]
+tag = master
+protocol = git
+repo_url = https://github.com/bertinia/ocean_remap
+local_path = ocean_remap/ocean_remap
+required = True
+
 [externals_description]
 schema_version = 1.0.0
 

Machines/cheyenne_modules

@@ -2,18 +2,19 @@
 
 echo "Python boot-strap modules for machine cheyenne"
 
-module load python/2.7.14
-module load intel/17.0.1
+module load python/2.7.16
+##module load intel/17.0.1
+module load gnu/8.3.0
 module load ncarenv
 module load ncarcompilers
 module load mpt/2.19
-module load netcdf/4.6.1
-module load nco/4.7.4
-module load ncl/6.4.0
+module load netcdf/4.6.3
+module load nco/4.7.9
+module load ncl/6.6.2
 
 # clone the ncat virtualenv first with helper script ncar_pylib
 # use "ncar_pylib --help" to see all options
-ncar_pylib -c 20181024 ${pp_dir}/cesm-env2
+ncar_pylib -c 20190627 ${pp_dir}/cesm-env2
 
 export PYTHONPATH=${pp_dir}/cesm-env2/lib/python2.7/site-packages
 

Machines/dav_modules

@@ -2,18 +2,18 @@
 
 echo "Python boot-strap modules for NCAR DAV"
 
-module load python/2.7.14
+module load python/2.7.15
 module load intel/17.0.1
 module load ncarenv
 module load ncarcompilers
 module load impi
 module load netcdf/4.6.1
 module load nco/4.7.4
-module load ncl/6.4.0
+module load ncl/6.6.2
 
 # clone the ncat virtualenv first with helper script ncar_pylib
 # use "ncar_pylib --help" to see all options
-ncar_pylib -c 20181029 ${pp_dir}/cesm-env2
+ncar_pylib -c 20190326 ${pp_dir}/cesm-env2
 
 export PYTHONPATH=${pp_dir}/cesm-env2/lib/python2.7/site-packages
 

Machines/machine_postprocess.xml

@@ -5,6 +5,7 @@
   <machine name="dav" hostname="dav,geyser,caldera,pronghorn,casper">
     <timeseries_pes  nodes="4" pes_per_node="16" wallclock="04:00:00" memory="100G">64</timeseries_pes>
     <xconform_pes  nodes="4" pes_per_node="16" wallclock="06:00:00" memory="100G">64</xconform_pes>
+    <ocn_remap_pes  nodes="4" pes_per_node="9" wallclock="08:00:00" memory="100G">32</ocn_remap_pes>
     <mpi_command>srun</mpi_command>
     <pythonpath></pythonpath>
     <f2py fcompiler="intelem" f77exec="/glade/apps/opt/modulefiles/ys/cmpwrappers/ifort">f2py</f2py>
@@ -24,7 +25,7 @@
       <module>module load impi</module>
       <module>module load netcdf/4.6.1</module>
       <module>module load nco/4.7.4</module>
-      <module>module load ncl/6.4.0</module>
+      <module>module load ncl/6.6.2</module>
     </modules>
     <components>
       <component name="atm">
@@ -126,26 +127,27 @@
   <machine name="cheyenne" hostname="cheyenne">
     <timeseries_pes queue="regular" nodes="16" pes_per_node="9" wallclock="12:00:00">144</timeseries_pes>
     <xconform_pes queue="regular" nodes="16" pes_per_node="4" wallclock="12:00:00">64</xconform_pes>
+    <ocn_remap_pes queue="regular" nodes="4" pes_per_node="9" wallclock="08:00:00">32</ocn_remap_pes>
     <mpi_command>mpiexec_mpt dplace -s 1</mpi_command>
     <pythonpath></pythonpath>
     <f2py fcompiler="intelem" f77exec="/glade/u/apps/ch/opt/ncarcompilers/0.4.1/mpi/ifort">f2py</f2py>
     <za>
-      <compiler>ifort</compiler>
+      <compiler>gfortran</compiler>
       <flags>-c -g -O2</flags>
-      <include>-I/glade/u/apps/ch/opt/netcdf/4.6.1/intel/17.0.1/include</include>
-      <libs>-L/glade/u/apps/ch/opt/netcdf/4.6.1/intel/17.0.1/lib -lnetcdff -lnetcdf</libs>
+      <include>-I/glade/u/apps/ch/opt/netcdf/4.6.3/gnu/8.3.0/include</include>
+      <libs>-L/glade/u/apps/ch/opt/netcdf/4.6.3/gnu/8.3.0/lib -lnetcdff -lnetcdf</libs>
     </za>
     <reset_modules>
       <module>module purge</module>
     </reset_modules>
     <modules>
-      <module>module load intel/17.0.1</module>
+      <module>module load gnu/8.3.0</module>
       <module>module load ncarenv</module>
       <module>module load ncarcompilers</module>
       <module>module load mpt/2.19</module>
-      <module>module load netcdf/4.6.1</module>
-      <module>module load nco/4.7.4</module>
-      <module>module load ncl/6.4.0</module>
+      <module>module load netcdf/4.6.3</module>
+      <module>module load nco/4.7.9</module>
+      <module>module load ncl/6.6.2</module>
     </modules>
     <components>
       <component name="atm">

Machines/machine_postprocess.xsd

@@ -38,6 +38,34 @@
     </xs:complexType>
   </xs:element>
 
+  <xs:element name="xconform_pes">
+    <xs:complexType>
+      <xs:simpleContent>
+	<xs:extension base="xs:string">
+	  <xs:attribute name="queue" use="optional"/>
+	  <xs:attribute name="nodes" use="optional"/>
+	  <xs:attribute name="pes_per_node" use="optional"/>
+	  <xs:attribute name="wallclock" use="required"/>
+	  <xs:attribute name="memory" use="optional"/>
+	</xs:extension>
+      </xs:simpleContent>
+    </xs:complexType>
+  </xs:element>
+
+  <xs:element name="ocn_remap_pes">
+    <xs:complexType>
+      <xs:simpleContent>
+	<xs:extension base="xs:string">
+	  <xs:attribute name="queue" use="optional"/>
+	  <xs:attribute name="nodes" use="optional"/>
+	  <xs:attribute name="pes_per_node" use="optional"/>
+	  <xs:attribute name="wallclock" use="required"/>
+	  <xs:attribute name="memory" use="optional"/>
+	</xs:extension>
+      </xs:simpleContent>
+    </xs:complexType>
+  </xs:element>
+
   <xs:element name="averages_pes">
     <xs:complexType>
       <xs:simpleContent>
@@ -158,6 +186,8 @@
     <xs:complexType>
       <xs:sequence>
 	<xs:element ref="timeseries_pes" minOccurs="0" maxOccurs="1"/>
+	<xs:element ref="xconform_pes" minOccurs="0" maxOccurs="1"/>
+	<xs:element ref="ocn_remap_pes" minOccurs="0" maxOccurs="1"/>
 	<xs:element ref="mpi_command" minOccurs="0" maxOccurs="1"/>
 	<xs:element ref="pythonpath" minOccurs="0" maxOccurs="1"/>
 	<xs:element ref="f2py" minOccurs="0" maxOccurs="1"/>

Makefile

@@ -27,7 +27,8 @@ SUBDIRS = \
         conformer \
         conform \
 	ilamb \
-	diagnostics 
+	diagnostics \
+	ocean_remap
 
 # MAKECMDGOALS is the make option: make 'clobber' or 'all'
 TARGET = $(MAKECMDGOALS)

Tools/copy_html

@@ -261,6 +261,10 @@ def read_paths(env, comp_data):
                             env[values[-2]].append(values[-1])
                     else:
                         env[values[-2]] = []
+
+    if 'OCNDIAG_WEBDIR' not in env:
+        env['OCNDIAG_WEBDIR'] = list()
+
     return env
 
 

Tools/ration_script

@@ -23,8 +23,7 @@ pwd
 . activate
 
 ## load the boot-strap modules
-##module load python/2.7.14
-module load intel/17.0.1
+module load gnu/8.3.0
 module load ncarenv
 module load ncarcompilers
 module load mpt/2.19

cesm_utils/cesm_utils/create_postprocess

@@ -280,7 +280,8 @@ def read_machine_xml(machineName, xmlFile):
         if machineName.lower() == xmlmachine.get('name').lower():
 
             found = True
-            # get the timeseries pes first
+
+            # get the timeseries pes 
             tseries_pes = xmlmachine.find('timeseries_pes')
             machine['timeseries_pes'] = tseries_pes.text
             machine['timeseries_queue'] = ''
@@ -295,7 +296,6 @@ def read_machine_xml(machineName, xmlFile):
             if 'memory' in tseries_pes.attrib:
                 machine['timeseries_memory'] = tseries_pes.get('memory')
 
-
             # get the conform pes 
             xconform_pes = xmlmachine.find('xconform_pes')
             machine['xconform_pes'] = xconform_pes.text
@@ -311,6 +311,21 @@ def read_machine_xml(machineName, xmlFile):
             if 'memory' in xconform_pes.attrib:
                 machine['conform_memory'] = xconform_pes.get('memory')
 
+            # get the ocn_remap pes 
+            ocn_remap_pes = xmlmachine.find('ocn_remap_pes')
+            machine['ocn_remap_pes'] = ocn_remap_pes.text
+            machine['ocn_remap_queue'] = ''
+            if 'queue' in ocn_remap_pes.attrib:
+                machine['ocn_remap_queue'] = ocn_remap_pes.get('queue').lower()
+            machine['ocn_remap_ppn'] = ocn_remap_pes.get('pes_per_node').lower()
+            machine['ocn_remap_wallclock'] = ocn_remap_pes.get('wallclock').lower()
+            machine['ocn_remap_nodes'] = ''
+            if 'nodes' in ocn_remap_pes.attrib:
+                machine['ocn_remap_nodes'] = ocn_remap_pes.get('nodes').lower()
+            machine['ocn_remap_memory'] = ''
+            if 'memory' in ocn_remap_pes.attrib:
+                machine['ocn_remap_memory'] = ocn_remap_pes.get('memory')
+
             # get the mpi run command
             machine['mpi_command'] = xmlmachine.find('mpi_command').text
 
@@ -685,6 +700,13 @@ def main(options):
     create_env_file(envDict=envDict, configFile=config_file, tmplFile=tmpl_file, 
                     envFile=env_file, obs_root='', comp='', standalone=standalone)
 
+    # generate the env_ocn_remap.xml file
+    config_file = '{0}/diagnostics/diagnostics/ocn/Config/config_ocn_remap.xml'.format(envDict['POSTPROCESS_PATH'])
+    env_file = '{0}/env_ocn_remap.xml'.format(envDict['PP_CASE_PATH'])
+    tmpl_file = 'env_postprocess.tmpl'
+    create_env_file(envDict=envDict, configFile=config_file, tmplFile=tmpl_file, 
+                    envFile=env_file, obs_root='', comp='', standalone=standalone)
+
     # generate the env_diags_[component].xml files
     for comp in itertools.chain(compList, imbList):
         dir_name = comp
@@ -769,6 +791,30 @@ def main(options):
                  memory=machine['conform_memory'],
                  options=options, standalone=standalone)
 
+    # generate the ocn_remap batch submit script from template files
+    postProcessCmd = 'ocn_remap_generator.py'
+    processName = 'ocn_remap'
+    outFile = '{0}/{1}'.format(envDict['PP_CASE_PATH'],processName)
+    ocn_remap_tmpl = 'postprocess.tmpl'
+    create_batch(ppDir=envDict['POSTPROCESS_PATH'], 
+                 pes=machine['ocn_remap_pes'], 
+                 batchTmpl=batch_tmpl, runTmpl=ocn_remap_tmpl, 
+                 postProcessCmd=postProcessCmd, 
+                 mpiCmd=machine['mpi_command'], outFile=outFile, 
+                 processName=processName, 
+                 project=envDict['PROJECT'], 
+                 pythonpath=machine['pythonpath'], 
+                 caseRoot=envDict['PP_CASE_PATH'], 
+                 reset_modules=machine['reset_modules'], 
+                 modules=machine['modules'], 
+                 queue=machine['ocn_remap_queue'], 
+                 ppn=machine['ocn_remap_ppn'], 
+                 nodes=machine['ocn_remap_nodes'], 
+                 wallclock=machine['ocn_remap_wallclock'], 
+                 memory=machine['ocn_remap_memory'],
+                 options=options, standalone=standalone)
+
+
     # generate the diagnostics batch submit scripts from template files
     for comp in compList:
         # generate the averages batch submit script
@@ -980,6 +1026,28 @@ def main(options):
                      memory=machine['conform_memory'],
                      options=options, standalone=standalone)
 
+        # generate the ocn_remap batch submit script from template files
+        postProcessCmd = 'ocn_remap_generator.py'
+        processName = 'ocn_remap_dav'
+        outFile = '{0}/{1}'.format(envDict['PP_CASE_PATH'],processName)
+        create_batch(ppDir=pp_dav,
+                     pes=machine['ocn_remap_pes'], 
+                     batchTmpl=batch_tmpl, runTmpl=run_tmpl, 
+                     postProcessCmd=postProcessCmd, 
+                     mpiCmd=machine['mpi_command'], outFile=outFile, 
+                     processName=processName, 
+                     project=envDict['PROJECT'], 
+                     pythonpath=machine['pythonpath'], 
+                     caseRoot=envDict['PP_CASE_PATH'], 
+                     reset_modules=machine['reset_modules'], 
+                     modules=machine['modules'], 
+                     queue=machine['ocn_remap_queue'],
+                     ppn=machine['ocn_remap_ppn'],
+                     nodes=machine['ocn_remap_nodes'],
+                     wallclock=machine['ocn_remap_wallclock'],
+                     memory=machine['ocn_remap_memory'],
+                     options=options, standalone=standalone)
+
         # generate the diagnostics batch submit scripts from template files
         for comp in compList:
             # generate the averages batch submit script

diagnostics/diagnostics/ocn/Config/config_ocn_remap.xml

@@ -0,0 +1,70 @@
+<?xml version="1.0"?>
+
+<config_definition version="1.0">
+
+  <groups>
+    <group name="global">
+      <order>0</order>
+      <comment>Settings for controlling ocean remapping of CMIP6 variables.</comment>
+
+      <entry id="OCNREMAP_matrix_2d_fname"
+	     type="char"
+	     valid_values=""
+	     value="$POSTPROCESS_PATH/ocean_remap/ocean_remap/POP_gx1v7_to_latlon_1x1_0E_mask_conserve_20181015.nc"
+	     group="global"
+	     desc="Full path to matrix 2d file"
+	     ></entry>
+
+      <entry id="OCNREMAP_matrix_3d_fname"
+	     type="char"
+	     valid_values=""
+	     value="$POSTPROCESS_PATH/ocean_remap/ocean_remap/POP_gx1v7_to_latlon_1x1_0E_fulldepth_conserve_20181015.nc"
+	     group="global"
+	     desc="Full path to matrix 3d file"
+	     ></entry>
+
+      <entry id="OCNREMAP_cmip6"
+	     type="char"
+	     valid_values=""
+	     value="$POSTPROCESS_PATH/ocean_remap/ocean_remap/ocn_vars.txt"
+	     group="global"
+	     desc="Full path to a text file that contains per line MIP_TABLE:VARNAME.  This list and the data path can be pieced together to find all files to be regridded."
+	     ></entry>
+
+      <entry id="OCNREMAP_filelist"
+	     type="char"
+	     valid_values=""
+	     value="$POSTPROCESS_PATH/ocean_remap/ocean_remap/file_list.txt"
+	     group="global"
+	     desc="Full path to a text file that contains a full path file name per line that needs to be regridded."
+	     ></entry>
+
+      <entry id="OCNREMAP_indir"
+	     type="char"
+	     valid_values=""
+	     value="/glade/collections/cdg/cmip6/[case_id]"
+	     group="global"
+	     desc="Full path to a base directory to find files to regrid.  All files with the *.nc found under this path will be regridded, unless restricted with the OCNREMAP_cmip6 or OCNREMAP_filelist options."
+	     ></entry>
+
+      <entry id="OCNREMAP_outdir"
+	     type="char"
+	     valid_values=""
+	     value="/glade/collections/cdg/cmip6/[case_id]"
+	     group="global"
+	     desc="Full path to a output directory (base directory if CMIP6)"
+	     ></entry>
+
+      <entry id="OCNREMAP_chunk"
+	     type="char"
+	     valid_values=""
+	     value="10"
+	     group="global"
+	     desc="The amount of time slices to operate on at once."
+	     ></entry>
+
+    </group>
+  </groups>
+
+</config_definition>
+