Merge pull request #615 from NCAR/update_from_develop_2024-11-21

Update `main` with latest commits from `develop`

Author: climbfuji

.github/workflows/capgen_unit_tests.yaml

@@ -3,7 +3,7 @@ name: Capgen Unit Tests
 on:
   workflow_dispatch:
   pull_request:
-    branches: [feature/capgen, main]
+    branches: [develop, main]
 
 jobs:
   unit_tests:
@@ -15,7 +15,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: update repos and install dependencies
-      run: sudo apt-get update && sudo apt-get install -y build-essential ${{matrix.fortran-compiler}} cmake python3 git
+      run: sudo apt-get update && sudo apt-get install -y build-essential ${{matrix.fortran-compiler}} cmake python3 git libxml2-utils
     - name: Run unit tests
       run: cd test && ./run_fortran_tests.sh
 

.github/workflows/python.yaml

@@ -3,7 +3,7 @@ name: Python package
 on:
   workflow_dispatch:
   pull_request:
-    branches: [feature/capgen, main]
+    branches: [develop, main]
 
 jobs:
   build:

scripts/ccpp_capgen.py

@@ -400,7 +400,7 @@ def compare_fheader_to_mheader(meta_header, fort_header, logger):
 ###############################################################################
 def check_fortran_against_metadata(meta_headers, fort_headers,
                                    mfilename, ffilename, logger,
-                                   dyn_routines=None, fortran_routines=None):
+                                   fortran_routines=None):
 ###############################################################################
     """Compare a set of metadata headers from <mfilename> against the
     code in the associated Fortran file, <ffilename>.
@@ -452,17 +452,6 @@ def check_fortran_against_metadata(meta_headers, fort_headers,
                                       's' if num_errors > 1 else '',
                                       mfilename, ffilename))
     # end if
-    # Check that any dynamic constituent routines declared in the metadata are
-    # present in the Fortran
-    if dyn_routines:
-        for routine in dyn_routines:
-            if routine not in fortran_routines:
-                # throw an error - it's not in the Fortran
-                errmsg = f"Dynamic constituent routine {routine} not found in fortran {ffilename}"
-                raise CCPPError(errmsg)
-            # end if
-        # end for
-    # end if
     # No return, an exception is raised on error
 
 ###############################################################################
@@ -561,15 +550,8 @@ def parse_scheme_files(scheme_filenames, run_env, skip_ddt_check=False):
         for sect in [x.sections() for x in ftables]:
             fheaders.extend(sect)
         # end for
-        dyn_routines = []
-        for table in mtables:
-            if table.dyn_const_routine:
-                dyn_routines.append(table.dyn_const_routine)
-            # end if
-        # end for
         check_fortran_against_metadata(mheaders, fheaders,
                                        filename, fort_file, logger,
-                                       dyn_routines=dyn_routines,
                                        fortran_routines=additional_routines)
         # Check for duplicate tables, then add to dict
         for table in mtables:
@@ -593,22 +575,6 @@ def parse_scheme_files(scheme_filenames, run_env, skip_ddt_check=False):
             # end if
         # end for
     # end for
-    # Check for duplicate dynamic constituent routine names
-    dyn_val_dict = {}
-    for table in table_dict:
-        routine_name = table_dict[table].dyn_const_routine
-        if routine_name:
-            if routine_name in dyn_val_dict:
-                # dynamic constituent routines must have unique names
-                scheme_name = dyn_val_dict[routine_name]
-                errmsg = f"ERROR: Dynamic constituent routine names must be unique. Cannot add " \
-                         f"{routine_name} for {table}. Routine already exists in {scheme_name}. "
-                raise CCPPError(errmsg)
-            else:
-                dyn_val_dict[routine_name] = table
-            # end if
-        # end if
-    # end for
 
     return header_dict.values(), table_dict
 
@@ -674,24 +640,12 @@ def capgen(run_env, return_db=False):
     # First up, handle the host files
     host_model = parse_host_model_files(host_files, host_name, run_env)
     # Next, parse the scheme files
-    # We always need to parse the ccpp_constituent_prop_ptr_t DDT
+    # We always need to parse the constituent DDTs
     const_prop_mod = os.path.join(src_dir, "ccpp_constituent_prop_mod.meta")
     if const_prop_mod not in scheme_files:
         scheme_files= [const_prop_mod] + scheme_files
     # end if
     scheme_headers, scheme_tdict = parse_scheme_files(scheme_files, run_env)
-    # Pull out the dynamic constituent routines, if any
-    dyn_const_dict = {}
-    dyn_val_dict = {}
-    for table in scheme_tdict:
-        routine_name = scheme_tdict[table].dyn_const_routine
-        if routine_name is not None:
-            if routine_name not in dyn_val_dict:
-               dyn_const_dict[table] = routine_name
-               dyn_val_dict[routine_name] = table
-            # end if
-        # end if
-    # end for
     if run_env.verbose:
         ddts = host_model.ddt_lib.keys()
         if ddts:
@@ -722,7 +676,7 @@ def capgen(run_env, return_db=False):
         # end if
         os.makedirs(outtemp_dir)
     # end if
-    ccpp_api = API(sdfs, host_model, scheme_headers, run_env, dyn_const_dict)
+    ccpp_api = API(sdfs, host_model, scheme_headers, run_env)
     cap_filenames = ccpp_api.write(outtemp_dir, run_env)
     if run_env.generate_host_cap:
         # Create a cap file

scripts/ccpp_datafile.py

@@ -58,8 +58,6 @@
                   {"report" : "dependencies", "type" : bool,
                    "help" : ("Return a list of scheme and host "
                              "dependency module names")},
-                  {"report" : "dyn_const_routines", "type" : bool,
-                   "help" : ("Return the constituent routines for a suite")},
                   {"report" : "suite_list", "type" : bool,
                    "help" : "Return a list of configured suite names"},
                   {"report" : "required_variables", "type" : str,
@@ -106,8 +104,6 @@ def __init__(self, action, value=True):
         # Test a valid action
         >>> DatatableReport('input_variables', False).action
         'input_variables'
-        >>> DatatableReport('dyn_const_routines', True).value
-        True
 
         # Test an invalid action
         >>> DatatableReport('banana', True).value
@@ -400,40 +396,6 @@ def _retrieve_dependencies(table):
     # end for
     return sorted(result)
 
-###############################################################################
-def _retrieve_dyn_const_routines(table):
-###############################################################################
-    """Find and return a list of all scheme constituent routines.
-    # Test valid dynamic constituent routines
-    >>> table = ET.fromstring("<ccpp_datatable version='1.0'><dyn_const_routines>" \
-               "<dyn_const_routine parent='banana'>dyn_const_get"    \
-               "</dyn_const_routine><dyn_const_routine>dyn_const_2"  \
-               "</dyn_const_routine></dyn_const_routines></ccpp_datatable>")
-    >>> _retrieve_dyn_const_routines(table)
-    ['dyn_const_2', 'dyn_const_get']
-
-    # Test no dynamic constituent routines
-    >>> table = ET.fromstring("<ccpp_datatable version='1.0'><dyn_const_routines>" \
-                              "</dyn_const_routines></ccpp_datatable>")
-    >>> _retrieve_dyn_const_routines(table)
-    []
-
-    # Test missing dynamic constituent routines tag
-    >>> table = ET.fromstring("<ccpp_datatable version='1.0'></ccpp_datatable>")
-    >>> _retrieve_dyn_const_routines(table)
-    Traceback (most recent call last):
-    ...
-    ccpp_datafile.CCPPDatatableError: Could not find 'dyn_const_routines' element
-
-    """
-    routines = table.find("dyn_const_routines")
-    if routines is None:
-        raise CCPPDatatableError("Could not find 'dyn_const_routines' element")
-    # end if
-    routine_names = [routine.text for routine in routines if routine.text]
-    # end for
-    return sorted(routine_names)
-
 ###############################################################################
 def _find_var_dictionary(table, dict_name=None, dict_type=None):
 ###############################################################################
@@ -746,8 +708,6 @@ def datatable_report(datatable, action, sep, exclude_protected=False):
         result = _retrieve_module_list(table)
     elif action.action_is("dependencies"):
         result = _retrieve_dependencies(table)
-    elif action.action_is("dyn_const_routines"):
-        result = _retrieve_dyn_const_routines(table)
     elif action.action_is("suite_list"):
         result = _retrieve_suite_list(table)
     elif action.action_is("required_variables"):
@@ -1093,20 +1053,6 @@ def _add_dependencies(parent, scheme_depends, host_depends):
         entry.text = sfile
     # end for
 
-###############################################################################
-def _add_dyn_const_routine(file_entry, routine, scheme):
-###############################################################################
-    """Add a section to <parent> that lists all the constituent routines
-    for the suite
-    >>> file_entry = ET.fromstring("<ccpp_datatable><dyn_const_routines></dyn_const_routines></ccpp_datatable>")
-    >>> _add_dyn_const_routine(file_entry, 'test_dyn_const', 'test_scheme')
-    >>> table_entry_pretty_print(file_entry, 0)
-    '<ccpp_datatable>\\n  <dyn_const_routines />\\n  <dyn_const_routine parent=test_scheme>\\n    test_dyn_const\\n  </dyn_const_routine>\\n</ccpp_datatable>\\n'
-    """
-    entry = ET.SubElement(file_entry, "dyn_const_routine")
-    entry.text = routine
-    entry.set("parent", scheme)
-
 ###############################################################################
 def _add_generated_files(parent, host_files, suite_files, ccpp_kinds, src_dir):
 ###############################################################################
@@ -1233,17 +1179,6 @@ def generate_ccpp_datatable(run_env, host_model, api, scheme_headers,
         # end for
     # end for
     _add_dependencies(datatable, scheme_depends, host_depends)
-    # Add in all constituent routines
-    first_const_routine = True
-    for table in scheme_tdict:
-        if scheme_tdict[table].dyn_const_routine is not None:
-            if first_const_routine:
-                file_entry = ET.SubElement(datatable, "dyn_const_routines")
-                first_const_routine = False
-            # end if
-            _add_dyn_const_routine(file_entry, scheme_tdict[table].dyn_const_routine, table)
-        # end if
-    # end for
     # Write tree
     datatable_tree = PrettyElementTree(datatable)
     datatable_tree.write(run_env.datatable_file)

scripts/ccpp_prebuild.py

@@ -55,7 +55,7 @@ def parse_arguments():
     verbose = args.verbose
     debug = args.debug
     if args.suites:
-        sdfs = ['suite_{0}.xml'.format(x) for x in args.suites.split(',')]
+        sdfs = ['{0}.xml'.format(x) for x in args.suites.split(',')]
     else:
         sdfs = None
     builddir = args.builddir
@@ -181,8 +181,22 @@ def parse_suites(suites_dir, sdfs):
     logging.info('Parsing suite definition files ...')
     suites = []
     for sdf in sdfs:
-        logging.info('Parsing suite definition file {0} ...'.format(os.path.join(suites_dir, sdf)))
-        suite = Suite(sdf_name=os.path.join(suites_dir, sdf))
+        sdf_file=os.path.join(suites_dir, sdf)
+        if not os.path.exists(sdf_file):
+            # If suite file not found, check old filename convention (suite_[suitename].xml)
+            sdf_file_legacy=os.path.join(suites_dir, f"suite_{sdf}")
+            if os.path.exists(sdf_file_legacy):
+                logging.warning("Parsing suite definition file using legacy naming convention")
+                logging.warning(f"Filename {os.path.basename(sdf_file_legacy)}")
+                logging.warning(f"Suite name {sdf}")
+                sdf_file=sdf_file_legacy
+            else:
+                logging.critical(f"Suite definition file {sdf_file} not found.")
+                success = False
+                return (success, suites)
+
+        logging.info(f'Parsing suite definition file {sdf_file} ...')
+        suite = Suite(sdf_name=sdf_file)
         success = suite.parse()
         if not success:
             logging.error('Parsing suite definition file {0} failed.'.format(sdf))

scripts/ccpp_state_machine.py

@@ -3,6 +3,7 @@
 # CCPP framework imports
 from state_machine import StateMachine
 
+_REG_ST = r"(?:register)"
 _INIT_ST = r"(?:init(?:ial(?:ize)?)?)"
 _FINAL_ST = r"(?:final(?:ize)?)"
 _RUN_ST = r"(?:run)"
@@ -12,7 +13,9 @@
 # Allowed CCPP transitions
 # pylint: disable=bad-whitespace
 RUN_PHASE_NAME = 'run'
-CCPP_STATE_MACH = StateMachine((('initialize', 'uninitialized',
+CCPP_STATE_MACH = StateMachine((('register', 'uninitialized',
+                                 'uninitialized', _REG_ST),
+                                ('initialize', 'uninitialized',
                                  'initialized', _INIT_ST),
                                 ('timestep_initial', 'initialized',
                                  'in_time_step', _TS_INIT_ST),

scripts/ccpp_suite.py

@@ -70,6 +70,8 @@ class Suite(VarDictionary):
 '''
 
     # Note that these group names need to match CCPP_STATE_MACH
+    __register_group_name = 'register'
+
     __initial_group_name = 'initialize'
 
     __final_group_name = 'finalize'
@@ -205,6 +207,8 @@ def parse(self, run_env):
         if run_env.logger and run_env.logger.isEnabledFor(logging.INFO):
             run_env.logger.info(lmsg.format(self.name))
         # end if
+        gname = Suite.__register_group_name
+        self.__suite_reg_group = self.new_group_from_name(gname, run_env)
         gname = Suite.__initial_group_name
         self.__suite_init_group = self.new_group_from_name(gname, run_env)
         gname = Suite.__final_group_name
@@ -214,11 +218,13 @@ def parse(self, run_env):
         gname = Suite.__timestep_final_group_name
         self.__timestep_final_group = self.new_group_from_name(gname, run_env)
         # Set up some groupings for later efficiency
-        self._beg_groups = [self.__suite_init_group.name,
+        self._beg_groups = [self.__suite_reg_group.name,
+                            self.__suite_init_group.name,
                             self.__timestep_init_group.name]
         self._end_groups = [self.__suite_final_group.name,
                             self.__timestep_final_group.name]
         # Build hierarchical structure as in SDF
+        self.__groups.append(self.__suite_reg_group)
         self.__groups.append(self.__suite_init_group)
         self.__groups.append(self.__timestep_init_group)
         for suite_item in suite_xml:
@@ -560,8 +566,13 @@ def write(self, output_dir, run_env):
             outfile.end_module_header()
             for group in self.__groups:
                 if group.name in self._beg_groups:
-                    group.write(outfile, self.__host_arg_list_noloop,
-                                1, const_mod, suite_vars=self, allocate=True)
+                    if group.name == self.__suite_reg_group.name:
+                        group.write(outfile, self.__host_arg_list_noloop,
+                                    1, const_mod, suite_vars=self)
+                    else:
+                        group.write(outfile, self.__host_arg_list_noloop,
+                                    1, const_mod, suite_vars=self, allocate=True)
+                    # end if
                 elif group.name in self._end_groups:
                     group.write(outfile, self.__host_arg_list_noloop,
                                 1, const_mod, suite_vars=self, deallocate=True)
@@ -615,7 +626,7 @@ class API(VarDictionary):
                         'kind':'len=*', 'units':'',
                         'dimensions':'()'}, _API_SOURCE, _API_DUMMY_RUN_ENV)
 
-    def __init__(self, sdfs, host_model, scheme_headers, run_env, dyn_const_dict={}):
+    def __init__(self, sdfs, host_model, scheme_headers, run_env):
         """Initialize this API.
         <sdfs> is the list of Suite Definition Files to be parsed for
             data needed by the CCPP cap.
@@ -624,14 +635,11 @@ def __init__(self, sdfs, host_model, scheme_headers, run_env, dyn_const_dict={})
         <scheme_headers> is the list of parsed physics scheme metadata files.
             Every scheme referenced by an SDF in <sdfs> MUST be in this list,
             however, unused schemes are allowed.
-        <dyn_const_dict> is the dictionary (key = scheme name) of dynamic
-            constituent routine names
         <run_env> is the CCPPFrameworkEnv object for this framework run.
         """
         self.__module = 'ccpp_physics_api'
         self.__host = host_model
         self.__suites = list()
-        self.__dyn_const_dict = dyn_const_dict
         super().__init__(self.module, run_env, parent_dict=self.host_model)
         # Create a usable library out of scheme_headers
         # Structure is dictionary of dictionaries
@@ -1187,11 +1195,6 @@ def suites(self):
         "Return the list of this API's suites"
         return self.__suites
 
-    @property
-    def dyn_const_dict(self):
-        """Return the dynamic constituent routine dictionary"""
-        return self.__dyn_const_dict
-
 ###############################################################################
 if __name__ == "__main__":
     try:

scripts/ccpp_track_variables.py

@@ -69,7 +69,7 @@ def create_metadata_filename_dict(metapath):
        with that scheme"""
 
     metadata_dict = {}
-    scheme_filenames = glob.glob(os.path.join(metapath, "*.meta"))
+    scheme_filenames = glob.glob(os.path.join(metapath, "*.meta"), recursive=True)
     if not scheme_filenames:
         raise Exception(f'No files found in {metapath} with ".meta" extension')
 

scripts/common.py

@@ -73,7 +73,7 @@
 CCPP_STATIC_SUBROUTINE_NAME = 'ccpp_physics_{stage}'
 
 # Filename pattern for suite definition files
-SUITE_DEFINITION_FILENAME_PATTERN = re.compile('^suite_(.*)\.xml$')
+SUITE_DEFINITION_FILENAME_PATTERN = re.compile('^(.*)\.xml$')
 
 # Maximum number of concurrent CCPP instances per MPI task
 CCPP_NUM_INSTANCES = 200