Merge pull request #2971 from ekluzek/merge-b4bdev-20250220

Merge b4bdev 20250220

.git-blame-ignore-revs

@@ -58,3 +58,4 @@ bd535c710db78420b8e8b9d71d88d8339e899c59
 4b20bbd7003e6f77dab4e3268cc4a43f9b5a3b5d
 cf433215b58ba8776ec5edfb0b0d80c0836ed3a0
 16d57ff37859b34dab005693e3085d64e2bcd95a
+e8fc526e0d7818d45f171488c78392c4ff63902a

doc/ChangeLog

@@ -1,4 +1,88 @@
 ===============================================================
+Tag name: ctsm5.3.026
+Originator(s): erik (Erik Kluzek,UCAR/TSS,303-497-1326)
+Date: Wed 26 Feb 2025 09:55:48 AM MST
+One-line Summary: Merge b4b-dev to master: run_tower updates, reduce log noise
+
+Purpose and description of changes
+----------------------------------
+
+Merge b4b-dev to master.
+
+Some run_tower updatesm, fix some issues, and improve code health. Also reduces some log noise.
+
+
+Significant changes to scientifically-supported configurations
+--------------------------------------------------------------
+
+Does this tag change answers significantly for any of the following physics configurations?
+(Details of any changes will be given in the "Answer changes" section below.)
+
+    [Put an [X] in the box for any configuration with significant answer changes.]
+
+[ ] clm6_0
+
+[ ] clm5_0
+
+[ ] ctsm5_0-nwp
+
+[ ] clm4_5
+
+
+Bugs fixed
+----------
+List of CTSM issues fixed (include CTSM Issue # and description) [one per line]:
+  Resolves #2884 -- run_neon overwrite ignored
+  Resolves #2885 -- run_neon setup_only ignored
+  Resolves #2946 -- python tests much longer
+  Resolves #2717 -- cleanup cesm log
+  Resolves #2737 -- reduce log noise
+
+Notes of particular relevance for users
+---------------------------------------
+Changes to CTSM's user interface (e.g., new/renamed XML or namelist variables):
+  Two new options to run_tower:
+  --no-inputdata-check option implies --setup-only but skips the check/download of input data. This is used instead of --setup-only
+  in run_tower system testing for a speedup of ~20%.
+  --xmlchange option allows user to specify xmlchange settings to apply. E.g., --xmlchange CLM_CO2_TYPE=constant,CCSM_CO2_PPMV=850.
+
+Notes of particular relevance for developers:
+---------------------------------------------
+
+Caveats for developers (e.g., code that is duplicated that requires double maintenance):
+
+Testing summary: Regular
+----------------
+ [PASS means all tests PASS; OK means tests PASS other than expected fails.]
+
+  python testing (if python code has changed; see instructions in python/README.md; document testing done):
+
+    derecho - PASS
+
+  regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing):
+
+    derecho ----- OK
+    izumi ------- OK
+
+If the tag used for baseline comparisons was NOT the previous tag, note that here:
+
+
+Answer changes
+--------------
+
+Changes answers relative to baseline: No bit-for-bit
+
+Other details
+-------------
+
+Pull Requests that document the changes (include PR ids):
+(https://github.com/ESCOMP/ctsm/pull)
+  #2969 -- Simplify tower types
+  #2962 -- Run tower fix and improvements
+  #2927 -- Remove log noise
+
+===============================================================
+===============================================================
 Tag name: ctsm5.3.025
 Originator(s): glemieux (Gregory Lemieux, LBNL, glemieux@lbl.gov)
 Date: Thu Feb 20 14:24:45 MST 2025

doc/ChangeSum

@@ -1,5 +1,6 @@
 Tag                   Who      Date  Summary
 ============================================================================================================================
+       ctsm5.3.026 samrabin 02/26/2025 Merge b4b-dev to master: run_tower updates, reduce log noise
        ctsm5.3.025 glemieux 02/20/2025 FATES default parameter file update
        ctsm5.3.024 xinchang 02/11/2025 Change choice of pressure in CLMU building energy model
        ctsm5.3.023  afoster 02/08/2025 merge b4b-dev

python/ctsm/site_and_regional/neon_site.py

@@ -34,83 +34,8 @@ class NeonSite(TowerSite):
     A class for encapsulating neon sites.
     """
 
-    def build_base_case(
-        self,
-        cesmroot,
-        output_root,
-        res,
-        compset,
-        user_mods_dirs=None,
-        overwrite=False,
-        setup_only=False,
-    ):
-        if user_mods_dirs is None:
-            user_mods_dirs = [
-                os.path.join(
-                    self.cesmroot, "cime_config", "usermods_dirs", "clm", "NEON", self.name
-                )
-            ]
-        case_path = super().build_base_case(cesmroot, output_root, res, compset, user_mods_dirs)
-
-        return case_path
-
-    # pylint: disable=too-many-statements
-    def run_case(
-        self,
-        base_case_root,
-        run_type,
-        prism,
-        user_version,
-        tower_type=None,
-        user_mods_dirs=None,
-        overwrite=False,
-        setup_only=False,
-        no_batch=False,
-        rerun=False,
-        experiment=False,
-    ):
-        """
-        Run case.
-
-        Args:
-        self
-        base_case_root: str, opt
-            file path of base case
-        run_type: str, opt
-            transient, post_ad, or ad case, default transient
-        prism: bool, opt
-            if True, use PRISM precipitation, default False
-        user_version: str, opt
-            default 'latest'
-        overwrite: bool, opt
-            default False
-        setup_only: bool, opt
-            default False; if True, set up but do not run case
-        no_batch: bool, opt
-            default False
-        rerun: bool, opt
-            default False
-        experiment: str, opt
-            name of experiment, default False
-        """
-        user_mods_dirs = [
-            os.path.join(self.cesmroot, "cime_config", "usermods_dirs", "clm", "NEON", self.name)
-        ]
-        tower_type = "NEON"
-
-        super().run_case(
-            base_case_root,
-            run_type,
-            prism,
-            user_version,
-            tower_type,
-            user_mods_dirs,
-            overwrite,
-            setup_only,
-            no_batch,
-            rerun,
-            experiment,
-        )
+    def __init__(self, *args, **kwargs):
+        super().__init__("NEON", *args, **kwargs)
 
     def modify_user_nl(self, case_root, run_type, rundir, site_lines=None):
         # TODO: include neon-specific user namelist lines, using this as just an example currently

python/ctsm/site_and_regional/plumber_site.py

@@ -33,87 +33,8 @@ class Plumber2Site(TowerSite):
     A class for encapsulating plumber sites.
     """
 
-    def build_base_case(
-        self,
-        cesmroot,
-        output_root,
-        res,
-        compset,
-        user_mods_dirs=None,
-        overwrite=False,
-        setup_only=False,
-    ):
-        if user_mods_dirs is None:
-            user_mods_dirs = [
-                os.path.join(
-                    self.cesmroot, "cime_config", "usermods_dirs", "clm", "PLUMBER2", self.name
-                )
-            ]
-        case_path = super().build_base_case(cesmroot, output_root, res, compset, user_mods_dirs)
-
-        return case_path
-
-    # pylint: disable=too-many-statements
-    def run_case(
-        self,
-        base_case_root,
-        run_type,
-        prism,
-        user_version,
-        tower_type=None,
-        user_mods_dirs=None,
-        overwrite=False,
-        setup_only=False,
-        no_batch=False,
-        rerun=False,
-        experiment=False,
-    ):
-        """
-        Run case.
-
-        Args:
-        self
-        base_case_root: str, opt
-            file path of base case
-        run_type: str, opt
-            transient, post_ad, or ad case, default ad
-            (ad case is default because PLUMBER requires spinup)
-        prism: bool, opt
-            if True, use PRISM precipitation, default False
-            Note: only supported for NEON sites
-        user_version: str, opt
-            default 'latest'; this could be useful later
-            This is currently only implemented with neon (not plumber) sites
-        overwrite: bool, opt
-            default False
-        setup_only: bool, opt
-            default False; if True, set up but do not run case
-        no_batch: bool, opt
-            default False
-        rerun: bool, opt
-            default False
-        experiment: str, opt
-            name of experiment, default False
-        """
-        user_mods_dirs = [
-            os.path.join(
-                self.cesmroot, "cime_config", "usermods_dirs", "clm", "PLUMBER2", self.name
-            )
-        ]
-        tower_type = "PLUMBER"
-        super().run_case(
-            base_case_root,
-            run_type,
-            prism,
-            user_version,
-            tower_type,
-            user_mods_dirs,
-            overwrite,
-            setup_only,
-            no_batch,
-            rerun,
-            experiment,
-        )
+    def __init__(self, *args, **kwargs):
+        super().__init__("PLUMBER2", *args, **kwargs)
 
     def set_ref_case(self, case):
         super().set_ref_case(case)

python/ctsm/site_and_regional/run_tower.py

@@ -233,7 +233,9 @@ def main(description):
         setup_only,
         no_batch,
         rerun,
+        no_input_data_check,
         user_version,
+        xmlchange,
     ) = get_parser(sys.argv, description, valid_neon_sites, valid_plumber_sites)
 
     if output_root:
@@ -261,9 +263,9 @@ def main(description):
                 if run_from_postad:
                     neon_site.finidat = None
                 if not base_case_root:
-                    user_mods_dirs = None
+                    neon_site.set_default_user_mods_dirs()
                     base_case_root = neon_site.build_base_case(
-                        cesmroot, output_root, res, compset, user_mods_dirs, overwrite, setup_only
+                        cesmroot, output_root, res, compset, overwrite, setup_only
                     )
                 logger.info("-----------------------------------")
                 logger.info("Running CTSM for neon site : %s", neon_site.name)
@@ -278,6 +280,8 @@ def main(description):
                     no_batch=no_batch,
                     rerun=rerun,
                     experiment=experiment,
+                    no_input_data_check=no_input_data_check,
+                    xmlchange=xmlchange,
                 )
 
     # -- check for available plumber data:
@@ -290,9 +294,9 @@ def main(description):
                 if run_from_postad:
                     plumber_site.finidat = None
                 if not base_case_root:
-                    user_mods_dirs = None
+                    plumber_site.set_default_user_mods_dirs()
                     base_case_root = plumber_site.build_base_case(
-                        cesmroot, output_root, res, compset, user_mods_dirs, overwrite, setup_only
+                        cesmroot, output_root, res, compset, overwrite, setup_only
                     )
                 logger.info("-----------------------------------")
                 logger.info("Running CTSM for plumber site : %s", plumber_site.name)
@@ -306,4 +310,6 @@ def main(description):
                     no_batch=no_batch,
                     rerun=rerun,
                     experiment=experiment,
+                    no_input_data_check=no_input_data_check,
+                    xmlchange=xmlchange,
                 )

python/ctsm/site_and_regional/tower_arg_parse.py

@@ -104,6 +104,19 @@ def get_parser(args, description, valid_neon_sites, valid_plumber_sites):
         default=False,
     )
 
+    parser.add_argument(
+        "--no-input-data-check",
+        "--no-check-input-data",
+        help="""
+                Don't check for input data. Implies --setup-only.
+                [default: %(default)s]
+                """,
+        action="store_true",
+        dest="no_input_data_check",
+        required=False,
+        default=False,
+    )
+
     parser.add_argument(
         "--rerun",
         help="""
@@ -185,6 +198,17 @@ def get_parser(args, description, valid_neon_sites, valid_plumber_sites):
         choices=["v1", "v2", "v3"],
     )
 
+    parser.add_argument(
+        "--xmlchange",
+        help="""
+                Any xmlchanges (e.g., CLM_CO2_TYPE=constant,CCSM_CO2_PPMV=500)
+                [default: %(default)s]
+                """,
+        required=False,
+        type=str,
+        default=None,
+    )
+
     args = parse_args_and_handle_standard_logging_options(args, parser)
 
     if args.neon_sites:
@@ -230,6 +254,10 @@ def get_parser(args, description, valid_neon_sites, valid_plumber_sites):
         root_logger = logging.getLogger()
         root_logger.setLevel(logging.WARN)
 
+    # --no-input-data-check implies --setup-only
+    if args.no_input_data_check and not args.setup_only:
+        args.setup_only = True
+
     return (
         neon_sites,
         plumber_sites,
@@ -243,5 +271,7 @@ def get_parser(args, description, valid_neon_sites, valid_plumber_sites):
         args.setup_only,
         args.no_batch,
         args.rerun,
+        args.no_input_data_check,
         args.user_version,
+        args.xmlchange,
     )