Merge branch 'development' into develop/implement-dyn-run

Author: kuanchihwang

.gitmodules

@@ -1,9 +1,15 @@
 [submodule "ccpp-framework"]
         path          = ccpp_framework
         url           = https://github.com/NCAR/ccpp-framework
-        fxtag         = 2024-07-19-dev
+        fxtag         = 2024-10-31-dev
         fxrequired    = AlwaysRequired
-        fxDONOTUSEurl = https://github.com/NCAR/ccpp-framework
+	fxDONOTUSEurl = https://github.com/NCAR/ccpp-framework
+[submodule "history"]
+	path = src/history/buffers
+	url = https://github.com/ESMCI/history_output
+	fxtag         = history01_00
+	fxrequired    = AlwaysRequired
+	fxDONOTUSEurl = https://github.com/ESMCI/history_output
 [submodule "mpas"]
         path          = src/dynamics/mpas/dycore
         url           = https://github.com/MPAS-Dev/MPAS-Model.git
@@ -13,8 +19,8 @@
         fxDONOTUSEurl = https://github.com/MPAS-Dev/MPAS-Model.git
 [submodule "ncar-physics"]
         path          = src/physics/ncar_ccpp
-        url           = https://github.com/ESCOMP/atmospheric_physics
-        fxtag         = 098585940ad763be58ebab849bb8eaf325fda42a
+	url           = https://github.com/ESCOMP/atmospheric_physics
+	fxtag         = atmos_phys0_06_000
         fxrequired    = AlwaysRequired
         fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics
 [submodule "ccs_config"]

ccpp_framework

@@ -1447,8 +1447,8 @@ def append_user_nl_file(self, user_nl_file):
                             #Notify loop to check the next line for a comma:
                             is_continue_line = False
                         #End if
-
-                    else:
+                    elif ('hist_' not in line_s[0]):
+                        #Raise parsing error; ignore hist lines to be processed by hist_config.py
                         emsg = "Cannot parse the following line in '{}' :\n'{}'"
                         raise AtmInParamGenError(emsg.format(user_nl_file, line))
                     #End if ("=" sign check)

cime_config/atm_in_paramgen.py

@@ -97,6 +97,10 @@ def _build_cam():
                               case.get_value("COMP_INTERFACE")),
                  os.path.join(atm_root, "src", "dynamics", "utils"),
                  os.path.join(atm_root, "src", "physics", "utils"),
+                 os.path.join(atm_root, "src", "history"),
+                 os.path.join(atm_root, "src", "history", "buffers", "src"),
+                 os.path.join(atm_root, "src", "history", "buffers", "src", "hash"),
+                 os.path.join(atm_root, "src", "history", "buffers", "src", "util"),
                  os.path.join(atm_root, "src", "utils")]
         for path in phys_dirs:
             if path not in paths:

cime_config/buildlib

@@ -5,6 +5,7 @@ CAM namelist creator
 """
 import sys
 import os
+import re
 import shutil
 import logging
 import glob
@@ -31,6 +32,8 @@ sys.path.append(_CIME_CONFIG_PATH)
 
 # Import CAM's configure structure:
 from cam_config import ConfigCAM
+# HistoryConfig allows translation from user_nl_cam into Fortran namelists
+from hist_config import HistoryConfig
 
 #Import CAM's ParamGen class:
 from atm_in_paramgen import AtmInParamGen
@@ -51,6 +54,7 @@ class CamBuildnmlError(ValueError):
 # This simplifies the filename mangling for different cases.
 def _create_ic_filename(inst_string, i_or_r,
                         run_refcase, run_refdate, run_reftod):
+    """Simplify the filename mangling for different cases."""
     return f"{run_refcase}.cam{inst_string}.{i_or_r}.{run_refdate}-{run_reftod}.nc"
 
 ##################
@@ -297,7 +301,8 @@ def buildnml(case, caseroot, compname):
             # End if
 
         # Determine location and name of "user_nl_cam" files:
-        user_nl_file = os.path.join(caseroot, "user_nl_cam" + inst_string)
+        user_nl_fname = "user_nl_cam" + inst_string
+        user_nl_file = os.path.join(caseroot, user_nl_fname)
 
         # Check that file actually exists.  If not then throw an error:
         if not os.path.exists(user_nl_file):
@@ -362,6 +367,16 @@ def buildnml(case, caseroot, compname):
         # Create CAM namelist using CIME's nmlgen routine:
         pg_atm.write(namelist_file)
 
+        # Add history namelists to atm_in
+        hist_configs = HistoryConfig(filename=user_nl_file, logger=_LOGGER)
+        with open(namelist_file, 'a', encoding='utf-8') as nl_file:
+            hist_configs.output_class_namelist(nl_file)
+            for key in sorted(hist_configs.keys()):
+                hist_configs[key].output_config_namelist(nl_file, logger=_LOGGER)
+            # end for
+        # end with
+
+
 ###############################################################################
 def _main_func():
 

cime_config/buildnml

@@ -435,22 +435,31 @@ def generate_physics_suites(build_cache, preproc_defs, host_name,
     if not os.path.exists(physics_blddir):
         os.makedirs(physics_blddir)
     # End if
-    # Collect all source directories
-    atm_phys_src_dir = os.path.join(atm_root, "src", "physics", "ncar_ccpp")
-    source_search = [source_mods_dir, atm_phys_src_dir]
-    # Find all metadata files, organize by scheme name
+    # Set top-level CCPP physics directory
+    atm_phys_top_dir = os.path.join(atm_root, "src", "physics", "ncar_ccpp")
+    # Collect all possible Suite Definition File (SDF) locations
+    atm_suites_path      = os.path.join(atm_phys_top_dir, "suites")
+    atm_test_suites_path = os.path.join(atm_phys_top_dir, "test", "test_suites")
+    suite_search         = [source_mods_dir, atm_suites_path, atm_test_suites_path]
+    # Find all scheme metadata files, organized by scheme name
+    atm_schemes_path = os.path.join(atm_phys_top_dir, "schemes")
+    source_search    = [source_mods_dir, atm_schemes_path]
     all_scheme_files = _find_metadata_files(source_search, find_scheme_names)
 
     # Find the SDFs specified for this model build
     sdfs = []
     scheme_files = []
     xml_files = {} # key is scheme, value is xml file path
     for sdf in phys_suites_str.split(';'):
-        sdf_path = _find_file(f"suite_{sdf}.xml", source_search)
+        sdf_path = _find_file(f"suite_{sdf}.xml", suite_search)
         if not sdf_path:
             emsg = f"ERROR: Unable to find SDF for suite '{sdf}'"
             raise CamAutoGenError(emsg)
         # End if
+        if os.path.dirname(os.path.abspath(sdf_path)) == atm_test_suites_path:
+          #Notify user that a test suite is being used
+          _LOGGER.info("Using non-standard test suite: %s", sdf)
+        # End if
         sdfs.append(sdf_path)
         # Given an SDF, find all the schemes it calls
         _, suite = read_xml_file(sdf_path)
@@ -587,13 +596,13 @@ def generate_physics_suites(build_cache, preproc_defs, host_name,
     # there to the bld directory:
     if do_gen_ccpp:
         # Set CCPP physics "utilities" path
-        atm_phys_util_dir = os.path.join(atm_phys_src_dir, "utilities")
+        atm_phys_util_dir = os.path.join(atm_schemes_path, "utilities")
 
         # Check that directory exists
         if not os.path.isdir(atm_phys_util_dir):
-            #CAM-SIMA will likely not run without this, so raise an error
+            # CAM-SIMA will likely not run without this, so raise an error
             emsg = "ERROR: Unable to find CCPP physics utilities directory:\n"
-            emsg += f" {atm_phys_util_dir}\n Have you run 'checkout_externals'?"
+            emsg += f" {atm_phys_util_dir}\n Have you run 'git-fleximod'?"
             raise CamAutoGenError(emsg)
         # end if
 
@@ -602,6 +611,25 @@ def generate_physics_suites(build_cache, preproc_defs, host_name,
         for util_file in atm_phys_util_files:
             shutil.copy(util_file, physics_blddir)
         # end for
+
+        # Copy to_be_ccppized utility modules to the build directory,
+        # as SIMA cam_constituents depends on them.
+        # Note: to_be_ccppized utility modules to be removed once functionality is migrated
+        # to SIMA or CCPPized in atmospheric_physics.
+        atm_phys_to_be_ccppized_dir = os.path.join(atm_phys_top_dir, "to_be_ccppized")
+
+        # Check that the directory exists
+        if not os.path.isdir(atm_phys_to_be_ccppized_dir):
+            # CAM-SIMA will likely not run without this, so raise an error
+            emsg = "ERROR: Unable to find CCPP physics to_be_ccppized directory:\n"
+            emsg += f" {atm_phys_to_be_ccppized_dir}\n Have you run 'git-fleximod'?"
+            raise CamAutoGenError(emsg)
+        # end if
+
+        atm_phys_to_be_ccppized_files = glob.glob(os.path.join(atm_phys_to_be_ccppized_dir, "*.F90"))
+        for to_be_ccppized_file in atm_phys_to_be_ccppized_files:
+           shutil.copy(to_be_ccppized_file, physics_blddir)
+        # end for
     # end if
 
     if do_gen_ccpp or do_gen_nl:

cime_config/cam_autogen.py

@@ -159,12 +159,10 @@
       <value compset="_CAM\d0%WX.*%SDYN">-nlev 145</value> -->
 
       <!-- Simple models -->
-<!--      <value compset="_CAM%HS94" grid="(%ne[0-9])|(%[0-9.]+x)">-analytic_ic</value>
-      <value compset="_CAM%ADIAB">-phys adiabatic</value>
-      <value compset="_CAM%DABIP04">-phys adiabatic</value>
-      <value compset="_CAM%TJ16">-phys tj2016 -analytic_ic</value>
-      <value compset="_CAM%HS94">-phys held_suarez</value>
-      <value compset="_CAM%KESSLER">-phys kessler -chem terminator -analytic_ic</value> -->
+<!--      <value compset="_CAM%ADIAB">-phys adiabatic</value>
+      <value compset="_CAM%DABIP04">-phys adiabatic</value> -->
+      <value compset="_CAM%TJ16">--physics-suites tj2016 --analytic_ic</value>
+<!--      <value compset="_CAM%KESSLER">-phys kessler -chem terminator -analytic_ic</value> -->
       <value compset="_CAM%KESSLER">--physics-suites kessler --analytic_ic</value>
       <value compset="_CAM%HS94">--physics-suites held_suarez_1994 --analytic_ic</value>
       <value compset="_CAM%PHYSTEST">--dyn none --physics-suites adiabatic</value>