Run MUSICA in CAM-SIMA

.gitmodules

@@ -20,7 +20,7 @@
 [submodule "ncar-physics"]
         path          = src/physics/ncar_ccpp
 	url           = https://github.com/ESCOMP/atmospheric_physics
-	fxtag         = 36f336e87
+	fxtag         = ac83803062a5d38b92e416e1cf737d3811e20cd1
         fxrequired    = AlwaysRequired
         fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics
 [submodule "ccs_config"]

cime_config/atm_musica_config.py

@@ -4,7 +4,8 @@
 """
 
 MUSICA_CCPP_SCHEME_NAME = "musica_ccpp"
+MUSICA_CONFIG_DIR_NAME = "musica_configurations"
 MUSICA_REPO_URL = "https://github.com/NCAR/musica.git"
 MUSICA_TAG = "cc39bb00d2220fc81c85b22d3ceea4a39bd2bacf"
 CHEMISTRY_DATA_REPO_URL = "https://github.com/NCAR/cam-sima-chemistry-data.git"
-CHEMISTRY_DATA_TAG = "2b58f2410ec7a565bcf80dee16ec20f6bc35d78b"
+CHEMISTRY_DATA_TAG = "71ed143c54b0d5d6e3e70f3d05d413fddcf8d59e"

cime_config/buildlib

@@ -12,7 +12,7 @@ import subprocess
 import logging
 
 from cam_config import ConfigCAM # CAM's configure structure
-from atm_musica_config import MUSICA_CCPP_SCHEME_NAME
+from atm_musica_config import MUSICA_CCPP_SCHEME_NAME, MUSICA_CONFIG_DIR_NAME
 from atm_musica_config import MUSICA_REPO_URL, MUSICA_TAG
 from atm_musica_config import CHEMISTRY_DATA_REPO_URL, CHEMISTRY_DATA_TAG
 
@@ -176,21 +176,22 @@ def _build_cam():
         # If MUSICA-CCPP scheme is used in a suite, download
         # the MUSICA configuration and build the MUSICA library
         if MUSICA_CCPP_SCHEME_NAME in scheme_names:
-            _download_musica_configuration(caseroot)
+            _ = _download_musica_configuration(caseroot)
             musica_install_path = _build_musica(caseroot)
 
             cam_linked_libs = case.get_value("CAM_LINKED_LIBS")
             musica_libs = "-lmusica-fortran -lmusica -lyaml-cpp"
             if not musica_libs in cam_linked_libs:
                 _set_musica_lib_path(musica_install_path, caseroot)
 
-            cmd += ' USER_INCLDIR="'\
-                    f'-I{os.path.join(musica_install_path, "include", "micm")} '\
-                    f'-I{os.path.join(musica_install_path, "include", "musica")} '\
-                    f'-I{os.path.join(musica_install_path, "include", "musica", "micm")} '\
-                    f'-I{os.path.join(musica_install_path, "include", "musica", "tuvx")} '\
-                    f'-I{os.path.join(musica_install_path, "include", "musica", "fortran")} '\
+            cmd += (' USER_INCLDIR="'
+                    f'-I{os.path.join(musica_install_path, "include", "micm")} '
+                    f'-I{os.path.join(musica_install_path, "include", "musica")} '
+                    f'-I{os.path.join(musica_install_path, "include", "musica", "micm")} '
+                    f'-I{os.path.join(musica_install_path, "include", "musica", "tuvx")} '
+                    f'-I{os.path.join(musica_install_path, "include", "musica", "fortran")} '
                     '"'
+                    )
 
         retcode, out, err = run_cmd(cmd)
         _LOGGER.info("Command %s:\n\nstdout:\n%s\n\nstderr:\n%s\n", cmd, out, err)
@@ -284,8 +285,8 @@ def _build_musica(clone_dest: str) -> str:
         subprocess.run(command, cwd=bld_path, stdout=subprocess.PIPE,
                        stderr=subprocess.PIPE, text=True, check=False)
     except subprocess.CalledProcessError as e:
-        raise subprocess.CalledProcessError(e.returncode, e.cmd, "The subprocess \
-            for cmake to configure the MUSICA CMake project failed.") from e
+        raise subprocess.CalledProcessError(e.returncode, e.cmd,
+            "The subprocess for cmake to configure the MUSICA CMake project failed.") from e
     except FileNotFoundError as e:
         raise FileNotFoundError("The 'cmake' command was not found.") from e
     except OSError as e:
@@ -294,10 +295,10 @@ def _build_musica(clone_dest: str) -> str:
     command = ["cmake", "--build", ".", "--target", "install"]
     try:
         subprocess.run(command, cwd=bld_path, stdout=subprocess.PIPE,
-                       stderr=subprocess.PIPE, text=True, check=False)
+                       text=True, check=False)
     except subprocess.CalledProcessError as e:
-        raise subprocess.CalledProcessError(e.returncode, e.cmd, "The subprocess \
-            for cmake to build the MUSICA library failed.") from e
+        raise subprocess.CalledProcessError(e.returncode, e.cmd,
+            "The subprocess for cmake to build the MUSICA library failed.") from e
     except FileNotFoundError as e:
         raise FileNotFoundError("The 'cmake' command was not found.") from e
     except OSError as e:
@@ -308,7 +309,7 @@ def _build_musica(clone_dest: str) -> str:
     return musica_install_path
 
 ###############################################################################
-def _download_musica_configuration(download_dest: str) -> None:
+def _download_musica_configuration(download_dest: str) -> str:
 ###############################################################################
     """
     Downloads the MUSICA configuration and renames the configuration
@@ -321,13 +322,14 @@ def _download_musica_configuration(download_dest: str) -> None:
         Exception: If the directory to be renamed is not found or
             any other exceptions occur during the renaming process,
             an exception is raised with the error message.
+    Returns:
+        musica_config_path: path to the MUSICA configuration directory
     """
-    musica_config_dir_name = "musica_configurations"
 
     _clone_and_checkout(CHEMISTRY_DATA_REPO_URL, CHEMISTRY_DATA_TAG, download_dest)
 
     original_dir = os.path.join(download_dest, "cam-sima-chemistry-data", "mechanisms")
-    renamed_dir = os.path.join(download_dest, "cam-sima-chemistry-data", musica_config_dir_name)
+    renamed_dir = os.path.join(download_dest, "cam-sima-chemistry-data", MUSICA_CONFIG_DIR_NAME)
     try:
         os.rename(original_dir, renamed_dir)
     except FileNotFoundError as e:
@@ -339,12 +341,14 @@ def _download_musica_configuration(download_dest: str) -> None:
     except OSError as e:
         raise OSError("An error occurred while renaming.") from e
 
-    musica_config_path = os.path.join(download_dest, musica_config_dir_name)
+    musica_config_path = os.path.join(download_dest, MUSICA_CONFIG_DIR_NAME)
     if os.path.exists(musica_config_path):
         shutil.rmtree(musica_config_path)
 
     shutil.move(renamed_dir, download_dest)
 
+    return musica_config_path
+
 ###############################################################################
 def _set_musica_lib_path(musica_install_path: str, caseroot: str) -> None:
 ###############################################################################
@@ -374,8 +378,8 @@ def _set_musica_lib_path(musica_install_path: str, caseroot: str) -> None:
         subprocess.run(command, cwd=caseroot, stdout=subprocess.PIPE,
                        stderr=subprocess.PIPE, text=True, check=False)
     except subprocess.CalledProcessError as e:
-        raise subprocess.CalledProcessError(e.returncode, e.cmd, "The subprocess \
-            for xmlchange to set the MUSICA library path failed.") from e
+        raise subprocess.CalledProcessError(e.returncode, e.cmd,
+            "The subprocess for xmlchange to set the MUSICA library path failed.") from e
     except FileNotFoundError as e:
         raise FileNotFoundError("The 'xmlchange' command was not found.") from e
     except OSError as e:
@@ -408,8 +412,8 @@ def _clone_and_checkout(repo_url: str, tag_name: str, clone_dest: str) -> None:
         subprocess.run(["git", "clone", repo_url, repo_path],
            stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False)
     except subprocess.CalledProcessError as e:
-        raise subprocess.CalledProcessError(e.returncode, e.cmd, f"The subprocess \
-            for git to clone the repository {repo_url} failed.") from e
+        raise subprocess.CalledProcessError(e.returncode, e.cmd,
+            f"The subprocess for git to clone the repository {repo_url} failed.") from e
     except FileNotFoundError as e:
         raise FileNotFoundError("The 'git' command was not found.") from e
     except OSError as e:
@@ -419,8 +423,8 @@ def _clone_and_checkout(repo_url: str, tag_name: str, clone_dest: str) -> None:
         subprocess.run(["git", "-C", repo_path, "checkout", tag_name],
            stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False)
     except subprocess.CalledProcessError as e:
-        raise subprocess.CalledProcessError(e.returncode, e.cmd, f"The subprocess \
-            for git to checkout the branch {tag_name} failed.") from e
+        raise subprocess.CalledProcessError(e.returncode, e.cmd,
+            f"The subprocess for git to checkout the branch {tag_name} failed.") from e
     except FileNotFoundError as e:
         raise FileNotFoundError("The 'git' command was not found.") from e
     except OSError as e:

cime_config/buildnml

@@ -22,7 +22,7 @@ from CIME.Tools.standard_script_setup import *
 from CIME.XML.standard_module_setup   import *
 from CIME.buildnml                    import create_namelist_infile, parse_input
 from CIME.case                        import Case
-from CIME.utils                       import expect
+from CIME.utils                       import expect, symlink_force
 
 # Save local (cime_config) directory path:
 _CIME_CONFIG_PATH = os.path.dirname(os.path.abspath(__file__))
@@ -35,8 +35,10 @@ from cam_config import ConfigCAM
 # HistoryConfig allows translation from user_nl_cam into Fortran namelists
 from hist_config import HistoryConfig
 
-#Import CAM's ParamGen class:
+# Import CAM's ParamGen class:
 from atm_in_paramgen import AtmInParamGen
+# Import specific names associated with the MUSICA scheme
+from atm_musica_config import MUSICA_CCPP_SCHEME_NAME, MUSICA_CONFIG_DIR_NAME
 
 # Open CIME case log:
 _LOGGER = logging.getLogger(__name__)
@@ -166,7 +168,7 @@ def buildnml(case, caseroot, compname):
     gen_indent = 3
 
     #Generate model code and meta-data:
-    _ = config.generate_cam_src(gen_indent)
+    scheme_names = config.generate_cam_src(gen_indent)
 
     #----------------------------------------------------------------
     # Create namelist attribute dictionary (to set namelist defaults):
@@ -376,6 +378,21 @@ def buildnml(case, caseroot, compname):
             # end for
         # end with
 
+        # If MUSICA-CCPP scheme is used in a suite, create symlinks in the run
+        # directory pointing to the MUSICA configuration data located in the caseroot
+        if MUSICA_CCPP_SCHEME_NAME in scheme_names:
+            musica_config_src_dir = os.path.join(caseroot, MUSICA_CONFIG_DIR_NAME)
+            musica_config_dest_dir = os.path.join(rundir, MUSICA_CONFIG_DIR_NAME)
+
+            if os.path.exists(musica_config_src_dir):
+                os.makedirs(musica_config_dest_dir, exist_ok=True)
+                for root, _, files in os.walk(musica_config_src_dir):
+                    rel_path = os.path.relpath(root, musica_config_src_dir)
+                    dest_dir = os.path.join(musica_config_dest_dir, rel_path)
+
+                    os.makedirs(dest_dir, exist_ok=True)
+                    for file in files:
+                        symlink_force(os.path.join(root, file), os.path.join(dest_dir, file))
 
 ###############################################################################
 def _main_func():

docker/Dockerfile.musica

@@ -77,11 +77,29 @@ RUN /home/cam_sima_user/CAM-SIMA/cime/scripts/create_newcase --case $CASE_NAME \
 
 WORKDIR $CASE_NAME
 
-RUN ./case.setup
-
+RUN ./xmlchange COMPILER=gnu
+RUN ./xmlchange DEBUG=true
 RUN ./xmlchange CAM_CONFIG_OPTS="--dyn none --physics-suites musica"
-RUN ./xmlchange ROF_NCPL=48
 RUN ./xmlchange STOP_OPTION=nsteps
 RUN ./xmlchange STOP_N=5
 
+# Match the GLC timestep to atmosphere timestep
+RUN ./xmlchange ATM_NCPL=48
+RUN ./xmlchange GLC_NCPL=48
+
+# Avoid writing restart files
+RUN ./xmlchange REST_N=100
+
+RUN ./case.setup
+
+# Specify the path to the MUSICA configuraiton files by adding the following lines to the `user_nl_cam` file.
+# For example, to configure the Chapman mechanisms, include these lines. 
+# filename_of_micm_configuration=/home/cam_sima_user/case_name/test-case/musica_configurations/chapman/micm/config.json
+# filename_of_tuvx_configuration=/home/cam_sima_user/case_name/test-case/musica_configurations/chapman/tuvx/config.json
+# filename_of_tuvx_micm_mapping_configuration=/home/cam_sima_user/case_name/test-case/musica_configurations/chapman/tuvx_micm_mapping.json
+
+# You can run the model with the snapshot file by adding this line to the end of 'user_nl_cam' file on Derecho.
+# Note that this snapshot contains only ten time slices.
+# ncdata=/glade/campaign/cesm/community/amwg/sima_baselines/cam_sima_test_snapshots/cam_ne3pg3_kessler_snapshot_derecho_gnu_before_c20240412.nc
+
 RUN ./case.build

src/control/cam_comp.F90

@@ -84,27 +84,31 @@ subroutine cam_init(caseid, ctitle, model_doi_url,                         &
       !
       !-----------------------------------------------------------------------
 
-      use cam_initfiles,            only: cam_initfiles_open
-      use dyn_grid,                 only: model_grid_init
-      use phys_comp,                only: phys_init
-      use phys_comp,                only: phys_register
-      use dyn_comp,                 only: dyn_init
-!      use cam_restart,              only: cam_read_restart
-      use camsrfexch,               only: hub2atm_alloc, atm2hub_alloc
-      use cam_history,              only: history_init_files
-!      use history_scam,             only: scm_intht
-      use cam_pio_utils,            only: init_pio_subsystem
-      use cam_instance,             only: inst_suffix
-!      use history_defaults,         only: initialize_iop_history
-      use stepon,                   only: stepon_init
-      use air_composition,          only: air_composition_init
-      use cam_ccpp_cap,             only: cam_ccpp_initialize_constituents
-      use physics_grid,             only: columns_on_task
-      use vert_coord,               only: pver
-      use phys_vars_init_check,     only: mark_as_initialized
-      use tropopause_climo_read,    only: tropopause_climo_read_file
-      use musica_ccpp_dependencies, only: musica_ccpp_dependencies_init
-      use orbital_data,             only: orbital_data_init
+      use cam_initfiles,             only: cam_initfiles_open
+      use dyn_grid,                  only: model_grid_init
+      use phys_comp,                 only: phys_init
+      use phys_comp,                 only: phys_register
+      use dyn_comp,                  only: dyn_init
+!      use cam_restart,               only: cam_read_restart
+      use camsrfexch,                only: hub2atm_alloc, atm2hub_alloc
+      use cam_history,               only: history_init_files
+!      use history_scam,              only: scm_intht
+      use cam_pio_utils,             only: init_pio_subsystem
+      use cam_instance,              only: inst_suffix
+!      use history_defaults,          only: initialize_iop_history
+      use stepon,                    only: stepon_init
+      use air_composition,           only: air_composition_init
+      use cam_ccpp_cap,              only: cam_ccpp_initialize_constituents
+      use cam_ccpp_cap,              only: cam_model_const_properties
+      use cam_ccpp_cap,              only: cam_constituents_array
+      use physics_grid,              only: columns_on_task
+      use vert_coord,                only: pver
+      use phys_vars_init_check,      only: mark_as_initialized
+      use tropopause_climo_read,     only: tropopause_climo_read_file
+      use orbital_data,              only: orbital_data_init
+      use ccpp_kinds,                only: kind_phys
+      use ccpp_constituent_prop_mod, only: ccpp_constituent_prop_ptr_t
+      use musica_ccpp_dependencies,  only: musica_ccpp_dependencies_init
 
       ! Arguments
       character(len=cl), intent(in) :: caseid                ! case ID
@@ -146,10 +150,13 @@ subroutine cam_init(caseid, ctitle, model_doi_url,                         &
       type(cam_out_t), pointer :: cam_out       ! Output from CAM to surface
       type(cam_in_t),  pointer :: cam_in        ! Merged input state to CAM
 
-                                                ! Local variables
+      ! Local variables
       character(len=cs)        :: filein        ! Input namelist filename
       integer                  :: errflg
       character(len=cx)        :: errmsg
+
+      type(ccpp_constituent_prop_ptr_t), pointer :: constituent_properties(:)
+      real(kind_phys),                   pointer :: constituents_array(:,:,:)
       !-----------------------------------------------------------------------
 
       call init_pio_subsystem()
@@ -248,6 +255,21 @@ subroutine cam_init(caseid, ctitle, model_doi_url,                         &
       ! Read tropopause climatology
       call tropopause_climo_read_file()
 
+      ! Temporary:  Prescribe realistic but inaccurate physical quantities
+      ! necessary for MUSICA that are currently unavailable in CAM-SIMA.
+      ! It also initializes the MUSICA constituent values until the file
+      ! I/O object is implemented.
+      !
+      ! Remove this when MUSICA input data are available from CAM-SIMA or
+      ! other physics schemes.
+      constituent_properties => cam_model_const_properties()
+      constituents_array => cam_constituents_array()
+      call musica_ccpp_dependencies_init(columns_on_task, pver, &
+           constituent_properties, constituents_array)
+
+      ! Initialize orbital data
+      call orbital_data_init(columns_on_task)
+
       call phys_init()
 
 !!XXgoldyXX: v need to import this
@@ -261,16 +283,6 @@ subroutine cam_init(caseid, ctitle, model_doi_url,                         &
       ! end if
       call history_init_files(model_doi_url, caseid, ctitle)
 
-      ! Temporary:  Prescribe realistic but inaccurate physical quantities
-      ! necessary for MUSICA that are currently unavailable in CAM-SIMA.
-      !
-      ! Remove this when MUSICA input data are available from CAM-SIMA or
-      ! other physics schemes.
-      call musica_ccpp_dependencies_init(columns_on_task, pver, iulog)
-
-      ! Initialize orbital data
-      call orbital_data_init(columns_on_task)
-
    end subroutine cam_init
 
    !