update docker to build case

Author: boulderdaze

cime_config/atm_musica_config.py

@@ -1,9 +1,9 @@
 """
-The URLs and tags provided in this script are read by buildlib.py to build 
+The URLs and tags provided in this script are read by buildlib to build
 and install the MUSICA library, as well as to download the MUSICA configuration."
 """
 
 musica_repo_url = "https://github.com/NCAR/musica.git"
-musica_tag = "e0c3d219fc8bbfd388cd3c95a3646d2e3aa1e288" # main
+musica_tag = "802bc9041f964a7edd5ec896bec8cdc0c90de50b"
 chemistry_data_repo_url = "https://github.com/NCAR/cam-sima-chemistry-data.git"
 chemistry_data_tag = "b81cbc2e61c41ecd2d09167d6736daf1bf1be149"

cime_config/buildlib

@@ -79,7 +79,7 @@ def _build_cam():
         config.generate_cam_src(gen_indent)
 
         dycore = config.get_value('dyn')
-        physics_suites = config.get_value('physics_suites')
+        phys_suites = config.get_value('physics_suites')
         reg_dir = config.get_value('reg_dir')
         init_dir = config.get_value('init_dir')
         phys_dirs_str = config.get_value('phys_dirs')
@@ -125,6 +125,7 @@ def _build_cam():
             paths.append(os.path.join(atm_root, "src", "dynamics", "tests",
                                       "initial_conditions"))
 
+        # TODO(jiwon); this path is not found
         # If using the CMEPS/NUOPC coupler, then add additional path:
         if case.get_value("COMP_INTERFACE") == "nuopc":
             paths.append(os.path.join(__CIMEROOT, "src", "drivers",
@@ -135,7 +136,7 @@ def _build_cam():
         with open(filepath_src, "w", encoding='utf-8') as filepath:
             filepath.write("\n".join(paths))
             filepath.write("\n")
-        # End with
+        # End if
 
         # Move Filepath to the bld directory unless it has not changed
         if os.path.isfile(filepath_dst):
@@ -154,17 +155,25 @@ def _build_cam():
         if dycore == "mpas":
             _setup_mpas(case)
 
-        if physics_suites == "musica":
+        # If the physics suite is MUSICA, build the MUSICA library and configuration
+        if phys_suites == "musica":
             _build_musica_library(case)
-            _download_musica_configuration(case)
-            # clone_dest = os.path.normpath(case.get_value("CASEROOT"))
-            # bld_path = os.path.normpath(os.path.join(clone_dest, "musica", "build", "install", "include"))
-            # paths.append(bld_path)
+            _download_musica_configuration(caseroot)
 
         complib = os.path.join(libroot, "libatm.a")
         makefile = os.path.join(casetools, "Makefile")
 
         cmd = f"{gmake} complib -j {gmake_j} MODEL=cam COMPLIB={complib}"
+
+        if phys_suites == "musica":
+            cmd += ' USER_INCLDIR="'\
+                    f'-I{os.path.join(bldroot, "musica", "include", "micm")} '\
+                    f'-I{os.path.join(bldroot, "musica", "include", "musica")} '\
+                    f'-I{os.path.join(bldroot, "musica", "include", "musica", "micm")} '\
+                    f'-I{os.path.join(bldroot, "musica", "include", "musica", "tuvx")} '\
+                    f'-I{os.path.join(bldroot, "musica", "include", "musica", "fortran")} '\
+                    '"'
+
         cmd += f" -f {makefile} {get_standard_makefile_args(case)} "
 
         # Add C Pre-Processor (CPP) definitions, if present:
@@ -175,26 +184,10 @@ def _build_cam():
         retcode, out, err = run_cmd(cmd)
         _LOGGER.info("%s: \n\n output:\n %s \n\n err:\n\n%s\n", cmd, out, err)
         expect(retcode == 0, f"Command {cmd} failed with rc={retcode}")
-        
-
-        # JIWON
-        if physics_suites == "musica":
-            clone_dest = os.path.normpath(case.get_value("CASEROOT"))
-            # bld_path = os.path.normpath(os.path.join(clone_dest, "musica", "build", "install", "include"))
-            bld_path = os.path.normpath(os.path.join(case.get_value("EXEROOT"), "atm", "obj", "musica", "include"))
-            cmd = f"{gmake} complib -j {gmake_j} MODEL=cam USER_INCLDIR={bld_path}"
-            cmd += f" -f {makefile} {get_standard_makefile_args(case)} "
-
-            # Add C Pre-Processor (CPP) definitions, if present:
-            if config.cpp_defs:
-                ccpp_defs_str = ' '.join(config.cpp_defs)
-                cmd += f" USER_CPPDEFS='{ccpp_defs_str}'"
-
-            retcode, out, err = run_cmd(cmd)
-            _LOGGER.info("%s: \n\n output:\n %s \n\n err:\n\n%s\n", cmd, out, err)
-            expect(retcode == 0, f"Command {cmd} failed with rc={retcode}")
 
+###############################################################################
 def _setup_mpas(case: Case) -> None:
+###############################################################################
     """
     Setup MPAS build infrastructure.
     """
@@ -222,7 +215,9 @@ def _setup_mpas(case: Case) -> None:
     shutil.copytree(os.path.normpath(os.path.join(mpas_dycore_src_root, os.pardir, os.pardir, "assets")), mpas_dycore_bld_root, copy_function=_copy2_as_needed, dirs_exist_ok=True)
     shutil.copytree(os.path.normpath(os.path.join(mpas_dycore_src_root, os.pardir, os.pardir, "driver")), os.path.join(mpas_dycore_bld_root, "driver"), copy_function=_copy2_as_needed, dirs_exist_ok=True)
 
+###############################################################################
 def _copy2_as_needed(src: str, dst: str) -> None:
+###############################################################################
     """
     Wrapper around `shutil.copy2`. Copy the file `src` to the file or directory `dst` as needed.
     """
@@ -242,25 +237,34 @@ def _copy2_as_needed(src: str, dst: str) -> None:
                 # Example scenario: User added some new source code files.
                 shutil.copy2(src, dst)
 
+###############################################################################
 def _build_musica_library(case: Case) -> None:
+###############################################################################
+    """
+    Builds and installs the MUSICA library.
+
+    Args:
+        case (Case)
 
+    Raises:
+        Exception: If configuring the CMake MUSICA project fails or
+            the MUSICA library build fails, an exception is raised.
+
+    """
     install_dir_name = "install"
     build_type = "Release"
+    caseroot = os.path.normpath(case.get_value("CASEROOT"))
+    _clone_and_checkout(musica_repo_url, musica_tag, caseroot)
 
-    clone_dest = os.path.normpath(case.get_value("CASEROOT"))
-    _clone_and_checkout(musica_repo_url, musica_tag, clone_dest)
-
-    bld_path = os.path.normpath(os.path.join(clone_dest, "musica", "build"))
-    # Ensure the build directory is clean
+    bld_path = os.path.join(caseroot, "musica", "build")
     if os.path.exists(bld_path):
-        print(f"Removing existing build directory: {bld_path}")
         shutil.rmtree(bld_path)
-
     os.makedirs(bld_path)
+
+    # To install the target, the working directory must be the build directory.
+    current_dir = os.getcwd()
     os.chdir(bld_path)
 
-    # Run cmake to configure the project
-    print("Configuring the project with CMake...")
     command = [
         "cmake",
         f"-DCMAKE_INSTALL_PREFIX={install_dir_name}",
@@ -269,92 +273,84 @@ def _build_musica_library(case: Case) -> None:
         f"-DMUSICA_BUILD_FORTRAN_INTERFACE=ON",
         ".."
     ]
-    print(f"Running command: {' '.join(command)}")
     result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
     if result.returncode != 0:
         raise Exception(f"Unable to configure the MUSICA CMake project. Error: {result.stderr}")
 
-    # Build the project
-    print("Building the project...")
     command = ["cmake", "--build", ".", "--target", "install"]
     result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
     if result.returncode != 0:
         raise Exception(f"Unable to build the MUSICA library. Error: {result.stderr}")
-    
-    print(f"MUSICA installed to {install_dir_name}")
 
-    atm_bld_root = os.path.normpath(os.path.join(case.get_value("EXEROOT"), "atm", "obj"))
+    # Create symlinks pointing to the MUSICA libraries in the ATM build directory
+    atm_bld_root = os.path.join(case.get_value("EXEROOT"), "atm", "obj")
     musica_lib_root = os.path.join(atm_bld_root, "musica", "lib")
     os.makedirs(musica_lib_root, exist_ok=True)
-    musica_install_path = os.path.join(bld_path, install_dir_name, "lib64")
-
-    root, _, files = next(os.walk(musica_install_path, topdown=True))
-    for f in files:
-        symlink_force(os.path.join(root, f), os.path.join(musica_lib_root, f))
-
-    src_dir = os.path.join(bld_path, install_dir_name, "include")
-    dest_dir = os.path.join(atm_bld_root, "musica", "include")
-    # os.makedirs(dest_dir, exist_ok=True)
-        # Walk through the source directory
-    # for root, dirs, files in os.walk(src_dir):
-    #     # Calculate the relative path from the source directory
-    #     rel_path = os.path.relpath(root, src_dir)
-    #     # Define the corresponding path in the destination
-    #     dest_root = os.path.join(dest_dir, rel_path)
-        
-    #     # Create the destination directory structure if it doesn't exist
-    #     os.makedirs(dest_root, exist_ok=True)
-
-    #     # Create symlinks for all files in the current directory
-    #     for file in files:
-    #         src_file = os.path.join(root, file)
-    #         dest_file = os.path.join(dest_root, file)
-
-    #         # Create a symlink if it doesn't already exist
-    #         if not os.path.exists(dest_file):
-    #             os.symlink(src_file, dest_file)
-    #             try:
-    #                 os.symlink(src_file, dest_file)
-    #                 print(f"Created symlink: {dest_file} -> {src_file}")
-    #             except OSError as e:
-    #                 print(f"Error creating symlink for {src_file}: {e}")
-    #         else:
-    #             print(f"Symlink already exists: {dest_file}")
-                
-
-def _download_musica_configuration(case: Case) -> None:
-    """_summary_
 
+    installed_lib_path = os.path.join(bld_path, install_dir_name, "lib64")
+    root, _, files = next(os.walk(installed_lib_path, topdown=True))
+    for file in files:
+        symlink_force(os.path.join(root, file), os.path.join(musica_lib_root, file))
+
+    # Create symlinks pointing to the MUSICA include files in the ATM build directory
+    musica_include_root = os.path.join(atm_bld_root, "musica", "include")
+    os.makedirs(musica_include_root, exist_ok=True)
+
+    installed_include_path = os.path.join(bld_path, install_dir_name, "include")
+    for root, _, files in os.walk(installed_include_path):
+        rel_path = os.path.relpath(root, installed_include_path)
+        dest_dir = os.path.join(musica_include_root, rel_path)
+
+        os.makedirs(dest_dir, exist_ok=True)
+        for file in files:
+            symlink_force(os.path.join(root, file), os.path.join(dest_dir, file))
+
+    os.chdir(current_dir)
+
+###############################################################################
+def _download_musica_configuration(download_dest: str) -> None:
+###############################################################################
+    """
+    Downloads the MUSICA configuration and renames the configuration
+    directory to match the name in the MUSICA-CCPP configuration.
+
+    Args:
+        download_dest: destination where configuration will be downloaded.
+
+    Raises:
+        Exception: If the directory to be renamed is not found or
+            any other exceptions occur during the renaming process,
+            an exception is raised with the error message.
     """
     musica_config_dir_name = "musica_configurations"
 
-    case_root = os.path.normpath(case.get_value("CASEROOT"))
-    _clone_and_checkout(chemistry_data_repo_url, chemistry_data_tag, case_root)
+    _clone_and_checkout(chemistry_data_repo_url, chemistry_data_tag, download_dest)
 
-    original_config_dir = os.path.join(case_root, "cam-sima-chemistry-data", "mechanisms")
-    renamed_config_dir = os.path.join(case_root, "cam-sima-chemistry-data", musica_config_dir_name)
+    original_dir = os.path.join(download_dest, "cam-sima-chemistry-data", "mechanisms")
+    renamed_dir = os.path.join(download_dest, "cam-sima-chemistry-data", musica_config_dir_name)
     try:
-        os.rename(original_config_dir, renamed_config_dir)
+        os.rename(original_dir, renamed_dir)
     except FileNotFoundError:
-        raise Exception(f"The directory {original_config_dir} was not found.")
+        raise Exception(f"The directory {original_dir} was not found.")
     except Exception as e:
         raise Exception(f"An error occurred while renaming: {e}")
 
-    musica_config_path = os.path.join(case_root, musica_config_dir_name)
+    musica_config_path = os.path.join(download_dest, musica_config_dir_name)
     if os.path.exists(musica_config_path):
         shutil.rmtree(musica_config_path)
 
-    shutil.move(renamed_config_dir, case_root)
-
+    shutil.move(renamed_dir, download_dest)
 
-def _clone_and_checkout(repo_url, tag_name, clone_dest):
+###############################################################################
+def _clone_and_checkout(repo_url: str, tag_name: str, clone_dest: str):
+###############################################################################
     """
-    Clones a Git repository from the provided URL and checks out a specific branch.
+    Clones a Git repository from the URL and checks out a specific branch.
 
     Args:
-        repo_url (str): The URL of the Git repository to clone
-        tag_name (str): The tag name to check out
-        clone_dest (str): destination where the repository will be cloned. (absolute path)
+        repo_url: The URL of the Git repository to clone
+        tag_name: The tag name to check out
+        clone_dest: destination where the repository will be cloned.
 
     Raises:
         Exception: If the `git clone` or `git checkout` commands fail,
@@ -366,15 +362,19 @@ def _clone_and_checkout(repo_url, tag_name, clone_dest):
     if os.path.exists(repo_path):
         shutil.rmtree(repo_path)
 
-    result = subprocess.run(["git", "clone", repo_url, repo_path], stderr=subprocess.PIPE, text=True)
+    result = subprocess.run(["git", "clone", repo_url, repo_path],
+                            stderr=subprocess.PIPE, text=True)
     if result.returncode != 0:
-        raise Exception(f"Unable to clone the repository: {repo_url}. Error: {result.stderr}")
+        raise Exception(f"Unable to clone the repository: {repo_url}. \
+                        Error: {result.stderr}")
 
-    result = subprocess.run(["git", "-C", repo_path, "checkout", tag_name], stderr=subprocess.PIPE, text=True)
+    result = subprocess.run(["git", "-C", repo_path, "checkout", tag_name],
+                            stderr=subprocess.PIPE, text=True)
     if result.returncode != 0:
-        raise Exception(f"Unable to checkout the branch: {tag_name}. Error: {result.stderr}")
+        raise Exception(f"Unable to checkout the branch: {tag_name}. \
+                        Error: {result.stderr}")
 
 ###############################################################################
 
 if __name__ == "__main__":
-    _build_cam()
+    _build_cam()