Update externals (including git-fleximod) to match cam6_4_117

.github/workflows/fleximod_test.yaml

@@ -13,7 +13,7 @@ jobs:
         uses: actions/checkout@v4
       - id: run-fleximod
         run: |
-          $GITHUB_WORKSPACE/bin/git-fleximod update
+          $GITHUB_WORKSPACE/bin/git-fleximod update -o
           echo
           echo "Update complete, checking status"
           echo
@@ -23,6 +23,4 @@ jobs:
           echo
           echo "Checking if git fleximod matches expected externals"
           echo
-          git diff --exit-code
-
-
+          git add . && git diff --exit-code && git diff --cached --exit-code

.gitmodules

@@ -20,7 +20,7 @@
 [submodule "ncar-physics"]
         path          = src/physics/ncar_ccpp
 	url           = https://github.com/ESCOMP/atmospheric_physics
-	fxtag         = 178c1833d72a4027c6162a6c11b52bbbdcb272b6
+	fxtag         = atmos_phys0_17_002
         fxrequired    = AlwaysRequired
         fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics
 [submodule "rrtmgp-data"]
@@ -32,13 +32,13 @@
 [submodule "ccs_config"]
         path          = ccs_config
         url           = https://github.com/ESMCI/ccs_config_cesm.git
-        fxtag         = ccs_config_cesm1.0.53
+        fxtag         = ccs_config_cesm1.0.61
         fxrequired    = ToplevelRequired
         fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git
 [submodule "cdeps"]
         path          = components/cdeps
         url           = https://github.com/ESCOMP/CDEPS.git
-        fxtag         = cdeps1.0.73
+        fxtag         = cdeps1.0.80
         fxrequired    = ToplevelRequired
         fxDONOTUSEurl = https://github.com/ESCOMP/CDEPS.git
 [submodule "cice"]
@@ -50,25 +50,25 @@
 [submodule "cime"]
         path          = cime
         url           = https://github.com/ESMCI/cime
-        fxtag         = cime6.1.113
+        fxtag         = cime6.1.126
         fxrequired    = ToplevelRequired
 	fxDONOTUSEurl = https://github.com/ESMCI/cime
 [submodule "cism"]
         path          = components/cism
         url           = https://github.com/ESCOMP/CISM-wrapper
-        fxtag         = cismwrap_2_2_006
+        fxtag         = cismwrap_2_2_010
         fxrequired    = ToplevelRequired
         fxDONOTUSEurl = https://github.com/ESCOMP/CISM-wrapper
 [submodule "clm"]
         path          = components/clm
         url           = https://github.com/ESCOMP/CTSM
-        fxtag         = ctsm5.3.029
+        fxtag         = alpha-ctsm5.4.CMIP7.09.ctsm5.3.068
         fxrequired    = ToplevelRequired
         fxDONOTUSEurl = https://github.com/ESCOMP/CTSM
 [submodule "cmeps"]
         path          = components/cmeps
         url           = https://github.com/ESCOMP/CMEPS.git
-        fxtag         = cmeps1.0.47
+        fxtag         = cmeps1.1.16
         fxrequired    = ToplevelRequired
 	fxDONOTUSEurl = https://github.com/ESCOMP/CMEPS.git
 [submodule "fms"]
@@ -80,25 +80,25 @@
 [submodule "mizuRoute"]
         path          = components/mizuRoute
         url           = https://github.com/ESCOMP/mizuRoute
-        fxtag         = cesm-coupling.n03_v2.2.0
+        fxtag         = v3.0.0
         fxrequired    = ToplevelRequired
         fxDONOTUSEurl = https://github.com/ESCOMP/mizuRoute
 [submodule "mosart"]
         path          = components/mosart
         url           = https://github.com/ESCOMP/MOSART
-        fxtag         = mosart1.1.08
+        fxtag         = mosart1.1.12
         fxrequired    = ToplevelRequired
         fxDONOTUSEurl = https://github.com/ESCOMP/MOSART
 [submodule "parallelio"]
         path          = libraries/parallelio
         url           = https://github.com/NCAR/ParallelIO
-        fxtag         = pio2_6_4
+        fxtag         = pio2_6_6
         fxrequired    = ToplevelRequired
 	fxDONOTUSEurl = https://github.com/NCAR/ParallelIO
 [submodule "rtm"]
         path          = components/rtm
         url           = https://github.com/ESCOMP/RTM
-        fxtag         = rtm1_0_86
+        fxtag         = rtm1_0_89
         fxrequired    = ToplevelRequired
         fxDONOTUSEurl = https://github.com/ESCOMP/RTM
 [submodule "share"]
@@ -110,6 +110,6 @@
 [submodule "tools/CUPiD"]
 	path = tools/CUPiD
 	url = https://github.com/NCAR/CUPiD.git
-	fxtag         = v0.2.1
+	fxtag         = v0.3.1
 	fxrequired    = ToplevelRequired
 	fxDONOTUSEurl = https://github.com/NCAR/CUPiD.git

.lib/git-fleximod/git_fleximod/cli.py

@@ -1,38 +1,50 @@
 from pathlib import Path
-import argparse
+import argparse, os, sys
 from git_fleximod import utils
 
-__version__ = "0.9.4"
+__version__ = "1.0.2"
+
+class CustomArgumentParser(argparse.ArgumentParser):
+    def print_help(self, file=None):
+        # First print the default help message
+        super().print_help(file)
+
+        # Then append the contents of README.md
+        candidate_paths = [
+            Path(sys.prefix) / "share" / "git_fleximod" / "README.md",
+            Path(__file__).resolve().parent.parent / "README.md",  # fallback for dev
+        ]
+        for path in candidate_paths:
+            if os.path.exists(path):
+                with open(path) as f:
+                    print( f.read(), file=file)
+                    return
+        print( "README.md not found.", file=file)
 
 def find_root_dir(filename=".gitmodules"):
     """ finds the highest directory in tree
     which contains a file called filename """
-    try:
-        root = utils.execute_subprocess(["git","rev-parse", "--show-toplevel"],
-                                        output_to_caller=True ).rstrip()
-    except:
-        d = Path.cwd()
-        root = Path(d.root)
-        dirlist = []
-        dl = d
-        while dl != root:
-            dirlist.append(dl)
-            dl = dl.parent
-        dirlist.append(root)
-        dirlist.reverse()
-
-        for dl in dirlist:
-            attempt = dl / filename
-            if attempt.is_file():
-                return str(dl)
-        return None
-    return Path(root)
+    d = Path.cwd()
+    root = Path(d.root)
+    dirlist = []
+    dl = d
+    while dl != root:
+        dirlist.append(dl)
+        dl = dl.parent
+    dirlist.append(root)
+    dirlist.reverse()
+
+    for dl in dirlist:
+        attempt = dl / filename
+        if attempt.is_file():
+            return str(dl)
+    return None
 
 def get_parser():
     description = """
     %(prog)s manages checking out groups of gitsubmodules with additional support for Earth System Models
     """
-    parser = argparse.ArgumentParser(
+    parser = CustomArgumentParser(
         description=description, formatter_class=argparse.RawDescriptionHelpFormatter
     )
 

.lib/git-fleximod/git_fleximod/git_fleximod.py

@@ -9,6 +9,7 @@
 import shutil
 import logging
 import textwrap
+import asyncio
 from git_fleximod import utils
 from git_fleximod import cli
 from git_fleximod.gitinterface import GitInterface
@@ -218,10 +219,10 @@ def git_toplevelroot(root_dir, logger):
     _, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
     return superroot
 
-def submodules_update(gitmodules, root_dir, requiredlist, force):
-    for name in gitmodules.sections():
+async def submodules_update(gitmodules, root_dir, requiredlist, force):
+    async def update_submodule(name, requiredlist, force):
         submod = init_submodule_from_gitmodules(gitmodules, name, root_dir, logger)
-    
+
         _, needsupdate, localmods, testfails = submod.status()
         if not submod.fxrequired:
             submod.fxrequired = "AlwaysRequired"
@@ -239,11 +240,11 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
             if "Optional" in fxrequired and "Optional" not in requiredlist:
                 if fxrequired.startswith("Always"):
                     print(f"Skipping optional component {name:>20}")
-                continue
+                return # continue to next submodule
         optional = "AlwaysOptional" in requiredlist
 
         if fxrequired in requiredlist:
-            submod.update()
+            await submod.update()
             repodir = os.path.join(root_dir, submod.path)
             if os.path.exists(os.path.join(repodir, ".gitmodules")):
                 # recursively handle this checkout
@@ -252,7 +253,10 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
                 newrequiredlist = ["AlwaysRequired"]
                 if optional:
                     newrequiredlist.append("AlwaysOptional")
-                submodules_update(gitsubmodules, repodir, newrequiredlist, force=force)
+                await submodules_update(gitsubmodules, repodir, newrequiredlist, force=force)
+
+    tasks = [update_submodule(name, requiredlist, force) for name in gitmodules.sections()]
+    await asyncio.gather(*tasks)
 
 def local_mods_output():
     text = """\
@@ -346,7 +350,7 @@ def main():
         sys.exit(f"No submodule components found, root_dir={root_dir}")
     retval = 0
     if action == "update":
-        submodules_update(gitmodules, root_dir, fxrequired, force)
+        asyncio.run(submodules_update(gitmodules, root_dir, fxrequired, force))
     elif action == "status":
         tfails, lmods, updates = submodules_status(gitmodules, root_dir, toplevel=True)
         if tfails + lmods + updates > 0:

.lib/git-fleximod/git_fleximod/gitinterface.py

@@ -2,6 +2,7 @@
 import sys
 from . import utils
 from pathlib import Path
+import asyncio
 
 class GitInterface:
     def __init__(self, repo_path, logger):
@@ -47,6 +48,16 @@ def _init_git_repo(self):
             command = ("git", "-C", str(self.repo_path), "init")
             utils.execute_subprocess(command)
 
+    def _git_operation_command(self, operation, args):
+        newargs = []
+        for a in args:
+            # Do not use ssh interface
+            if isinstance(a, str):
+                a = a.replace("git@github.com:", "https://github.com/")
+            newargs.append(a)
+
+        return self._git_command(operation, *newargs)
+
     # pylint: disable=unused-argument
     def git_operation(self, operation, *args, **kwargs):
         newargs = []
@@ -66,6 +77,25 @@ def git_operation(self, operation, *args, **kwargs):
         else:
             return 0, command
 
+    # pylint: disable=unused-argument
+    async def git_operation_async(self, operation, *args, **kwargs):
+        command = self._git_operation_command(operation, args)
+        if isinstance(command, list):
+            try:
+                process = await asyncio.create_subprocess_exec(
+                    *command,
+                    stdout=asyncio.subprocess.PIPE,
+                    stderr=asyncio.subprocess.PIPE
+                )
+                stdout, stderr = await process.communicate()
+                status = process.returncode
+                output = stdout.decode().strip() if stdout else stderr.decode().strip()
+                return status, output
+            except Exception as e:
+                sys.exit(e)
+        else:
+            return 0, command
+
     def config_get_value(self, section, name):
         if self._use_module:
             config = self.repo.config_reader()