Merge branch 'development' into feature/pfunit-integration

Author: mwaxmonsky

.gitmodules

@@ -1,7 +1,7 @@
 [submodule "ccpp-framework"]
         path          = ccpp_framework
         url           = https://github.com/NCAR/ccpp-framework
-        fxtag         = 2024-11-07-dev
+        fxtag         = 2025-01-06-dev
         fxrequired    = AlwaysRequired
 	fxDONOTUSEurl = https://github.com/NCAR/ccpp-framework
 [submodule "history"]
@@ -20,7 +20,7 @@
 [submodule "ncar-physics"]
         path          = src/physics/ncar_ccpp
 	url           = https://github.com/ESCOMP/atmospheric_physics
-	fxtag         = 491e56247815ef23bfd8dba65d1e3c3b78ba164a
+	fxtag         = 74e905b7a0ee5b2d2bfc3e3dd942eb9963398373
         fxrequired    = AlwaysRequired
         fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics
 [submodule "ccs_config"]

.lib/git-fleximod/git_fleximod/cli.py

@@ -2,27 +2,31 @@
 import argparse
 from git_fleximod import utils
 
-__version__ = "0.8.4"
+__version__ = "0.9.4"
 
 def find_root_dir(filename=".gitmodules"):
     """ finds the highest directory in tree
     which contains a file called filename """
-    d = Path.cwd()
-    root = Path(d.root)
-    dirlist = []
-    dl = d
-    while dl != root:
-        dirlist.append(dl)
-        dl = dl.parent
-    dirlist.append(root)
-    dirlist.reverse()
-
-    for dl in dirlist:
-        attempt = dl / filename
-        if attempt.is_file():
-            return str(dl)
-    return None
-
+    try:
+        root = utils.execute_subprocess(["git","rev-parse", "--show-toplevel"],
+                                        output_to_caller=True ).rstrip()
+    except:
+        d = Path.cwd()
+        root = Path(d.root)
+        dirlist = []
+        dl = d
+        while dl != root:
+            dirlist.append(dl)
+            dl = dl.parent
+        dirlist.append(root)
+        dirlist.reverse()
+
+        for dl in dirlist:
+            attempt = dl / filename
+            if attempt.is_file():
+                return str(dl)
+        return None
+    return Path(root)
 
 def get_parser():
     description = """

.lib/git-fleximod/git_fleximod/git_fleximod.py

@@ -124,8 +124,8 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
     # set the repository remote
 
     logger.info("Setting remote origin in {}/{}".format(root_dir, path))
-    status = sprepo_git.git_operation("remote", "-v")
-    if url not in status:
+    _, remotelist = sprepo_git.git_operation("remote", "-v")
+    if url not in remotelist:
         sprepo_git.git_operation("remote", "add", "origin", url)
 
     topgit = os.path.join(gitroot, ".git")
@@ -181,6 +181,8 @@ def init_submodule_from_gitmodules(gitmodules, name, root_dir, logger):
     url = gitmodules.get(name, "url")
     assert path and url, f"Malformed .gitmodules file {path} {url}"
     tag = gitmodules.get(name, "fxtag")
+    if not tag:
+        tag = gitmodules.get(name, "hash")
     fxurl = gitmodules.get(name, "fxDONOTUSEurl")
     fxsparse = gitmodules.get(name, "fxsparse")
     fxrequired = gitmodules.get(name, "fxrequired")
@@ -213,7 +215,7 @@ def submodules_status(gitmodules, root_dir, toplevel=False, depth=0):
 
 def git_toplevelroot(root_dir, logger):
     rgit = GitInterface(root_dir, logger)
-    superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
+    _, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
     return superroot
 
 def submodules_update(gitmodules, root_dir, requiredlist, force):
@@ -250,7 +252,6 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
                 newrequiredlist = ["AlwaysRequired"]
                 if optional:
                     newrequiredlist.append("AlwaysOptional")
-
                 submodules_update(gitsubmodules, repodir, newrequiredlist, force=force)
 
 def local_mods_output():
@@ -342,7 +343,7 @@ def main():
         excludelist=excludelist,
     )
     if not gitmodules.sections():
-        sys.exit("No submodule components found")
+        sys.exit(f"No submodule components found, root_dir={root_dir}")
     retval = 0
     if action == "update":
         submodules_update(gitmodules, root_dir, fxrequired, force)

.lib/git-fleximod/git_fleximod/gitinterface.py

@@ -59,11 +59,12 @@ def git_operation(self, operation, *args, **kwargs):
         command = self._git_command(operation, *newargs)
         if isinstance(command, list):
             try:
-                return utils.execute_subprocess(command, output_to_caller=True)
+                status, output = utils.execute_subprocess(command, status_to_caller=True, output_to_caller=True)
+                return status, output.rstrip()
             except Exception as e:
                 sys.exit(e)
         else:
-            return command
+            return 0, command
 
     def config_get_value(self, section, name):
         if self._use_module:
@@ -81,6 +82,8 @@ def config_get_value(self, section, name):
     def config_set_value(self, section, name, value):
         if self._use_module:
             with self.repo.config_writer() as writer:
+                if "." in section:  
+                    section = section.replace("."," \"")+'"'           
                 writer.set_value(section, name, value)
             writer.release()  # Ensure changes are saved
         else:

.lib/git-fleximod/git_fleximod/submodule.py

@@ -60,8 +60,9 @@ def status(self):
         if not os.path.exists(os.path.join(smpath, ".git")):
             rootgit = GitInterface(self.root_dir, self.logger)
             # submodule commands use path, not name
-            tags = rootgit.git_operation("ls-remote", "--tags", self.url)
-            result = rootgit.git_operation("submodule","status",smpath).split()
+            status, tags = rootgit.git_operation("ls-remote", "--tags", self.url)
+            status, result = rootgit.git_operation("submodule","status",smpath)
+            result = result.split()
 
             if result:
                 ahash = result[0][1:]
@@ -80,9 +81,9 @@ def status(self):
                 result = f"e {self.name:>20} not checked out, aligned at tag {self.fxtag}{optional}"
                 needsupdate = True
             elif self.fxtag:
-                ahash = rootgit.git_operation(
+                status, ahash = rootgit.git_operation(
                     "submodule", "status", "{}".format(self.path)
-                ).rstrip()
+                )
                 ahash = ahash[1 : len(self.fxtag) + 1]
                 if self.fxtag == ahash:
                     result = f"e {self.name:>20} not checked out, aligned at hash {ahash}{optional}"
@@ -96,14 +97,15 @@ def status(self):
         else:
             with utils.pushd(smpath):
                 git = GitInterface(smpath, self.logger)
-                remote = git.git_operation("remote").rstrip()
+                status, remote = git.git_operation("remote")
                 if remote == '':
                     result = f"e {self.name:>20} has no associated remote"
                     testfails = True
                     needsupdate = True
                     return result, needsupdate, localmods, testfails                    
-                rurl = git.git_operation("ls-remote","--get-url").rstrip()
-                line = git.git_operation("log", "--pretty=format:\"%h %d\"").partition('\n')[0]
+                status, rurl = git.git_operation("ls-remote","--get-url")
+                status, lines = git.git_operation("log", "--pretty=format:\"%h %d\"")
+                line = lines.partition('\n')[0]
                 parts = line.split()
                 ahash = parts[0][1:]
                 atag = None
@@ -120,7 +122,7 @@ def status(self):
 
 
                 #print(f"line is {line} ahash is {ahash} atag is {atag} {parts}")
-                #                atag = git.git_operation("describe", "--tags", "--always").rstrip()
+                #                atag = git.git_operation("describe", "--tags", "--always")
                 # ahash =  git.git_operation("rev-list", "HEAD").partition("\n")[0]
 
                 recurse = False
@@ -149,10 +151,10 @@ def status(self):
                         result = f"e {self.name:>20} has no fxtag defined in .gitmodules, module at {ahash}"
                     testfails = False
 
-                status = git.git_operation("status", "--ignore-submodules", "-uno")
-                if "nothing to commit" not in status:
+                status, output = git.git_operation("status", "--ignore-submodules", "-uno")
+                if "nothing to commit" not in output:
                     localmods = True
-                    result = "M" + textwrap.indent(status, "                      ")
+                    result = "M" + textwrap.indent(output, "                      ")
 #        print(f"result {result} needsupdate {needsupdate} localmods {localmods} testfails {testfails}")
         return result, needsupdate, localmods, testfails
 
@@ -171,10 +173,11 @@ def _add_remote(self, git):
         Returns:
             str: The name of the new remote if added, or the name of the existing remote that matches the submodule's URL.
         """ 
-        remotes = git.git_operation("remote", "-v").splitlines()
+        status, remotes = git.git_operation("remote", "-v")
+        remotes = remotes.splitlines()
         upstream = None
         if remotes:
-            upstream = git.git_operation("ls-remote", "--get-url").rstrip()
+            status, upstream = git.git_operation("ls-remote", "--get-url")
             newremote = "newremote.00"
             tmpurl = self.url.replace("git@github.com:", "https://github.com/")
             line = next((s for s in remotes if self.url in s or tmpurl in s), None)
@@ -183,7 +186,7 @@ def _add_remote(self, git):
                 return newremote
             else:
                 i = 0
-                while "newremote" in remotes:
+                while newremote in remotes:
                     i = i + 1
                     newremote = f"newremote.{i:02d}"
         else:
@@ -214,12 +217,19 @@ def sparse_checkout(self):
         """ 
         self.logger.info("Called sparse_checkout for {}".format(self.name))
         rgit = GitInterface(self.root_dir, self.logger)
-        superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
+        status, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
         if superroot:
             gitroot = superroot.strip()
         else:
-            gitroot = self.root_dir.strip()
-        assert os.path.isdir(os.path.join(gitroot, ".git"))
+            gitroot = self.root_dir
+        # Now need to move the .git dir to the submodule location
+        rootdotgit = os.path.join(self.root_dir, ".git")
+        while os.path.isfile(rootdotgit):
+            with open(rootdotgit) as f:
+                line = f.readline().rstrip()
+                if line.startswith("gitdir: "):
+                    rootdotgit = os.path.abspath(os.path.join(self.root_dir,line[8:]))
+        assert os.path.isdir(rootdotgit)
         # first create the module directory
         if not os.path.isdir(os.path.join(self.root_dir, self.path)):
             os.makedirs(os.path.join(self.root_dir, self.path))
@@ -244,8 +254,8 @@ def sparse_checkout(self):
         # set the repository remote
 
         self.logger.info("Setting remote origin in {}/{}".format(self.root_dir, self.path))
-        status = sprepo_git.git_operation("remote", "-v")
-        if self.url not in status:
+        status, remotes = sprepo_git.git_operation("remote", "-v")
+        if self.url not in remotes:
             sprepo_git.git_operation("remote", "add", "origin", self.url)
 
         topgit = os.path.join(gitroot, ".git")
@@ -256,46 +266,46 @@ def sparse_checkout(self):
                     os.path.join(self.root_dir, f.read().split()[1]),
                     start=os.path.join(self.root_dir, self.path),
                 )
-                topgit = os.path.join(gitpath, "modules")
+                rootdotgit = os.path.join(gitpath, "modules", self.name)
         else:
-            topgit = os.path.relpath(
-                os.path.join(self.root_dir, ".git", "modules"),
+            rootdotgit = os.path.relpath(
+                os.path.join(self.root_dir, ".git", "modules", self.name),
                 start=os.path.join(self.root_dir, self.path),
             )
 
-        with utils.pushd(sprep_repo):
-            if not os.path.isdir(topgit):
-                os.makedirs(topgit)
-        topgit += os.sep + self.name
-
         if os.path.isdir(os.path.join(self.root_dir, self.path, ".git")):
             with utils.pushd(sprep_repo):
-                if os.path.isdir(os.path.join(topgit,".git")):
-                    shutil.rmtree(os.path.join(topgit,".git"))
-                shutil.move(".git", topgit)
+                if os.path.isdir(os.path.join(rootdotgit,".git")):
+                    shutil.rmtree(os.path.join(rootdotgit,".git"))
+                shutil.move(".git", rootdotgit)
                 with open(".git", "w") as f:
-                    f.write("gitdir: " + os.path.relpath(topgit))
-                #    assert(os.path.isdir(os.path.relpath(topgit, start=sprep_repo)))
-                gitsparse = os.path.abspath(os.path.join(topgit, "info", "sparse-checkout"))
+                    f.write("gitdir: " + os.path.relpath(rootdotgit))
+                infodir = os.path.join(rootdotgit, "info")
+                if not os.path.isdir(infodir):
+                    os.makedirs(infodir)
+                gitsparse = os.path.abspath(os.path.join(infodir, "sparse-checkout"))
             if os.path.isfile(gitsparse):
                 self.logger.warning(
-                    "submodule {} is already initialized {}".format(self.name, topgit)
+                    "submodule {} is already initialized {}".format(self.name, rootdotgit)
                 )
                 return
 
             with utils.pushd(sprep_repo):
                 if os.path.isfile(self.fxsparse):
+                    
                     shutil.copy(self.fxsparse, gitsparse)
 
 
         # Finally checkout the repo
         sprepo_git.git_operation("fetch", "origin", "--tags")
-        sprepo_git.git_operation("checkout", self.fxtag)
-
-        print(f"Successfully checked out {self.name:>20} at {self.fxtag}")
-        rgit.config_set_value(f'submodule "{self.name}"', "active", "true")
-        rgit.config_set_value(f'submodule "{self.name}"', "url", self.url)
-        rgit.config_set_value(f'submodule "{self.name}"', "path", self.path)
+        status,_ = sprepo_git.git_operation("checkout", self.fxtag)
+        if status:
+            print(f"Error checking out {self.name:>20} at {self.fxtag}")
+        else:
+            print(f"Successfully checked out {self.name:>20} at {self.fxtag}")
+        rgit.config_set_value('submodule.' + self.name, "active", "true")
+        rgit.config_set_value('submodule.' + self.name, "url", self.url)
+        rgit.config_set_value('submodule.' + self.name, "path", self.path)
 
     def update(self):
         """
@@ -342,15 +352,15 @@ def update(self):
                     git.git_operation("clone", self.url, self.path)
                     smgit = GitInterface(repodir, self.logger)
                     if not tag:
-                        tag = smgit.git_operation("describe", "--tags", "--always").rstrip()
+                        status, tag = smgit.git_operation("describe", "--tags", "--always")
                     smgit.git_operation("checkout", tag)
                     # Now need to move the .git dir to the submodule location
                     rootdotgit = os.path.join(self.root_dir, ".git")
                     if os.path.isfile(rootdotgit):
                         with open(rootdotgit) as f:
                             line = f.readline()
                             if line.startswith("gitdir: "):
-                                rootdotgit = line[8:].rstrip()
+                                rootdotgit = line[8:]
 
                     newpath = os.path.abspath(os.path.join(self.root_dir, rootdotgit, "modules", self.name))
                     if os.path.exists(newpath):
@@ -393,15 +403,16 @@ def update(self):
                 git = GitInterface(submoddir, self.logger)
                 # first make sure the url is correct
                 newremote = self._add_remote(git)
-                tags = git.git_operation("tag", "-l")
+                status, tags = git.git_operation("tag", "-l")
                 fxtag = self.fxtag
                 if fxtag and fxtag not in tags:
                     git.git_operation("fetch", newremote, "--tags")
-                atag = git.git_operation("describe", "--tags", "--always").rstrip()
+                status, atag = git.git_operation("describe", "--tags", "--always")
                 if fxtag and fxtag != atag:
                     try:
-                        git.git_operation("checkout", fxtag)
-                        print(f"{self.name:>20} updated to {fxtag}")
+                        status, _ = git.git_operation("checkout", fxtag)
+                        if not status:
+                            print(f"{self.name:>20} updated to {fxtag}")
                     except Exception as error:
                         print(error)
 

.lib/git-fleximod/git_fleximod/utils.py

@@ -307,12 +307,12 @@ def execute_subprocess(commands, status_to_caller=False, output_to_caller=False)
         # simple status check. If returning, it is the callers
         # responsibility determine if an error occurred and handle it
         # appropriately.
+        msg_context = (
+            "Process did not run successfully; "
+            "returned status {0}".format(error.returncode)
+        )
+        msg = failed_command_msg(msg_context, commands, output=error.output)
         if not return_to_caller:
-            msg_context = (
-                "Process did not run successfully; "
-                "returned status {0}".format(error.returncode)
-            )
-            msg = failed_command_msg(msg_context, commands, output=error.output)
             logging.error(error)
             logging.error(msg)
             log_process_output(error.output)