Fix CAM7 regression tests (ESCOMP#348)

Tag name (required for release branches): sima0_00_003
Originator(s): peverwhee

Description (include the issue title, and the keyword ['closes',
'fixes', 'resolves'] followed by the issue number):

Updates framework tag to include bugfix for groups - closes ESCOMP#335 
Updates atmospheric_physics external to use apply_constituent_tendencies
Updates git-fleixmod to 0.9.4

Describe any changes made to build system: n/a

Describe any changes made to the namelist: n/a

List any changes to the defaults for the input datasets (e.g. boundary
datasets): n/a

List all files eliminated and why: n/a

List all files added and what they do: n/a

List all existing files that have been modified, and describe the
changes:
(Helpful git command: `git diff --name-status
development...<your_branch_name>`)

M .gitmodules
- update ccpp-framework and atmospheric_physics tags

M .lib/git-fleixmod/*
- update git-fleixmod to 0.9.4

M test/existing-test-failures.txt
- only MPAS tests still fail

If there are new failures (compared to the
`test/existing-test-failures.txt` file),
have them OK'd by the gatekeeper, note them here, and add them to the
file.
If there are baseline differences, include the test and the reason for
the
diff. What is the nature of the change? Roundoff?

derecho/intel/aux_sima: All PASS except MPAS; new baselines for CAM7
(test never passed before)

derecho/gnu/aux_sima: All PASS except MPAS; new baselines for CAM7 (test
never passed before)

If this changes climate describe any run(s) done to evaluate the new
climate in enough detail that it(they) could be reproduced:

CAM-SIMA date used for the baseline comparison tests if different than
latest:

Author: peverwhee

.gitmodules

@@ -1,7 +1,7 @@
 [submodule "ccpp-framework"]
         path          = ccpp_framework
         url           = https://github.com/NCAR/ccpp-framework
-        fxtag         = 2024-11-07-dev
+        fxtag         = 2025-01-06-dev
         fxrequired    = AlwaysRequired
 	fxDONOTUSEurl = https://github.com/NCAR/ccpp-framework
 [submodule "history"]
@@ -20,7 +20,7 @@
 [submodule "ncar-physics"]
         path          = src/physics/ncar_ccpp
 	url           = https://github.com/ESCOMP/atmospheric_physics
-	fxtag         = 491e56247815ef23bfd8dba65d1e3c3b78ba164a
+	fxtag         = 74e905b7a0ee5b2d2bfc3e3dd942eb9963398373
         fxrequired    = AlwaysRequired
         fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics
 [submodule "ccs_config"]

.lib/git-fleximod/git_fleximod/cli.py

@@ -2,27 +2,31 @@
 import argparse
 from git_fleximod import utils
 
-__version__ = "0.8.4"
+__version__ = "0.9.4"
 
 def find_root_dir(filename=".gitmodules"):
     """ finds the highest directory in tree
     which contains a file called filename """
-    d = Path.cwd()
-    root = Path(d.root)
-    dirlist = []
-    dl = d
-    while dl != root:
-        dirlist.append(dl)
-        dl = dl.parent
-    dirlist.append(root)
-    dirlist.reverse()
-
-    for dl in dirlist:
-        attempt = dl / filename
-        if attempt.is_file():
-            return str(dl)
-    return None
-
+    try:
+        root = utils.execute_subprocess(["git","rev-parse", "--show-toplevel"],
+                                        output_to_caller=True ).rstrip()
+    except:
+        d = Path.cwd()
+        root = Path(d.root)
+        dirlist = []
+        dl = d
+        while dl != root:
+            dirlist.append(dl)
+            dl = dl.parent
+        dirlist.append(root)
+        dirlist.reverse()
+
+        for dl in dirlist:
+            attempt = dl / filename
+            if attempt.is_file():
+                return str(dl)
+        return None
+    return Path(root)
 
 def get_parser():
     description = """

.lib/git-fleximod/git_fleximod/git_fleximod.py

@@ -124,8 +124,8 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
     # set the repository remote
 
     logger.info("Setting remote origin in {}/{}".format(root_dir, path))
-    status = sprepo_git.git_operation("remote", "-v")
-    if url not in status:
+    _, remotelist = sprepo_git.git_operation("remote", "-v")
+    if url not in remotelist:
         sprepo_git.git_operation("remote", "add", "origin", url)
 
     topgit = os.path.join(gitroot, ".git")
@@ -181,6 +181,8 @@ def init_submodule_from_gitmodules(gitmodules, name, root_dir, logger):
     url = gitmodules.get(name, "url")
     assert path and url, f"Malformed .gitmodules file {path} {url}"
     tag = gitmodules.get(name, "fxtag")
+    if not tag:
+        tag = gitmodules.get(name, "hash")
     fxurl = gitmodules.get(name, "fxDONOTUSEurl")
     fxsparse = gitmodules.get(name, "fxsparse")
     fxrequired = gitmodules.get(name, "fxrequired")
@@ -213,7 +215,7 @@ def submodules_status(gitmodules, root_dir, toplevel=False, depth=0):
 
 def git_toplevelroot(root_dir, logger):
     rgit = GitInterface(root_dir, logger)
-    superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
+    _, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
     return superroot
 
 def submodules_update(gitmodules, root_dir, requiredlist, force):
@@ -250,7 +252,6 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
                 newrequiredlist = ["AlwaysRequired"]
                 if optional:
                     newrequiredlist.append("AlwaysOptional")
-
                 submodules_update(gitsubmodules, repodir, newrequiredlist, force=force)
 
 def local_mods_output():
@@ -342,7 +343,7 @@ def main():
         excludelist=excludelist,
     )
     if not gitmodules.sections():
-        sys.exit("No submodule components found")
+        sys.exit(f"No submodule components found, root_dir={root_dir}")
     retval = 0
     if action == "update":
         submodules_update(gitmodules, root_dir, fxrequired, force)

.lib/git-fleximod/git_fleximod/gitinterface.py

@@ -59,11 +59,12 @@ def git_operation(self, operation, *args, **kwargs):
         command = self._git_command(operation, *newargs)
         if isinstance(command, list):
             try:
-                return utils.execute_subprocess(command, output_to_caller=True)
+                status, output = utils.execute_subprocess(command, status_to_caller=True, output_to_caller=True)
+                return status, output.rstrip()
             except Exception as e:
                 sys.exit(e)
         else:
-            return command
+            return 0, command
 
     def config_get_value(self, section, name):
         if self._use_module:
@@ -81,6 +82,8 @@ def config_get_value(self, section, name):
     def config_set_value(self, section, name, value):
         if self._use_module:
             with self.repo.config_writer() as writer:
+                if "." in section:  
+                    section = section.replace("."," \"")+'"'           
                 writer.set_value(section, name, value)
             writer.release()  # Ensure changes are saved
         else:

.lib/git-fleximod/git_fleximod/submodule.py

@@ -60,8 +60,9 @@ def status(self):
         if not os.path.exists(os.path.join(smpath, ".git")):
             rootgit = GitInterface(self.root_dir, self.logger)
             # submodule commands use path, not name
-            tags = rootgit.git_operation("ls-remote", "--tags", self.url)
-            result = rootgit.git_operation("submodule","status",smpath).split()
+            status, tags = rootgit.git_operation("ls-remote", "--tags", self.url)
+            status, result = rootgit.git_operation("submodule","status",smpath)
+            result = result.split()
 
             if result:
                 ahash = result[0][1:]
@@ -80,9 +81,9 @@ def status(self):
                 result = f"e {self.name:>20} not checked out, aligned at tag {self.fxtag}{optional}"
                 needsupdate = True
             elif self.fxtag:
-                ahash = rootgit.git_operation(
+                status, ahash = rootgit.git_operation(
                     "submodule", "status", "{}".format(self.path)
-                ).rstrip()
+                )
                 ahash = ahash[1 : len(self.fxtag) + 1]
                 if self.fxtag == ahash:
                     result = f"e {self.name:>20} not checked out, aligned at hash {ahash}{optional}"
@@ -96,14 +97,15 @@ def status(self):
         else:
             with utils.pushd(smpath):
                 git = GitInterface(smpath, self.logger)
-                remote = git.git_operation("remote").rstrip()
+                status, remote = git.git_operation("remote")
                 if remote == '':
                     result = f"e {self.name:>20} has no associated remote"
                     testfails = True
                     needsupdate = True
                     return result, needsupdate, localmods, testfails                    
-                rurl = git.git_operation("ls-remote","--get-url").rstrip()
-                line = git.git_operation("log", "--pretty=format:\"%h %d\"").partition('\n')[0]
+                status, rurl = git.git_operation("ls-remote","--get-url")
+                status, lines = git.git_operation("log", "--pretty=format:\"%h %d\"")
+                line = lines.partition('\n')[0]
                 parts = line.split()
                 ahash = parts[0][1:]
                 atag = None
@@ -120,7 +122,7 @@ def status(self):
 
 
                 #print(f"line is {line} ahash is {ahash} atag is {atag} {parts}")
-                #                atag = git.git_operation("describe", "--tags", "--always").rstrip()
+                #                atag = git.git_operation("describe", "--tags", "--always")
                 # ahash =  git.git_operation("rev-list", "HEAD").partition("\n")[0]
 
                 recurse = False
@@ -149,10 +151,10 @@ def status(self):
                         result = f"e {self.name:>20} has no fxtag defined in .gitmodules, module at {ahash}"
                     testfails = False
 
-                status = git.git_operation("status", "--ignore-submodules", "-uno")
-                if "nothing to commit" not in status:
+                status, output = git.git_operation("status", "--ignore-submodules", "-uno")
+                if "nothing to commit" not in output:
                     localmods = True
-                    result = "M" + textwrap.indent(status, "                      ")
+                    result = "M" + textwrap.indent(output, "                      ")
 #        print(f"result {result} needsupdate {needsupdate} localmods {localmods} testfails {testfails}")
         return result, needsupdate, localmods, testfails
 
@@ -171,10 +173,11 @@ def _add_remote(self, git):
         Returns:
             str: The name of the new remote if added, or the name of the existing remote that matches the submodule's URL.
         """ 
-        remotes = git.git_operation("remote", "-v").splitlines()
+        status, remotes = git.git_operation("remote", "-v")
+        remotes = remotes.splitlines()
         upstream = None
         if remotes:
-            upstream = git.git_operation("ls-remote", "--get-url").rstrip()
+            status, upstream = git.git_operation("ls-remote", "--get-url")
             newremote = "newremote.00"
             tmpurl = self.url.replace("git@github.com:", "https://github.com/")
             line = next((s for s in remotes if self.url in s or tmpurl in s), None)
@@ -183,7 +186,7 @@ def _add_remote(self, git):
                 return newremote
             else:
                 i = 0
-                while "newremote" in remotes:
+                while newremote in remotes:
                     i = i + 1
                     newremote = f"newremote.{i:02d}"
         else:
@@ -214,12 +217,19 @@ def sparse_checkout(self):
         """ 
         self.logger.info("Called sparse_checkout for {}".format(self.name))
         rgit = GitInterface(self.root_dir, self.logger)
-        superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
+        status, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
         if superroot:
             gitroot = superroot.strip()
         else:
-            gitroot = self.root_dir.strip()
-        assert os.path.isdir(os.path.join(gitroot, ".git"))
+            gitroot = self.root_dir
+        # Now need to move the .git dir to the submodule location
+        rootdotgit = os.path.join(self.root_dir, ".git")
+        while os.path.isfile(rootdotgit):
+            with open(rootdotgit) as f:
+                line = f.readline().rstrip()
+                if line.startswith("gitdir: "):
+                    rootdotgit = os.path.abspath(os.path.join(self.root_dir,line[8:]))
+        assert os.path.isdir(rootdotgit)
         # first create the module directory
         if not os.path.isdir(os.path.join(self.root_dir, self.path)):
             os.makedirs(os.path.join(self.root_dir, self.path))
@@ -244,8 +254,8 @@ def sparse_checkout(self):
         # set the repository remote
 
         self.logger.info("Setting remote origin in {}/{}".format(self.root_dir, self.path))
-        status = sprepo_git.git_operation("remote", "-v")
-        if self.url not in status:
+        status, remotes = sprepo_git.git_operation("remote", "-v")
+        if self.url not in remotes:
             sprepo_git.git_operation("remote", "add", "origin", self.url)
 
         topgit = os.path.join(gitroot, ".git")
@@ -256,46 +266,46 @@ def sparse_checkout(self):
                     os.path.join(self.root_dir, f.read().split()[1]),
                     start=os.path.join(self.root_dir, self.path),
                 )
-                topgit = os.path.join(gitpath, "modules")
+                rootdotgit = os.path.join(gitpath, "modules", self.name)
         else:
-            topgit = os.path.relpath(
-                os.path.join(self.root_dir, ".git", "modules"),
+            rootdotgit = os.path.relpath(
+                os.path.join(self.root_dir, ".git", "modules", self.name),
                 start=os.path.join(self.root_dir, self.path),
             )
 
-        with utils.pushd(sprep_repo):
-            if not os.path.isdir(topgit):
-                os.makedirs(topgit)
-        topgit += os.sep + self.name
-
         if os.path.isdir(os.path.join(self.root_dir, self.path, ".git")):
             with utils.pushd(sprep_repo):
-                if os.path.isdir(os.path.join(topgit,".git")):
-                    shutil.rmtree(os.path.join(topgit,".git"))
-                shutil.move(".git", topgit)
+                if os.path.isdir(os.path.join(rootdotgit,".git")):
+                    shutil.rmtree(os.path.join(rootdotgit,".git"))
+                shutil.move(".git", rootdotgit)
                 with open(".git", "w") as f:
-                    f.write("gitdir: " + os.path.relpath(topgit))
-                #    assert(os.path.isdir(os.path.relpath(topgit, start=sprep_repo)))
-                gitsparse = os.path.abspath(os.path.join(topgit, "info", "sparse-checkout"))
+                    f.write("gitdir: " + os.path.relpath(rootdotgit))
+                infodir = os.path.join(rootdotgit, "info")
+                if not os.path.isdir(infodir):
+                    os.makedirs(infodir)
+                gitsparse = os.path.abspath(os.path.join(infodir, "sparse-checkout"))
             if os.path.isfile(gitsparse):
                 self.logger.warning(
-                    "submodule {} is already initialized {}".format(self.name, topgit)
+                    "submodule {} is already initialized {}".format(self.name, rootdotgit)
                 )
                 return
 
             with utils.pushd(sprep_repo):
                 if os.path.isfile(self.fxsparse):
+                    
                     shutil.copy(self.fxsparse, gitsparse)
 
 
         # Finally checkout the repo
         sprepo_git.git_operation("fetch", "origin", "--tags")
-        sprepo_git.git_operation("checkout", self.fxtag)
-
-        print(f"Successfully checked out {self.name:>20} at {self.fxtag}")
-        rgit.config_set_value(f'submodule "{self.name}"', "active", "true")
-        rgit.config_set_value(f'submodule "{self.name}"', "url", self.url)
-        rgit.config_set_value(f'submodule "{self.name}"', "path", self.path)
+        status,_ = sprepo_git.git_operation("checkout", self.fxtag)
+        if status:
+            print(f"Error checking out {self.name:>20} at {self.fxtag}")
+        else:
+            print(f"Successfully checked out {self.name:>20} at {self.fxtag}")
+        rgit.config_set_value('submodule.' + self.name, "active", "true")
+        rgit.config_set_value('submodule.' + self.name, "url", self.url)
+        rgit.config_set_value('submodule.' + self.name, "path", self.path)
 
     def update(self):
         """
@@ -342,15 +352,15 @@ def update(self):
                     git.git_operation("clone", self.url, self.path)
                     smgit = GitInterface(repodir, self.logger)
                     if not tag:
-                        tag = smgit.git_operation("describe", "--tags", "--always").rstrip()
+                        status, tag = smgit.git_operation("describe", "--tags", "--always")
                     smgit.git_operation("checkout", tag)
                     # Now need to move the .git dir to the submodule location
                     rootdotgit = os.path.join(self.root_dir, ".git")
                     if os.path.isfile(rootdotgit):
                         with open(rootdotgit) as f:
                             line = f.readline()
                             if line.startswith("gitdir: "):
-                                rootdotgit = line[8:].rstrip()
+                                rootdotgit = line[8:]
 
                     newpath = os.path.abspath(os.path.join(self.root_dir, rootdotgit, "modules", self.name))
                     if os.path.exists(newpath):
@@ -393,15 +403,16 @@ def update(self):
                 git = GitInterface(submoddir, self.logger)
                 # first make sure the url is correct
                 newremote = self._add_remote(git)
-                tags = git.git_operation("tag", "-l")
+                status, tags = git.git_operation("tag", "-l")
                 fxtag = self.fxtag
                 if fxtag and fxtag not in tags:
                     git.git_operation("fetch", newremote, "--tags")
-                atag = git.git_operation("describe", "--tags", "--always").rstrip()
+                status, atag = git.git_operation("describe", "--tags", "--always")
                 if fxtag and fxtag != atag:
                     try:
-                        git.git_operation("checkout", fxtag)
-                        print(f"{self.name:>20} updated to {fxtag}")
+                        status, _ = git.git_operation("checkout", fxtag)
+                        if not status:
+                            print(f"{self.name:>20} updated to {fxtag}")
                     except Exception as error:
                         print(error)
 

.lib/git-fleximod/git_fleximod/utils.py

@@ -307,12 +307,12 @@ def execute_subprocess(commands, status_to_caller=False, output_to_caller=False)
         # simple status check. If returning, it is the callers
         # responsibility determine if an error occurred and handle it
         # appropriately.
+        msg_context = (
+            "Process did not run successfully; "
+            "returned status {0}".format(error.returncode)
+        )
+        msg = failed_command_msg(msg_context, commands, output=error.output)
         if not return_to_caller:
-            msg_context = (
-                "Process did not run successfully; "
-                "returned status {0}".format(error.returncode)
-            )
-            msg = failed_command_msg(msg_context, commands, output=error.output)
             logging.error(error)
             logging.error(msg)
             log_process_output(error.output)