Merge pull request #4660 from ESMCI/add_case_git

Add case git

Author: jedwards4b

CIME/SystemTests/funit.py

@@ -5,7 +5,8 @@
 from CIME.XML.standard_module_setup import *
 from CIME.SystemTests.system_tests_common import SystemTestsCommon
 from CIME.build import post_build
-from CIME.utils import append_testlog, get_cime_root
+from CIME.status import append_testlog
+from CIME.utils import get_cime_root
 from CIME.test_status import *
 
 logger = logging.getLogger(__name__)

CIME/SystemTests/hommebaseclass.py

@@ -4,11 +4,11 @@
 from CIME.XML.standard_module_setup import *
 from CIME.SystemTests.system_tests_common import SystemTestsCommon
 from CIME.build import post_build
-from CIME.utils import append_testlog, SharedArea
+from CIME.status import append_testlog
+from CIME.utils import SharedArea
 from CIME.test_status import *
 
 import shutil
-from distutils import dir_util
 
 logger = logging.getLogger(__name__)
 
@@ -97,10 +97,9 @@ def run_phase(self):
                     shutil.rmtree(full_baseline_dir)
 
                 with SharedArea():
-                    dir_util.copy_tree(
+                    shutil.copytree(
                         os.path.join(exeroot, "tests", "baseline"),
                         full_baseline_dir,
-                        preserve_mode=False,
                     )
 
         elif compare:

CIME/SystemTests/mvk.py

@@ -10,10 +10,11 @@
 import json
 import logging
 
-from distutils import dir_util
+from shutils import copytree
 
 import CIME.test_status
 import CIME.utils
+from CIME.status import append_testlog
 from CIME.SystemTests.system_tests_common import SystemTestsCommon
 from CIME.case.case_setup import case_setup
 from CIME.XML.machines import Machines
@@ -173,10 +174,9 @@ def _compare_baseline(self):
             urlroot = CIME.utils.get_urlroot(mach_obj)
             if htmlroot is not None:
                 with CIME.utils.SharedArea():
-                    dir_util.copy_tree(
+                    copytree(
                         evv_out_dir,
                         os.path.join(htmlroot, "evv", case_name),
-                        preserve_mode=False,
                     )
                 if urlroot is None:
                     urlroot = "[{}_URL]".format(mach_name.capitalize())
@@ -203,4 +203,4 @@ def _compare_baseline(self):
                 )
             )
 
-            CIME.utils.append_testlog(comments, self._orig_caseroot)
+            append_testlog(comments, self._orig_caseroot)

CIME/SystemTests/pgn.py

@@ -16,14 +16,15 @@
 import logging
 
 from collections import OrderedDict
-from distutils import dir_util
+from shutils import copytree
 
 import pandas as pd
 import numpy as np
 
 
 import CIME.test_status
 import CIME.utils
+from CIME.status import append_testlog
 from CIME.SystemTests.system_tests_common import SystemTestsCommon
 from CIME.case.case_setup import case_setup
 from CIME.XML.machines import Machines
@@ -80,7 +81,7 @@ def build_phase(self, sharedlib_only=False, model_only=False):
             if not model_only:
                 # Lay all of the components out concurrently
                 logger.debug(
-                    "PGN_INFO: Updating NINST for multi-instance in " "env_mach_pes.xml"
+                    "PGN_INFO: Updating NINST for multi-instance in env_mach_pes.xml"
                 )
                 for comp in ["ATM", "OCN", "WAV", "GLC", "ICE", "ROF", "LND"]:
                     ntasks = self._case.get_value("NTASKS_{}".format(comp))
@@ -224,10 +225,9 @@ def _compare_baseline(self):
             urlroot = CIME.utils.get_urlroot(mach_obj)
             if htmlroot is not None:
                 with CIME.utils.SharedArea():
-                    dir_util.copy_tree(
+                    copytree(
                         evv_out_dir,
                         os.path.join(htmlroot, "evv", case_name),
-                        preserve_mode=False,
                     )
                 if urlroot is None:
                     urlroot = "[{}_URL]".format(mach_name.capitalize())
@@ -253,7 +253,7 @@ def _compare_baseline(self):
                 )
             )
 
-            CIME.utils.append_testlog(comments, self._orig_caseroot)
+            append_testlog(comments, self._orig_caseroot)
 
     def run_phase(self):
         logger.debug("PGN_INFO: RUN PHASE")

CIME/SystemTests/system_tests_common.py

@@ -4,8 +4,8 @@
 from CIME.XML.standard_module_setup import *
 from CIME.XML.env_run import EnvRun
 from CIME.XML.env_test import EnvTest
+from CIME.status import append_testlog
 from CIME.utils import (
-    append_testlog,
     get_model,
     safe_copy,
     get_timestamp,

CIME/SystemTests/tsc.py

@@ -11,10 +11,11 @@
 import json
 import logging
 
-from distutils import dir_util
+from shutil import copytree
 
 import CIME.test_status
 import CIME.utils
+from CIME.status import append_testlog
 from CIME.SystemTests.system_tests_common import SystemTestsCommon
 from CIME.case.case_setup import case_setup
 from CIME.hist_utils import rename_all_hist_files
@@ -213,10 +214,9 @@ def _compare_baseline(self):
             urlroot = CIME.utils.get_urlroot(mach_obj)
             if htmlroot is not None:
                 with CIME.utils.SharedArea():
-                    dir_util.copy_tree(
+                    copytree(
                         evv_out_dir,
                         os.path.join(htmlroot, "evv", case_name),
-                        preserve_mode=False,
                     )
                 if urlroot is None:
                     urlroot = "[{}_URL]".format(mach_name.capitalize())
@@ -243,7 +243,7 @@ def _compare_baseline(self):
                 )
             )
 
-            CIME.utils.append_testlog(comments, self._orig_caseroot)
+            append_testlog(comments, self._orig_caseroot)
 
     def _generate_baseline(self):
         super(TSC, self)._generate_baseline()

CIME/Tools/xmlchange

@@ -55,9 +55,10 @@ from standard_script_setup import *
 from CIME.utils import (
     expect,
     convert_to_type,
-    append_case_status,
     get_batch_script_for_job,
+    Timeout,
 )
+from CIME.status import append_case_status
 from CIME.case import Case
 from CIME.locked_files import check_lockedfiles
 
@@ -246,6 +247,40 @@ def xmlchange_single_value(
     if xmlid == "JOB_QUEUE":
         case.set_value("USER_REQUESTED_QUEUE", xmlval, subgroup)
 
+    # if CASE_GIT_REPOSITORY is updated set the remote to the new repo
+    # note that if origin exists and is different than CASE_GIT_REPOSITORY it
+    # will be deleted and replaced
+    if xmlid == "CASE_GIT_REPOSITORY":
+        if not case._gitinterface:
+            logger.warning(
+                "You must run case.setup before you can set CASE_GIT_REPOSITORY"
+            )
+            return
+        remote = case._gitinterface.git_operation("remote")
+        if remote:
+            if remote == xmlval:
+                return
+            else:
+                case._gitinterface.git_operation("remote", "rm", remote)
+        branch = case._gitinterface.git_operation("branch").split()[1]
+        remote_branches = ""
+        with Timeout(30):
+            remote_branches = case._gitinterface.git_operation(
+                "ls-remote", "--heads", xmlval
+            )
+        chk = remote_branches.find("refs/heads/" + branch)
+
+        if chk >= 0:
+            expect(
+                False,
+                " Could not set {} to {}, remote branch {} already exists, Rename case and try again.".format(
+                    xmlid, xmlval, branch
+                ),
+            )
+        with Timeout(30):
+            case._gitinterface.git_operation("remote", "add", "origin", xmlval)
+            case._gitinterface.git_operation("push", "--set-upstream", "origin", branch)
+
 
 def xmlchange(
     caseroot,
@@ -321,7 +356,13 @@ def xmlchange(
         for arg in sys.argv:
             argstr += "%s " % arg
         msg = "<command> %s </command>" % (argstr)
-        append_case_status("xmlchange", "success", msg=msg, caseroot=caseroot)
+        append_case_status(
+            "xmlchange",
+            "success",
+            msg=msg,
+            caseroot=caseroot,
+            gitinterface=case._gitinterface,
+        )
 
 
 def _main_func(description):

CIME/XML/env_batch.py

@@ -919,7 +919,6 @@ def _submit_single_job(
         resubmit_immediate=False,
         workflow=True,
     ):
-
         if not dry_run:
             logger.warning("Submit job {}".format(job))
         batch_system = self.get_value("BATCH_SYSTEM", subgroup=None)
@@ -1080,19 +1079,19 @@ def _submit_single_job(
                 submitargs,
                 run_args,
                 batchredirect,
-                get_batch_script_for_job(job),
+                os.path.join(self._caseroot, get_batch_script_for_job(job)),
             )
         else:
             sequence = (
                 batchsubmit,
                 submitargs,
                 batchredirect,
-                get_batch_script_for_job(job),
+                os.path.join(self._caseroot, get_batch_script_for_job(job)),
                 run_args,
             )
 
         submitcmd = " ".join(s.strip() for s in sequence if s is not None)
-        if submitcmd.startswith("ssh"):
+        if submitcmd.startswith("ssh") and "$CASEROOT" in submitcmd:
             # add ` before cd $CASEROOT and at end of command
             submitcmd = submitcmd.replace("cd $CASEROOT", "'cd $CASEROOT") + "'"
 
@@ -1385,7 +1384,6 @@ def make_all_batch_files(self, case):
             template = case.get_resolved_value(
                 env_workflow.get_value("template", subgroup=job)
             )
-
             if os.path.isabs(template):
                 input_batch_script = template
             else:

CIME/build.py

@@ -4,11 +4,11 @@
 import glob, shutil, time, threading, subprocess
 from pathlib import Path
 from CIME.XML.standard_module_setup import *
+from CIME.status import run_and_log_case_status
 from CIME.utils import (
     get_model,
     analyze_build_log,
     stringify_bool,
-    run_and_log_case_status,
     get_timestamp,
     run_sub_or_cmd,
     run_cmd,
@@ -1318,13 +1318,18 @@ def case_build(
         cb = cb + " (SHAREDLIB_BUILD)"
     if model_only == True:
         cb = cb + " (MODEL_BUILD)"
-    return run_and_log_case_status(functor, cb, caseroot=caseroot)
+    return run_and_log_case_status(
+        functor, cb, caseroot=caseroot, gitinterface=case._gitinterface
+    )
 
 
 ###############################################################################
 def clean(case, cleanlist=None, clean_all=False, clean_depends=None):
     ###############################################################################
     functor = lambda: _clean_impl(case, cleanlist, clean_all, clean_depends)
     return run_and_log_case_status(
-        functor, "build.clean", caseroot=case.get_value("CASEROOT")
+        functor,
+        "build.clean",
+        caseroot=case.get_value("CASEROOT"),
+        gitinterface=case._gitinterface,
     )

CIME/case/case.py

@@ -13,10 +13,12 @@
 # pylint: disable=import-error,redefined-builtin
 from CIME import utils
 from CIME.config import Config
-from CIME.utils import expect, get_cime_root, append_status
+from CIME.status import append_status
+from CIME.utils import expect, get_cime_root
 from CIME.utils import convert_to_type, get_model, set_model
 from CIME.utils import get_project, get_charge_account, check_name
 from CIME.utils import get_current_commit, safe_copy, get_cime_default_driver
+from CIME.gitinterface import GitInterface
 from CIME.locked_files import LOCKED_DIR, lock_file
 from CIME.XML.machines import Machines
 from CIME.XML.pes import Pes
@@ -77,7 +79,7 @@ class Case(object):
 
     """
 
-    from CIME.case.case_setup import case_setup
+    from CIME.case.case_setup import case_setup, _create_case_repo
     from CIME.case.case_clone import create_clone, _copy_user_modified_to_clone
     from CIME.case.case_test import case_test
     from CIME.case.case_submit import check_DA_settings, check_case, submit
@@ -98,7 +100,6 @@ class Case(object):
     )
 
     def __init__(self, case_root=None, read_only=True, record=False, non_local=False):
-
         if case_root is None:
             case_root = os.getcwd()
         expect(
@@ -164,7 +165,7 @@ def __init__(self, case_root=None, read_only=True, record=False, non_local=False
         self._component_description = {}
         self._is_env_loaded = False
         self._loaded_envs = None
-
+        self._gitinterface = None
         # these are user_mods as defined in the compset
         # Command Line user_mods are handled seperately
 
@@ -199,6 +200,10 @@ def __init__(self, case_root=None, read_only=True, record=False, non_local=False
                             mach == probed_machine,
                             f"Current machine {probed_machine} does not match case machine {mach}.",
                         )
+                if os.path.exists(os.path.join(self.get_value("CASEROOT"), ".git")):
+                    self._gitinterface = GitInterface(
+                        self.get_value("CASEROOT"), logger
+                    )
 
             self.initialize_derived_attributes()
 
@@ -682,7 +687,6 @@ def _set_compset(self, compset_name, files):
         # Loop through all of the files listed in COMPSETS_SPEC_FILE and find the file
         # that has a match for either the alias or the longname in that order
         for component in components:
-
             # Determine the compsets file for this component
             compsets_filename = files.get_value(
                 "COMPSETS_SPEC_FILE", {"component": component}
@@ -1284,7 +1288,6 @@ def configure(
         gpu_type=None,
         gpu_offload=None,
     ):
-
         expect(
             check_name(compset_name, additional_chars="."),
             "Invalid compset name {}".format(compset_name),