Add case git

CIME/BuildTools/configure.py

@@ -28,7 +28,7 @@
 from CIME.XML.files import Files
 from CIME.build import CmakeTmpBuildDir
 
-import shutil
+import shutil, glob
 
 logger = logging.getLogger(__name__)
 
@@ -72,6 +72,12 @@ def configure(
         output_cmake_macros_dir = os.path.join(output_dir, "cmake_macros")
         if not os.path.exists(output_cmake_macros_dir):
             shutil.copytree(new_cmake_macros_dir, output_cmake_macros_dir)
+            ccs_mach_dir = os.path.join(
+                new_cmake_macros_dir, "..", machobj.get_machine_name()
+            )
+            for f in glob.iglob(os.path.join(ccs_mach_dir, "*.cmake")):
+                print(f"copying {f} to {output_dir}")
+                safe_copy(f, output_dir)
 
         copy_local_macros_to_dir(
             output_cmake_macros_dir, extra_machdir=extra_machines_dir

CIME/SystemTests/funit.py

@@ -5,7 +5,8 @@
 from CIME.XML.standard_module_setup import *
 from CIME.SystemTests.system_tests_common import SystemTestsCommon
 from CIME.build import post_build
-from CIME.utils import append_testlog, get_cime_root
+from CIME.status import append_testlog
+from CIME.utils import get_cime_root
 from CIME.test_status import *
 
 logger = logging.getLogger(__name__)

CIME/SystemTests/hommebaseclass.py

@@ -4,11 +4,11 @@
 from CIME.XML.standard_module_setup import *
 from CIME.SystemTests.system_tests_common import SystemTestsCommon
 from CIME.build import post_build
-from CIME.utils import append_testlog, SharedArea
+from CIME.status import append_testlog
+from CIME.utils import SharedArea
 from CIME.test_status import *
 
 import shutil
-from distutils import dir_util
 
 logger = logging.getLogger(__name__)
 
@@ -97,10 +97,9 @@ def run_phase(self):
                     shutil.rmtree(full_baseline_dir)
 
                 with SharedArea():
-                    dir_util.copy_tree(
+                    shutil.copytree(
                         os.path.join(exeroot, "tests", "baseline"),
                         full_baseline_dir,
-                        preserve_mode=False,
                     )
 
         elif compare:

CIME/SystemTests/mvk.py

@@ -10,10 +10,11 @@
 import json
 import logging
 
-from distutils import dir_util
+from shutils import copytree
 
 import CIME.test_status
 import CIME.utils
+from CIME.status import append_testlog
 from CIME.SystemTests.system_tests_common import SystemTestsCommon
 from CIME.case.case_setup import case_setup
 from CIME.XML.machines import Machines
@@ -173,10 +174,9 @@ def _compare_baseline(self):
             urlroot = CIME.utils.get_urlroot(mach_obj)
             if htmlroot is not None:
                 with CIME.utils.SharedArea():
-                    dir_util.copy_tree(
+                    copytree(
                         evv_out_dir,
                         os.path.join(htmlroot, "evv", case_name),
-                        preserve_mode=False,
                     )
                 if urlroot is None:
                     urlroot = "[{}_URL]".format(mach_name.capitalize())
@@ -203,4 +203,4 @@ def _compare_baseline(self):
                 )
             )
 
-            CIME.utils.append_testlog(comments, self._orig_caseroot)
+            append_testlog(comments, self._orig_caseroot)

CIME/SystemTests/pgn.py

@@ -16,14 +16,15 @@
 import logging
 
 from collections import OrderedDict
-from distutils import dir_util
+from shutils import copytree
 
 import pandas as pd
 import numpy as np
 
 
 import CIME.test_status
 import CIME.utils
+from CIME.status import append_testlog
 from CIME.SystemTests.system_tests_common import SystemTestsCommon
 from CIME.case.case_setup import case_setup
 from CIME.XML.machines import Machines
@@ -80,7 +81,7 @@ def build_phase(self, sharedlib_only=False, model_only=False):
             if not model_only:
                 # Lay all of the components out concurrently
                 logger.debug(
-                    "PGN_INFO: Updating NINST for multi-instance in " "env_mach_pes.xml"
+                    "PGN_INFO: Updating NINST for multi-instance in env_mach_pes.xml"
                 )
                 for comp in ["ATM", "OCN", "WAV", "GLC", "ICE", "ROF", "LND"]:
                     ntasks = self._case.get_value("NTASKS_{}".format(comp))
@@ -224,10 +225,9 @@ def _compare_baseline(self):
             urlroot = CIME.utils.get_urlroot(mach_obj)
             if htmlroot is not None:
                 with CIME.utils.SharedArea():
-                    dir_util.copy_tree(
+                    copytree(
                         evv_out_dir,
                         os.path.join(htmlroot, "evv", case_name),
-                        preserve_mode=False,
                     )
                 if urlroot is None:
                     urlroot = "[{}_URL]".format(mach_name.capitalize())
@@ -253,7 +253,7 @@ def _compare_baseline(self):
                 )
             )
 
-            CIME.utils.append_testlog(comments, self._orig_caseroot)
+            append_testlog(comments, self._orig_caseroot)
 
     def run_phase(self):
         logger.debug("PGN_INFO: RUN PHASE")

CIME/SystemTests/system_tests_common.py

@@ -4,8 +4,8 @@
 from CIME.XML.standard_module_setup import *
 from CIME.XML.env_run import EnvRun
 from CIME.XML.env_test import EnvTest
+from CIME.status import append_testlog
 from CIME.utils import (
-    append_testlog,
     get_model,
     safe_copy,
     get_timestamp,

CIME/SystemTests/tsc.py

@@ -11,10 +11,11 @@
 import json
 import logging
 
-from distutils import dir_util
+from shutil import copytree
 
 import CIME.test_status
 import CIME.utils
+from CIME.status import append_testlog
 from CIME.SystemTests.system_tests_common import SystemTestsCommon
 from CIME.case.case_setup import case_setup
 from CIME.hist_utils import rename_all_hist_files
@@ -213,10 +214,9 @@ def _compare_baseline(self):
             urlroot = CIME.utils.get_urlroot(mach_obj)
             if htmlroot is not None:
                 with CIME.utils.SharedArea():
-                    dir_util.copy_tree(
+                    copytree(
                         evv_out_dir,
                         os.path.join(htmlroot, "evv", case_name),
-                        preserve_mode=False,
                     )
                 if urlroot is None:
                     urlroot = "[{}_URL]".format(mach_name.capitalize())
@@ -243,7 +243,7 @@ def _compare_baseline(self):
                 )
             )
 
-            CIME.utils.append_testlog(comments, self._orig_caseroot)
+            append_testlog(comments, self._orig_caseroot)
 
     def _generate_baseline(self):
         super(TSC, self)._generate_baseline()

CIME/Tools/xmlchange

@@ -20,6 +20,10 @@ Examples:
    compatibility; only works for a single variable at a time):
       ./xmlchange --id REST_N --val 4
 
+   To set a variable in which the value has commas, you'll need to use
+   the alternative syntax:
+      ./xmlchange --id VARNAME --val "one,two"
+
    Several xml variables that have settings for each component have somewhat special treatment.
    The variables that this currently applies to are:
     NTASKS, NTHRDS, ROOTPE, PIO_TYPENAME, PIO_STRIDE, PIO_NUMTASKS, PIO_ASYNC_INTERFACE
@@ -51,9 +55,10 @@ from standard_script_setup import *
 from CIME.utils import (
     expect,
     convert_to_type,
-    append_case_status,
     get_batch_script_for_job,
+    Timeout,
 )
+from CIME.status import append_case_status
 from CIME.case import Case
 from CIME.locked_files import check_lockedfiles
 
@@ -62,6 +67,7 @@ import re
 # Set logger
 logger = logging.getLogger("xmlchange")
 
+
 ###############################################################################
 def parse_command_line(args, description):
     ###############################################################################
@@ -241,6 +247,40 @@ def xmlchange_single_value(
     if xmlid == "JOB_QUEUE":
         case.set_value("USER_REQUESTED_QUEUE", xmlval, subgroup)
 
+    # if CASE_GIT_REPOSITORY is updated set the remote to the new repo
+    # note that if origin exists and is different than CASE_GIT_REPOSITORY it
+    # will be deleted and replaced
+    if xmlid == "CASE_GIT_REPOSITORY":
+        if not case._gitinterface:
+            logger.warning(
+                "You must run case.setup before you can set CASE_GIT_REPOSITORY"
+            )
+            return
+        remote = case._gitinterface.git_operation("remote")
+        if remote:
+            if remote == xmlval:
+                return
+            else:
+                case._gitinterface.git_operation("remote", "rm", remote)
+        branch = case._gitinterface.git_operation("branch").split()[1]
+        remote_branches = ""
+        with Timeout(30):
+            remote_branches = case._gitinterface.git_operation(
+                "ls-remote", "--heads", xmlval
+            )
+        chk = remote_branches.find("refs/heads/" + branch)
+
+        if chk >= 0:
+            expect(
+                False,
+                " Could not set {} to {}, remote branch {} already exists, Rename case and try again.".format(
+                    xmlid, xmlval, branch
+                ),
+            )
+        with Timeout(30):
+            case._gitinterface.git_operation("remote", "add", "origin", xmlval)
+            case._gitinterface.git_operation("push", "--set-upstream", "origin", branch)
+
 
 def xmlchange(
     caseroot,
@@ -255,7 +295,6 @@ def xmlchange(
     dryrun,
     non_local,
 ):
-
     with Case(caseroot, read_only=False, record=True, non_local=non_local) as case:
         comp_classes = case.get_values("COMP_CLASSES")
         env_test = None
@@ -275,7 +314,6 @@ def xmlchange(
 
             # Change values
             for setting in listofsettings:
-
                 pair = setting.split("=", 1)
                 expect(
                     len(pair) == 2,
@@ -310,18 +348,21 @@ def xmlchange(
             )
 
         check_lockedfiles(
-            case,
-            skip=["env_case"],
-            caseroot=caseroot,
-            whitelist=xmlfile,
+            case, skip=["env_case"], caseroot=caseroot, whitelist=xmlfile, quiet=True
         )
 
     if not noecho:
         argstr = ""
         for arg in sys.argv:
             argstr += "%s " % arg
         msg = "<command> %s </command>" % (argstr)
-        append_case_status("xmlchange", "success", msg=msg, caseroot=caseroot)
+        append_case_status(
+            "xmlchange",
+            "success",
+            msg=msg,
+            caseroot=caseroot,
+            gitinterface=case._gitinterface,
+        )
 
 
 def _main_func(description):

CIME/XML/batch.py

@@ -34,6 +34,8 @@ def __init__(
         if infile is None:
             infile = files.get_value("BATCH_SPEC_FILE")
 
+        config_dir = os.path.dirname(infile)
+
         schema = files.get_schema("BATCH_SPEC_FILE")
 
         GenericXML.__init__(self, infile, schema=schema)
@@ -50,12 +52,20 @@ def __init__(
         #
         # This could cause problems if node matches are repeated when only one is expected.
         infile = os.path.join(os.environ.get("HOME"), ".cime", "config_batch.xml")
+        usehome = False
         if os.path.exists(infile):
             GenericXML.read(self, infile)
+            usehome = True
+        useextra = False
         if extra_machines_dir:
             infile = os.path.join(extra_machines_dir, "config_batch.xml")
             if os.path.exists(infile):
                 GenericXML.read(self, infile)
+                useextra = True
+        if not usehome and not useextra:
+            batchfile = os.path.join(config_dir, self.machine, "config_batch.xml")
+            if os.path.exists(batchfile):
+                GenericXML.read(self, batchfile)
 
         if self.batch_system is not None:
             self.set_batch_system(self.batch_system, machine=machine)

CIME/XML/env_batch.py

@@ -919,7 +919,6 @@ def _submit_single_job(
         resubmit_immediate=False,
         workflow=True,
     ):
-
         if not dry_run:
             logger.warning("Submit job {}".format(job))
         batch_system = self.get_value("BATCH_SYSTEM", subgroup=None)
@@ -1080,19 +1079,19 @@ def _submit_single_job(
                 submitargs,
                 run_args,
                 batchredirect,
-                get_batch_script_for_job(job),
+                os.path.join(self._caseroot, get_batch_script_for_job(job)),
             )
         else:
             sequence = (
                 batchsubmit,
                 submitargs,
                 batchredirect,
-                get_batch_script_for_job(job),
+                os.path.join(self._caseroot, get_batch_script_for_job(job)),
                 run_args,
             )
 
         submitcmd = " ".join(s.strip() for s in sequence if s is not None)
-        if submitcmd.startswith("ssh"):
+        if submitcmd.startswith("ssh") and "$CASEROOT" in submitcmd:
             # add ` before cd $CASEROOT and at end of command
             submitcmd = submitcmd.replace("cd $CASEROOT", "'cd $CASEROOT") + "'"
 
@@ -1385,7 +1384,6 @@ def make_all_batch_files(self, case):
             template = case.get_resolved_value(
                 env_workflow.get_value("template", subgroup=job)
             )
-
             if os.path.isabs(template):
                 input_batch_script = template
             else:

CIME/build.py

@@ -4,11 +4,11 @@
 import glob, shutil, time, threading, subprocess
 from pathlib import Path
 from CIME.XML.standard_module_setup import *
+from CIME.status import run_and_log_case_status
 from CIME.utils import (
     get_model,
     analyze_build_log,
     stringify_bool,
-    run_and_log_case_status,
     get_timestamp,
     run_sub_or_cmd,
     run_cmd,
@@ -1318,13 +1318,18 @@ def case_build(
         cb = cb + " (SHAREDLIB_BUILD)"
     if model_only == True:
         cb = cb + " (MODEL_BUILD)"
-    return run_and_log_case_status(functor, cb, caseroot=caseroot)
+    return run_and_log_case_status(
+        functor, cb, caseroot=caseroot, gitinterface=case._gitinterface
+    )
 
 
 ###############################################################################
 def clean(case, cleanlist=None, clean_all=False, clean_depends=None):
     ###############################################################################
     functor = lambda: _clean_impl(case, cleanlist, clean_all, clean_depends)
     return run_and_log_case_status(
-        functor, "build.clean", caseroot=case.get_value("CASEROOT")
+        functor,
+        "build.clean",
+        caseroot=case.get_value("CASEROOT"),
+        gitinterface=case._gitinterface,
     )