Add support for branch run type in EAMxx

components/eamxx/cime_config/namelist_defaults_scream.xml

@@ -521,7 +521,6 @@ be lost if SCREAM_HACK_XML is not enabled.
     <!-- CA RRM grid -->
     <topography_filename hgrid="ne0np4_CAx32v1">${DIN_LOC_ROOT}/atm/cam/topo/USGS-gtopo30_CA_ne32_x32_v1_pg2_16xdel2.nc</topography_filename>
 
-    <restart_casename>${CASE}.scream</restart_casename>
     <!-- Fields that we need to initialize, but are not in an initial condition file need to be inited here -->
     <surf_evap>0.0</surf_evap> <!-- TODO, Delete this when the IC fixes come in -->
     <precip_liq_surf_mass>0.0</precip_liq_surf_mass>
@@ -607,7 +606,6 @@ be lost if SCREAM_HACK_XML is not enabled.
   <Scorpio>
     <output_yaml_files type="array(string)"/>
     <model_restart>
-      <filename_prefix>./${CASE}.scream</filename_prefix>
       <iotype>default</iotype>
       <output_control locked="true">
         <Frequency>${REST_N}</Frequency>

components/eamxx/src/control/atmosphere_driver.cpp

@@ -153,17 +153,31 @@ init_scorpio(const int atm_id)
 }
 
 void AtmosphereDriver::
-init_time_stamps (const util::TimeStamp& run_t0, const util::TimeStamp& case_t0)
+init_time_stamps (const util::TimeStamp& run_t0, const util::TimeStamp& case_t0, int run_type)
 {
   m_atm_logger->info("  [EAMxx] Run  start time stamp: " + run_t0.to_string());
   m_atm_logger->info("  [EAMxx] Case start time stamp: " + case_t0.to_string());
 
+  EKAT_REQUIRE_MSG (case_t0<=run_t0,
+      "Error! Case t0 time stamp must precede the run t0 time stamp.\n"
+      "  - case t0: " + case_t0.to_string() + "\n"
+      "  - run  t0: " + run_t0.to_string() + "\n");
+
   // Initialize time stamps
   m_run_t0 = m_current_ts = run_t0;
   m_case_t0 = case_t0;
-}
-
 
+  switch (run_type) {
+    case 0:
+      m_run_type = RunType::Initial; break;
+    case 1:
+      m_run_type = RunType::Restart; break;
+    case -1:
+      m_run_type = case_t0==run_t0 ? RunType::Initial : RunType::Restart; break;
+    default:
+      EKAT_ERROR_MSG ("Unsupported/unrecognized run_type: " + std::to_string(run_type) + "\n");
+  }
+}
 
 void AtmosphereDriver::
 setup_iop ()
@@ -243,10 +257,11 @@ void AtmosphereDriver::create_grids()
   // To avoid having to pass the same data twice in the input file,
   // we have the AD add the IC file name to the GM params
   const auto& ic_pl = m_atm_params.sublist("initial_conditions");
-  if (m_case_t0<m_run_t0) {
+  if (m_run_type==RunType::Restart) {
     // Restarted run -> read geo data from restart file
-    const auto& casename = ic_pl.get<std::string>("restart_casename");
-    auto filename = find_filename_in_rpointer (casename,true,m_atm_comm,m_run_t0);
+    const auto& provenance = m_atm_params.sublist("provenance");
+    const auto& casename = provenance.get<std::string>("rest_caseid");
+    auto filename = find_filename_in_rpointer (casename+".scream",true,m_atm_comm,m_run_t0);
     gm_params.set("ic_filename", filename);
     m_atm_params.sublist("provenance").set("initial_conditions_file",filename);
   } else if (ic_pl.isParameter("Filename")) {
@@ -697,6 +712,7 @@ void AtmosphereDriver::create_output_managers () {
   if (io_params.isSublist("model_restart")) {
     // Create model restart manager
     auto params = io_params.sublist("model_restart");
+    params.set<std::string>("filename_prefix",m_casename+".scream");
     params.set<std::string>("Averaging Type","Instant");
     params.sublist("provenance") = m_atm_params.sublist("provenance");
 
@@ -789,16 +805,19 @@ void AtmosphereDriver::initialize_output_managers () {
 
 void AtmosphereDriver::
 set_provenance_data (std::string caseid,
+                     std::string rest_caseid,
                      std::string hostname,
                      std::string username)
 {
 #ifdef SCREAM_CIME_BUILD
   // Check the inputs are valid
   EKAT_REQUIRE_MSG (caseid!="", "Error! Invalid case id: " + caseid + "\n");
+  EKAT_REQUIRE_MSG (m_run_type==RunType::Initial or rest_caseid!="",
+      "Error! Invalid restart case id: " + rest_caseid + "\n");
   EKAT_REQUIRE_MSG (hostname!="", "Error! Invalid hostname: " + hostname + "\n");
   EKAT_REQUIRE_MSG (username!="", "Error! Invalid username: " + username + "\n");
 #else
-  caseid = "EAMxx standalone";
+  caseid = rest_caseid = m_casename;
   char* user = new char[32];
   char* host = new char[256];
   int err;
@@ -819,6 +838,7 @@ set_provenance_data (std::string caseid,
 #endif
   auto& provenance = m_atm_params.sublist("provenance");
   provenance.set("caseid",caseid);
+  provenance.set("rest_caseid",rest_caseid);
   provenance.set("hostname",hostname);
   provenance.set("username",username);
   provenance.set("version",std::string(EAMXX_GIT_VERSION));
@@ -834,7 +854,9 @@ initialize_fields ()
   start_timer("EAMxx::initialize_fields");
 
   // See the [rrtmgp active gases] note in share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp
-  if (fvphyshack) fv_phys_rrtmgp_active_gases_set_restart(m_case_t0 < m_run_t0);
+  if (fvphyshack) {
+    TraceGasesWorkaround::singleton().run_type = m_run_type;
+  }
 
   // See if we need to print a DAG. We do this first, cause if any input
   // field is missing from the initial condition file, an error will be thrown.
@@ -859,7 +881,7 @@ initialize_fields ()
   }
 
   // Initialize fields
-  if (m_case_t0<m_run_t0) {
+  if (m_run_type==RunType::Restart) {
     restart_model ();
   } else {
     set_initial_conditions ();
@@ -924,8 +946,9 @@ void AtmosphereDriver::restart_model ()
   m_atm_logger->info("  [EAMxx] restart_model ...");
 
   // First, figure out the name of the netcdf file containing the restart data
-  const auto& casename = m_atm_params.sublist("initial_conditions").get<std::string>("restart_casename");
-  auto filename = find_filename_in_rpointer (casename,true,m_atm_comm,m_run_t0);
+  const auto& provenance = m_atm_params.sublist("provenance");
+  const auto& casename = provenance.get<std::string>("rest_caseid");
+  auto filename = find_filename_in_rpointer (casename+".scream",true,m_atm_comm,m_run_t0);
 
   m_atm_logger->info("    [EAMxx] Restart filename: " + filename);
 
@@ -1555,15 +1578,13 @@ void AtmosphereDriver::initialize_atm_procs ()
   m_memory_buffer->allocate();
   m_atm_process_group->init_buffers(*m_memory_buffer);
 
-  const bool restarted_run = m_case_t0 < m_run_t0;
-
   // Setup SurfaceCoupling import and export (if they exist)
   if (m_surface_coupling_import_data_manager || m_surface_coupling_export_data_manager) {
     setup_surface_coupling_processes();
   }
 
   // Initialize the processes
-  m_atm_process_group->initialize(m_current_ts, restarted_run ? RunType::Restarted : RunType::Initial);
+  m_atm_process_group->initialize(m_current_ts, m_run_type);
 
   // Create and add energy and mass conservation check to appropriate atm procs
   setup_column_conservation_checks();

components/eamxx/src/control/atmosphere_driver.hpp

@@ -66,7 +66,8 @@ class AtmosphereDriver
   void set_params (const ekat::ParameterList& params);
 
   // Init time stamps
-  void init_time_stamps (const util::TimeStamp& run_t0, const util::TimeStamp& case_t0);
+  // run_type: -1: deduce from run/case t0, 0: initial, 1: restart
+  void init_time_stamps (const util::TimeStamp& run_t0, const util::TimeStamp& case_t0, int run_type = -1);
 
   // Set AD params
   void init_scorpio (const int atm_id = 0);
@@ -113,6 +114,7 @@ class AtmosphereDriver
   void add_additional_column_data_to_property_checks ();
 
   void set_provenance_data (std::string caseid = "",
+                            std::string rest_caseid = "",
                             std::string hostname = "",
                             std::string username = "");
 
@@ -224,6 +226,7 @@ class AtmosphereDriver
   // restarted runs, the latter is "older" than the former
   util::TimeStamp                           m_run_t0;
   util::TimeStamp                           m_case_t0;
+  RunType                                   m_run_type;
 
   // This is the comm containing all (and only) the processes assigned to the atmosphere
   ekat::Comm                                m_atm_comm;

components/eamxx/src/dynamics/homme/eamxx_homme_fv_phys.cpp

@@ -233,9 +233,12 @@ void HommeDynamics::remap_fv_phys_to_dyn () const {
 
 // See the [rrtmgp active gases] note in share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp
 void HommeDynamics
-::fv_phys_rrtmgp_active_gases_init (const std::shared_ptr<const GridsManager>& gm) {
+::fv_phys_rrtmgp_active_gases_init (const std::shared_ptr<const GridsManager>& gm)
+{
+  // NOTE: we would like to avoid this if it's a restart run, but at this point of the
+  //       init sequence we still don't know the run type. So we must add the trace gases
+  //       fields, and we will deal with them later
   auto& trace_gases_workaround = TraceGasesWorkaround::singleton();
-  if (trace_gases_workaround.is_restart()) return; // always false b/c it hasn't been set yet
   using namespace ekat::units;
   using namespace ShortFieldTagsNames;
   const auto& rgn = m_cgll_grid->name();
@@ -254,15 +257,15 @@ ::fv_phys_rrtmgp_active_gases_init (const std::shared_ptr<const GridsManager>& g
 }
 
 // See the [rrtmgp active gases] note in share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp
-void HommeDynamics::fv_phys_rrtmgp_active_gases_remap () {
+void HommeDynamics::fv_phys_rrtmgp_active_gases_remap (const RunType run_type) {
   // Note re: restart: Ideally, we'd know if we're restarting before having to
   // call add_field above. However, we only find out after. Because the pg2
   // field was declared Updated, it will read the restart data. But we don't
   // actually want to remap from CGLL to pg2 now. So if restarting, just do the
   // cleanup part at the end.
   auto& trace_gases_workaround = TraceGasesWorkaround::singleton();
   const auto& rgn = m_cgll_grid->name();
-  if (not trace_gases_workaround.is_restart()) {
+  if (run_type==RunType::Initial) {
     using namespace ShortFieldTagsNames;
     const auto& dgn = m_dyn_grid ->name();
     const auto& pgn = m_phys_grid->name();

components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp

@@ -224,8 +224,6 @@ apply_iop_forcing(const Real dt)
   using ESU = ekat::ExeSpaceUtils<KT::ExeSpace>;
   using PF = PhysicsFunctions<DefaultDevice>;
   using ColOps = ColumnOps<DefaultDevice, Real>;
-  using C = physics::Constants<Real>;
-  constexpr Real Rair = C::Rair;
 
   // Homme objects
   const auto& c = Homme::Context::singleton();
@@ -239,7 +237,6 @@ apply_iop_forcing(const Real dt)
   const auto nelem    = m_dyn_grid->get_num_local_dofs()/(NGP*NGP);
   const auto total_levels = m_dyn_grid->get_num_vertical_levels();
   const auto qsize = params.qsize;
-  const auto theta_hydrostatic_mode = params.theta_hydrostatic_mode;
 
   // Sanity checks since we will be switching between ekat::Pack
   // and Homme::Scalar view types

components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp

@@ -444,7 +444,7 @@ void HommeDynamics::initialize_impl (const RunType run_type)
   prim_init_model_f90 ();
 
   if (fv_phys_active()) {
-    fv_phys_dyn_to_fv_phys(run_type != RunType::Initial);
+    fv_phys_dyn_to_fv_phys(run_type == RunType::Restart);
     // [CGLL ICs in pg2] Remove the CGLL fields from the process. The AD has a
     // separate fvphyshack-based line to remove the whole CGLL FM. The intention
     // is to clear the view memory on the device, but I don't know if these two
@@ -457,7 +457,7 @@ void HommeDynamics::initialize_impl (const RunType run_type)
     for (const auto& f : {"horiz_winds", "T_mid", "ps", "phis", "pseudo_density"})
       remove_field(f, rgn);
     remove_group("tracers", rgn);
-    fv_phys_rrtmgp_active_gases_remap();
+    fv_phys_rrtmgp_active_gases_remap(run_type);
   }
 
   // Set up field property checks

components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp

@@ -103,7 +103,7 @@ class HommeDynamics : public AtmosphereProcess
   void fv_phys_post_process();
   // See [rrtmgp active gases] in eamxx_homme_fv_phys.cpp.
   void fv_phys_rrtmgp_active_gases_init(const std::shared_ptr<const GridsManager>& gm);
-  void fv_phys_rrtmgp_active_gases_remap();
+  void fv_phys_rrtmgp_active_gases_remap (const RunType run_type);
 
   // Rayleigh friction functions
   void rayleigh_friction_init ();

components/eamxx/src/mct_coupling/atm_comp_mct.F90

@@ -95,7 +95,8 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename )
     use ekat_string_utils,  only: string_f2c
     use mct_mod,            only: mct_aVect_init, mct_gsMap_lsize
     use seq_flds_mod,       only: seq_flds_a2x_fields, seq_flds_x2a_fields
-    use seq_infodata_mod,   only: seq_infodata_start_type_start, seq_infodata_start_type_cont
+    use seq_infodata_mod,   only: seq_infodata_start_type_start, seq_infodata_start_type_cont, &
+                                  seq_infodata_start_type_brnch
     use seq_comm_mct,       only: seq_comm_inst, seq_comm_name, seq_comm_suffix
     use shr_file_mod,       only: shr_file_getunit, shr_file_setIO
     use shr_sys_mod,        only: shr_sys_abort
@@ -136,9 +137,9 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename )
     ! TODO: read this from the namelist?
     character(len=256)                :: yaml_fname = "./data/scream_input.yaml"
     character(kind=c_char,len=256), target :: yaml_fname_c, atm_log_fname_c
-    character(len=256) :: caseid, username, hostname
-    character(kind=c_char,len=256), target :: caseid_c, username_c, hostname_c, calendar_c
-    logical (kind=c_bool) :: restarted_run
+    character(len=256) :: caseid, username, hostname, rest_caseid
+    character(kind=c_char,len=256), target :: caseid_c, username_c, hostname_c, calendar_c, rest_caseid_c
+    integer (kind=c_int) :: run_type_c
     !-------------------------------------------------------------------------------
 
     ! Grab some data from the cdata structure (coming from the coupler)
@@ -149,7 +150,8 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename )
          dom=dom_atm, &
          infodata=infodata)
     call seq_infodata_getData(infodata, atm_phase=phase, start_type=run_type, &
-                              username=username, case_name=caseid, hostname=hostname)
+                              username=username, case_name=caseid, &
+                              rest_case_name=rest_caseid, hostname=hostname)
     call seq_infodata_PutData(infodata, atm_aero=.true.)
     call seq_infodata_PutData(infodata, atm_prognostic=.true.)
 
@@ -189,6 +191,17 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename )
     ! Initialize atm
     !----------------------------------------------------------------------------
 
+    if (trim(run_type) == trim(seq_infodata_start_type_start)) then
+      run_type_c = 0
+    else if (trim(run_type) == trim(seq_infodata_start_type_cont) ) then
+      run_type_c = 1
+    else if (trim(run_type) == trim(seq_infodata_start_type_brnch) ) then
+      run_type_c = 1
+    else
+      print *, "[eamxx] ERROR! Unsupported run type: "//trim(run_type)
+      call mpi_abort(mpicom_atm,ierr,mpi_ierr)
+    endif
+
     ! Init the AD
     call seq_timemgr_EClockGetData(EClock, calendar=calendar, &
                                    curr_ymd=cur_ymd, curr_tod=cur_tod, &
@@ -197,13 +210,13 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename )
     call string_f2c(calendar,calendar_c)
     call string_f2c(trim(atm_log_fname),atm_log_fname_c)
     call string_f2c(trim(caseid),caseid_c)
+    call string_f2c(trim(rest_caseid),rest_caseid_c)
     call string_f2c(trim(hostname),hostname_c)
     call string_f2c(trim(username),username_c)
-    call scream_create_atm_instance (mpicom_atm, ATM_ID, yaml_fname_c, atm_log_fname_c, &
+    call scream_create_atm_instance (mpicom_atm, ATM_ID, yaml_fname_c, atm_log_fname_c, run_type_c, &
                           INT(cur_ymd,kind=C_INT),  INT(cur_tod,kind=C_INT), &
                           INT(case_start_ymd,kind=C_INT), INT(case_start_tod,kind=C_INT), &
-                          calendar_c, caseid_c, hostname_c, username_c)
-
+                          calendar_c, caseid_c, rest_caseid_c, hostname_c, username_c)
 
     ! Init MCT gsMap
     call atm_Set_gsMap_mct (mpicom_atm, ATM_ID, gsMap_atm)
@@ -224,16 +237,6 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename )
     call mct_aVect_init(x2a, rList=seq_flds_x2a_fields, lsize=lsize)
     call mct_aVect_init(a2x, rList=seq_flds_a2x_fields, lsize=lsize)
 
-    ! Complete AD initialization based on run type
-    if (trim(run_type) == trim(seq_infodata_start_type_start)) then
-      restarted_run = .false.
-    else if (trim(run_type) == trim(seq_infodata_start_type_cont) ) then
-      restarted_run = .true.
-    else
-      print *, "[eamxx] ERROR! Unsupported starttype: "//trim(run_type)
-      call mpi_abort(mpicom_atm,ierr,mpi_ierr)
-    endif
-
     ! Init surface coupling stuff in the AD
     call scream_set_cpl_indices (x2a, a2x)
 

components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp

@@ -101,12 +101,14 @@ extern "C"
 void scream_create_atm_instance (const MPI_Fint f_comm, const int atm_id,
                                  const char* input_yaml_file,
                                  const char* atm_log_file,
+                                 const int run_type,
                                  const int run_start_ymd,
                                  const int run_start_tod,
                                  const int case_start_ymd,
                                  const int case_start_tod,
                                  const char* calendar_name,
                                  const char* caseid,
+                                 const char* rest_caseid,
                                  const char* hostname,
                                  const char* username)
 {
@@ -173,8 +175,8 @@ void scream_create_atm_instance (const MPI_Fint f_comm, const int atm_id,
     ad.set_comm(atm_comm);
     ad.set_params(scream_params);
     ad.init_scorpio(atm_id);
-    ad.init_time_stamps(run_t0,case_t0);
-    ad.set_provenance_data (caseid,hostname,username);
+    ad.init_time_stamps(run_t0,case_t0,run_type);
+    ad.set_provenance_data (caseid,rest_caseid,hostname,username);
     ad.create_output_managers ();
     ad.create_atm_processes ();
     ad.create_grids ();
@@ -293,7 +295,7 @@ void scream_finalize (/* args ? */) {
 
 // Get the local (i.e., on current atm rank only) number of physics columns
 int scream_get_num_local_cols () {
-  int ncols;
+  int ncols = -1;
   fpe_guard_wrapper([&]() {
     const auto& ad = get_ad();
     const auto& gm = ad.get_grids_manager();
@@ -307,7 +309,7 @@ int scream_get_num_local_cols () {
 
 // Get the global (i.e., the whole earth) number of physics columns
 int scream_get_num_global_cols () {
-  int ncols;
+  int ncols = -1;
   fpe_guard_wrapper([&]() {
     const auto& ad = get_ad();
     const auto& gm = ad.get_grids_manager();