From 3422a999eda3bc16f1ac68e75ece95bb7f99e502 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 24 Oct 2024 14:28:44 -0600 Subject: [PATCH 1/7] EAMxx: enable branch run type It's just like a restart run, but we may have a different case name --- .../cime_config/namelist_defaults_scream.xml | 1 - .../eamxx/src/control/atmosphere_driver.cpp | 34 +++++++++++----- .../eamxx/src/control/atmosphere_driver.hpp | 5 ++- .../eamxx/src/mct_coupling/atm_comp_mct.F90 | 39 ++++++++++--------- .../mct_coupling/scream_cxx_f90_interface.cpp | 6 ++- components/eamxx/src/share/scream_types.hpp | 2 +- 6 files changed, 55 insertions(+), 32 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 1b84aef2273..6d8558160b1 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -521,7 +521,6 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/cam/topo/USGS-gtopo30_CA_ne32_x32_v1_pg2_16xdel2.nc - ${CASE}.scream 0.0 0.0 diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index 0f59639a4b1..f6f9aa48ce0 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -153,17 +153,31 @@ init_scorpio(const int atm_id) } void AtmosphereDriver:: -init_time_stamps (const util::TimeStamp& run_t0, const util::TimeStamp& case_t0) +init_time_stamps (const util::TimeStamp& run_t0, const util::TimeStamp& case_t0, int run_type) { m_atm_logger->info(" [EAMxx] Run start time stamp: " + run_t0.to_string()); m_atm_logger->info(" [EAMxx] Case start time stamp: " + case_t0.to_string()); + EKAT_REQUIRE_MSG (case_t0<=run_t0, + "Error! Case t0 time stamp must precede the run t0 time stamp.\n" + " - case t0: " + case_t0.to_string() + "\n" + " - run t0: " + run_t0.to_string() + "\n"); + // Initialize time stamps m_run_t0 = m_current_ts = run_t0; m_case_t0 = case_t0; -} - + switch (run_type) { + case 0: + m_run_type = RunType::Initial; break; + case 1: + m_run_type = RunType::Restart; break; + case -1: + m_run_type = case_t0==run_t0 ? RunType::Initial : RunType::Restart; break; + default: + EKAT_ERROR_MSG ("Unsupported/unrecognized run_type: " + std::to_string(run_type) + "\n"); + } +} void AtmosphereDriver:: setup_iop () @@ -243,7 +257,7 @@ void AtmosphereDriver::create_grids() // To avoid having to pass the same data twice in the input file, // we have the AD add the IC file name to the GM params const auto& ic_pl = m_atm_params.sublist("initial_conditions"); - if (m_case_t0 read geo data from restart file const auto& casename = ic_pl.get("restart_casename"); auto filename = find_filename_in_rpointer (casename,true,m_atm_comm,m_run_t0); @@ -789,16 +803,19 @@ void AtmosphereDriver::initialize_output_managers () { void AtmosphereDriver:: set_provenance_data (std::string caseid, + std::string rest_caseid, std::string hostname, std::string username) { #ifdef SCREAM_CIME_BUILD // Check the inputs are valid EKAT_REQUIRE_MSG (caseid!="", "Error! Invalid case id: " + caseid + "\n"); + EKAT_REQUIRE_MSG (m_run_type==RunType::Initial or rest_caseid!="", + "Error! Invalid restart case id: " + rest_caseid + "\n"); EKAT_REQUIRE_MSG (hostname!="", "Error! Invalid hostname: " + hostname + "\n"); EKAT_REQUIRE_MSG (username!="", "Error! Invalid username: " + username + "\n"); #else - caseid = "EAMxx standalone"; + caseid = rest_caseid = "EAMxx standalone"; char* user = new char[32]; char* host = new char[256]; int err; @@ -819,6 +836,7 @@ set_provenance_data (std::string caseid, #endif auto& provenance = m_atm_params.sublist("provenance"); provenance.set("caseid",caseid); + provenance.set("rest_caseid",rest_caseid); provenance.set("hostname",hostname); provenance.set("username",username); provenance.set("version",std::string(EAMXX_GIT_VERSION)); @@ -859,7 +877,7 @@ initialize_fields () } // Initialize fields - if (m_case_t0allocate(); m_atm_process_group->init_buffers(*m_memory_buffer); - const bool restarted_run = m_case_t0 < m_run_t0; - // Setup SurfaceCoupling import and export (if they exist) if (m_surface_coupling_import_data_manager || m_surface_coupling_export_data_manager) { setup_surface_coupling_processes(); } // Initialize the processes - m_atm_process_group->initialize(m_current_ts, restarted_run ? RunType::Restarted : RunType::Initial); + m_atm_process_group->initialize(m_current_ts, m_run_type); // Create and add energy and mass conservation check to appropriate atm procs setup_column_conservation_checks(); diff --git a/components/eamxx/src/control/atmosphere_driver.hpp b/components/eamxx/src/control/atmosphere_driver.hpp index 5c4f12d6cc9..41801745ea2 100644 --- a/components/eamxx/src/control/atmosphere_driver.hpp +++ b/components/eamxx/src/control/atmosphere_driver.hpp @@ -66,7 +66,8 @@ class AtmosphereDriver void set_params (const ekat::ParameterList& params); // Init time stamps - void init_time_stamps (const util::TimeStamp& run_t0, const util::TimeStamp& case_t0); + // run_type: -1: deduce from run/case t0, 0: initial, 1: restart + void init_time_stamps (const util::TimeStamp& run_t0, const util::TimeStamp& case_t0, int run_type = -1); // Set AD params void init_scorpio (const int atm_id = 0); @@ -113,6 +114,7 @@ class AtmosphereDriver void add_additional_column_data_to_property_checks (); void set_provenance_data (std::string caseid = "", + std::string rest_caseid = "", std::string hostname = "", std::string username = ""); @@ -224,6 +226,7 @@ class AtmosphereDriver // restarted runs, the latter is "older" than the former util::TimeStamp m_run_t0; util::TimeStamp m_case_t0; + RunType m_run_type; // This is the comm containing all (and only) the processes assigned to the atmosphere ekat::Comm m_atm_comm; diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index 620873603bb..eb9165c2acb 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -95,7 +95,8 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) use ekat_string_utils, only: string_f2c use mct_mod, only: mct_aVect_init, mct_gsMap_lsize use seq_flds_mod, only: seq_flds_a2x_fields, seq_flds_x2a_fields - use seq_infodata_mod, only: seq_infodata_start_type_start, seq_infodata_start_type_cont + use seq_infodata_mod, only: seq_infodata_start_type_start, seq_infodata_start_type_cont, & + seq_infodata_start_type_brnch use seq_comm_mct, only: seq_comm_inst, seq_comm_name, seq_comm_suffix use shr_file_mod, only: shr_file_getunit, shr_file_setIO use shr_sys_mod, only: shr_sys_abort @@ -136,9 +137,9 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) ! TODO: read this from the namelist? character(len=256) :: yaml_fname = "./data/scream_input.yaml" character(kind=c_char,len=256), target :: yaml_fname_c, atm_log_fname_c - character(len=256) :: caseid, username, hostname - character(kind=c_char,len=256), target :: caseid_c, username_c, hostname_c, calendar_c - logical (kind=c_bool) :: restarted_run + character(len=256) :: caseid, username, hostname, rest_caseid + character(kind=c_char,len=256), target :: caseid_c, username_c, hostname_c, calendar_c, rest_caseid_c + integer (kind=c_int) :: run_type_c !------------------------------------------------------------------------------- ! Grab some data from the cdata structure (coming from the coupler) @@ -149,7 +150,8 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) dom=dom_atm, & infodata=infodata) call seq_infodata_getData(infodata, atm_phase=phase, start_type=run_type, & - username=username, case_name=caseid, hostname=hostname) + username=username, case_name=caseid, & + rest_case_name=rest_caseid, hostname=hostname) call seq_infodata_PutData(infodata, atm_aero=.true.) call seq_infodata_PutData(infodata, atm_prognostic=.true.) @@ -189,6 +191,17 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) ! Initialize atm !---------------------------------------------------------------------------- + if (trim(run_type) == trim(seq_infodata_start_type_start)) then + run_type_c = 0 + else if (trim(run_type) == trim(seq_infodata_start_type_cont) ) then + run_type_c = 1 + else if (trim(run_type) == trim(seq_infodata_start_type_brnch) ) then + run_type_c = 1 + else + print *, "[eamxx] ERROR! Unsupported run type: "//trim(run_type) + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + ! Init the AD call seq_timemgr_EClockGetData(EClock, calendar=calendar, & curr_ymd=cur_ymd, curr_tod=cur_tod, & @@ -197,13 +210,13 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) call string_f2c(calendar,calendar_c) call string_f2c(trim(atm_log_fname),atm_log_fname_c) call string_f2c(trim(caseid),caseid_c) + call string_f2c(trim(rest_caseid),rest_caseid_c) call string_f2c(trim(hostname),hostname_c) call string_f2c(trim(username),username_c) - call scream_create_atm_instance (mpicom_atm, ATM_ID, yaml_fname_c, atm_log_fname_c, & + call scream_create_atm_instance (mpicom_atm, ATM_ID, yaml_fname_c, atm_log_fname_c, run_type_c, & INT(cur_ymd,kind=C_INT), INT(cur_tod,kind=C_INT), & INT(case_start_ymd,kind=C_INT), INT(case_start_tod,kind=C_INT), & - calendar_c, caseid_c, hostname_c, username_c) - + calendar_c, caseid_c, rest_caseid_c, hostname_c, username_c) ! Init MCT gsMap call atm_Set_gsMap_mct (mpicom_atm, ATM_ID, gsMap_atm) @@ -224,16 +237,6 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) call mct_aVect_init(x2a, rList=seq_flds_x2a_fields, lsize=lsize) call mct_aVect_init(a2x, rList=seq_flds_a2x_fields, lsize=lsize) - ! Complete AD initialization based on run type - if (trim(run_type) == trim(seq_infodata_start_type_start)) then - restarted_run = .false. - else if (trim(run_type) == trim(seq_infodata_start_type_cont) ) then - restarted_run = .true. - else - print *, "[eamxx] ERROR! Unsupported starttype: "//trim(run_type) - call mpi_abort(mpicom_atm,ierr,mpi_ierr) - endif - ! Init surface coupling stuff in the AD call scream_set_cpl_indices (x2a, a2x) diff --git a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp index 4f6b34afde9..733acf0ecbb 100644 --- a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp +++ b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp @@ -101,12 +101,14 @@ extern "C" void scream_create_atm_instance (const MPI_Fint f_comm, const int atm_id, const char* input_yaml_file, const char* atm_log_file, + const int run_type, const int run_start_ymd, const int run_start_tod, const int case_start_ymd, const int case_start_tod, const char* calendar_name, const char* caseid, + const char* rest_caseid, const char* hostname, const char* username) { @@ -173,8 +175,8 @@ void scream_create_atm_instance (const MPI_Fint f_comm, const int atm_id, ad.set_comm(atm_comm); ad.set_params(scream_params); ad.init_scorpio(atm_id); - ad.init_time_stamps(run_t0,case_t0); - ad.set_provenance_data (caseid,hostname,username); + ad.init_time_stamps(run_t0,case_t0,run_type); + ad.set_provenance_data (caseid,rest_caseid,hostname,username); ad.create_output_managers (); ad.create_atm_processes (); ad.create_grids (); diff --git a/components/eamxx/src/share/scream_types.hpp b/components/eamxx/src/share/scream_types.hpp index 5d0dacf8621..941ae81ac8a 100644 --- a/components/eamxx/src/share/scream_types.hpp +++ b/components/eamxx/src/share/scream_types.hpp @@ -34,7 +34,7 @@ enum class RepoState { // The type of this run enum class RunType { Initial, - Restarted + Restart }; // We cannot expect BFB results between f90 and cxx if optimizations are on. From 25d940b28fc0cfb95243dd18c91f3a1863bc9e76 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 24 Oct 2024 14:31:34 -0600 Subject: [PATCH 2/7] EAMxx: simplified a bit TraceGasesWorkaround struct --- components/eamxx/src/control/atmosphere_driver.cpp | 7 +++++-- .../eamxx/src/dynamics/homme/eamxx_homme_fv_phys.cpp | 11 +++++++---- .../dynamics/homme/eamxx_homme_process_interface.cpp | 4 ++-- .../dynamics/homme/eamxx_homme_process_interface.hpp | 2 +- .../eamxx_fv_phys_rrtmgp_active_gases_workaround.cpp | 8 +------- .../eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp | 11 ++--------- 6 files changed, 18 insertions(+), 25 deletions(-) diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index f6f9aa48ce0..e9abc95faa7 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -852,7 +852,9 @@ initialize_fields () start_timer("EAMxx::initialize_fields"); // See the [rrtmgp active gases] note in share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp - if (fvphyshack) fv_phys_rrtmgp_active_gases_set_restart(m_case_t0 < m_run_t0); + if (fvphyshack) { + TraceGasesWorkaround::singleton().run_type = m_run_type; + } // See if we need to print a DAG. We do this first, cause if any input // field is missing from the initial condition file, an error will be thrown. @@ -942,7 +944,8 @@ void AtmosphereDriver::restart_model () m_atm_logger->info(" [EAMxx] restart_model ..."); // First, figure out the name of the netcdf file containing the restart data - const auto& casename = m_atm_params.sublist("initial_conditions").get("restart_casename"); + const auto& provenance = m_atm_params.sublist("provenance"); + const auto& casename = provenance.get("rest_caseid"); auto filename = find_filename_in_rpointer (casename,true,m_atm_comm,m_run_t0); m_atm_logger->info(" [EAMxx] Restart filename: " + filename); diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_fv_phys.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_fv_phys.cpp index a121600536b..aae0e549cd7 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_fv_phys.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_fv_phys.cpp @@ -233,9 +233,12 @@ void HommeDynamics::remap_fv_phys_to_dyn () const { // See the [rrtmgp active gases] note in share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp void HommeDynamics -::fv_phys_rrtmgp_active_gases_init (const std::shared_ptr& gm) { +::fv_phys_rrtmgp_active_gases_init (const std::shared_ptr& gm) +{ + // NOTE: we would like to avoid this if it's a restart run, but at this point of the + // init sequence we still don't know the run type. So we must add the trace gases + // fields, and we will deal with them later auto& trace_gases_workaround = TraceGasesWorkaround::singleton(); - if (trace_gases_workaround.is_restart()) return; // always false b/c it hasn't been set yet using namespace ekat::units; using namespace ShortFieldTagsNames; const auto& rgn = m_cgll_grid->name(); @@ -254,7 +257,7 @@ ::fv_phys_rrtmgp_active_gases_init (const std::shared_ptr& g } // See the [rrtmgp active gases] note in share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp -void HommeDynamics::fv_phys_rrtmgp_active_gases_remap () { +void HommeDynamics::fv_phys_rrtmgp_active_gases_remap (const RunType run_type) { // Note re: restart: Ideally, we'd know if we're restarting before having to // call add_field above. However, we only find out after. Because the pg2 // field was declared Updated, it will read the restart data. But we don't @@ -262,7 +265,7 @@ void HommeDynamics::fv_phys_rrtmgp_active_gases_remap () { // cleanup part at the end. auto& trace_gases_workaround = TraceGasesWorkaround::singleton(); const auto& rgn = m_cgll_grid->name(); - if (not trace_gases_workaround.is_restart()) { + if (run_type==RunType::Initial) { using namespace ShortFieldTagsNames; const auto& dgn = m_dyn_grid ->name(); const auto& pgn = m_phys_grid->name(); diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp index bb238feef74..8b7495ffd73 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp @@ -444,7 +444,7 @@ void HommeDynamics::initialize_impl (const RunType run_type) prim_init_model_f90 (); if (fv_phys_active()) { - fv_phys_dyn_to_fv_phys(run_type != RunType::Initial); + fv_phys_dyn_to_fv_phys(run_type == RunType::Restart); // [CGLL ICs in pg2] Remove the CGLL fields from the process. The AD has a // separate fvphyshack-based line to remove the whole CGLL FM. The intention // is to clear the view memory on the device, but I don't know if these two @@ -457,7 +457,7 @@ void HommeDynamics::initialize_impl (const RunType run_type) for (const auto& f : {"horiz_winds", "T_mid", "ps", "phis", "pseudo_density"}) remove_field(f, rgn); remove_group("tracers", rgn); - fv_phys_rrtmgp_active_gases_remap(); + fv_phys_rrtmgp_active_gases_remap(run_type); } // Set up field property checks diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp index c6fa150b10b..93dff0cd72e 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp @@ -103,7 +103,7 @@ class HommeDynamics : public AtmosphereProcess void fv_phys_post_process(); // See [rrtmgp active gases] in eamxx_homme_fv_phys.cpp. void fv_phys_rrtmgp_active_gases_init(const std::shared_ptr& gm); - void fv_phys_rrtmgp_active_gases_remap(); + void fv_phys_rrtmgp_active_gases_remap (const RunType run_type); // Rayleigh friction functions void rayleigh_friction_init (); diff --git a/components/eamxx/src/share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.cpp b/components/eamxx/src/share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.cpp index 7f3f7817ea8..876ab819c7d 100644 --- a/components/eamxx/src/share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.cpp +++ b/components/eamxx/src/share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.cpp @@ -1,11 +1,5 @@ -#include "share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp" - namespace scream { bool fvphyshack = false; -void fv_phys_rrtmgp_active_gases_set_restart (const bool restart) { - TraceGasesWorkaround::singleton().set_restart(restart); -} - -} +} // namespace scream diff --git a/components/eamxx/src/share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp b/components/eamxx/src/share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp index 68154c09338..bdfdb4cbd38 100644 --- a/components/eamxx/src/share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp +++ b/components/eamxx/src/share/util/eamxx_fv_phys_rrtmgp_active_gases_workaround.hpp @@ -1,7 +1,5 @@ #include "share/grid/remap/abstract_remapper.hpp" -#include "ekat/ekat_parameter_list.hpp" - namespace scream { // TODO [rrtmgp active gases] This is to address issue #1782. It supports option @@ -12,22 +10,19 @@ struct TraceGasesWorkaround { private: - TraceGasesWorkaround() { restart = false; } + TraceGasesWorkaround () = default; - bool restart; std::shared_ptr remapper; std::vector active_gases; // other than h2o public: + RunType run_type = RunType::Initial; static TraceGasesWorkaround& singleton() { static TraceGasesWorkaround self; return self; } - void set_restart (const bool is_restart) { - restart = is_restart; - } void set_remapper (const std::shared_ptr& remap_ptr) { remapper = remap_ptr; } @@ -36,12 +31,10 @@ struct TraceGasesWorkaround active_gases.push_back(gas_name); } - bool is_restart() const { return restart; } std::shared_ptr get_remapper() const { return remapper; } std::vector get_active_gases() const { return active_gases; } }; extern bool fvphyshack; -void fv_phys_rrtmgp_active_gases_set_restart(const bool restart); } // namespace scream From 82a85c84b94b00909912b8ef328965cf8d89c51d Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 24 Oct 2024 15:51:11 -0600 Subject: [PATCH 3/7] EAMxx: pass run_type to OM setup as well --- .../src/share/io/scream_output_manager.cpp | 16 +++++++++++----- .../src/share/io/scream_output_manager.hpp | 17 +++++++++++++---- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 3e436657a61..bb5d60f7365 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -16,6 +16,7 @@ namespace scream { + OutputManager:: ~OutputManager () { @@ -25,7 +26,7 @@ OutputManager:: void OutputManager:: initialize(const ekat::Comm& io_comm, const ekat::ParameterList& params, const util::TimeStamp& run_t0, const util::TimeStamp& case_t0, - const bool is_model_restart_output) + const bool is_model_restart_output, const RunType run_type) { // Sanity checks EKAT_REQUIRE_MSG (run_t0.is_valid(), @@ -41,7 +42,7 @@ initialize(const ekat::Comm& io_comm, const ekat::ParameterList& params, m_params = params; m_run_t0 = run_t0; m_case_t0 = case_t0; - m_is_restarted_run = (case_t0>& field_mgrs, // Note: the user might decide *not* to restart the output, so give the option // of disabling the restart. Also, the user might want to change the // filename_prefix, so allow to specify a different filename_prefix for the restart file. - if (m_is_restarted_run and not m_is_model_restart_output) { + if (m_run_type==RunType::Restart and not m_is_model_restart_output) { // Allow to skip history restart, or to specify a filename_prefix for the restart file // that is different from the filename_prefix of the current output. auto& restart_pl = m_params.sublist("Restart"); @@ -251,7 +252,7 @@ setup (const std::map>& field_mgrs, m_time_bnds.resize(2); m_time_bnds[0] = m_run_t0.days_from(m_case_t0); } else if (m_output_control.output_enabled() and - m_run_t0==m_case_t0 and + m_run_type==RunType::Initial and not m_is_model_restart_output and not m_params.sublist("output_control").get("skip_t0_output",false)) // This will be true for ERS/ERP tests { @@ -928,13 +929,18 @@ push_to_logger() return y; }; + auto rt_to_string = [](RunType rt) { + std::string s = rt==RunType::Initial ? "Initial" : "Restart"; + return s; + }; + m_atm_logger->info("[EAMxx::output_manager] - New Output stream"); m_atm_logger->info(" Filename prefix: " + m_filename_prefix); m_atm_logger->info(" Run t0: " + m_run_t0.to_string()); m_atm_logger->info(" Case t0: " + m_case_t0.to_string()); m_atm_logger->info(" Reference t0: " + m_output_control.last_write_ts.to_string()); m_atm_logger->info(" Is Restart File ?: " + bool_to_string(m_is_model_restart_output)); - m_atm_logger->info(" Is Restarted Run ?: " + bool_to_string(m_is_restarted_run)); + m_atm_logger->info(" Run type : " + rt_to_string(m_run_type)); m_atm_logger->info(" Averaging Type: " + e2str(m_avg_type)); m_atm_logger->info(" Output Frequency: " + std::to_string(m_output_control.frequency) + " " + m_output_control.frequency_units); switch (m_output_file_specs.storage.type) { diff --git a/components/eamxx/src/share/io/scream_output_manager.hpp b/components/eamxx/src/share/io/scream_output_manager.hpp index d59363f3c32..1e4f2dba569 100644 --- a/components/eamxx/src/share/io/scream_output_manager.hpp +++ b/components/eamxx/src/share/io/scream_output_manager.hpp @@ -79,12 +79,21 @@ class OutputManager // - is_model_restart_output: whether this output stream is to write a model restart file void initialize (const ekat::Comm& io_comm, const ekat::ParameterList& params, const util::TimeStamp& run_t0, const util::TimeStamp& case_t0, - const bool is_model_restart_output); + const bool is_model_restart_output, const RunType run_type); + // This overloads are to make certain unit tests easier + void initialize (const ekat::Comm& io_comm, const ekat::ParameterList& params, + const util::TimeStamp& run_t0, const util::TimeStamp& case_t0, + const bool is_model_restart_output) + { + auto run_type = case_t0 Date: Thu, 24 Oct 2024 16:31:19 -0600 Subject: [PATCH 4/7] EAMxx: fix init_timestep logic in OM --- .../src/share/io/scream_output_manager.cpp | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index bb5d60f7365..0628a889be9 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -286,10 +286,12 @@ void OutputManager::init_timestep (const util::TimeStamp& start_of_step, const R return; } + const bool is_first_step = m_output_control.dt==0 and dt>0; + // Make sure dt is in the control m_output_control.set_dt(dt); - if (start_of_step==m_case_t0 and m_avg_type==OutputAvgType::Instant and + if (m_run_type==RunType::Initial and is_first_step and m_avg_type==OutputAvgType::Instant and m_output_file_specs.storage.type!=NumSnaps and m_output_control.frequency_units=="nsteps") { // This is the 1st step of the whole run, and a very sneaky corner case. Bear with me. // When we call run, we also compute next_write_ts. Then, we use next_write_ts to see if the @@ -304,17 +306,11 @@ void OutputManager::init_timestep (const util::TimeStamp& start_of_step, const R close_or_flush_if_needed (m_output_file_specs,m_output_control); } - // Check if the end of this timestep will correspond to an output step. If not, there's nothing to do - const auto& end_of_step = start_of_step+dt; - - // Note: a full checkpoint not only writes globals in the restart file, but also all the history variables. - // Since we *always* write a history restart file, we can have a non-full checkpoint, if the average - // type is Instant and/or the frequency is every step. A non-full checkpoint will simply write some - // global attribute, such as the time of last write. - const bool is_output_step = m_output_control.is_write_step(end_of_step) || end_of_step==m_case_t0; - const bool is_checkpoint_step = m_checkpoint_control.is_write_step(end_of_step); - const bool has_checkpoint_data = m_avg_type!=OutputAvgType::Instant; - if (not is_output_step and not (is_checkpoint_step and has_checkpoint_data) ) { + // Note: we need to "init" the timestep if we are going to do something this step, which means we either + // have INST output and it's a write step, or we have AVG output. + const auto end_of_step = start_of_step+dt; + const bool is_output_step = m_output_control.is_write_step(end_of_step) || end_of_step==m_case_t0; + if (not is_output_step and m_avg_type==OutputAvgType::Instant) { return; } @@ -739,7 +735,6 @@ setup_internals (const std::map>& field_mgr // 1. use_case_as_start_reference: TRUE - implies we want to calculate frequency from the beginning of the whole simulation, even if this is a restarted run. // 2. use_case_as_start_reference: FALSE - implies we want to base the frequency of output on when this particular simulation started. // Note, (2) is needed for restarts since the restart frequency in CIME assumes a reference of when this run began. - // NOTE: m_is_restarted_run and not m_is_model_restart_output, these timestamps will be corrected once we open the hist restart file const bool use_case_as_ref = out_control_pl.get("use_case_as_start_reference",!m_is_model_restart_output); const bool perform_history_restart = m_params.sublist("Restart").get("Perform Restart",true); const auto& start_ref = use_case_as_ref and perform_history_restart ? m_case_t0 : m_run_t0; From e1467c60bd86bd1da96e0f8dfe7aa6f47fe2d4f2 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 24 Oct 2024 16:13:13 -0600 Subject: [PATCH 5/7] EAMxx: fix compilation errors in mct interface --- components/eamxx/src/mct_coupling/scream_f2c_mod.F90 | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 index 2bfbe3612b4..5adb0825041 100644 --- a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 +++ b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 @@ -14,19 +14,20 @@ module scream_f2c_mod ! any structure related to the component coupler. Other subroutines ! will have to be called *after* this one, to achieve that. subroutine scream_create_atm_instance (f_comm,atm_id,yaml_fname,atm_log_fname, & + run_type_c, & run_start_ymd,run_start_tod, & case_start_ymd,case_start_tod, & calendar_name, & - caseid, hostname, username) bind(c) + caseid, rest_caseid, hostname, username) bind(c) use iso_c_binding, only: c_int, c_char ! ! Input(s) ! - integer (kind=c_int), value, intent(in) :: f_comm, atm_id + integer (kind=c_int), value, intent(in) :: f_comm, atm_id, run_type_c integer (kind=c_int), value, intent(in) :: run_start_tod, run_start_ymd integer (kind=c_int), value, intent(in) :: case_start_tod, case_start_ymd character(kind=c_char), target, intent(in) :: yaml_fname(*), atm_log_fname(*), calendar_name(*), & - caseid(*), hostname(*), username(*) + caseid(*), rest_caseid(*), hostname(*), username(*) end subroutine scream_create_atm_instance subroutine scream_get_cols_latlon (lat, lon) bind(c) From 6f3afbb486948ef66a7c18afc049a9645ea3711f Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 24 Oct 2024 16:32:38 -0600 Subject: [PATCH 6/7] EAMxx: fix a couple of warnings --- components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp | 3 --- .../eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp | 4 ++-- components/eamxx/src/physics/mam/srf_emission_impl.hpp | 7 ------- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index cf70dd49f56..9c04a3f1ba6 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -224,8 +224,6 @@ apply_iop_forcing(const Real dt) using ESU = ekat::ExeSpaceUtils; using PF = PhysicsFunctions; using ColOps = ColumnOps; - using C = physics::Constants; - constexpr Real Rair = C::Rair; // Homme objects const auto& c = Homme::Context::singleton(); @@ -239,7 +237,6 @@ apply_iop_forcing(const Real dt) const auto nelem = m_dyn_grid->get_num_local_dofs()/(NGP*NGP); const auto total_levels = m_dyn_grid->get_num_vertical_levels(); const auto qsize = params.qsize; - const auto theta_hydrostatic_mode = params.theta_hydrostatic_mode; // Sanity checks since we will be switching between ekat::Pack // and Homme::Scalar view types diff --git a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp index 733acf0ecbb..1e51aa47a36 100644 --- a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp +++ b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp @@ -295,7 +295,7 @@ void scream_finalize (/* args ? */) { // Get the local (i.e., on current atm rank only) number of physics columns int scream_get_num_local_cols () { - int ncols; + int ncols = -1; fpe_guard_wrapper([&]() { const auto& ad = get_ad(); const auto& gm = ad.get_grids_manager(); @@ -309,7 +309,7 @@ int scream_get_num_local_cols () { // Get the global (i.e., the whole earth) number of physics columns int scream_get_num_global_cols () { - int ncols; + int ncols = -1; fpe_guard_wrapper([&]() { const auto& ad = get_ad(); const auto& gm = ad.get_grids_manager(); diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 64ae65714c5..48dc1fa7087 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -104,9 +104,6 @@ void srfEmissFunctions::perform_time_interpolation( // NOTE: we *assume* data_beg and data_end have the *same* hybrid v coords. // IF this ever ceases to be the case, you can interp those too. - using ExeSpace = typename KT::ExeSpace; - using ESU = ekat::ExeSpaceUtils; - // Gather time stamp info auto &t_now = time_state.t_now; auto &t_beg = time_state.t_beg_month; @@ -184,8 +181,6 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( const int time_index, // zero-based AbstractRemapper &srfEmiss_horiz_interp, srfEmissInput &srfEmiss_input) { using namespace ShortFieldTagsNames; - using ESU = ekat::ExeSpaceUtils; - using Member = typename KokkosTypes::MemberType; start_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file"); @@ -211,8 +206,6 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( .get_identifier() .get_layout(); - const int ncols = layout.dim(COL); - // Read fields from the file for(int i = 0; i < srfEmiss_horiz_interp.get_num_fields(); ++i) { auto sector = From 6fa67eb586d0f4eca1d90f869d8ea0adff3513ca Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 25 Oct 2024 11:02:00 -0600 Subject: [PATCH 7/7] EAMxx: remove namelist param for restart casename In CIME runs, use info from cpl, while in standalone use a fixed string --- .../eamxx/cime_config/namelist_defaults_scream.xml | 1 - components/eamxx/src/control/atmosphere_driver.cpp | 10 ++++++---- .../dynamics_physics/model_restart/input_initial.yaml | 1 - .../model_restart/input_restarted.yaml | 3 --- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 6d8558160b1..9be71e49242 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -606,7 +606,6 @@ be lost if SCREAM_HACK_XML is not enabled. - ./${CASE}.scream default ${REST_N} diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index e9abc95faa7..9cf2b48e358 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -259,8 +259,9 @@ void AtmosphereDriver::create_grids() const auto& ic_pl = m_atm_params.sublist("initial_conditions"); if (m_run_type==RunType::Restart) { // Restarted run -> read geo data from restart file - const auto& casename = ic_pl.get("restart_casename"); - auto filename = find_filename_in_rpointer (casename,true,m_atm_comm,m_run_t0); + const auto& provenance = m_atm_params.sublist("provenance"); + const auto& casename = provenance.get("rest_caseid"); + auto filename = find_filename_in_rpointer (casename+".scream",true,m_atm_comm,m_run_t0); gm_params.set("ic_filename", filename); m_atm_params.sublist("provenance").set("initial_conditions_file",filename); } else if (ic_pl.isParameter("Filename")) { @@ -711,6 +712,7 @@ void AtmosphereDriver::create_output_managers () { if (io_params.isSublist("model_restart")) { // Create model restart manager auto params = io_params.sublist("model_restart"); + params.set("filename_prefix",m_casename+".scream"); params.set("Averaging Type","Instant"); params.sublist("provenance") = m_atm_params.sublist("provenance"); @@ -815,7 +817,7 @@ set_provenance_data (std::string caseid, EKAT_REQUIRE_MSG (hostname!="", "Error! Invalid hostname: " + hostname + "\n"); EKAT_REQUIRE_MSG (username!="", "Error! Invalid username: " + username + "\n"); #else - caseid = rest_caseid = "EAMxx standalone"; + caseid = rest_caseid = m_casename; char* user = new char[32]; char* host = new char[256]; int err; @@ -946,7 +948,7 @@ void AtmosphereDriver::restart_model () // First, figure out the name of the netcdf file containing the restart data const auto& provenance = m_atm_params.sublist("provenance"); const auto& casename = provenance.get("rest_caseid"); - auto filename = find_filename_in_rpointer (casename,true,m_atm_comm,m_run_t0); + auto filename = find_filename_in_rpointer (casename+".scream",true,m_atm_comm,m_run_t0); m_atm_logger->info(" [EAMxx] Restart filename: " + filename); diff --git a/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_initial.yaml b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_initial.yaml index 77bd88dc0d7..d5339747fdb 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_initial.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_initial.yaml @@ -69,7 +69,6 @@ grids_manager: # List all the yaml files with the output parameters Scorpio: model_restart: - filename_prefix: model_restart output_control: Frequency: 30 frequency_units: nsecs diff --git a/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_restarted.yaml b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_restarted.yaml index 73e638e2ece..f7d1075e334 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_restarted.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_restarted.yaml @@ -9,9 +9,6 @@ time_stepping: run_t0: 2023-01-01-00030 # YYYY-MM-DD-XXXXX case_t0: 2023-01-01-00000 # YYYY-MM-DD-XXXXX -initial_conditions: - restart_casename: model_restart - atmosphere_processes: atm_procs_list: [homme,physics] schedule_type: Sequential