diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index 0f1cb1e31ab..0d589bb0bb6 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -685,6 +685,21 @@ void AtmosphereDriver::create_fields() fm->add_to_group(fid.name(),"RESTART"); } + auto& driver_options_pl = m_atm_params.sublist("driver_options"); + const int verb_lvl = driver_options_pl.get("atmosphere_dag_verbosity_level",-1); + if (verb_lvl>0) { + // now that we've got fields, generate a DAG with fields and dependencies + // NOTE: at this point, fields provided by initial conditions may (will) + // appear as unmet dependencies + AtmProcDAG dag; + // First, add all atm processes + dag.create_dag(*m_atm_process_group); + // Write a dot file for visualization + if (m_atm_comm.am_i_root()) { + dag.write_dag("scream_atm_createField_dag.dot", std::max(verb_lvl,0)); + } + } + m_ad_status |= s_fields_created; stop_timer("EAMxx::create_fields"); @@ -861,28 +876,6 @@ initialize_fields () TraceGasesWorkaround::singleton().run_type = m_run_type; } - // See if we need to print a DAG. We do this first, cause if any input - // field is missing from the initial condition file, an error will be thrown. - // By printing the DAG first, we give the user the possibility of seeing - // what fields are inputs to the atm time step, so he/she can fix the i.c. file. - // TODO: would be nice to do the IC input first, and mark the fields in the - // DAG node "Begin of atm time step" in red if there's no initialization - // mechanism set for them. That is, allow field XYZ to not be found in - // the IC file, and throw an error when the dag is created. - - auto& driver_options_pl = m_atm_params.sublist("driver_options"); - const int verb_lvl = driver_options_pl.get("atmosphere_dag_verbosity_level",-1); - if (verb_lvl>0) { - // Check the atm DAG for missing stuff - AtmProcDAG dag; - - // First, add all atm processes - dag.create_dag(*m_atm_process_group); - - // Write a dot file for visualization - dag.write_dag("scream_atm_dag.dot",std::max(verb_lvl,0)); - } - // Initialize fields if (m_run_type==RunType::Restart) { restart_model (); @@ -1060,7 +1053,6 @@ void AtmosphereDriver::set_initial_conditions () // Check which fields need to have an initial condition. std::map> ic_fields_names; std::vector ic_fields_to_copy; - std::map> fields_inited; // Check which fields should be loaded from the topography file std::map> topography_file_fields_names; @@ -1089,7 +1081,7 @@ void AtmosphereDriver::set_initial_conditions () EKAT_ERROR_MSG ("ERROR: invalid assignment for variable " + fname + ", only scalar " "double or string, or vector double arguments are allowed"); } - fields_inited[grid_name].push_back(fname); + m_fields_inited[grid_name].push_back(fname); } else if (fname == "phis" or fname == "sgh30") { // Both phis and sgh30 need to be loaded from the topography file auto& this_grid_topo_file_fnames = topography_file_fields_names[grid_name]; @@ -1105,7 +1097,7 @@ void AtmosphereDriver::set_initial_conditions () grid_name == "Point Grid") { this_grid_topo_file_fnames.push_back("PHIS_d"); this_grid_topo_eamxx_fnames.push_back(fname); - fields_inited[grid_name].push_back(fname); + m_fields_inited[grid_name].push_back(fname); } else { EKAT_ERROR_MSG ("Error! Requesting phis on an unknown grid: " + grid_name + ".\n"); } @@ -1117,7 +1109,7 @@ void AtmosphereDriver::set_initial_conditions () " topo file only has sgh30 for Physics PG2.\n"); topography_file_fields_names[grid_name].push_back("SGH30"); topography_eamxx_fields_names[grid_name].push_back(fname); - fields_inited[grid_name].push_back(fname); + m_fields_inited[grid_name].push_back(fname); } } else if (not (fvphyshack and grid_name == "Physics PG2")) { // The IC file is written for the GLL grid, so we only load @@ -1129,7 +1121,7 @@ void AtmosphereDriver::set_initial_conditions () // If this field is the parent of other subfields, we only read from file the subfields. if (not ekat::contains(this_grid_ic_fnames,fname)) { this_grid_ic_fnames.push_back(fname); - fields_inited[grid_name].push_back(fname); + m_fields_inited[grid_name].push_back(fname); } } else if (fvphyshack and grid_name == "Physics GLL") { // [CGLL ICs in pg2] I tried doing something like this in @@ -1146,7 +1138,7 @@ void AtmosphereDriver::set_initial_conditions () } else { this_grid_ic_fnames.push_back(fname); } - fields_inited[grid_name].push_back(fname); + m_fields_inited[grid_name].push_back(fname); } } } @@ -1192,7 +1184,7 @@ void AtmosphereDriver::set_initial_conditions () auto p = f.get_header().get_parent().lock(); if (p) { const auto& pname = p->get_identifier().name(); - if (ekat::contains(fields_inited[grid_name],pname)) { + if (ekat::contains(m_fields_inited[grid_name],pname)) { // The parent is already inited. No need to init this field as well. names.erase(it2); run_again = true; @@ -1210,15 +1202,14 @@ void AtmosphereDriver::set_initial_conditions () // lat/lon data in topo file is defined in terms of PG2 grid, // so if we have a topo field on GLL grid, we need to setup // io info using the IC file (which is always GLL). - for (const auto& it : m_field_mgrs) { - const auto& grid_name = it.first; + for (const auto &it : m_field_mgrs) { + const auto &grid_name = it.first; if (ic_fields_names[grid_name].size() > 0 or - topography_eamxx_fields_names[grid_name].size() > 0) { - const auto& file_name = grid_name == "Physics GLL" - ? - ic_pl.get("Filename") - : - ic_pl.get("topography_filename"); + topography_eamxx_fields_names[grid_name].size() > 0) { + const auto &file_name = + grid_name == "Physics GLL" + ? ic_pl.get("Filename") + : ic_pl.get("topography_filename"); m_iop->setup_io_info(file_name, it.second->get_grid()); } } @@ -1415,7 +1406,7 @@ void AtmosphereDriver::set_initial_conditions () // Loop through fields and apply perturbation. for (size_t f=0; fget_grid()->name()], fname), + EKAT_REQUIRE_MSG(ekat::contains(m_fields_inited[fm->get_grid()->name()], fname), "Error! Attempting to apply perturbation to field not in initial_conditions.\n" " - Field: "+fname+"\n" " - Grid: "+fm->get_grid()->name()+"\n"); @@ -1614,6 +1605,23 @@ void AtmosphereDriver::initialize_atm_procs () m_atm_logger->info("[EAMxx] initialize_atm_procs ... done!"); report_res_dep_memory_footprint (); + + auto& driver_options_pl = m_atm_params.sublist("driver_options"); + const int verb_lvl = driver_options_pl.get("atmosphere_dag_verbosity_level",-1); + if (verb_lvl>0) { + // now that we've got fields, generate a DAG with fields and dependencies + // NOTE: at this point, fields provided by initial conditions may (will) + // appear as unmet dependencies + AtmProcDAG dag; + // First, add all atm processes + dag.create_dag(*m_atm_process_group); + // process the initial conditions to maybe fulfill unmet dependencies + dag.process_initial_conditions(m_fields_inited); + // Write a dot file for visualization + if (m_atm_comm.am_i_root()) { + dag.write_dag("scream_atm_initProc_dag.dot", std::max(verb_lvl,0)); + } + } } void AtmosphereDriver:: diff --git a/components/eamxx/src/control/atmosphere_driver.hpp b/components/eamxx/src/control/atmosphere_driver.hpp index a3acfba5d94..f74f57d3cc3 100644 --- a/components/eamxx/src/control/atmosphere_driver.hpp +++ b/components/eamxx/src/control/atmosphere_driver.hpp @@ -264,6 +264,9 @@ class AtmosphereDriver // Current simulation casename std::string m_casename; + + // maps grid name to a vector of its initialized fields + std::map> m_fields_inited; }; } // namespace control diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_dag.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_dag.cpp index 7b6fc218a0b..b0338df690a 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_dag.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_dag.cpp @@ -5,6 +5,16 @@ namespace scream { +// this is an initial "DAG" with only atm_process nodes to at least provide +// *some* information if we crash soon after this +void AtmProcDAG:: +init_atm_proc_nodes(const group_type& atm_procs) +{ + cleanup (); + // Create the nodes + add_nodes(atm_procs); +} + void AtmProcDAG:: create_dag(const group_type& atm_procs) { @@ -171,64 +181,117 @@ void AtmProcDAG::write_dag (const std::string& fname, const int verbosity) const ofile.open (fname.c_str()); ofile << "strict digraph G {\n" - << "rankdir=\"LR\""; + << "rankdir=\"LR\"\n"; + bool has_IC_field = false; for (const auto& n : m_nodes) { const auto& unmet = m_unmet_deps.at(n.id); + std::string box_fmt; + int id_IC = -1; + int id_begin = -1; + int id_end = -1; + if (n.name == "Begin of atm time step") { + id_begin = n.id; + box_fmt = " color=\"#00667E\"\n fontcolor=\"#00667E\"\n style=filled\n" + " fillcolor=\"#b9d4dc\"\n"; + } else if (n.name == "Initial Conditions") { + id_IC = n.id; + box_fmt = " color=\"#006219\"\n fontcolor=\"#006219\"\n style=filled\n" + " fillcolor=\"#b9dcc2\"\n"; + } else if (n.name == "End of atm time step") { + id_end = n.id; + box_fmt = " color=\"#88621e\"\n fontcolor=\"#88621e\"\n style=filled\n" + " fillcolor=\"#dccfb9\"\n"; + } + // Write node, with computed/required fields ofile << n.id << " [\n" << " shape=box\n" + << box_fmt + << " penwidth=4\n" + << " fontsize=30\n" << " label=<\n" << " \n" - << " "; - if (verbosity>1) { + << " \n"; + + int sz_comp = n.computed.size(), sz_req = n.required.size(), + sz_grcomp = n.gr_computed.size(), sz_grreq = n.gr_required.size(); + int nfield = sz_comp + sz_req + sz_grcomp + sz_grreq; + if (verbosity > 1 && nfield > 0) { // FieldIntentifier prints bare min with verb 0. // DAG starts printing fids with verb 2, so fid verb is verb-2; int fid_verb = verbosity-2; - ofile << "
\n"; + ofile << "
\n"; + + if (sz_comp > 0) { + // Computed fields + if (n.id == id_begin) { + ofile << " \n"; + } else if (n.id == id_IC) { + ofile << " \n"; + } else if (n.id != id_end) { + ofile << " \n"; + } - // Computed fields - if (n.name=="Begin of atm time step") { - ofile << " \n"; - } else if (n.name!="End of atm time step"){ - ofile << " \n"; - } - for (const auto& fid : n.computed) { - std::string fc = " "; - ofile << " \n"; + for (const auto& fid : n.computed) { + std::string fc = " "; + ofile << " \n"; + } } - // Required fields - if (n.name=="End of atm time step") { - ofile << " \n"; - } else if (n.name!="Begin of atm time step") { - ofile << " \n"; - } - for (const auto& fid : n.required) { - std::string fc = " "; - ofile << " \n"; + } else if (n.id != id_begin && n.id != id_IC) { + ofile << " \n"; + } + for (const auto& fid : n.required) { + std::string fc = " "; + ofile << " \n"; } - ofile << "\n"; } // Computed groups - if (n.gr_computed.size()>0) { - if (n.name=="Begin of atm time step") { - ofile << " \n"; - } else if (n.name!="End of atm time step"){ - ofile << " \n"; + if (sz_grcomp > 0) { + if (n.id == id_begin) { + ofile << " \n"; + } else if (n.id != id_end){ + ofile << " \n"; } for (const auto& gr_fid : n.gr_computed) { std::string fc = " "; ofile << " \n"; @@ -242,10 +305,8 @@ void AtmProcDAG::write_dag (const std::string& fname, const int verbosity) const size_t i = 0; for (const auto& fn : members_names) { const auto f = members.at(fn); - const auto& mfid = f->get_header().get_identifier(); - const auto mfid_id = get_fid_index(mfid); std::string mfc = ""; if (len>0) { ofile << ","; @@ -269,18 +330,18 @@ void AtmProcDAG::write_dag (const std::string& fname, const int verbosity) const } // Required groups - if (n.gr_required.size()>0) { + if (sz_grreq > 0) { if (n.name=="End of atm time step") { - ofile << " \n"; + ofile << " \n"; } else if (n.name!="Begin of atm time step") { - ofile << " \n"; + ofile << " \n"; } for (const auto& gr_fid : n.gr_required) { std::string fc = " "; ofile << " \n"; @@ -327,10 +388,45 @@ void AtmProcDAG::write_dag (const std::string& fname, const int verbosity) const // Write all outgoing edges for (const auto c : n.children) { - ofile << n.id << "->" << c << "\n"; + ofile << n.id << "->" << c << "[penwidth=4];\n"; } } + if (!m_IC_processed && m_has_unmet_deps) { + int this_node_id = m_nodes.size() + 1; + ofile << this_node_id << " [\n" + << " shape=box\n" + << " color=\"#605d57\"\n" + << " fontcolor=\"#034a4a\"\n" + << " penwidth=8\n" + << " fontsize=40\n" + << " style=filled\n" + << " fillcolor=\"#999999\"\n" + << " align=\"center\"\n" + << " label=<NOTE: " + "Fields marked missing may be
provided by " + "the as-yet-unprocessed
initial condition>\n" + << "];\n"; + } + + if (m_IC_processed && has_IC_field) { + int this_node_id = m_nodes.size() + 1; + ofile << this_node_id << " [\n" + << " shape=box\n" + << " color=\"#605d57\"\n" + << " fontcolor=\"#031576\"\n" + << " penwidth=8\n" + << " fontsize=40\n" + << " style=filled\n" + << " fillcolor=\"#cccccc\"\n" + << " align=\"center\"\n" + << " label=<NOTE: Fields denoted " + "with green text " + "
indicate the field was provided by the " + "
initial conditions and never updated>\n" + << "];\n"; + } + // Close the file ofile << "}"; ofile.close(); @@ -341,6 +437,7 @@ void AtmProcDAG::cleanup () { m_fid_to_last_provider.clear(); m_unmet_deps.clear(); m_has_unmet_deps = false; + m_IC_processed = false; } void AtmProcDAG:: @@ -364,10 +461,9 @@ add_nodes (const group_type& atm_procs) add_nodes(*group); } else { // Create a node for the process - // Node& node = m_nodes[proc->name()]; int id = m_nodes.size(); m_nodes.push_back(Node()); - Node& node = m_nodes.back();; + Node& node = m_nodes.back(); node.id = id; node.name = proc->name(); m_unmet_deps[id].clear(); // Ensures an entry for this id is in the map @@ -507,6 +603,60 @@ void AtmProcDAG::add_edges () { } } +void AtmProcDAG::process_initial_conditions(const grid_field_map &ic_inited) { + // First, add the fields that were determined to come from the previous time + // step => IC for t = 0 + // get the begin_node since the IC is identical at first + const Node &begin_node = m_nodes[m_nodes.size() - 2]; + int id = m_nodes.size(); + // Create a node for the ICs by copying the begin_node + m_nodes.push_back(Node(begin_node)); + Node& ic_node = m_nodes.back(); + // now set/clear the basic data for the ic_node + ic_node.id = id; + ic_node.name = "Initial Conditions"; + m_unmet_deps[id].clear(); + ic_node.children.clear(); + // now add the begin_node as a child of the ic_node + ic_node.children.push_back(begin_node.id); + // return if there's nothing to process in the ic_inited vector + if (ic_inited.size() == 0) { + return; + } + for (auto &node : m_nodes) { + if (m_unmet_deps.at(node.id).empty()) { + continue; + } else { + // NOTE: node_unmet_fields is a std::set + auto &node_unmet_fields = m_unmet_deps.at(node.id); + // add the current node as a child of the IC node + ic_node.children.push_back(node.id); + for (auto um_fid : node_unmet_fields) { + for (auto &it1 : ic_inited) { + const auto &grid_name = it1.first; + // if this unmet-dependency field's name is in the ic_inited map for + // the provided grid_name key, then we flip its value negative and + // break from the for (ic_inited) and for (node_unmet_fields) loops; + // otherwise, keep trying for the next grid_name + if (ekat::contains(ic_inited.at(grid_name), m_fids[um_fid].name())) { + auto id_now_met = node_unmet_fields.extract(um_fid); + id_now_met.value() = -id_now_met.value(); + node_unmet_fields.insert(std::move(id_now_met)); + // add the fid of the formerly unmet dep to the initial condition + // node's computed list + ic_node.computed.insert(um_fid); + goto endloop; + } else { + continue; + } + } + endloop:; + } + } + } + m_IC_processed = true; +} + int AtmProcDAG::add_fid (const FieldIdentifier& fid) { auto it = ekat::find(m_fids,fid); if (it==m_fids.end()) { diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_dag.hpp b/components/eamxx/src/share/atm_process/atmosphere_process_dag.hpp index 1a88381ecc8..ac07444989f 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_dag.hpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_dag.hpp @@ -16,6 +16,11 @@ class AtmProcDAG { void create_dag (const group_type& atm_procs); + using grid_field_map = std::map>; + void process_initial_conditions(const grid_field_map &ic_inited); + + void init_atm_proc_nodes(const group_type& atm_procs); + void add_surface_coupling (const std::set& imports, const std::set& exports); @@ -57,15 +62,17 @@ class AtmProcDAG { // Assign an id to each field identifier std::vector m_fids; - // Store groups so we can print info of their members if need be + // Store each field's groups so we can print members' info if need be std::map m_gr_fid_to_group; // Map each field id to its last provider std::map m_fid_to_last_provider; // Map a node id to a set of unmet field dependencies + // if value is negative, then that field is provided by the initial condition std::map> m_unmet_deps; bool m_has_unmet_deps; + bool m_IC_processed; // The nodes in the atm DAG std::vector m_nodes;
" << html_fix(n.name) << "
" << html_fix(n.name) + << "
" + << "Atm input fields from previous time step:
" + << "Initial Fields:
" + << "Computed Fields:
Atm input fields from previous time step:
Computed Fields:
" << fc << html_fix(print_fid(m_fids[fid],fid_verb)) << "
" << fc + << html_fix(print_fid(m_fids[fid],fid_verb)) + << "
Atm output fields for next time step:
Required Fields:
" << fc << html_fix(print_fid(m_fids[fid],fid_verb)); - if (ekat::contains(m_unmet_deps.at(n.id),fid)) { - ofile << " *** MISSING ***"; + if (sz_req > 0) { + // Required fields + if (n.id == id_end) { + ofile << "
" + << "Atm output fields for next time step:
" + << "Required Fields:
" << fc << html_fix(print_fid(m_fids[fid],fid_verb)); + if (ekat::contains(unmet, fid)) { + ofile << " *** MISSING ***"; + } else if (ekat::contains(unmet, -fid)) { + ofile << " (Init. Cond.)"; + has_IC_field = true; + } + ofile << "
Atm Input groups:
Computed Groups:
Atm Input groups:
Computed Groups:
" << fc << html_fix(print_fid(m_fids[gr_fid],fid_verb)); ofile << "
Atm Output Groups:
Atm Output Groups:
Required Groups:
Required Groups:
" << fc << html_fix(print_fid(m_fids[gr_fid],fid_verb)); - if (ekat::contains(m_unmet_deps.at(n.id),gr_fid)) { + if (ekat::contains(unmet, gr_fid)) { ofile << " *** MISSING ***"; } ofile << "