Merge remote-tracking branch 'tcclevenger/eamxx/turb_advect_default_mode' (PR #7173)

If all processes register a tracer with "NoPreference", the tracer will be
both dynamics and shoc advected.
    
Also, moves tracer consistency check to AtmosphereProcessGroup.
    
[BFB]

Author: tcclevenger

components/eamxx/src/control/atmosphere_driver.cpp

@@ -534,50 +534,8 @@ void AtmosphereDriver::create_fields()
   m_field_mgr->registration_begins();
 
   // Before registering fields, check that Field Requests for tracers are compatible
-  {
-    // Create map from tracer name to a vector which contains the field requests for that tracer.
-    std::map<std::string, std::set<FieldRequest>> tracer_requests;
-    auto gather_tracer_requests = [&] (FieldRequest req) {
-      if (not ekat::contains(req.groups, "tracers")) return;
-
-      std::string fname = req.fid.name();
-      if (tracer_requests.find(fname) == tracer_requests.end()) {
-        tracer_requests[fname] = {req};
-      } else {
-        tracer_requests[fname].emplace(req);
-      }
-    };
-    for (const auto& req : m_atm_process_group->get_required_field_requests()){
-      gather_tracer_requests(req);
-    }
-    for (const auto& req : m_atm_process_group->get_computed_field_requests()) {
-      gather_tracer_requests(req);
-    }
-
-    // Go through the map entry for each tracer and check that every one
-    // has the same request for turbulence advection.
-    for (auto fr : tracer_requests) {
-      const auto reqs = fr.second;
-
-      std::set<bool> turb_advect_types;
-      for (auto req : reqs) {
-        turb_advect_types.emplace(ekat::contains(req.groups, "turbulence_advected_tracers"));
-      }
-
-      if (turb_advect_types.size()!=1) {
-        std::ostringstream ss;
-        ss << "Error! Incompatible tracer request. Turbulence advection requests not consistent among processes.\n"
-              "  - Tracer name: " + fr.first + "\n"
-              "  - Requests (process name, grid name, is tracers turbulence advected):\n";
-        for (auto req : reqs) {
-          const auto grid_name = req.fid.get_grid_name();
-          const bool turb_advect = ekat::contains(req.groups, "turbulence_advected_tracers");
-          ss << "    - (" + req.calling_process + ", " + grid_name + ", " + (turb_advect ? "true" : "false") + ")\n";
-        }
-        EKAT_ERROR_MSG(ss.str());
-      }
-    }
-  }
+  // and store the correct type of turbulence advection for each tracer
+  m_atm_process_group->pre_process_tracer_requests();
 
   // Register required/computed fields. By now, the processes should have
   // fully built the ids of their required/computed fields and groups

components/eamxx/src/share/atm_process/atmosphere_process.hpp

@@ -176,10 +176,10 @@ class AtmosphereProcess : public ekat::enable_shared_from_this<AtmosphereProcess
   // These methods allow the AD to figure out what each process needs, with very fine
   // grain detail. See field_request.hpp for more info on what FieldRequest and GroupRequest
   // are, and field_group.hpp for what groups of fields are.
-  const std::set<FieldRequest>& get_required_field_requests () const { return m_required_field_requests; }
-  const std::set<FieldRequest>& get_computed_field_requests () const { return m_computed_field_requests; }
-  const std::set<GroupRequest>& get_required_group_requests () const { return m_required_group_requests; }
-  const std::set<GroupRequest>& get_computed_group_requests () const { return m_computed_group_requests; }
+  const std::list<FieldRequest>& get_required_field_requests () const { return m_required_field_requests; }
+  const std::list<FieldRequest>& get_computed_field_requests () const { return m_computed_field_requests; }
+  const std::list<GroupRequest>& get_required_group_requests () const { return m_required_group_requests; }
+  const std::list<GroupRequest>& get_computed_group_requests () const { return m_computed_group_requests; }
 
   // These sets allow to get all the actual in/out fields stored by the atm proc
   // Note: if an atm proc requires a group, then all the fields in the group, as well as
@@ -356,12 +356,14 @@ class AtmosphereProcess : public ekat::enable_shared_from_this<AtmosphereProcess
   void add_tracer (const std::string& name, std::shared_ptr<const AbstractGrid> grid,
                    const ekat::units::Units& u,
                    const int ps = 1,
-                   const TracerAdvection tracer_advection = TracerAdvection::DynamicsAndTurbulence)
+                   const TracerAdvection tracer_advection = TracerAdvection::NoPreference)
   {
     std::list<std::string> tracer_groups;
     tracer_groups.push_back("tracers");
     if (tracer_advection==TracerAdvection::DynamicsAndTurbulence) {
       tracer_groups.push_back("turbulence_advected_tracers");
+    } else if (tracer_advection==TracerAdvection::DynamicsOnly) {
+      tracer_groups.push_back("non_turbulence_advected_tracers");
     }
 
     FieldIdentifier fid(name, grid->get_3d_scalar_layout(true), u, grid->name());
@@ -391,14 +393,14 @@ class AtmosphereProcess : public ekat::enable_shared_from_this<AtmosphereProcess
 
     switch (RT) {
       case Required:
-        m_required_field_requests.emplace(req);
+        m_required_field_requests.push_back(req);
         break;
       case Computed:
-        m_computed_field_requests.emplace(req);
+        m_computed_field_requests.push_back(req);
         break;
       case Updated:
-        m_required_field_requests.emplace(req);
-        m_computed_field_requests.emplace(req);
+        m_required_field_requests.push_back(req);
+        m_computed_field_requests.push_back(req);
         break;
     }
   }
@@ -411,14 +413,14 @@ class AtmosphereProcess : public ekat::enable_shared_from_this<AtmosphereProcess
         "Error! Invalid request type in call to add_group.\n");
     switch (RT) {
       case Required:
-        m_required_group_requests.emplace(req);
+        m_required_group_requests.push_back(req);
         break;
       case Computed:
-        m_computed_group_requests.emplace(req);
+        m_computed_group_requests.push_back(req);
         break;
       case Updated:
-        m_required_group_requests.emplace(req);
-        m_computed_group_requests.emplace(req);
+        m_required_group_requests.push_back(req);
+        m_computed_group_requests.push_back(req);
         break;
     }
   }
@@ -558,12 +560,6 @@ class AtmosphereProcess : public ekat::enable_shared_from_this<AtmosphereProcess
   strmap_t<strmap_t<Field*>> m_fields_out_pointers;
   strmap_t<strmap_t<Field*>> m_internal_fields_pointers;
 
-  // The list of in/out field/group requests.
-  std::set<FieldRequest>   m_required_field_requests;
-  std::set<FieldRequest>   m_computed_field_requests;
-  std::set<GroupRequest>   m_required_group_requests;
-  std::set<GroupRequest>   m_computed_group_requests;
-
   // List of property checks for fields
   std::list<std::pair<CheckFailHandling,prop_check_ptr>> m_precondition_checks;
   std::list<std::pair<CheckFailHandling,prop_check_ptr>> m_postcondition_checks;
@@ -608,6 +604,12 @@ class AtmosphereProcess : public ekat::enable_shared_from_this<AtmosphereProcess
 
   // IOP object
   iop_data_ptr m_iop_data_manager;
+
+  // The list of in/out field/group requests.
+  std::list<FieldRequest>   m_required_field_requests;
+  std::list<FieldRequest>   m_computed_field_requests;
+  std::list<GroupRequest>   m_required_group_requests;
+  std::list<GroupRequest>   m_computed_group_requests;
 };
 
 // ================= IMPLEMENTATION ================== //

components/eamxx/src/share/atm_process/atmosphere_process_group.cpp

@@ -364,6 +364,79 @@ void AtmosphereProcessGroup::add_additional_data_fields_to_property_checks (cons
   }
 }
 
+void AtmosphereProcessGroup::pre_process_tracer_requests () {
+  // Create map from tracer name to a vector which contains the field requests for that tracer.
+  std::map<std::string, std::list<FieldRequest>> tracer_requests;
+  auto gather_tracer_requests = [&] (FieldRequest& req) {
+    if (ekat::contains(req.groups, "tracers")) {
+      tracer_requests[req.fid.name()].push_back(req);
+    }
+  };
+  for (auto& req : m_required_field_requests){
+    gather_tracer_requests(req);
+  }
+  for (auto& req : m_computed_field_requests) {
+    gather_tracer_requests(req);
+  }
+
+  // Go through the map entry for each tracer and check that every one
+  // has the same request for turbulence advection, or listed no preference.
+  std::map<std::string, std::string> tracer_advection_type;
+  for (auto& fr : tracer_requests) {
+    auto& reqs = fr.second;
+
+    bool turb_advect_req = false;
+    bool non_turb_advect_req = false;
+    for (auto& req : reqs) {
+      if (ekat::contains(req.groups, "turbulence_advected_tracers")) turb_advect_req = true;
+      if (ekat::contains(req.groups, "non_turbulence_advected_tracers")) non_turb_advect_req = true;
+      if (turb_advect_req and non_turb_advect_req) {
+        // All the info we need to error out, just break request loop
+        break;
+      }
+    }
+
+    if (turb_advect_req and non_turb_advect_req) {
+      std::ostringstream ss;
+      ss << "Error! Incompatible tracer request. Turbulence advection requests not consistent among processes.\n"
+            "  - Tracer name: " + fr.first + "\n"
+            "  - Requests (process name, grid name, turbulence advected request type):\n";
+      for (auto& req : reqs) {
+        const auto turb_advect = ekat::contains(req.groups, "turbulence_advected_tracers");
+        const auto non_turb_advect = ekat::contains(req.groups, "non_turbulence_advected_tracers");
+        std::string turb_advect_info =
+          (turb_advect ?       "DynamicsAndTurbulence" :
+            (non_turb_advect ? "DynamicsOnly" :
+                                "NoPreference"));
+        const auto grid_name = req.fid.get_grid_name();
+        ss << "    - (" + req.calling_process + ", " + grid_name + ", " + turb_advect_info + ")\n";
+      }
+      EKAT_ERROR_MSG(ss.str());
+    } else if (non_turb_advect_req) {
+      tracer_advection_type[fr.first] = "non_turbulence_advected_tracers";
+    } else {
+      tracer_advection_type[fr.first] = "turbulence_advected_tracers";
+    }
+  }
+
+  // Set correct tracer advection type (if not set)
+  auto add_correct_tracer_group = [&] (FieldRequest& req, const std::string& advect_type) {
+    if (not ekat::contains(req.groups, advect_type)) {
+      req.groups.push_back(advect_type);
+    }
+  };
+  for (auto& req : m_required_field_requests){
+    if (ekat::contains(req.groups, "tracers")) {
+      add_correct_tracer_group(req, tracer_advection_type.at(req.fid.name()));
+    }
+  }
+  for (auto& req : m_computed_field_requests) {
+    if (ekat::contains(req.groups, "tracers")) {
+      add_correct_tracer_group(req, tracer_advection_type.at(req.fid.name()));
+    }
+  }
+}
+
 void AtmosphereProcessGroup::initialize_impl (const RunType run_type) {
   for (auto& atm_proc : m_atm_processes) {
     atm_proc->initialize(timestamp(),run_type);

components/eamxx/src/share/atm_process/atmosphere_process_group.hpp

@@ -112,6 +112,11 @@ class AtmosphereProcessGroup : public AtmosphereProcess
     }
   }
 
+  // Pre-process tracer requests by checking for
+  // consistency among processes and correctly
+  // determining turbulence advection property
+  void pre_process_tracer_requests ();
+
 protected:
 
   // Adds fid to the list of required/computed fields of the group (as a whole).

components/eamxx/src/share/field/field_request.hpp

@@ -16,7 +16,9 @@ enum RequestType {
 // Whether a tracer should be advected by both Dynamics
 // and Turbulance, or only by Dynamics
 enum TracerAdvection {
-  DynamicsAndTurbulence, // Default use case
+  NoPreference, // Default. In the case that no process gives a preference,
+                // the tracer is advected by dynamics and turbulence
+  DynamicsAndTurbulence,
   DynamicsOnly,
 };