EAMxx: update EKAT submodule and adapt EAMxx

components/eam/src/physics/crm/pam/CMakeLists.txt

@@ -69,7 +69,6 @@ endif()
 
 
 if (PAM_SCREAM_USE_CXX)
-  set(EKAT_ENABLE_YAML_PARSER OFF CACHE BOOL "" FORCE)
   add_subdirectory(external/physics/scream_cxx_p3_shoc)
   add_subdirectory(external/physics/scream_cxx_interfaces)
   # add these here to allow driver to intialize P3
@@ -126,7 +125,7 @@ target_compile_options(pam_driver PUBLIC
 )
 
 if (PAM_SCREAM_USE_CXX)
-  target_link_libraries(pam_driver pam_core physics dynamics pam_scream_cxx_interfaces ekat p3 shoc physics_share scream_share)
+  target_link_libraries(pam_driver pam_core physics dynamics pam_scream_cxx_interfaces ekat::AllLibs p3 shoc physics_share scream_share)
 else()
   target_link_libraries(pam_driver pam_core physics dynamics )
 endif()

components/eamxx/CMakeLists.txt

@@ -237,9 +237,6 @@ set(SCREAM_FPE ${DEFAULT_FPE} CACHE BOOL "Enable floating point error exception"
 option (SCREAM_ENABLE_GETRUSAGE "Whether getrusage can be used to get memory usage." OFF)
 option (SCREAM_ENABLE_STATM "Whether /proc/self/statm can be used to get memory usage." OFF)
 
-# Whether to disable warnings from tpls.
-set (SCREAM_DISABLE_TPL_WARNINGS ON CACHE BOOL "")
-
 # Dycore settings
 set(DEFAULT_SCREAM_DYNAMICS_DYCORE "NONE")
 if (SCREAM_CIME_BUILD AND SCREAM_DYN_TARGET STREQUAL "theta-l_kokkos")
@@ -462,6 +459,8 @@ endif()
 #               Configure all tpls and subfolders                  #
 ####################################################################
 
+set (EKAT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../externals/ekat)
+
 if (DEFINED ENV{SCREAM_FAKE_ONLY})
   # We don't really need to build ekat, but we do need to configure the test-launcher
 
@@ -486,8 +485,6 @@ if (DEFINED ENV{SCREAM_FAKE_ONLY})
     set (TEST_LAUNCHER_ON_GPU False)
   endif()
 
-  set (EKAT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../externals/ekat)
-
   if (EKAT_ENABLE_MPI)
     find_package(MPI REQUIRED COMPONENTS C)
 
@@ -516,11 +513,29 @@ if (DEFINED ENV{SCREAM_FAKE_ONLY})
   endif()
 
 else()
-  include(EkatBuildEkat)
-  BuildEkat(PREFIX "SCREAM"
-    ENABLE_TESTS OFF
-    ENABLE_FPE   ON
-    ENABLE_FPE_DEFAULT_MASK OFF)
+  # Enable all EKAT packages
+  set(EKAT_ENABLE_ALL_PACKAGES ON)
+  set(EKAT_DISABLE_TPL_WARNINGS ON)
+
+  # Disable tests by default (user can override via -D EKAT_ENABLE_TESTS=ON)
+  set (EKAT_ENABLE_TESTS OFF)
+
+  # Ensure testing options are consistent with EAMxx settings (threads, ranks, packs, etc)
+  set (EKAT_TEST_MAX_THREADS ${SCREAM_TEST_MAX_THREADS} CACHE STRING "Max number of threads for tests")
+  set (EKAT_TEST_THREAD_INC ${SCREAM_TEST_THREAD_INC} CACHE STRING "Thread count increment for threaded tests")
+  set (EKAT_TEST_MAX_RANKS ${SCREAM_TEST_MAX_RANKS} CACHE STRING "Max number of ranks for tests")
+  set (EKAT_TEST_PACK_SIZE ${SCREAM_PACK_SIZE} CACHE STRING
+    " The number of scalars in a pack::Pack and Mask. Larger packs have good performance on conditional-free loops due to improved caching.")
+
+  if (SCREAM_DOUBLE_PRECISION)
+    set (EKAT_TEST_SINGLE_PRECISION OFF)
+    set (EKAT_TEST_DOUBLE_PRECISION ON)
+  else()
+    set (EKAT_TEST_SINGLE_PRECISION ON)
+    set (EKAT_TEST_DOUBLE_PRECISION OFF)
+  endif()
+
+  add_subdirectory(${EKAT_SOURCE_DIR} ${CMAKE_BINARY_DIR}/externals/ekat)
 endif()
 
 # Set compiler-specific flags

components/eamxx/cmake/ScreamUtils.cmake

@@ -81,14 +81,14 @@ set(SCREAM_CUT_TEST_MV_ARGS ${CUT_TEST_MV_ARGS})
 #
 
 # Scream always excludes the ekat test session since it has its own
-list(REMOVE_ITEM SCREAM_CUT_EXEC_OPTIONS EXCLUDE_TEST_SESSION)
+list(REMOVE_ITEM SCREAM_CUT_EXEC_OPTIONS USER_DEFINED_TEST_SESSION)
 
 ###############################################################################
 function(CreateUnitTestExec exec_name test_srcs)
 ###############################################################################
   # Call Ekat function, with a couple of extra params
   EkatCreateUnitTestExec("${exec_name}" "${test_srcs}" ${ARGN}
-    EXCLUDE_TEST_SESSION LIBS scream_share scream_test_support)
+    USER_DEFINED_TEST_SESSION LIBS scream_share scream_test_support)
 endfunction(CreateUnitTestExec)
 
 ###############################################################################

components/eamxx/docs/developer/kokkos_ekat.md

@@ -57,8 +57,7 @@ where
   run-time dimensions). E.g., a 2D view of doubles will have `DataType =
   double**`. There is also an ability to define compile-time dimensions by
   using `[]`, see
-  [Kokkos wiki section on views]
-  (<https://kokkos.org/kokkos-core-wiki/API/core/view/view.html).
+  [Kokkos wiki section on views](https://kokkos.org/kokkos-core-wiki/API/core/view/view.html).
 - `LayoutType`: mapping of indices into the underlying 1D memory storage. Types
   are:
   - `LayoutRight` (used in EAMxx): strides increase from the right most to the
@@ -242,7 +241,9 @@ will _not_ error out, but race conditions will exist among threads.
 
 ### KokkosTypes
 
-### ExeSpaceUtils
+### TeamPolicyFactory
+
+### ReductionUtils
 
 ### Vectorization: Packs
 

components/eamxx/scripts/gen_boiler.py

@@ -15,13 +15,14 @@
 f"""#include "catch2/catch.hpp"
 
 #include "share/eamxx_types.hpp"
-#include "ekat/ekat_pack.hpp"
-#include "ekat/kokkos/ekat_kokkos_utils.hpp"
 #include "physics/{phys}/{phys}_functions.hpp"
 #include "physics/{phys}/tests/infra/{phys}_test_data.hpp"
 
 #include "{phys}_unit_tests_common.hpp"
 
+#include <ekat_pack.hpp>
+#include <ekat_team_policy_utils.hpp>
+
 namespace scream {{
 namespace {phys} {{
 namespace unit_test {{
@@ -1903,7 +1904,7 @@ def gen_cxx_f2c_bind_impl(self, phys, sub, force_arg_data=None):
             vals_d(btemp_d_1[0]);
         <BLANKLINE>
           const Int nk_pack = ekat::npack<Spack>(nlev);
-          const auto policy = ekat::ExeSpaceUtils<ExeSpace>::get_default_team_policy(shcol, nk_pack);
+          const auto policy = ekat::TeamPolicyFactory<ExeSpace>::get_default_team_policy(shcol, nk_pack);
           Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) {
             const Int i = team.league_rank();
         <BLANKLINE>
@@ -2111,7 +2112,7 @@ def gen_cxx_f2c_bind_impl(self, phys, sub, force_arg_data=None):
             # 4) Get nk_pack and policy, launch kernel
             #
             impl += "  const Int nk_pack = ekat::npack<Spack>(nlev);\n"
-            impl += "  const auto policy = ekat::ExeSpaceUtils<ExeSpace>::get_default_team_policy(shcol, nk_pack);\n"
+            impl += "  const auto policy = ekat::TeamPolicyFactory<ExeSpace>::get_default_team_policy(shcol, nk_pack);\n"
             impl += "  Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) {\n"
             impl += "    const Int i = team.league_rank();\n\n"
 

components/eamxx/scripts/query-cf-database/CMakeLists.txt

@@ -11,5 +11,4 @@ target_compile_definitions(query-cf-database PUBLIC
                            CF_SCREAM_NAME_FILE=${CF_SCREAM_NAME_FILE})
 
 find_package (ekat HINTS ${EKAT_ROOT})
-find_package (yaml-cpp HINTS ${YAML_CPP_ROOT})
-target_link_libraries(query-cf-database ekat yaml-cpp)
+target_link_libraries(query-cf-database ekat::AllLibs)

components/eamxx/src/control/atmosphere_driver.cpp

@@ -14,11 +14,11 @@
 #include "share/io/eamxx_io_utils.hpp"
 #include "share/property_checks/mass_and_energy_column_conservation_check.hpp"
 
-#include "ekat/ekat_assert.hpp"
-#include "ekat/util/ekat_string_utils.hpp"
-#include "ekat/ekat_parameter_list.hpp"
-#include "ekat/ekat_parse_yaml_file.hpp"
-#include "ekat/std_meta/ekat_std_utils.hpp"
+#include <ekat_assert.hpp>
+#include <ekat_string_utils.hpp>
+#include <ekat_parameter_list.hpp>
+#include <ekat_yaml.hpp>
+#include <ekat_std_utils.hpp>
 
 // The global variable fvphyshack is used to help the initial pgN implementation
 // work around some current AD constraints. Search the code for "fvphyshack" to
@@ -955,23 +955,23 @@ void AtmosphereDriver::restart_model ()
 
   for (auto& it : m_atm_process_group->get_restart_extra_data()) {
     const auto& name = it.first;
-          auto& any  = it.second;
-
-    if (any.isType<int>()) {
-      ekat::any_cast<int>(any) = scorpio::get_attribute<int>(filename,"GLOBAL",name);
-    } else if (any.isType<std::int64_t>()) {
-      ekat::any_cast<std::int64_t>(any) = scorpio::get_attribute<std::int64_t>(filename,"GLOBAL",name);
-    } else if (any.isType<float>()) {
-      ekat::any_cast<float>(any) = scorpio::get_attribute<float>(filename,"GLOBAL",name);
-    } else if (any.isType<double>()) {
-      ekat::any_cast<double>(any) = scorpio::get_attribute<double>(filename,"GLOBAL",name);
-    } else if (any.isType<std::string>()) {
-      ekat::any_cast<std::string>(any) = scorpio::get_attribute<std::string>(filename,"GLOBAL",name);
+          auto& any  = *it.second;
+
+    if (any.type()==typeid(int)) {
+      std::any_cast<int&>(any) = scorpio::get_attribute<int>(filename,"GLOBAL",name);
+    } else if (any.type()==typeid(std::int64_t)) {
+      std::any_cast<std::int64_t&>(any) = scorpio::get_attribute<std::int64_t>(filename,"GLOBAL",name);
+    } else if (any.type()==typeid(float)) {
+      std::any_cast<float&>(any) = scorpio::get_attribute<float>(filename,"GLOBAL",name);
+    } else if (any.type()==typeid(double)) {
+      std::any_cast<double&>(any) = scorpio::get_attribute<double>(filename,"GLOBAL",name);
+    } else if (any.type()==typeid(std::string)) {
+      std::any_cast<std::string&>(any) = scorpio::get_attribute<std::string>(filename,"GLOBAL",name);
     } else {
       EKAT_ERROR_MSG (
           "Error! Unrecognized/unsupported concrete type for restart extra data.\n"
           " - extra data name  : " + name + "\n"
-          " - extra data typeid: " + any.content().type().name() + "\n");
+          " - extra data typeid: " + std::string(any.type().name()) + "\n");
     }
   }
 

components/eamxx/src/control/atmosphere_driver.hpp

@@ -12,9 +12,9 @@
 #include "share/atm_process/SCDataManager.hpp"
 #include "share/atm_process/IOPDataManager.hpp"
 
-#include "ekat/logging/ekat_logger.hpp"
-#include "ekat/mpi/ekat_comm.hpp"
-#include "ekat/ekat_parameter_list.hpp"
+#include <ekat_logger.hpp>
+#include <ekat_comm.hpp>
+#include <ekat_parameter_list.hpp>
 
 #include <memory>
 

components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp

@@ -1,7 +1,8 @@
 #include "atmosphere_surface_coupling_exporter.hpp"
 
-#include "ekat/ekat_assert.hpp"
-#include "ekat/util/ekat_units.hpp"
+#include <ekat_team_policy_utils.hpp>
+#include <ekat_assert.hpp>
+#include <ekat_units.hpp>
 
 #include <iomanip>
 
@@ -87,7 +88,7 @@ void SurfaceCouplingExporter::create_helper_field (const std::string& name,
   Field f(id);
   f.get_header().get_alloc_properties().request_allocation();
   f.allocate_view();
-  f.deep_copy(ekat::ScalarTraits<Real>::invalid());
+  f.deep_copy(ekat::invalid<Real>());
 
   m_helper_fields[name] = f;
 }
@@ -266,7 +267,7 @@ void SurfaceCouplingExporter::initialize_impl (const RunType /* run_type */)
         ++m_num_from_file_exports;
         --m_num_from_model_exports;
         auto& f_helper = m_helper_fields.at(fname);
-	// We want to add the field as a deep copy so that the helper_fields are automatically updated.
+        // We want to add the field as a deep copy so that the helper_fields are automatically updated.
         m_time_interp.add_field(f_helper.alias(rname), true);
         m_export_from_file_field_names.push_back(fname);
       }
@@ -288,9 +289,9 @@ void SurfaceCouplingExporter::initialize_impl (const RunType /* run_type */)
       for (size_t ii=0; ii<export_constant_fields.size(); ii++) {
         auto fname = export_constant_fields[ii];
         // Find the index for this field in the list of export fields.
-	auto v_loc = std::find(m_export_field_names_vector.begin(),m_export_field_names_vector.end(),fname);
-	EKAT_REQUIRE_MSG(v_loc != m_export_field_names_vector.end(), "ERROR!! surface_coupling_exporter::init - prescribed_constants has field with name " << fname << " which can't be found in set of exported fields\n.");
-	auto idx = v_loc - m_export_field_names_vector.begin();
+        auto v_loc = std::find(m_export_field_names_vector.begin(),m_export_field_names_vector.end(),fname);
+        EKAT_REQUIRE_MSG(v_loc != m_export_field_names_vector.end(), "ERROR!! surface_coupling_exporter::init - prescribed_constants has field with name " << fname << " which can't be found in set of exported fields\n.");
+        auto idx = v_loc - m_export_field_names_vector.begin();
         // This field should not have been set to anything else yet (recall FROM_MODEL is the default)
         EKAT_REQUIRE_MSG(m_export_source_h(idx)==FROM_MODEL,"Error! surface_coupling_exporter::init - attempting to set field " + fname + " export type, which has already been set.  Please check namelist options");
         m_export_source_h(idx) = CONSTANT;
@@ -364,7 +365,8 @@ void SurfaceCouplingExporter::set_from_file_exports()
 // index query in the below.
 void SurfaceCouplingExporter::compute_eamxx_exports(const double dt, const bool called_during_initialization)
 {
-  using PC = physics::Constants<Real>;
+  using PC  = physics::Constants<Real>;
+  using TPF = ekat::TeamPolicyFactory<KT::ExeSpace>;
 
   const auto& p_int                = get_field_in("p_int").get_view<const Real**>();
   const auto& pseudo_density       = get_field_in("pseudo_density").get_view<const Spack**>();
@@ -432,7 +434,7 @@ void SurfaceCouplingExporter::compute_eamxx_exports(const double dt, const bool
 
   // Preprocess exports
   auto export_source = m_export_source;
-  const auto setup_policy = ekat::ExeSpaceUtils<KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(num_cols, num_levs);
+  const auto setup_policy = TPF::get_thread_range_parallel_scan_team_policy(num_cols, num_levs);
   Kokkos::parallel_for(setup_policy, KOKKOS_LAMBDA(const Kokkos::TeamPolicy<KT::ExeSpace>::member_type& team) {
     const int i = team.league_rank();
 

components/eamxx/src/control/atmosphere_surface_coupling_exporter.hpp

@@ -9,7 +9,8 @@
 #include "share/atm_process/ATMBufferManager.hpp"
 #include "share/atm_process/SCDataManager.hpp"
 
-#include <ekat/ekat_parameter_list.hpp>
+#include <ekat_parameter_list.hpp>
+
 #include <string>
 
 namespace scream

components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp

@@ -3,8 +3,8 @@
 #include "share/property_checks/field_within_interval_check.hpp"
 #include "physics/share/physics_constants.hpp"
 
-#include "ekat/ekat_assert.hpp"
-#include "ekat/util/ekat_units.hpp"
+#include <ekat_assert.hpp>
+#include <ekat_units.hpp>
 
 #include <array>
 

components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp

@@ -1,11 +1,12 @@
 #ifndef SCREAM_IMPORTER_HPP
 #define SCREAM_IMPORTER_HPP
 
+#include "surface_coupling_utils.hpp"
+
 #include "share/atm_process/atmosphere_process.hpp"
-#include "ekat/ekat_parameter_list.hpp"
 #include "share/atm_process/SCDataManager.hpp"
 
-#include "surface_coupling_utils.hpp"
+#include <ekat_parameter_list.hpp>
 
 #include <string>
 

components/eamxx/src/control/tests/ad_tests.cpp

@@ -4,8 +4,8 @@
 #include "share/atm_process/atmosphere_process_group.hpp"
 #include "share/field/field_utils.hpp"
 
-#include "ekat/ekat_parameter_list.hpp"
-#include "ekat/ekat_parse_yaml_file.hpp"
+#include <ekat_parameter_list.hpp>
+#include <ekat_yaml.hpp>
 
 #include <catch2/catch.hpp>
 

components/eamxx/src/control/tests/dummy_atm_proc.hpp

@@ -2,8 +2,6 @@
 
 #include "share/atm_process/atmosphere_process.hpp"
 
-#include "ekat/ekat_pack.hpp"
-
 namespace scream {
 
 // === A dummy atm process, on Physics grid === //

components/eamxx/src/diagnostics/aerocom_cld.cpp

@@ -1,11 +1,11 @@
 #include "diagnostics/aerocom_cld.hpp"
 
-#include <ekat/kokkos/ekat_kokkos_utils.hpp>
-#include <string>
-
 #include "diagnostics/aerocom_cld_util.hpp"
 #include "share/util/eamxx_common_physics_functions.hpp"
 
+#include <ekat_team_policy_utils.hpp>
+#include <string>
+
 namespace scream {
 
 AeroComCld::AeroComCld(const ekat::Comm &comm,
@@ -98,7 +98,7 @@ void AeroComCld::compute_diagnostic_impl() {
    */
   using KT  = KokkosTypes<DefaultDevice>;
   using MT  = typename KT::MemberType;
-  using ESU = ekat::ExeSpaceUtils<typename KT::ExeSpace>;
+  using TPF = ekat::TeamPolicyFactory<typename KT::ExeSpace>;
 
   using PF = scream::PhysicsFunctions<DefaultDevice>;
 
@@ -124,7 +124,7 @@ void AeroComCld::compute_diagnostic_impl() {
   auto q_threshold           = q_thresh_set();
   auto cldfrac_tot_threshold = cldfrac_tot_thresh_set();
 
-  const auto policy = ESU::get_default_team_policy(m_ncols, m_nlevs);
+  const auto policy = TPF::get_default_team_policy(m_ncols, m_nlevs);
 
   const auto out = m_diagnostic_output.get_view<Real **>();