Merge pull request #3058 from ambrad/ambrad/eamxx/shoc-0set-bugfix

EAMxx/SHOC: Fix a subtle GPU bug.

Author: ambrad

cime_config/tests.py

@@ -705,6 +705,7 @@
             "PEM_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-spa_remap--scream-output-preset-4",
             "ERS_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5",
             "ERP_Ln22.conusx4v1pg2_r05_oECv3.F2010-SCREAMv1-noAero.scream-bfbhash--scream-output-preset-6",
+            "ERS_Ln22.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-L128--scream-output-preset-4"
             )
     },
 

components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp

@@ -138,8 +138,9 @@ class SHOCMacrophysics : public scream::AtmosphereProcess
 
         // Dry static energy
         shoc_s(i,k) = PF::calculate_dse(T_mid(i,k),z_mid(i,k),phis(i));
+
+        if (k+1 == nlev_packs) zi_grid(i,nlevi_v)[nlevi_p] = 0;
       });
-      zi_grid(i,nlevi_v)[nlevi_p] = 0;
       team.team_barrier();
 
       const auto zt_grid_s = ekat::subview(zt_grid, i);

components/eamxx/src/share/atm_process/atmosphere_process.hpp

@@ -274,7 +274,8 @@ class AtmosphereProcess : public ekat::enable_shared_from_this<AtmosphereProcess
 
   // For internal diagnostics and debugging.
   void print_global_state_hash(const std::string& label, const bool in = true,
-                               const bool out = true, const bool internal = true) const;
+                               const bool out = true, const bool internal = true,
+                               const Real* mem = nullptr, const int nmem = 0) const;
   // For BFB tracking in production simulations.
   void print_fast_global_state_hash(const std::string& label) const;
 

components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp

@@ -113,19 +113,32 @@ void hash (const std::list<FieldGroup>& fgs, HashType& accum) {
 
 } // namespace anon
 
+// (mem, nmem) describe an arbitrary device array. If non-0, the array will be
+// hashed and reported as a fourth line.
 void AtmosphereProcess
 ::print_global_state_hash (const std::string& label, const bool in, const bool out,
-                           const bool internal) const {
-  static constexpr int nslot = 3;
+                           const bool internal, const Real* mem, const int nmem) const {
+  static constexpr int nslot = 4;
   HashType laccum[nslot] = {0};
   hash(m_fields_in, laccum[0]);
   hash(m_groups_in, laccum[0]);
   hash(m_fields_out, laccum[1]);
   hash(m_groups_out, laccum[1]);
   hash(m_internal_fields, laccum[2]);
+  const bool hash_array = mem != nullptr;
+  if (hash_array) {
+    HashType accum = 0;
+    Kokkos::parallel_reduce(
+      Kokkos::RangePolicy<ExeSpace>(0, nmem),
+      KOKKOS_LAMBDA(const int i, HashType& accum) { bfbhash::hash(mem[i], accum); },
+      bfbhash::HashReducer<>(accum));
+    Kokkos::fence();
+    laccum[3] = accum;
+  }
   HashType gaccum[nslot];
-  bfbhash::all_reduce_HashType(m_comm.mpi_comm(), laccum, gaccum, nslot);
-  const bool show[] = {in, out, internal};
+  const int nr = hash_array ? nslot : nslot-1;
+  bfbhash::all_reduce_HashType(m_comm.mpi_comm(), laccum, gaccum, nr);
+  const bool show[] = {in, out, internal, hash_array};
   if (m_comm.am_i_root())
     for (int i = 0; i < nslot; ++i)
       if (show[i])