Merge branch 'bartgol/eamxx/homme-pd-remap-fix' into next (PR #7477)

When updating the views after subview info changes, we also need to update the stored subview (for later check).

Also, modify model_restart test, so that we don't re-run the 1st part of the run. Instead, make the base run also
write the restart file (like CIME does). Note: this requires running less than 2*rest_freq steps, or else the rpointer
file will be nuked, and the rest run will NOT find the first .r file. Hence, run 2+1 steps (with restart every 2) rather
than 1+1.

Author: bartgol

components/eamxx/src/dynamics/homme/physics_dynamics_remapper.cpp

@@ -181,7 +181,7 @@ subfields_info_has_changed (const std::map<int,SubviewInfo>& subfield_info,
 }
 
 void PhysicsDynamicsRemapper::
-update_subfields_views (const std::map<int,SubviewInfo>& subfield_info,
+update_subfields_views (std::map<int,SubviewInfo>& subfield_info,
                         const ViewsRepo& repo,
                         const std::vector<Field>& fields) const
 {
@@ -205,10 +205,12 @@ update_subfields_views (const std::map<int,SubviewInfo>& subfield_info,
     }
   };
 
-  for (const auto& it : subfield_info) {
-    const auto& f = fields[it.first];
-    if ( not(it.second==f.get_header().get_alloc_properties().get_subview_info()) ){
-      get_view(it.first,fields[it.first]);
+  for (auto& [fname, svi] : subfield_info) {
+    const auto& f = fields[fname];
+    const auto& new_svi = f.get_header().get_alloc_properties().get_subview_info();
+    if ( not(svi==new_svi) ) {
+      get_view(fname,f);
+      svi = new_svi;
     }
   }
   Kokkos::deep_copy(repo.views,  repo.h_views);

components/eamxx/src/dynamics/homme/physics_dynamics_remapper.hpp

@@ -151,7 +151,7 @@ class PhysicsDynamicsRemapper : public AbstractRemapper
 
   bool subfields_info_has_changed (const std::map<int,SubviewInfo>& subfield_info,
                                    const std::vector<Field>& fields) const;
-  void update_subfields_views (const std::map<int,SubviewInfo>& subfield_info,
+  void update_subfields_views (std::map<int,SubviewInfo>& subfield_info,
                                const ViewsRepo& repo,
                                const std::vector<Field>& fields) const;
 

components/eamxx/src/share/field/field_utils_impl.hpp

@@ -1005,6 +1005,32 @@ void print_field_hyperslab (const Field& f,
         }
         break;
       }
+      case 5:
+      {
+        dims_str[dims_left[4]] = ":";
+        auto v = f.get_strided_view<const T*****,Host>();
+        for (int i=0; i<layout.dim(0); ++i) {
+          dims_str[dims_left[0]] = std::to_string(i);
+          for (int j=0; j<layout.dim(1); ++j) {
+            dims_str[dims_left[1]] = std::to_string(j);
+            for (int k=0; k<layout.dim(2); ++k) {
+              dims_str[dims_left[2]] = std::to_string(k);
+              for (int l=0; l<layout.dim(3); ++l) {
+                dims_str[dims_left[3]] = std::to_string(l);
+                out << "  " << f.name() << "(" << ekat::join(dims_str,",") << ")";
+                for (int m=0; m<layout.dim(3); ++m) {
+                  if (m%max_per_line==0) {
+                    out << "\n    ";
+                  }
+                  out << v(i,j,k,l,m) << ", ";
+                }
+                out << "\n";
+              }
+            }
+          }
+        }
+        break;
+      }
       default:
         EKAT_ERROR_MSG (
             "Unsupported rank in print_field_hyperslab.\n"

components/eamxx/tests/multi-process/dynamics_physics/model_restart/CMakeLists.txt

@@ -8,68 +8,66 @@ BuildCprnc()
 #                 HOMME_TARGET   NP PLEV QSIZE_D
 CreateDynamicsLib("theta-l_kokkos"  4   72   10)
 
-# We have 3 runs:
-#  1) run for 2*N time steps starting from t=0 (baseline run)
-#  2) run for N time steps starting from t=0 (init run)
-#  3) run for N time steps re-starting from t=N*dt (restarted run)
-# We can use the same namelist for all tests, using 3 different input yaml files
+# We have 2 runs:
+#  1) run for 2*N time steps starting from t=0 (base run), write restart files at Nth step
+#  3) run for N time steps re-starting from t=N*dt (rest run)
+# We can use the same namelist for all tests. For input/output yaml files, we use the same
+# yaml template, but need to configure them b/c the run_t0 and the output file prefix differ
+# in the 2 runs.
 
 # Create a single executable for all the 3 runs
 CreateADUnitTestExec(model_restart
   LIBS cld_fraction ${dynLibName} shoc p3 scream_rrtmgp)
 
 # Set time integration options
 set (CASE_T0 2023-01-01-00000)
-set (CASE_TN 2023-01-01-00060)
+set (RUN_DT 30)
 
-# Create the baseline (run all 6 timsteps in a single run)
-CreateUnitTestFromExec(model_baseline model_restart
-                        EXE_ARGS "--args -ifile=input_baseline.yaml"
-                        MPI_RANKS ${SCREAM_TEST_MAX_RANKS}
-                        FIXTURES_SETUP baseline_run)
+set (HIST_FREQ 90)
+set (HIST_FREQ_UNITS nsecs)
+set (REST_FREQ 60)
+set (REST_FREQ_UNITS nsecs)
 
-# Start a simulation, but only run half of the time steps
-CreateUnitTestFromExec(model_initial model_restart
-                        EXE_ARGS "--args -ifile=input_initial.yaml"
+# Create the baseline (run all timsteps in a single run) as well as the restart files
+set(SUFFIX "base")
+set (RUN_T0 2023-01-01-00000)
+set (RUN_NSTEPS 3)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml
+               ${CMAKE_CURRENT_BINARY_DIR}/input_base.yaml)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml
+               ${CMAKE_CURRENT_BINARY_DIR}/output_base.yaml)
+CreateUnitTestFromExec(model_restart_base model_restart
+                        EXE_ARGS "--args -ifile=input_base.yaml"
                         MPI_RANKS ${SCREAM_TEST_MAX_RANKS}
-                        FIXTURES_SETUP initial_run
+                        FIXTURES_SETUP base_run
                         PROPERTIES RESOURCE_LOCK rpointer_file)
 
 # Restart the simulation, and run the second half of the time steps
-CreateUnitTestFromExec(model_restart model_restart
-                        EXE_ARGS "--args -ifile=input_restarted.yaml"
+set(SUFFIX "rest")
+set (RUN_T0 2023-01-01-00060)
+set (RUN_NSTEPS 1)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml
+               ${CMAKE_CURRENT_BINARY_DIR}/input_rest.yaml)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml
+               ${CMAKE_CURRENT_BINARY_DIR}/output_rest.yaml)
+CreateUnitTestFromExec(model_restart_rest model_restart
+                        EXE_ARGS "--args -ifile=input_rest.yaml"
                         MPI_RANKS ${SCREAM_TEST_MAX_RANKS}
-                        FIXTURES_REQUIRED initial_run
-                        FIXTURES_SETUP restarted_run
+                        FIXTURES_REQUIRED base_run
+                        FIXTURES_SETUP rest_run
                         PROPERTIES RESOURCE_LOCK rpointer_file)
 
-# Finally, compare the nc outputs generated by the basline and restarted runs
-# IMPORTANT: make sure these file names match what baseline/restarted runs produce
-set (SRC_FILE model_output_baseline.AVERAGE.nmins_x1.np${SCREAM_TEST_MAX_RANKS}.${CASE_T0}.nc)
-set (TGT_FILE model_output.AVERAGE.nmins_x1.np${SCREAM_TEST_MAX_RANKS}.${CASE_T0}.nc)
+# Finally, compare the nc outputs generated by the base and rest runs
+# IMPORTANT: make sure these file names match what base/rest runs produced
+set (SRC_FILE model_output_base.AVERAGE.${HIST_FREQ_UNITS}_x${HIST_FREQ}.np${SCREAM_TEST_MAX_RANKS}.${CASE_T0}.nc)
+set (TGT_FILE model_output_rest.AVERAGE.${HIST_FREQ_UNITS}_x${HIST_FREQ}.np${SCREAM_TEST_MAX_RANKS}.${CASE_T0}.nc)
 
-add_test (NAME restarted_vs_monolithic_check_np${SCREAM_TEST_MAX_RANKS}
+add_test (NAME model_restart_check_np${SCREAM_TEST_MAX_RANKS}
           COMMAND cmake -P ${CMAKE_BINARY_DIR}/bin/CprncTest.cmake ${SRC_FILE} ${TGT_FILE}
           WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
-set_tests_properties (restarted_vs_monolithic_check_np${SCREAM_TEST_MAX_RANKS} PROPERTIES
+set_tests_properties (model_restart_check_np${SCREAM_TEST_MAX_RANKS} PROPERTIES
                       RESOURCE_GROUPS "devices:1"
-                      FIXTURES_REQUIRED "baseline_run;restarted_run")
-
-# Configure yaml input file to run directory
-set (RUN_T0 2023-01-01-00000)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_baseline.yaml
-               ${CMAKE_CURRENT_BINARY_DIR}/input_baseline.yaml)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_initial.yaml
-               ${CMAKE_CURRENT_BINARY_DIR}/input_initial.yaml)
-set (RUN_T0 2023-01-01-00030)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_restarted.yaml
-               ${CMAKE_CURRENT_BINARY_DIR}/input_restarted.yaml)
-
-# The two yaml files that control the output streams (for the baseline and restart runs)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/model_output.yaml
-               ${CMAKE_CURRENT_BINARY_DIR}/model_output.yaml COPYONLY)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/model_restart_output.yaml
-               ${CMAKE_CURRENT_BINARY_DIR}/model_restart_output.yaml COPYONLY)
+                      FIXTURES_REQUIRED "base_run;rest_run")
 
 # Set homme's test options, so that we can configure the namelist correctly
 # Discretization/algorithm settings

components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_initial.yaml renamed to components/eamxx/tests/multi-process/dynamics_physics/model_restart/input.yaml

@@ -4,9 +4,9 @@ driver_options:
   atmosphere_dag_verbosity_level: 5
 
 time_stepping:
-  time_step: 30
-  number_of_steps: 1
-  run_t0: 2023-01-01-00000  # YYYY-MM-DD-XXXXX
+  time_step: ${RUN_DT}
+  number_of_steps: ${RUN_NSTEPS}
+  run_t0: ${RUN_T0}  # YYYY-MM-DD-XXXXX
   case_t0: 2023-01-01-00000  # YYYY-MM-DD-XXXXX
 
 initial_conditions:
@@ -68,13 +68,13 @@ grids_manager:
   type: homme
   physics_grid_type: gll
   dynamics_namelist_file_name: namelist.nl
-  vertical_coordinate_filename: IC_FILE
+  vertical_coordinate_filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev}
 
 # List all the yaml files with the output parameters
 scorpio:
   model_restart:
     output_control:
-      frequency:       30
-      frequency_units: nsecs
-  output_yaml_files: ["model_restart_output.yaml"]
+      frequency:       ${REST_FREQ}
+      frequency_units: ${REST_FREQ_UNITS}
+  output_yaml_files: ["output_${SUFFIX}.yaml"]
 ...