Add SCREAM_SYSTEM_WORKAROUND_P3_PART2. Return support for crayclang-scream. Rename craygnuamdgpu.

Author: trey-ornl

cime_config/machines/cmake_macros/craygnu-hipcc.cmake renamed to cime_config/machines/cmake_macros/craygnuamdgpu.cmake

@@ -5,7 +5,7 @@ set(SCC "cc")
 set(SCXX "hipcc")
 set(SFC "ftn")
 
-string(APPEND CPPDEFS " -DLINUX -DFORTRANUNDERSCORE -DNO_R16 -DCPRGNU")
+string(APPEND CPPDEFS " -DLINUX -DFORTRANUNDERSCORE -DNO_R16 -DCPRGNU -DSCREAM_SYSTEM_WORKAROUND_P3_PART2")
 if (COMP_NAME STREQUAL gptl)
     string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
 endif()

cime_config/machines/config_machines.xml

@@ -1098,7 +1098,7 @@
     <DESC>Frontier. AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs.</DESC>
     <NODENAME_REGEX>.*frontier.*</NODENAME_REGEX>
     <OS>CNL</OS>
-    <COMPILERS>craygnu-hipcc</COMPILERS>
+    <COMPILERS>craygnuamdgpu,crayclang-scream</COMPILERS>
     <MPILIBS>mpich</MPILIBS>
     <PROJECT>cli115</PROJECT>
     <SAVE_TIMING_DIR>/lustre/orion/proj-shared/cli115</SAVE_TIMING_DIR>
@@ -1135,7 +1135,7 @@
       <cmd_path lang="sh">module</cmd_path>
       <cmd_path lang="csh">module</cmd_path>
       <cmd_path lang="python">/usr/share/lmod/lmod/libexec/lmod python</cmd_path>
-      <modules>
+      <modules compiler="craygnuamdgpu">
         <command name="reset"></command>
         <!-- PrgEnv-gnu before cpe, or cray-mpich module doesn't update correctly -->
         <command name="load">PrgEnv-gnu</command>
@@ -1152,6 +1152,25 @@
         <command name="load">cray-parallel-netcdf</command>
         <command name="unload">darshan-runtime</command>
       </modules>
+      <modules compiler="crayclang-scream">
+        <command name="reset"></command>
+        <command name="load">cpe/22.12</command>
+        <command name="load">craype-accel-amd-gfx90a</command>
+        <command name="load">rocm/5.4.0</command>
+        <command name="load">libunwind/1.6.2</command>
+        <command name="load">cce/15.0.1</command>
+        <command name="load">libfabric/1.15.2.0</command>
+        <command name="load">craype/2.7.20</command>
+        <command name="load">cray-mpich/8.1.26</command>
+        <command name="load">cray-python/3.9.13.1</command>
+        <command name="load">subversion/1.14.1</command>
+        <command name="load">git/2.36.1</command>
+        <command name="load">cmake/3.21.3</command>
+        <command name="load">cray-hdf5-parallel/1.12.2.1</command>
+        <command name="load">cray-netcdf-hdf5parallel/4.9.0.1</command>
+        <command name="load">cray-parallel-netcdf/1.12.3.1</command>
+        <command name="unload">darshan-runtime</command>
+      </modules>
     </module_system>
 
     <RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
@@ -1176,9 +1195,13 @@
       <env name="OMP_PLACES">threads</env>
     </environment_variables>
 
-    <environment_variables compiler="craygnu-hipcc" mpilib="mpich">
+    <environment_variables compiler="craygnuamdgpu" mpilib="mpich">
       <env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /sw/frontier/spack-envs/cpe24.07-gpu/opt/gcc-13.2/adios2-2.10.0-cyq7kvlfq5nixmw5unih2gjdkhvitava; else echo "$ADIOS2_ROOT"; fi}</env>
     </environment_variables>
+    <environment_variables compiler="crayclang-scream" mpilib="mpich">
+      <env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.26/crayclang-scream-14.0.0; else echo "$ADIOS2_ROOT"; fi}</env>
+    </environment_variables>
+
   </machine>
 
   <!-- Skylake nodes of Stampede2 at TACC -->

components/eamxx/CMakeLists.txt

@@ -550,9 +550,8 @@ DisableMpiCxxBindings()
 if (SCREAM_DOUBLE_PRECISION)
   if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel")
     set(SCREAM_Fortran_FLAGS -real-size 64)
-    #elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "[Cc]lang" OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "HIPCC")
-    #find what is crayclang id in cmake
-    #set(SCREAM_Fortran_FLAGS -s default32 -eZ)
+  elseif("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "Cray")
+    set(SCREAM_Fortran_FLAGS -s default32 -eZ)
   else()
     set(SCREAM_Fortran_FLAGS -fdefault-real-8 -fdefault-double-8)
   endif()

components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp

@@ -10,6 +10,9 @@ namespace p3 {
  * this file, #include p3_functions.hpp instead.
  */
 
+#ifdef SCREAM_SYSTEM_WORKAROUND_P3_PART2
+#pragma clang optimize off
+#endif
 template <>
 void Functions<Real,DefaultDevice>
 ::p3_main_part2_disp(
@@ -130,7 +133,9 @@ ::p3_main_part2_disp(
     if (!hydrometeorsPresent(i)) return;
   });
 }
-
+#ifdef SCREAM_SYSTEM_WORKAROUND_P3_PART2
+#pragma clang optimize on
+#endif
 } // namespace p3
 } // namespace scream