Remove YAKL backend support for eamxx/rrtmgp

components/eamxx/CMakeLists.txt

@@ -68,7 +68,7 @@ else()
 endif()
 
 ####################################################################
-#                  Kokkos/YAKL-related settings                    #
+#                  Kokkos related settings                         #
 ####################################################################
 
 if (Kokkos_ENABLE_CUDA)
@@ -95,31 +95,25 @@ endif()
 option (Kokkos_ENABLE_SERIAL "" ON)
 
 set (EAMXX_ENABLE_GPU FALSE CACHE BOOL "")
-set (CUDA_BUILD FALSE CACHE BOOL "") #needed for yakl if kokkos vars are not visible there?
-set (HIP_BUILD FALSE CACHE BOOL "") #needed for yakl if kokkos vars are not visible there?
-set (SYCL_BUILD FALSE CACHE BOOL "") #needed for yakl if kokkos vars are not visible there?
 
 # Determine if this is a Cuda build.
 if (Kokkos_ENABLE_CUDA)
   # Add CUDA as a language for CUDA builds
   enable_language(CUDA)
   set (EAMXX_ENABLE_GPU TRUE  CACHE BOOL "" FORCE)
-  set (CUDA_BUILD TRUE CACHE BOOL "" FORCE) #needed for yakl if kokkos vars are not visible there?
 endif ()
 
 # Determine if this is a HIP build.
 if (Kokkos_ENABLE_HIP)
   # Add CUDA as a language for CUDA builds
   enable_language(HIP)
   set (EAMXX_ENABLE_GPU TRUE CACHE BOOL "" FORCE)
-  set (HIP_BUILD TRUE CACHE BOOL "" FORCE) #needed for yakl if kokkos vars are not visible there?
 endif ()
 
 # Determine if this is a sycl build.
 if (Kokkos_ENABLE_SYCL)
   #enable_language(SYCL)
   set (EAMXX_ENABLE_GPU TRUE CACHE BOOL "" FORCE)
-  set (SYCL_BUILD TRUE CACHE BOOL "" FORCE) #needed for yakl if kokkos vars are not visible there?
 endif ()
 
 if( NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "[Cc]lang" )
@@ -226,17 +220,6 @@ endif()
 #     #cmakedefine RRTMGP_EXPENSIVE_CHECKS
 option (SCREAM_RRTMGP_DEBUG "Turn on extra debug checks in RRTMGP" ${SCREAM_DEBUG})
 
-option(SCREAM_RRTMGP_ENABLE_YAKL "Use YAKL under rrtmgp" FALSE)
-option(SCREAM_RRTMGP_ENABLE_KOKKOS "Use Kokkos under rrtmgp" TRUE)
-if (SCREAM_RRTMGP_ENABLE_YAKL)
-  add_definitions("-DRRTMGP_ENABLE_YAKL")
-endif()
-
-if (SCREAM_RRTMGP_ENABLE_KOKKOS)
-  add_definitions("-DRRTMGP_ENABLE_KOKKOS")
-endif()
-
-
 set(SCREAM_DOUBLE_PRECISION TRUE CACHE BOOL "Set to double precision (default True)")
 
 # For now, only used in share/grid/remap/refining_remapper_rma.*pp
@@ -623,8 +606,6 @@ function (print_var var)
 endfunction ()
 
 print_var(EAMXX_ENABLE_GPU)
-print_var(CUDA_BUILD)
-print_var(HIP_BUILD)
 print_var(SCREAM_MACHINE)
 print_var(SCREAM_DYNAMICS_DYCORE)
 print_var(SCREAM_DOUBLE_PRECISION)

components/eamxx/src/physics/rrtmgp/CMakeLists.txt

@@ -2,153 +2,21 @@ include(EkatUtils)
 include(EkatSetCompilerFlags)
 include(ScreamUtils)
 
-# Copied from EKAT, YAKL is an interface target so requires special
-# handling. Get rid of this once RRTMGP is using kokkos.
-macro (SetCudaFlagsYakl targetName)
-  if (Kokkos_ENABLE_CUDA)
-    # We must find CUDA
-    find_package(CUDA REQUIRED)
-
-    # Still check if CUDA_FOUND is true, since we don't know if the particular
-    # FindCUDA.cmake module being used is checking _FIND_REQUIRED
-    if (NOT CUDA_FOUND)
-      message (FATAL_ERROR "Error! Unable to find CUDA.")
-    endif()
-
-    set(options CUDA_LANG)
-    set(args1v)
-    set(argsMv FLAGS)
-    cmake_parse_arguments(SCF "${options}" "${args1v}" "${argsMv}" ${ARGN})
-
-    if (SCF_FLAGS)
-      set (FLAGS ${SCF_FLAGS})
-    else ()
-      # We need host-device lambdas
-      set (FLAGS --expt-extended-lambda)
-
-      IsDebugBuild (SCF_DEBUG)
-      if (SCF_DEBUG)
-        # Turn off fused multiply add for debug so we can stay BFB with host
-        list (APPEND FLAGS --fmad=false)
-      endif()
-    endif()
-
-    # Set the flags on the target
-    if (SCF_CUDA_LANG)
-      # User is setting the src files language to CUDA
-      target_compile_options (${targetName} INTERFACE
-        "$<$<COMPILE_LANGUAGE:CUDA>:${FLAGS}>")
-    else()
-      # We assume the user is setting the src files lang to CXX
-      target_compile_options (${targetName} INTERFACE
-        "$<$<COMPILE_LANGUAGE:CXX>:${FLAGS}>")
-    endif()
-  endif()
-endmacro()
-
-##################################
-#             YAKL               #
-##################################
-
-# RRTMGP++ requires YAKL
-if (SCREAM_RRTMGP_ENABLE_YAKL)
-  string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_ci)
-  if (TARGET yakl)
-    # Other E3SM components are building YAKL...
-    message ("It appears some other part of E3SM is building YAKL.\n"
-      "We will reuse that, but if this is a debug build we will\n"
-      "add the --fmad=false flag to the cuda flags used by YAKL\n")
-  else ()
-    # Prepare CUDA/HIP flags for YAKL
-    if (CUDA_BUILD)
-      string(REPLACE ";" " " KOKKOS_CUDA_OPTIONS_STR "${KOKKOS_CUDA_OPTIONS}")
-      set(YAKL_ARCH "CUDA")
-      set(YAKL_CUDA_FLAGS "-DYAKL_ARCH_CUDA ${KOKKOS_CUDA_OPTIONS_STR} --expt-relaxed-constexpr -ccbin ${CMAKE_CXX_COMPILER}")
-      string (REPLACE " " ";" YAKL_CUDA_FLAGS_LIST ${YAKL_CUDA_FLAGS})
-    endif()
-    if (HIP_BUILD)
-      set(YAKL_ARCH "HIP")
-      set(YAKL_HIP_FLAGS "-DYAKL_ARCH_HIP -O3 -D__HIP_ROCclr__ -D__HIP_ARCH_GFX90A__=1 --rocm-path=${ROCM_PATH} --offload-arch=gfx90a -x hip")
-      string (REPLACE " " ";" YAKL_HIP_FLAGS_LIST ${YAKL_HIP_FLAGS})
-    endif()
-    if (SYCL_BUILD)
-      set(YAKL_ARCH "SYCL")
-      set(YAKL_SYCL_FLAGS " -fp-model precise -DYAKL_ARCH_SYCL -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64")
-      string (REPLACE " " ";" YAKL_SYCL_FLAGS_LIST ${YAKL_SYCL_FLAGS})
-    endif()
-
-    set (YAKL_SOURCE_DIR ${SCREAM_BASE_DIR}/../../externals/YAKL)
-    add_subdirectory(${YAKL_SOURCE_DIR} ${CMAKE_BINARY_DIR}/externals/YAKL)
-
-    # Set some additional flag/cpp option on the yakl target
-
-    cmake_policy (SET CMP0079 NEW) # Allow to link to a tgt from a different directory
-
-    # EAMxx *requires* MPI, so simply look for it, then link against it
-    find_package(MPI REQUIRED COMPONENTS C)
-    target_link_libraries (yakl INTERFACE MPI::MPI_C)
-
-    # For debug builds, set -DYAKL_DEBUG
-    if (CMAKE_BUILD_TYPE_ci STREQUAL "debug")
-      target_compile_definitions(yakl INTERFACE YAKL_DEBUG)
-    endif()
-  endif()
-
-  # See eamxx/src/dynamics/homme/CMakeLists.txt for an explanation of this
-  # workaround.
-  if ((SCREAM_MACHINE STREQUAL "ascent" OR SCREAM_MACHINE STREQUAL "pm-gpu") AND CMAKE_BUILD_TYPE_ci STREQUAL "debug")
-    SetCudaFlagsYakl(yakl CUDA_LANG FLAGS -UNDEBUG)
-  else()
-    SetCudaFlagsYakl(yakl CUDA_LANG)
-  endif()
-
-  list(APPEND CMAKE_MODULE_PATH ${YAKL_SOURCE_DIR})
-  include (yakl_utils)
-endif()
-
 ##################################
 #           RRTMGP               #
 ##################################
 
 set(EAM_RRTMGP_DIR ${SCREAM_BASE_DIR}/../eam/src/physics/rrtmgp)
 # Build RRTMGP library; this builds the core RRTMGP external source as a library named "rrtmgp"
 # NOTE: The external RRTMGP build needs some fixes to work with CUDA in a library build, so for now we will build these ourselves
-set(EXTERNAL_SRC
-  ${EAM_RRTMGP_DIR}/external/cpp/rrtmgp/kernels/mo_gas_optics_kernels.cpp
-  ${EAM_RRTMGP_DIR}/external/cpp/rrtmgp/mo_rrtmgp_util_reorder.cpp
-  ${EAM_RRTMGP_DIR}/external/cpp/rte/expand_and_transpose.cpp
-  ${EAM_RRTMGP_DIR}/external/cpp/rte/kernels/mo_fluxes_broadband_kernels.cpp
-  ${EAM_RRTMGP_DIR}/external/cpp/rte/kernels/mo_optical_props_kernels.cpp
-  ${EAM_RRTMGP_DIR}/external/cpp/rte/kernels/mo_rte_solver_kernels.cpp
-  ${EAM_RRTMGP_DIR}/external/cpp/extensions/fluxes_byband/mo_fluxes_byband_kernels.cpp
-  ${EAM_RRTMGP_DIR}/external/cpp/examples/all-sky/mo_garand_atmos_io.cpp
-  ${EAM_RRTMGP_DIR}/external/cpp/examples/all-sky/mo_load_cloud_coefficients.cpp
-  ${EAM_RRTMGP_DIR}/external/cpp/examples/mo_load_coefficients.cpp
-)
-add_library(rrtmgp ${EXTERNAL_SRC})
-target_compile_definitions(rrtmgp PUBLIC EAMXX_HAS_RRTMGP)
-EkatDisableAllWarning(rrtmgp)
-if (SCREAM_RRTMGP_ENABLE_YAKL)
-  yakl_process_target(rrtmgp)
-else()
-  if (CUDA_BUILD)
-    target_compile_options(rrtmgp PUBLIC $<$<COMPILE_LANGUAGE:CXX>:--expt-relaxed-constexpr>)
-  endif()
-endif()
+add_library(rrtmgp INTERFACE)
+target_compile_definitions(rrtmgp INTERFACE EAMXX_HAS_RRTMGP)
 
-# NOTE: cannot use 'PUBLIC' in target_link_libraries,
-#       since yakl_process_target already used it
-#       with the "plain" signature
 if (NOT TARGET Kokkos::kokkos)
   find_package(Kokkos REQUIRED)
 endif ()
-if (SCREAM_RRTMGP_ENABLE_YAKL)
-  target_link_libraries(rrtmgp yakl Kokkos::kokkos)
-else()
-  target_link_libraries(rrtmgp Kokkos::kokkos)
-endif()
-target_include_directories(rrtmgp PUBLIC
-    ${SCREAM_BASE_DIR}/../../externals/YAKL
+target_link_libraries(rrtmgp INTERFACE Kokkos::kokkos)
+target_include_directories(rrtmgp INTERFACE
     ${EAM_RRTMGP_DIR}/external/cpp
     ${EAM_RRTMGP_DIR}/external/cpp/extensions/cloud_optics
     ${EAM_RRTMGP_DIR}/external/cpp/examples
@@ -166,29 +34,14 @@ target_include_directories(rrtmgp PUBLIC
 # separates out the code that comprises the core RRTMGP library from the extensions
 # and examples that we have modified for use in SCREAM specifically.
 
-# However, due to the mix of YAKL and Kokkos, we split the target in two:
-#  - scream_rrtmgp: kokkos-based interface to EAMxx
-#  - scream_rrtmgp_yakl: source codes to be built with YAKL flags/options
-
-##################################
-#       SCREAM_RRTMGP_YAKL       #
-##################################
-
 set(SCREAM_RRTMGP_SOURCES_INTERFACE
   eamxx_rrtmgp_interface.cpp
 )
 
 add_library(eamxx_rrtmgp_interface ${SCREAM_RRTMGP_SOURCES_INTERFACE})
-if (SCREAM_RRTMGP_ENABLE_YAKL)
-  yakl_process_target(eamxx_rrtmgp_interface)
-endif()
-
-# NOTE: cannot use 'PUBLIC' in target_link_libraries,
-#       since yakl_process_target already used it
-#       with the "plain" signature
 
 find_library(NETCDF_C netcdf HINTS ${NetCDF_C_PATH} PATH_SUFFIXES lib lib64)
-target_link_libraries(eamxx_rrtmgp_interface ${NETCDF_C} rrtmgp scream_share Kokkos::kokkos)
+target_link_libraries(eamxx_rrtmgp_interface PUBLIC ${NETCDF_C} rrtmgp scream_share Kokkos::kokkos)
 target_include_directories(eamxx_rrtmgp_interface PUBLIC
     ${CMAKE_CURRENT_SOURCE_DIR})
 target_include_directories(eamxx_rrtmgp_interface SYSTEM PUBLIC
@@ -213,19 +66,6 @@ target_include_directories(scream_rrtmgp PUBLIC
     ${CMAKE_CURRENT_SOURCE_DIR}
     ${CMAKE_CURRENT_BINARY_DIR}/modules)
 
-# If yakl builds with LANG!=CXX, then the yakl CPP defines don't transfer to scream
-# targets, b/c of the lang difference. So, if YAKL_ARCH is set, we add
-# ${YAKL_${YAKL_ARCH}_FLAGS} flags to the CXX flags of scream_rrtmgp.
-# In particular, this will ensure that all the yakl macros
-# are correctly defined in YAKL headers, depending on the backend
-if (SCREAM_RRTMGP_ENABLE_YAKL)
-  if (YAKL_ARCH)
-    target_compile_options(scream_rrtmgp PUBLIC
-      "$<$<COMPILE_LANGUAGE:CXX>:${YAKL_${YAKL_ARCH}_FLAGS_LIST}>")
-  endif()
-endif()
-
-
 # Ensure RRTMGP lookup tables are present in the data dir
 set (RRTMGP_TABLES
   scream/init/rrtmgp-data-sw-g112-210809.nc