Skip to content

Remove YAKL backend support for eamxx/rrtmgp #7345

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
May 15, 2025
Merged

Remove YAKL backend support for eamxx/rrtmgp #7345

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ccf65fb
Remove YAKL backend support for eamxx/rrtmgp
jgfouca May 12, 2025
612f34b
Remove legacy YAKL build settings (CUDA_BUILD etc)
jgfouca May 12, 2025
9db9236
Fix enable kokkos
jgfouca May 12, 2025
7cdc48d
Fix CUDA
jgfouca May 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 3 additions & 20 deletions components/eamxx/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ else()
endif()

####################################################################
# Kokkos/YAKL-related settings #
# Kokkos related settings #
####################################################################

if (Kokkos_ENABLE_CUDA)
Expand All @@ -95,31 +95,25 @@ endif()
option (Kokkos_ENABLE_SERIAL "" ON)

set (EAMXX_ENABLE_GPU FALSE CACHE BOOL "")
set (CUDA_BUILD FALSE CACHE BOOL "") #needed for yakl if kokkos vars are not visible there?
set (HIP_BUILD FALSE CACHE BOOL "") #needed for yakl if kokkos vars are not visible there?
set (SYCL_BUILD FALSE CACHE BOOL "") #needed for yakl if kokkos vars are not visible there?

# Determine if this is a Cuda build.
if (Kokkos_ENABLE_CUDA)
# Add CUDA as a language for CUDA builds
enable_language(CUDA)
set (EAMXX_ENABLE_GPU TRUE CACHE BOOL "" FORCE)
set (CUDA_BUILD TRUE CACHE BOOL "" FORCE) #needed for yakl if kokkos vars are not visible there?
endif ()

# Determine if this is a HIP build.
if (Kokkos_ENABLE_HIP)
# Add CUDA as a language for CUDA builds
enable_language(HIP)
set (EAMXX_ENABLE_GPU TRUE CACHE BOOL "" FORCE)
set (HIP_BUILD TRUE CACHE BOOL "" FORCE) #needed for yakl if kokkos vars are not visible there?
endif ()

# Determine if this is a sycl build.
if (Kokkos_ENABLE_SYCL)
#enable_language(SYCL)
set (EAMXX_ENABLE_GPU TRUE CACHE BOOL "" FORCE)
set (SYCL_BUILD TRUE CACHE BOOL "" FORCE) #needed for yakl if kokkos vars are not visible there?
endif ()

if( NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "[Cc]lang" )
Expand Down Expand Up @@ -225,17 +219,8 @@ endif()
# and then adding to eamxx_config.h:
# #cmakedefine RRTMGP_EXPENSIVE_CHECKS
option (SCREAM_RRTMGP_DEBUG "Turn on extra debug checks in RRTMGP" ${SCREAM_DEBUG})

option(SCREAM_RRTMGP_ENABLE_YAKL "Use YAKL under rrtmgp" FALSE)
option(SCREAM_RRTMGP_ENABLE_KOKKOS "Use Kokkos under rrtmgp" TRUE)
if (SCREAM_RRTMGP_ENABLE_YAKL)
add_definitions("-DRRTMGP_ENABLE_YAKL")
endif()

if (SCREAM_RRTMGP_ENABLE_KOKKOS)
add_definitions("-DRRTMGP_ENABLE_KOKKOS")
endif()

# This can be removed once rrtmgp is kokkos-only.
add_definitions("-DRRTMGP_ENABLE_KOKKOS")

set(SCREAM_DOUBLE_PRECISION TRUE CACHE BOOL "Set to double precision (default True)")

Expand Down Expand Up @@ -623,8 +608,6 @@ function (print_var var)
endfunction ()

print_var(EAMXX_ENABLE_GPU)
print_var(CUDA_BUILD)
print_var(HIP_BUILD)
print_var(SCREAM_MACHINE)
print_var(SCREAM_DYNAMICS_DYCORE)
print_var(SCREAM_DOUBLE_PRECISION)
Expand Down
3 changes: 0 additions & 3 deletions components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/kokkos_rrtmgp/shell_commands
View file
Open in desktop

This file was deleted.

2 changes: 0 additions & 2 deletions components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/yakl_rrtmgp/shell_commands
View file
Open in desktop

This file was deleted.

170 changes: 5 additions & 165 deletions components/eamxx/src/physics/rrtmgp/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,153 +2,21 @@ include(EkatUtils)
include(EkatSetCompilerFlags)
include(ScreamUtils)

# Copied from EKAT, YAKL is an interface target so requires special
# handling. Get rid of this once RRTMGP is using kokkos.
macro (SetCudaFlagsYakl targetName)
if (Kokkos_ENABLE_CUDA)
# We must find CUDA
find_package(CUDA REQUIRED)

# Still check if CUDA_FOUND is true, since we don't know if the particular
# FindCUDA.cmake module being used is checking _FIND_REQUIRED
if (NOT CUDA_FOUND)
message (FATAL_ERROR "Error! Unable to find CUDA.")
endif()

set(options CUDA_LANG)
set(args1v)
set(argsMv FLAGS)
cmake_parse_arguments(SCF "${options}" "${args1v}" "${argsMv}" ${ARGN})

if (SCF_FLAGS)
set (FLAGS ${SCF_FLAGS})
else ()
# We need host-device lambdas
set (FLAGS --expt-extended-lambda)

IsDebugBuild (SCF_DEBUG)
if (SCF_DEBUG)
# Turn off fused multiply add for debug so we can stay BFB with host
list (APPEND FLAGS --fmad=false)
endif()
endif()

# Set the flags on the target
if (SCF_CUDA_LANG)
# User is setting the src files language to CUDA
target_compile_options (${targetName} INTERFACE
"$<$<COMPILE_LANGUAGE:CUDA>:${FLAGS}>")
else()
# We assume the user is setting the src files lang to CXX
target_compile_options (${targetName} INTERFACE
"$<$<COMPILE_LANGUAGE:CXX>:${FLAGS}>")
endif()
endif()
endmacro()

##################################
# YAKL #
##################################

# RRTMGP++ requires YAKL
if (SCREAM_RRTMGP_ENABLE_YAKL)
string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_ci)
if (TARGET yakl)
# Other E3SM components are building YAKL...
message ("It appears some other part of E3SM is building YAKL.\n"
"We will reuse that, but if this is a debug build we will\n"
"add the --fmad=false flag to the cuda flags used by YAKL\n")
else ()
# Prepare CUDA/HIP flags for YAKL
if (CUDA_BUILD)
string(REPLACE ";" " " KOKKOS_CUDA_OPTIONS_STR "${KOKKOS_CUDA_OPTIONS}")
set(YAKL_ARCH "CUDA")
set(YAKL_CUDA_FLAGS "-DYAKL_ARCH_CUDA ${KOKKOS_CUDA_OPTIONS_STR} --expt-relaxed-constexpr -ccbin ${CMAKE_CXX_COMPILER}")
string (REPLACE " " ";" YAKL_CUDA_FLAGS_LIST ${YAKL_CUDA_FLAGS})
endif()
if (HIP_BUILD)
set(YAKL_ARCH "HIP")
set(YAKL_HIP_FLAGS "-DYAKL_ARCH_HIP -O3 -D__HIP_ROCclr__ -D__HIP_ARCH_GFX90A__=1 --rocm-path=${ROCM_PATH} --offload-arch=gfx90a -x hip")
string (REPLACE " " ";" YAKL_HIP_FLAGS_LIST ${YAKL_HIP_FLAGS})
endif()
if (SYCL_BUILD)
set(YAKL_ARCH "SYCL")
set(YAKL_SYCL_FLAGS " -fp-model precise -DYAKL_ARCH_SYCL -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64")
string (REPLACE " " ";" YAKL_SYCL_FLAGS_LIST ${YAKL_SYCL_FLAGS})
endif()

set (YAKL_SOURCE_DIR ${SCREAM_BASE_DIR}/../../externals/YAKL)
add_subdirectory(${YAKL_SOURCE_DIR} ${CMAKE_BINARY_DIR}/externals/YAKL)

# Set some additional flag/cpp option on the yakl target

cmake_policy (SET CMP0079 NEW) # Allow to link to a tgt from a different directory

# EAMxx *requires* MPI, so simply look for it, then link against it
find_package(MPI REQUIRED COMPONENTS C)
target_link_libraries (yakl INTERFACE MPI::MPI_C)

# For debug builds, set -DYAKL_DEBUG
if (CMAKE_BUILD_TYPE_ci STREQUAL "debug")
target_compile_definitions(yakl INTERFACE YAKL_DEBUG)
endif()
endif()

# See eamxx/src/dynamics/homme/CMakeLists.txt for an explanation of this
# workaround.
if ((SCREAM_MACHINE STREQUAL "ascent" OR SCREAM_MACHINE STREQUAL "pm-gpu") AND CMAKE_BUILD_TYPE_ci STREQUAL "debug")
SetCudaFlagsYakl(yakl CUDA_LANG FLAGS -UNDEBUG)
else()
SetCudaFlagsYakl(yakl CUDA_LANG)
endif()

list(APPEND CMAKE_MODULE_PATH ${YAKL_SOURCE_DIR})
include (yakl_utils)
endif()

##################################
# RRTMGP #
##################################

set(EAM_RRTMGP_DIR ${SCREAM_BASE_DIR}/../eam/src/physics/rrtmgp)
# Build RRTMGP library; this builds the core RRTMGP external source as a library named "rrtmgp"
# NOTE: The external RRTMGP build needs some fixes to work with CUDA in a library build, so for now we will build these ourselves
set(EXTERNAL_SRC
${EAM_RRTMGP_DIR}/external/cpp/rrtmgp/kernels/mo_gas_optics_kernels.cpp
${EAM_RRTMGP_DIR}/external/cpp/rrtmgp/mo_rrtmgp_util_reorder.cpp
${EAM_RRTMGP_DIR}/external/cpp/rte/expand_and_transpose.cpp
${EAM_RRTMGP_DIR}/external/cpp/rte/kernels/mo_fluxes_broadband_kernels.cpp
${EAM_RRTMGP_DIR}/external/cpp/rte/kernels/mo_optical_props_kernels.cpp
${EAM_RRTMGP_DIR}/external/cpp/rte/kernels/mo_rte_solver_kernels.cpp
${EAM_RRTMGP_DIR}/external/cpp/extensions/fluxes_byband/mo_fluxes_byband_kernels.cpp
${EAM_RRTMGP_DIR}/external/cpp/examples/all-sky/mo_garand_atmos_io.cpp
${EAM_RRTMGP_DIR}/external/cpp/examples/all-sky/mo_load_cloud_coefficients.cpp
${EAM_RRTMGP_DIR}/external/cpp/examples/mo_load_coefficients.cpp
)
add_library(rrtmgp ${EXTERNAL_SRC})
target_compile_definitions(rrtmgp PUBLIC EAMXX_HAS_RRTMGP)
EkatDisableAllWarning(rrtmgp)
if (SCREAM_RRTMGP_ENABLE_YAKL)
yakl_process_target(rrtmgp)
else()
if (CUDA_BUILD)
target_compile_options(rrtmgp PUBLIC $<$<COMPILE_LANGUAGE:CXX>:--expt-relaxed-constexpr>)
endif()
endif()
add_library(rrtmgp INTERFACE)
target_compile_definitions(rrtmgp INTERFACE EAMXX_HAS_RRTMGP)

# NOTE: cannot use 'PUBLIC' in target_link_libraries,
# since yakl_process_target already used it
# with the "plain" signature
if (NOT TARGET Kokkos::kokkos)
find_package(Kokkos REQUIRED)
endif ()
if (SCREAM_RRTMGP_ENABLE_YAKL)
target_link_libraries(rrtmgp yakl Kokkos::kokkos)
else()
target_link_libraries(rrtmgp Kokkos::kokkos)
endif()
target_include_directories(rrtmgp PUBLIC
${SCREAM_BASE_DIR}/../../externals/YAKL
target_link_libraries(rrtmgp INTERFACE Kokkos::kokkos)
target_include_directories(rrtmgp INTERFACE
${EAM_RRTMGP_DIR}/external/cpp
${EAM_RRTMGP_DIR}/external/cpp/extensions/cloud_optics
${EAM_RRTMGP_DIR}/external/cpp/examples
Expand All @@ -166,29 +34,14 @@ target_include_directories(rrtmgp PUBLIC
# separates out the code that comprises the core RRTMGP library from the extensions
# and examples that we have modified for use in SCREAM specifically.

# However, due to the mix of YAKL and Kokkos, we split the target in two:
# - scream_rrtmgp: kokkos-based interface to EAMxx
# - scream_rrtmgp_yakl: source codes to be built with YAKL flags/options

##################################
# SCREAM_RRTMGP_YAKL #
##################################

set(SCREAM_RRTMGP_SOURCES_INTERFACE
eamxx_rrtmgp_interface.cpp
)

add_library(eamxx_rrtmgp_interface ${SCREAM_RRTMGP_SOURCES_INTERFACE})
if (SCREAM_RRTMGP_ENABLE_YAKL)
yakl_process_target(eamxx_rrtmgp_interface)
endif()

# NOTE: cannot use 'PUBLIC' in target_link_libraries,
# since yakl_process_target already used it
# with the "plain" signature

find_library(NETCDF_C netcdf HINTS ${NetCDF_C_PATH} PATH_SUFFIXES lib lib64)
target_link_libraries(eamxx_rrtmgp_interface ${NETCDF_C} rrtmgp scream_share Kokkos::kokkos)
target_link_libraries(eamxx_rrtmgp_interface PUBLIC ${NETCDF_C} rrtmgp scream_share Kokkos::kokkos)
target_include_directories(eamxx_rrtmgp_interface PUBLIC
${CMAKE_CURRENT_SOURCE_DIR})
target_include_directories(eamxx_rrtmgp_interface SYSTEM PUBLIC
Expand All @@ -213,19 +66,6 @@ target_include_directories(scream_rrtmgp PUBLIC
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_BINARY_DIR}/modules)

# If yakl builds with LANG!=CXX, then the yakl CPP defines don't transfer to scream
# targets, b/c of the lang difference. So, if YAKL_ARCH is set, we add
# ${YAKL_${YAKL_ARCH}_FLAGS} flags to the CXX flags of scream_rrtmgp.
# In particular, this will ensure that all the yakl macros
# are correctly defined in YAKL headers, depending on the backend
if (SCREAM_RRTMGP_ENABLE_YAKL)
if (YAKL_ARCH)
target_compile_options(scream_rrtmgp PUBLIC
"$<$<COMPILE_LANGUAGE:CXX>:${YAKL_${YAKL_ARCH}_FLAGS_LIST}>")
endif()
endif()


# Ensure RRTMGP lookup tables are present in the data dir
set (RRTMGP_TABLES
scream/init/rrtmgp-data-sw-g112-210809.nc
Expand Down
Loading
Loading