added -DACC_KERNELS

Author: pletzer

CMakeLists.txt

@@ -8,6 +8,7 @@ set(CMAKE_CXX_STANDARD 11)
 
 option(OPENACC "Enable OpenACC support" OFF)
 option(ACC_MULTICORE "Whether for OpenACC to compile for multicore" OFF)
+option(ACC_KERNELS "Whether for OpenACC to use the kernels directives" OFF)
 option(OPENMP "Enable OpenMP support" ON)
 option(OPENMP_OFFLOAD "Enable OpenMP offloading to GPU support" OFF)
 option(ROCM "Enable ROCm, offloading to MI100 using OpenMPI 4.5" OFF)
@@ -110,23 +111,28 @@ if ((NOT OPENACC) AND OPENMP_FOUND)
 else ()
   # OpenACC is incompatible with OpenMP, either one or the other or none
   if (OPENACC)
-      add_definitions(-DHAVE_OPENACC)
-      if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "^NVHPC|PGI")
-	  if (ACC_MULTICORE)
-              message(STATUS "Apply OpenACC directives for offloading to multicore CPU")
-              set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -acc=multicore -Minfo=acc")
+    add_definitions(-DHAVE_OPENACC)
+    if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "^NVHPC|PGI")
+      if (ACC_KERNELS)
+        message(STATUS "Apply OpenACC kernels directives")
+        add_definitions(-DHAVE_OPENACC_KERNELS)
+      endif ()
+	    if (ACC_MULTICORE)
+        message(STATUS "Apply OpenACC directives for offloading to multicore CPU")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -acc=multicore -Minfo=acc")
 	      set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -acc=multicore -Minfo=acc")
-	  else ()
-              set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -acc -gpu=mem:managed -Minfo=acc")
+	    else ()
+        message(STATUS "Apply OpenACC directives for offloading to GPU")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -acc -gpu=mem:managed -Minfo=acc")
 	      set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -acc -gpu=mem:managed -Minfo=acc")
-	  endif ()
-	  message(STATUS "Using PGI or Nvidia C++ compiler, added OpenACC compiler switches")
-      elseif ("${CMAKE_CXX_COMPILER_ID}" MATCHES "^GNU")
-	  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenacc -fopt-info-optimized-omp")
-	  set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fopenacc -fopt-info-optimized-omp")
-	  message(STATUS "Using GNU C++ compiler, added OpenACC compiler switches")
-      endif ()
-      message(STATUS "OpenACC enabled")
+	    endif ()
+	    message(STATUS "Using PGI or Nvidia C++ compiler, added OpenACC compiler switches")
+    elseif ("${CMAKE_CXX_COMPILER_ID}" MATCHES "^GNU")
+	    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenacc -fopt-info-optimized-omp")
+	    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fopenacc -fopt-info-optimized-omp")
+	    message(STATUS "Using GNU C++ compiler, added OpenACC compiler switches")
+    endif ()
+    message(STATUS "OpenACC enabled")
   endif () 
 endif ()
 

upwind/fortran/upwindF08Gpu.F90

@@ -162,7 +162,11 @@ subroutine upwind_advect(this, deltaTime)
         enddo
 
 #if defined(HAVE_OPENACC)
+    #if defined(USE_OPENACC_KERNELS)
+        !$acc kernels
+    #else
         !$acc parallel loop
+    #endif
 #else
         !$omp target
         !$omp parallel do
@@ -171,15 +175,23 @@ subroutine upwind_advect(this, deltaTime)
             oldF(i) = fptr(i)
         enddo
 #if defined(HAVE_OPENACC)
+    #if defined(USE_OPENACC_KERNELS)
+        !$acc end kernels
+    #else
         !$acc end parallel loop
+    #endif
 #else
         !$omp end parallel do
         !$omp end target
 #endif
 
         ! iterate over the cells
 #if defined(HAVE_OPENACC)
+    #if defined(USE_OPENACC_KERNELS)
+        !$acc kernels
+    #else
         !$acc parallel loop private(inds, j, oldIndex, upI)
+    #endif
 #else
         !$omp target
         !$omp parallel do private(inds, j, oldIndex, upI)
@@ -215,7 +227,11 @@ subroutine upwind_advect(this, deltaTime)
             enddo
         enddo
 #if defined(HAVE_OPENACC)
+    #if defined(USE_OPENACC_KERNELS)
+        !$acc end kernels
+    #else
         !$acc end parallel loop
+    #endif
 #else   
         !$omp end parallel do
         !$omp end target