more

Author: WeiqunZhang

Src/Base/AMReX_BaseFab.H

@@ -1205,8 +1205,14 @@ BaseFab<T>::prefetchToHost () const noexcept
 #elif defined(AMREX_USE_CUDA) && !defined(_WIN32)
         if (Gpu::Device::devicePropMajor() >= 6) {
             std::size_t s = sizeof(T)*this->nvar*this->domain.numPts();
-            AMREX_CUDA_SAFE_CALL(cudaMemPrefetchAsync(this->dptr, s,
-                                                      cudaCpuDeviceId,
+#if defined(__CUDACC__) && (__CUDACC_VER_MAJOR__ >= 13)
+            cudaMemLocation location = {};
+            location.type = cudaMemLocationTypeDevice;
+            location.id = cudaCpuDeviceId;
+#else
+            auto location = cudaCpuDeviceId;
+#endif
+            AMREX_CUDA_SAFE_CALL(cudaMemPrefetchAsync(this->dptr, s, location,
                                                       Gpu::gpuStream()));
         }
 #elif defined(AMREX_USE_HIP)
@@ -1229,8 +1235,14 @@ BaseFab<T>::prefetchToDevice () const noexcept
 #elif defined(AMREX_USE_CUDA) && !defined(_WIN32)
         if (Gpu::Device::devicePropMajor() >= 6) {
             std::size_t s = sizeof(T)*this->nvar*this->domain.numPts();
-            AMREX_CUDA_SAFE_CALL(cudaMemPrefetchAsync(this->dptr, s,
-                                                      Gpu::Device::deviceId(),
+#if defined(__CUDACC__) && (__CUDACC_VER_MAJOR__ >= 13)
+            cudaMemLocation location = {};
+            location.type = cudaMemLocationTypeDevice;
+            location.id = Gpu::Device::deviceId();
+#else
+            auto location = Gpu::Device::deviceId();
+#endif
+            AMREX_CUDA_SAFE_CALL(cudaMemPrefetchAsync(this->dptr, s, location,
                                                       Gpu::gpuStream()));
         }
 #elif defined(AMREX_USE_HIP)

Src/Base/AMReX_GpuContainers.H

@@ -344,9 +344,16 @@ namespace amrex::Gpu {
         // Currently only implemented for CUDA.
 #if defined(AMREX_USE_CUDA) && !defined(_WIN32)
         if (Gpu::Device::devicePropMajor() >= 6) {
+#if defined(__CUDACC__) && (__CUDACC_VER_MAJOR__ >= 13)
+            cudaMemLocation location = {};
+            location.type = cudaMemLocationTypeDevice;
+            location.id = cudaCpuDeviceId;
+#else
+            auto location = cudaCpuDeviceId;
+#endif
             AMREX_CUDA_SAFE_CALL(cudaMemPrefetchAsync(&(*begin),
                                                       size*sizeof(value_type),
-                                                      cudaCpuDeviceId,
+                                                      location,
                                                       Gpu::gpuStream()));
         }
 #endif
@@ -375,9 +382,16 @@ namespace amrex::Gpu {
         // Currently only implemented for CUDA.
 #if defined(AMREX_USE_CUDA) && !defined(_WIN32)
         if (Gpu::Device::devicePropMajor() >= 6) {
+#if defined(__CUDACC__) && (__CUDACC_VER_MAJOR__ >= 13)
+            cudaMemLocation location = {};
+            location.type = cudaMemLocationTypeDevice;
+            location.id = Gpu::Device::deviceId();
+#else
+            auto location = Gpu::Device::deviceId();
+#endif
             AMREX_CUDA_SAFE_CALL(cudaMemPrefetchAsync(&(*begin),
                                                       size*sizeof(value_type),
-                                                      Gpu::Device::deviceId(),
+                                                      location,
                                                       Gpu::gpuStream()));
         }
 #endif

Src/Base/AMReX_GpuDevice.cpp

@@ -828,19 +828,20 @@ Device::mem_advise_set_preferred (void* p, std::size_t sz, int device)
 #if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
     if (device_prop.managedMemory == 1 && device_prop.concurrentManagedAccess == 1)
     {
+#if defined(AMREX_USE_CUDA)
 #if defined(__CUDACC__) && (__CUDACC_VER_MAJOR__ >= 13)
         cudaMemLocation location = {};
         location.type = cudaMemLocationTypeDevice;
         location.id = device;
-        AMREX_CUDA_SAFE_CALL(
-            cudaMemAdvise(p, sz, cudaMemAdviseSetPreferredLocation, location));
 #else
+        auto location = device;
+#endif
+#endif
         AMREX_HIP_OR_CUDA
             (AMREX_HIP_SAFE_CALL(
                  hipMemAdvise(p, sz, hipMemAdviseSetPreferredLocation, device)),
              AMREX_CUDA_SAFE_CALL(
-                 cudaMemAdvise(p, sz, cudaMemAdviseSetPreferredLocation, device)));
-#endif
+                 cudaMemAdvise(p, sz, cudaMemAdviseSetPreferredLocation, location)));
     }
 #elif defined(AMREX_USE_SYCL)
     // xxxxx SYCL todo: mem_advise
@@ -859,19 +860,20 @@ Device::mem_advise_set_readonly (void* p, std::size_t sz)
 #if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
     if (device_prop.managedMemory == 1 && device_prop.concurrentManagedAccess == 1)
     {
+#if defined(AMREX_USE_CUDA)
 #if defined(__CUDACC__) && (__CUDACC_VER_MAJOR__ >= 13)
         cudaMemLocation location = {};
         location.type = cudaMemLocationTypeDevice;
         location.id = cudaCpuDeviceId;
-        AMREX_CUDA_SAFE_CALL(
-            cudaMemAdvise(p, sz, cudaMemAdviseSetReadMostly, location));
 #else
+        auto location = cudaCpuDeviceId;
+#endif
+#endif
         AMREX_HIP_OR_CUDA
             (AMREX_HIP_SAFE_CALL(
                  hipMemAdvise(p, sz, hipMemAdviseSetReadMostly, hipCpuDeviceId)),
              AMREX_CUDA_SAFE_CALL(
-                 cudaMemAdvise(p, sz, cudaMemAdviseSetReadMostly, cudaCpuDeviceId)));
-#endif
+                 cudaMemAdvise(p, sz, cudaMemAdviseSetReadMostly, location)));
     }
 #elif defined(AMREX_USE_SYCL)
     // xxxxx SYCL todo: mem_advise