Skip to content

Commit 90397bb

Browse files
pdziekanpdziekan
committed
Merge branch 'master' of github.com:igfuw/libcloudphxx into th_pres_opts
2 parents abc7b57 + 65055aa commit 90397bb

20 files changed

+150
-85
lines changed

‎.github/workflows/test_libclouphxx.yml‎

Lines changed: 59 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ on:
88

99
jobs:
1010
build_CUDA:
11-
runs-on: ubuntu-20.04
11+
runs-on: ubuntu-24.04
1212
strategy:
1313
matrix:
1414
name: ["CUDA"]
@@ -18,20 +18,24 @@ jobs:
1818
disable_cuda: false
1919
# not enough RAM to compile Debug CUDA on 4 threads
2020
threads: 1
21+
tag: "uwlcm_ubuntu_24_04_cuda_12_9_0"
22+
app_opts: "--nv"
2123

2224
steps:
2325
- uses: actions/checkout@v2
2426

2527
- name: build libcloudph++
26-
uses: igfuw/libcloudphxx_build@v0.3
28+
uses: igfuw/libcloudphxx_build@master
2729
with:
2830
disable_cuda: ${{matrix.disable_cuda}}
2931
build_type: ${{matrix.build_type}}
3032
threads: ${{matrix.threads}}
3133
path: ${{ github.workspace }}
34+
tag: ${{ matrix.tag }}
35+
app_opts: ${{ matrix.app_opts }}
3236

3337
build:
34-
runs-on: ubuntu-20.04
38+
runs-on: ubuntu-24.04
3539
strategy:
3640
matrix:
3741
name: ["no_CUDA"]
@@ -43,17 +47,17 @@ jobs:
4347
disable_cuda: true
4448
threads: 4
4549
- mpi: "none"
46-
tag: "ubuntu_20_04_cuda_11_4"
50+
tag: "uwlcm_ubuntu_24_04_cuda_12_9_0"
4751
cxx: "g++"
4852
- mpi: "mvapich2"
49-
tag: "ubuntu_20_04_cuda_11_4_mvapich2"
53+
tag: "uwlcm_ubuntu_24_04_cuda_12_9_0_mvapich2"
5054
cxx: "mpic++"
5155

5256
steps:
5357
- uses: actions/checkout@v2
5458

5559
- name: build libcloudph++
56-
uses: igfuw/libcloudphxx_build@v0.3
60+
uses: igfuw/libcloudphxx_build@master
5761
with:
5862
disable_cuda: ${{matrix.disable_cuda}}
5963
build_type: ${{matrix.build_type}}
@@ -76,17 +80,17 @@ jobs:
7680
# test jobs
7781
unit_test:
7882
needs: build
79-
runs-on: ubuntu-20.04
83+
runs-on: ubuntu-24.04
8084

8185
strategy:
8286
matrix:
8387
build_type: ["RelWithDebInfoPortable", "Debug"]
8488
mpi: ["none", "mvapich2"]
8589
include:
8690
- mpi: "none"
87-
tag: "ubuntu_20_04_cuda_11_4"
91+
tag: "uwlcm_ubuntu_24_04_cuda_12_9_0"
8892
- mpi: "mvapich2"
89-
tag: "ubuntu_20_04_cuda_11_4_mvapich2"
93+
tag: "uwlcm_ubuntu_24_04_cuda_12_9_0_mvapich2"
9094

9195
steps:
9296
- uses: actions/checkout@v2
@@ -99,21 +103,21 @@ jobs:
99103
- name: Decompress libcloudph++ build
100104
run: tar -xvf build.tar
101105

102-
- name: load UWLCM Singularity image
103-
uses: igfuw/load_UWLCM_singularity_image@v0.1b
106+
- name: load UWLCM Apptainer image
107+
uses: igfuw/load_UWLCM_singularity_image@main
104108
with:
105-
path: ${{ github.workspace }}/singularity_images
109+
path: ${{ github.workspace }}/apptainer_images
106110
tag: ${{ matrix.tag }}
107111

108112
- name: Run unit tests
109113
working-directory: ${{github.workspace}}/build
110114
# See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
111-
run: OMP_NUM_THREADS=4 singularity exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code)
115+
run: OMP_NUM_THREADS=4 apptainer exec $SI ctest -C ${{matrix.build_type}} || cat Testing/Temporary/LastTest.log / # "/" intentional! (just to make cat exit with an error code)
112116

113117

114118
kinematic_2D_test:
115119
needs: build
116-
runs-on: ubuntu-20.04
120+
runs-on: ubuntu-24.04
117121

118122
strategy:
119123
matrix:
@@ -145,7 +149,7 @@ jobs:
145149
path: libmpdataxx
146150

147151
- name: Install libmpdata++
148-
uses: igfuw/libmpdataxx_install@v0.5
152+
uses: igfuw/libmpdataxx_install@master
149153
with:
150154
build_type: ${{matrix.build_type}}
151155
threads: 4
@@ -154,17 +158,17 @@ jobs:
154158

155159
- name: Configure kinematic_2D CMake
156160
working-directory: ${{github.workspace}}/models/kinematic_2D
157-
run: singularity exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++
161+
run: apptainer exec -B ${{ github.workspace }}/installed/ $SI cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -Dlibcloudph++_DIR=${{ github.workspace }}/installed/share/libcloudph++ -Dlibmpdata++_DIR=${{ github.workspace }}/installed/share/libmpdata++
158162

159163

160164
- name: Build kinematic_2D
161165
working-directory: ${{github.workspace}}/models/kinematic_2D
162-
run: VERBOSE=1 singularity exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}}
166+
run: VERBOSE=1 apptainer exec -B ${{ github.workspace }}/installed/ $SI cmake --build build --config ${{matrix.build_type}}
163167

164168
- name: Run kinematic_2D tests
165169
working-directory: ${{github.workspace}}/models/kinematic_2D/build
166170
run: |
167-
singularity exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian)
171+
apptainer exec -B ${{ github.workspace }}/installed/ $SI ctest -VV -R travis # compare icicle results against reference data (done for full simulation for bulk schemes and a couple of steps for lagrangian)
168172
cat Testing/Temporary/LastTest.log
169173
170174
# Debugging with a ssh session on failure
@@ -177,7 +181,7 @@ jobs:
177181
needs: build
178182
# do after kinematic_2D_test, because otherwise we occasionally get "illegal instruction" errors, probably because libcloudphxx is installed simultaneously on the same runner at the same location by two different jobs (?)
179183
# needs: kinematic_2D_test
180-
runs-on: ubuntu-20.04
184+
runs-on: ubuntu-24.04
181185

182186
strategy:
183187
matrix:
@@ -203,14 +207,27 @@ jobs:
203207
- name: Decompress libcloudph++ build
204208
run: tar -xvf build.tar
205209

206-
- name: load UWLCM Singularity image
207-
uses: igfuw/load_UWLCM_singularity_image@v0.1b
210+
- name: load UWLCM Apptainer image
211+
uses: igfuw/load_UWLCM_singularity_image@main
208212
with:
209-
path: ${{ github.workspace }}/singularity_images
213+
path: ${{ github.workspace }}/apptainer_images
210214

211215
- name: Install libcloudph++
212216
run: sudo cmake --install build
213217

218+
- name: add libcloudhphxx.so to PYTHONPATH
219+
run: echo "PYTHONPATH=${{ github.workspace }}/installed$(python3 -c "import sysconfig; print(sysconfig.get_path('platlib'))")" >> $GITHUB_ENV
220+
# uses: allenevans/set-env@v2.0.0
221+
# with:
222+
# PYTHONPATH: ${{ github.workspace }}/installed$(python3 -c "import sysconfig; print(sysconfig.get_path('platlib'))"):$PYTHONPATH
223+
# uses: alsosee/envset@v1
224+
# with:
225+
# name: PYTHONPATH
226+
# value: ${{ github.workspace }}/installed$(python3 -c "import sysconfig; print(sysconfig.get_path('platlib'))"):$PYTHONPATH
227+
228+
- name: check PYTHONPATH
229+
run: echo ${PYTHONPATH}
230+
214231
- name: checkout parcel repo
215232
uses: actions/checkout@v2
216233
with:
@@ -222,21 +239,22 @@ jobs:
222239
- name: run parcel unit_test
223240
working-directory: ${{github.workspace}}/parcel
224241
if: ${{matrix.long_tests}}
225-
run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test
242+
run: apptainer exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test
243+
#run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages apptainer exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test
226244
#run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed,/usr/lib/python3/dist-packages/Gnuplot $SI python3 -m pytest -v unit_test
227245

228246
- name: run parcel long_test
229247
working-directory: ${{github.workspace}}/parcel
230248
if: ${{matrix.long_tests}}
231-
run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v long_test
249+
run: apptainer exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v long_test
232250

233251
- name: run parcel unit_test_debug
234252
working-directory: ${{github.workspace}}/parcel
235253
if: ${{matrix.debug_tests}}
236-
run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test_debug
254+
run: apptainer exec -B${{ github.workspace }}/installed $SI python3 -m pytest -s -v unit_test_debug
237255

238256
build_and_test_KiD-A:
239-
runs-on: ubuntu-20.04
257+
runs-on: ubuntu-24.04
240258
strategy:
241259
matrix:
242260
name: ["no_CUDA"]
@@ -263,7 +281,7 @@ jobs:
263281
- run: grep diag_accr include/libcloudph++/lgrngn/particles.hpp
264282

265283
- name: build libcloudph++
266-
uses: igfuw/libcloudphxx_build@v0.3
284+
uses: igfuw/libcloudphxx_build@master
267285
with:
268286
disable_cuda: ${{matrix.disable_cuda}}
269287
build_type: ${{matrix.build_type}}
@@ -274,6 +292,19 @@ jobs:
274292
- name: Install libcloudph++
275293
run: sudo cmake --install build
276294

295+
- name: add libcloudhphxx.so to PYTHONPATH
296+
run: echo "PYTHONPATH=${{ github.workspace }}/installed$(python3 -c "import sysconfig; print(sysconfig.get_path('platlib'))")" >> $GITHUB_ENV
297+
# uses: allenevans/set-env@v2.0.0
298+
# with:
299+
# PYTHONPATH: ${{ github.workspace }}/installed$(python3 -c "import sysconfig; print(sysconfig.get_path('platlib'))"):$PYTHONPATH
300+
# uses: alsosee/envset@v1
301+
# with:
302+
# name: PYTHONPATH
303+
# value: ${{ github.workspace }}/installed$(python3 -c "import sysconfig; print(sysconfig.get_path('platlib'))"):$PYTHONPATH
304+
305+
- name: check PYTHONPATH
306+
run: echo ${PYTHONPATH}
307+
277308
- name: checkout KiD-libcloud repo
278309
uses: actions/checkout@v2
279310
with:
@@ -282,7 +313,7 @@ jobs:
282313

283314
- name: run KiD LWP test
284315
working-directory: ${{github.workspace}}/kid-libcloud
285-
run: PYTHONPATH=${{ github.workspace }}/installed/usr/lib/python3/dist-packages singularity exec -B${{ github.workspace }}/installed $SI bash ./.travis_scripts/lwp_test.sh
316+
run: apptainer exec -B${{ github.workspace }}/installed $SI bash ./.travis_scripts/lwp_test.sh
286317

287318
call_test_uwlcm_hlpr:
288319
uses: igfuw/UWLCM/.github/workflows/test_uwlcm_hlpr.yml@master

‎CMakeLists.txt‎

Lines changed: 47 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ target_include_directories(cloudphxx_lgrngn
117117
target_compile_features(cloudphxx_lgrngn PUBLIC cxx_std_14)
118118
# config-specific flags
119119
target_compile_options(cloudphxx_lgrngn PRIVATE $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:DEBUG>>: -g -Og -DTHRUST_DEBUG>)
120-
target_compile_options(cloudphxx_lgrngn PRIVATE $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:RELEASE>>: -Ofast -march=native -Winline -DNDEBUG >)
120+
target_compile_options(cloudphxx_lgrngn PRIVATE $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:RELEASE>>: -Ofast -march=native -DNDEBUG >)
121121
target_compile_options(cloudphxx_lgrngn PRIVATE $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:RELWITHDEBINFO>>: -Ofast -march=native>)
122122
target_compile_options(cloudphxx_lgrngn PRIVATE $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:RELWITHDEBINFOPORTABLE>>: -Ofast>)
123123

@@ -158,26 +158,48 @@ if (CMAKE_CUDA_COMPILER)
158158
message(STATUS "Trying to obtain CUDA capability of local hardware...")
159159
set(pfx "compute cabability check")
160160
execute_process(COMMAND "mktemp" "-d" "/tmp/tmp.XXX" RESULT_VARIABLE status OUTPUT_VARIABLE tmpdir)
161+
string(STRIP "${tmpdir}" tmpdir) # Removes leading/trailing whitespace and newlines
161162
if (NOT status EQUAL 0)
162163
message(FATAL_ERROR "${pfx}: mkdtemp failed")
163164
endif()
164165
file(WRITE "${tmpdir}/test.cu" "
165166
#include <iostream>
167+
#include <fstream>
166168
167169
int main()
168170
{
169171
int DeviceCount;
170-
cudaGetDeviceCount(&DeviceCount);
172+
std::ofstream fdev(\"${tmpdir}/device_capability.txt\");
173+
cudaError_t error_id = cudaGetDeviceCount(&DeviceCount);
174+
175+
if (error_id != cudaSuccess) { // error_id==100 is 'no CUDA-capable device is detected'
176+
std::cerr << \"cudaGetDeviceCount returned \" << static_cast<int>(error_id) << \" -> \" << cudaGetErrorString(error_id) << std::endl;
177+
return 0;
178+
// exit(EXIT_FAILURE);
179+
}
180+
181+
int driverVersion, runtimeVersion;
171182
for(int i = 0 ; i < DeviceCount; ++i)
172183
{
184+
cudaSetDevice(i);
173185
cudaDeviceProp prop;
174-
cudaError_t err = cudaGetDeviceProperties(&prop, i);
175-
if (err == cudaSuccess)
176-
std::cout << prop.major << prop.minor << \" \" ;
186+
cudaError_t error_id = cudaGetDeviceProperties(&prop, i);
187+
if (error_id == cudaSuccess)
188+
{
189+
std::cout << \" Device \" << i << \": \" << prop.name << std::endl;
190+
cudaDriverGetVersion(&driverVersion);
191+
cudaRuntimeGetVersion(&runtimeVersion);
192+
std::cout << \" CUDA Driver Version / Runtime Version \"
193+
<< driverVersion / 1000 << \".\" << (driverVersion % 100) / 10 << \" / \"
194+
<< runtimeVersion / 1000 << \".\" << (runtimeVersion % 100) / 10 << std::endl;
195+
std::cout << \" CUDA Capability Major/Minor version number: \" << prop.major << \".\" << prop.minor << std::endl;
196+
fdev << prop.major << prop.minor << \" \" ;
197+
}
177198
else
178199
{
179-
std::cerr << cudaGetErrorString(err);
180-
std::cout << \"? \";
200+
std::cerr << \"cudaGetDeviceProperties returned \" << static_cast<int>(error_id) << \" -> \" << cudaGetErrorString(error_id) << std::endl;
201+
// std::cout << \"? \";
202+
// exit(EXIT_FAILURE);
181203
}
182204
}
183205
}
@@ -186,13 +208,18 @@ if (CMAKE_CUDA_COMPILER)
186208
if (NOT status EQUAL 0)
187209
message(FATAL_ERROR "${pfx}: nvcc failed\n ${msg}")
188210
endif()
189-
execute_process(COMMAND "./a.out" WORKING_DIRECTORY ${tmpdir} RESULT_VARIABLE status OUTPUT_VARIABLE msg)
211+
execute_process(COMMAND "./a.out" WORKING_DIRECTORY ${tmpdir} RESULT_VARIABLE status OUTPUT_VARIABLE msg ERROR_VARIABLE err)
212+
message(STATUS "error output of CUDA devices detection: ${err}")
190213
if (NOT status EQUAL 0)
191214
message(FATAL_ERROR "${pfx}: test program failed")
192215
endif()
216+
message(STATUS "output of CUDA devices detection:\n ${msg}")
217+
# read device capabilites from the file created by test.cu
218+
file(READ "${tmpdir}/device_capability.txt" msg)
193219
separate_arguments(msg)
194220
list(REMOVE_ITEM msg "")
195221
list(LENGTH msg gpu_count)
222+
message(STATUS "number of CUDA devices detected: ${gpu_count}")
196223
if(gpu_count GREATER 1 OR LIBCLOUDPHXX_FORCE_MULTI_CUDA)
197224
# compile the multi_CUDA backend
198225
message(STATUS "Detected more than 1 GPU or LIBCLOUDPHXX_FORCE_MULTI_CUDA set, the multi_CUDA backend will be built.")
@@ -208,12 +235,15 @@ if (CMAKE_CUDA_COMPILER)
208235
# https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html#gpu-compilation
209236
list(SORT msg)
210237
list(GET msg 0 VIRT)
211-
if (VIRT STREQUAL "?")
212-
set (VIRT "35")
213-
endif()
214-
target_compile_options(cloudphxx_lgrngn PRIVATE $<$<COMPILE_LANGUAGE:CUDA>: --gpu-architecture=compute_${VIRT}>)
238+
# if (VIRT STREQUAL "?")
239+
# set (VIRT "35")
240+
# endif()
241+
set_target_properties(cloudphxx_lgrngn PROPERTIES CUDA_ARCHITECTURES ${VIRT}-virtual)
242+
# set_property(TARGET PROPERTY CUDA_ARCHITECTURES 70-real 72-virtual)
243+
244+
# target_compile_options(cloudphxx_lgrngn PRIVATE $<$<COMPILE_LANGUAGE:CUDA>: --gpu-architecture=compute_${VIRT}>)
215245

216-
set(GPU_CODE_OPTIONS --gpu-code=)
246+
#set(GPU_CODE_OPTIONS --gpu-code=)
217247
foreach(code ${msg})
218248
message(STATUS "CUDA capability: ${code}")
219249
if (code STREQUAL "?")
@@ -224,9 +254,11 @@ if (CMAKE_CUDA_COMPILER)
224254
if (arch EQUAL "21") # see https://devtalk.nvidia.com/default/topic/606195/-solved-nvcc-fatal-unsupported-gpu-architecture-compute_21-/
225255
set(arch "20")
226256
endif()
227-
set(GPU_CODE_OPTIONS ${GPU_CODE_OPTIONS}sm_${arch},)
257+
set_property(TARGET cloudphxx_lgrngn APPEND PROPERTY CUDA_ARCHITECTURES ${arch}-real)
258+
#set_target_properties(cloudphxx_lgrngn PROPERTIES CUDA_ARCHITECTURES ${arch}-real)
259+
#set(GPU_CODE_OPTIONS ${GPU_CODE_OPTIONS}sm_${arch},)
228260
endforeach(code)
229-
target_compile_options(cloudphxx_lgrngn PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:${GPU_CODE_OPTIONS}>)
261+
# target_compile_options(cloudphxx_lgrngn PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:${GPU_CODE_OPTIONS}>)
230262
else()
231263
message(STATUS "CUDA capability check didn't find any devices")
232264
endif()

‎bindings/python/CMakeLists.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,9 +90,9 @@ target_link_options(cloudphxx PRIVATE "-Wl,--disable-new-dtags") # otherwise BUI
9090
set_property(TARGET cloudphxx PROPERTY INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
9191
set_property(TARGET cloudphxx PROPERTY INSTALL_RPATH_USE_LINK_PATH TRUE)
9292

93-
# Python_SITELIB is an absolute path, we manually prepend install prefix
93+
# Python_SITEARCH is an absolute path, we manually prepend install prefix
9494
install ( TARGETS cloudphxx
9595
LIBRARY
96-
DESTINATION "${CMAKE_INSTALL_PREFIX}${Python3_SITELIB}"
96+
DESTINATION "${CMAKE_INSTALL_PREFIX}${Python3_SITEARCH}"
9797
COMPONENT library
9898
)

‎include/libcloudph++/common/unary_function.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ namespace libcloudphxx
99
{
1010
//
1111
template <typename real_t>
12-
struct unary_function : std::unary_function<real_t, real_t>
12+
struct unary_function // : std::unary_function<real_t, real_t>
1313
{
1414
BOOST_GPU_ENABLED
1515
virtual real_t funval(const real_t) const = 0;

‎src/impl/particles_impl_adve.ipp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ namespace libcloudphxx
1212
namespace detail
1313
{
1414
template<class T>
15-
struct add_val : public thrust::unary_function<T, T>
15+
struct add_val //: public thrust::unary_function<T, T>
1616
{
1717
T val;
1818
add_val(T val): val(val) {}

‎src/impl/particles_impl_bcnd.ipp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,8 @@ namespace libcloudphxx
203203
n.begin(), n.end(), // input 1
204204
z.begin(), // stencil
205205
n_filtered.begin(), // output
206-
thrust::identity<n_t>(), // operation
206+
cuda::std::identity(), // operation
207+
// cuda::std::identity(), // operation
207208
arg::_1 < opts_init.z0 // condition
208209
);
209210

0 commit comments

Comments
 (0)