Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cpp/include/cudf_test/testing_main.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ inline auto parse_cudf_test_opts(int argc, char** argv)
std::getenv("GTEST_CUDF_STREAM_MODE"); // Overridden by CLI options
char const* env_stream_error_mode =
std::getenv("GTEST_CUDF_STREAM_ERROR_MODE"); // Overridden by CLI options
auto default_rmm_mode = env_rmm_mode ? env_rmm_mode : "pool";
auto default_rmm_mode = env_rmm_mode ? env_rmm_mode : "async";
auto default_stream_mode = env_stream_mode ? env_stream_mode : "default";
auto default_stream_error_mode = env_stream_error_mode ? env_stream_error_mode : "error";
options.allow_unrecognised_options().add_options()(
Expand Down
19 changes: 19 additions & 0 deletions cpp/scripts/gtest_memory_usage.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,32 @@
export GTEST_CUDF_RMM_MODE=cuda
export GTEST_CUDF_MEMORY_PEAK=1
export GTEST_BRIEF=1

# Collect all test results
results=()
for gt in gtests/*_TEST ; do
test_name=$(basename "${gt}")
echo -n "$test_name"
# dependent on the string output from testing_main.hpp
bytes=$(${gt} 2>/dev/null | grep Peak | cut -d' ' -f4)
echo ",${bytes}"
results+=("$test_name,$bytes")
done

unset GTEST_BRIEF
unset GTEST_CUDF_RMM_MODE
unset GTEST_CUDF_MEMORY_PEAK

# Output tests using more than 1GB
echo ""
echo "Tests using more than 1GB of memory:"
threshold=1073741824 # 1GB in bytes
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
threshold=1073741824 # 1GB in bytes
threshold=$((1024 * 1024 * 1024))

for result in "${results[@]}" ; do
test_name=$(echo "$result" | cut -d',' -f1)
bytes=$(echo "$result" | cut -d',' -f2)
if [[ -n "$bytes" && "$bytes" -gt "$threshold" ]] ; then
# Convert bytes to GB with 2 decimal places
gb=$(awk "BEGIN {printf \"%.2f\", $bytes / 1073741824}")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
gb=$(awk "BEGIN {printf \"%.2f\", $bytes / 1073741824}")
gb=$(awk "BEGIN {printf \"%.2f\", $bytes / (1024 * 1024 * 1024)}")

echo "$test_name: ${gb} GB"
fi
done
71 changes: 36 additions & 35 deletions cpp/tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,29 @@ enable_testing()
include(rapids-test)
rapids_test_init()

# ##################################################################################################
# **Note about test memory usage:**
#
# We design the test suite to run in CI with max parallelism on GPUs with 16 GB of memory or more.
# Tests are assigned GPU memory percentages to enable safe parallel execution. We draw an arbitrary
# line that tests are allowed to run in parallel if they consume less than 1 GB of peak GPU memory,
# and are run serially otherwise. This allows up to 14 tests in parallel on a 16 GB GPU with a bit
# of room for overhead. (14 * 7% is 98%, and we are required to use integer percentages.) Therefore,
# tests are assigned 7% of the GPU by default, and 100% if they use >=1 GB of memory. ctest won't
# schedule more than 100% of the GPU, so at most 14 tests in parallel. This means on a 16 GB GPU, we
# can safely run `ctest -jN` with N>=14.
#
# **Guidelines:**
#
# * Tests using <=1 GB: Use default (no GPUS/PERCENT parameters, defaults to 7%)
# * Tests using >1 GB: Mark as "GPUS 1 PERCENT 100" to reserve the entire GPU
#
# Marking large tests with PERCENT 100 prevents memory oversubscription and ensures only one large
# test runs at a time, while smaller tests can still run in parallel.
#
# The peak memory usage of all tests can be determined with cpp/scripts/gtest_memory_usage.sh.
# ##################################################################################################

# This function takes in a test name and test source and handles setting all of the associated
# properties and linking to build the test
function(ConfigureTest CMAKE_TEST_NAME)
Expand All @@ -30,7 +53,7 @@ function(ConfigureTest CMAKE_TEST_NAME)

if(NOT DEFINED _CUDF_TEST_GPUS AND NOT DEFINED _CUDF_TEST_PERCENT)
set(_CUDF_TEST_GPUS 1)
set(_CUDF_TEST_PERCENT 15)
set(_CUDF_TEST_PERCENT 7)
endif()

if(NOT DEFINED _CUDF_TEST_GPUS)
Expand Down Expand Up @@ -164,15 +187,15 @@ ConfigureTest(
groupby/sum_tests.cpp
groupby/tdigest_tests.cpp
groupby/var_tests.cpp
GPUS 1
PERCENT 100
)

# ##################################################################################################
# * join tests ------------------------------------------------------------------------------------
ConfigureTest(
JOIN_TEST join/join_tests.cpp join/conditional_join_tests.cu join/cross_join_tests.cpp
join/semi_anti_join_tests.cpp join/mixed_join_tests.cu join/distinct_join_tests.cpp
GPUS 1
PERCENT 100
)

# ##################################################################################################
Expand Down Expand Up @@ -204,17 +227,13 @@ ConfigureTest(
ConfigureTest(
PARTITIONING_TEST partitioning/hash_partition_test.cpp partitioning/round_robin_test.cpp
partitioning/partition_test.cpp
GPUS 1
PERCENT 70
)

# ##################################################################################################
# * quantiles tests -------------------------------------------------------------------------------
ConfigureTest(
QUANTILES_TEST quantiles/percentile_approx_test.cpp quantiles/quantile_test.cpp
quantiles/quantiles_test.cpp
GPUS 1
PERCENT 70 EXTRA_LIBS ${ARROW_LIBRARIES}
quantiles/quantiles_test.cpp EXTRA_LIBS ${ARROW_LIBRARIES}
)

# ##################################################################################################
Expand All @@ -231,8 +250,6 @@ ConfigureTest(
reductions/scan_tests.cpp
reductions/segmented_reduction_tests.cpp
reductions/tdigest_tests.cpp
GPUS 1
PERCENT 70
)

# ##################################################################################################
Expand Down Expand Up @@ -268,6 +285,8 @@ ConfigureTest(
binaryop/binop-compiled-test.cpp
binaryop/binop-compiled-fixed_point-test.cpp
binaryop/binop-generic-ptx-test.cpp
GPUS 1
PERCENT 100
)

# ##################################################################################################
Expand Down Expand Up @@ -310,11 +329,7 @@ ConfigureTest(
ConfigureTest(COMPRESSION_TEST io/comp/comp_test.cpp)
ConfigureTest(ROW_SELECTION_TEST io/row_selection_test.cpp)

ConfigureTest(
CSV_TEST io/csv_test.cpp
GPUS 1
PERCENT 30 EXTRA_LIBS ${ARROW_LIBRARIES}
)
ConfigureTest(CSV_TEST io/csv_test.cpp EXTRA_LIBS ${ARROW_LIBRARIES})
ConfigureTest(
ORC_TEST io/orc_chunked_reader_test.cu io/orc_test.cpp
GPUS 1
Expand All @@ -332,7 +347,7 @@ ConfigureTest(
io/parquet_v2_test.cpp
io/parquet_writer_test.cpp
GPUS 1
PERCENT 30
PERCENT 100
)
ConfigureTest(
HYBRID_SCAN_TEST
Expand All @@ -356,9 +371,7 @@ ConfigureTest(
CROARING_COMPILER_SUPPORTS_AVX512=0
)
ConfigureTest(
JSON_TEST io/json/json_test.cpp io/json/json_chunked_reader.cu
GPUS 1
PERCENT 30 EXTRA_LIBS ${ARROW_LIBRARIES}
JSON_TEST io/json/json_test.cpp io/json/json_chunked_reader.cu EXTRA_LIBS ${ARROW_LIBRARIES}
)
ConfigureTest(JSON_WRITER_TEST io/json/json_writer.cpp)
ConfigureTest(JSON_TYPE_CAST_TEST io/json/json_type_cast_test.cu)
Expand All @@ -367,11 +380,7 @@ ConfigureTest(MULTIBYTE_SPLIT_TEST io/text/multibyte_split_test.cpp)
ConfigureTest(JSON_QUOTE_NORMALIZATION io/json/json_quote_normalization_test.cpp)
ConfigureTest(JSON_WHITESPACE_NORMALIZATION io/json/json_whitespace_normalization_test.cu)
ConfigureTest(JSON_TREE_CSR io/json/json_tree_csr.cu)
ConfigureTest(
DATA_CHUNK_SOURCE_TEST io/text/data_chunk_source_test.cpp
GPUS 1
PERCENT 100
)
ConfigureTest(DATA_CHUNK_SOURCE_TEST io/text/data_chunk_source_test.cpp)
target_link_libraries(DATA_CHUNK_SOURCE_TEST PRIVATE ZLIB::ZLIB)
ConfigureTest(LOGICAL_STACK_TEST io/fst/logical_stack_test.cu)
ConfigureTest(FST_TEST io/fst/fst_test.cu)
Expand All @@ -382,8 +391,6 @@ ConfigureTest(TYPE_INFERENCE_TEST io/type_inference_test.cu)
ConfigureTest(
SORT_TEST sort/rank_test.cpp sort/segmented_sort_tests.cpp sort/sort_nested_types_tests.cpp
sort/sort_test.cpp sort/stable_sort_tests.cpp sort/top_k_tests.cpp
GPUS 1
PERCENT 70
)

# ##################################################################################################
Expand Down Expand Up @@ -479,11 +486,7 @@ ConfigureTest(DEVICE_ATOMICS_TEST device_atomics/device_atomics_test.cu)

# ##################################################################################################
# * transpose tests -------------------------------------------------------------------------------
ConfigureTest(
TRANSPOSE_TEST transpose/transpose_test.cpp
GPUS 1
PERCENT 70
)
ConfigureTest(TRANSPOSE_TEST transpose/transpose_test.cpp)

# ##################################################################################################
# * table tests -----------------------------------------------------------------------------------
Expand Down Expand Up @@ -536,8 +539,6 @@ ConfigureTest(
rolling/range_window_bounds_test.cpp
rolling/range_window_type_test.cpp
rolling/rolling_test.cpp
GPUS 1
PERCENT 70
)

# ##################################################################################################
Expand All @@ -558,6 +559,8 @@ ConfigureTest(
ConfigureTest(
RESHAPE_TEST reshape/byte_cast_tests.cpp reshape/interleave_columns_tests.cpp
reshape/table_to_array_tests.cpp reshape/tile_tests.cpp
GPUS 1
PERCENT 100
)

# ##################################################################################################
Expand Down Expand Up @@ -706,8 +709,6 @@ ConfigureTest(
lists/sort_lists_tests.cpp
lists/stream_compaction/apply_boolean_mask_tests.cpp
lists/stream_compaction/distinct_tests.cpp
GPUS 1
PERCENT 70
)

# ##################################################################################################
Expand Down
Loading