From 1a3b14aff3d7cbbab9659fc23a5c6478d505f3c8 Mon Sep 17 00:00:00 2001 From: Wayne Franz Date: Thu, 16 Jan 2025 15:24:05 -0500 Subject: [PATCH 01/84] Avoid evaluating CMake LINK_LANGUAGE generator expressions in CMake <3.18 (#667) Recently, we added CMake HIP Language support. That change introduced generator expressions that use the LINK_LANGUAGE keyword. This keyword is only supported in CMake 3.18+. The minimum CMake version we require is 3.16. As a result, if you built with CMake 3.16 - 3.18, you'd get a build failure. This change adds checks around all the spots where the LINK_LANGUAGE keyword is used to avoid this problem. It also adds a description of the CMake HIP Language support to the changelog (previously overlooked). --- CHANGELOG.md | 1 + benchmark/CMakeLists.txt | 8 +++++--- example/CMakeLists.txt | 8 +++++--- test/CMakeLists.txt | 8 +++++--- test/rocprim/CMakeLists.txt | 8 +++++--- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af0bd2546..f503db5d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ Full documentation for rocPRIM is available at [https://rocm.docs.amd.com/projec * Removed HIP-CPU support. HIP-CPU support was experimental and broken. * Changed the C++ version from 14 to 17. C++14 will be deprecated in the next major release. +* You can use CMake HIP language support with CMake 3.18 and later. To use HIP language support, run `cmake` with `-DUSE_HIPCXX=ON` instead of setting the `CXX` variable to the path to a HIP-aware compiler. ### Resolved issues diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 64975916d..700c4bbb5 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -82,10 +82,12 @@ function(add_rocprim_benchmark BENCHMARK_SOURCE) benchmark::benchmark ) - target_link_libraries(${BENCHMARK_TARGET} - PRIVATE + if(USE_HIPCXX) + target_link_libraries(${BENCHMARK_TARGET} + PRIVATE $,hip::host,hip::device> - ) + ) + endif() target_compile_options(${BENCHMARK_TARGET} PRIVATE diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index b9edcae00..df3400283 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -33,10 +33,12 @@ function(add_rocprim_example EXAMPLE_SOURCE) PRIVATE rocprim ) - target_link_libraries(${EXAMPLE_TARGET} - PRIVATE + if(USE_HIPCXX) + target_link_libraries(${EXAMPLE_TARGET} + PRIVATE $,hip::host,hip::device> - ) + ) + endif() set_target_properties(${EXAMPLE_TARGET} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/example" diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 31538a58d..d313be0c9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -76,10 +76,12 @@ function(add_hip_test TEST_NAME TEST_SOURCES) PRIVATE rocprim ) - target_link_libraries(${TEST_TARGET} - PRIVATE + if(USE_HIPCXX) + target_link_libraries(${TEST_TARGET} + PRIVATE $,hip::host,hip::device> - ) + ) + endif() target_compile_options(${TEST_TARGET} PRIVATE diff --git a/test/rocprim/CMakeLists.txt b/test/rocprim/CMakeLists.txt index d31e8f9ce..990a9cdf3 100644 --- a/test/rocprim/CMakeLists.txt +++ b/test/rocprim/CMakeLists.txt @@ -55,10 +55,12 @@ function(add_rocprim_test_internal TEST_NAME TEST_SOURCES TEST_TARGET) GTest::Main ) - target_link_libraries(${TEST_TARGET} - PRIVATE + if(USE_HIPCXX) + target_link_libraries(${TEST_TARGET} + PRIVATE $,hip::host,hip::device> - ) + ) + endif() target_compile_options(${TEST_TARGET} PRIVATE From 25f3387c4e677d9b25313a32f2642279ae9681e1 Mon Sep 17 00:00:00 2001 From: Wayne Franz Date: Tue, 21 Jan 2025 17:37:41 -0500 Subject: [PATCH 02/84] Fix Windows CMake link issue (#675) A recent cmake change added a condition check that avoids evaluating a LINK_LANGUAGE generator expression if HIP_USECXX is OFF (since LINK_LANGUAGE is unsupported in CMake < 3.18). However, it didn't account for the fact that the LINK_LANGUAGE expression is embedded in a conditional generator expression. This change adds an "else" block that ensures the "false" side of the conditional generator expression is still evaluated when the LINK_LANGUAGE code is skipped. This fixes some build issues on Windows. Also updates the readme to include recent changes to the way projects must link to rocPRIM. --- README.md | 10 ++++++++-- benchmark/CMakeLists.txt | 4 ++++ example/CMakeLists.txt | 4 ++++ test/CMakeLists.txt | 4 ++++ test/rocprim/CMakeLists.txt | 4 ++++ 5 files changed, 24 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5517f9d63..329b17042 100644 --- a/README.md +++ b/README.md @@ -157,10 +157,16 @@ find_package(rocprim REQUIRED CONFIG PATHS "/opt/rocm/rocprim") # to be linked manually by user target_link_libraries( roc::rocprim) -# Includes rocPRIM headers and required HIP dependencies -target_link_libraries( roc::rocprim_hip) +# Include rocPRIM headers and required HIP dependencies +# - If using HIP language support (USE_HIPCXX=ON): +target_link_libraries( hip::host) + +# - Otherwise: +target_link_libraries( hip::device) ``` +For more information on `hip::host` and `hip::device`, please see the [ROCm documentation](https://rocm.docs.amd.com/en/latest/conceptual/cmake-packages.html#consuming-the-hip-api-in-c-code). + ## Running unit tests Unit tests are implemented in terms of GoogleTest. Collections of tests are wrapped and invoked from diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 700c4bbb5..94eeb1e76 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -87,6 +87,10 @@ function(add_rocprim_benchmark BENCHMARK_SOURCE) PRIVATE $,hip::host,hip::device> ) + else() + target_link_libraries(${BENCHMARK_TARGET} + PRIVATE + hip::device) endif() target_compile_options(${BENCHMARK_TARGET} diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index df3400283..b4020ae5f 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -38,6 +38,10 @@ function(add_rocprim_example EXAMPLE_SOURCE) PRIVATE $,hip::host,hip::device> ) + else() + target_link_libraries(${EXAMPLE_TARGET} + PRIVATE + hip::device) endif() set_target_properties(${EXAMPLE_TARGET} PROPERTIES diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d313be0c9..1f12cddb9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -81,6 +81,10 @@ function(add_hip_test TEST_NAME TEST_SOURCES) PRIVATE $,hip::host,hip::device> ) + else() + target_link_libraries(${TEST_TARGET} + PRIVATE + hip::device) endif() target_compile_options(${TEST_TARGET} diff --git a/test/rocprim/CMakeLists.txt b/test/rocprim/CMakeLists.txt index 990a9cdf3..24e87ebeb 100644 --- a/test/rocprim/CMakeLists.txt +++ b/test/rocprim/CMakeLists.txt @@ -60,6 +60,10 @@ function(add_rocprim_test_internal TEST_NAME TEST_SOURCES TEST_TARGET) PRIVATE $,hip::host,hip::device> ) + else() + target_link_libraries(${TEST_TARGET} + PRIVATE + hip::device) endif() target_compile_options(${TEST_TARGET} From 4903d4a75e72c4f7f9a70431d14e014cefbe96cc Mon Sep 17 00:00:00 2001 From: Beatriz Navidad Vilches Date: Thu, 21 Nov 2024 12:58:56 +0100 Subject: [PATCH 03/84] Resolve "Add hipGraph compatibility for device run length encode" --- CHANGELOG.md | 5 +- benchmark/ConfigAutotuneSettings.cmake | 4 + .../benchmark_device_run_length_encode.cpp | 296 ++---- ...k_device_run_length_encode.parallel.cpp.in | 34 + ...mark_device_run_length_encode.parallel.hpp | 259 ++++++ benchmark/benchmark_device_scan.parallel.hpp | 4 +- .../benchmark_device_scan_by_key.parallel.hpp | 4 +- benchmark/benchmark_utils.hpp | 23 +- .../device_run_length_encode_non_trivial.hpp | 599 ++++++++++++ .../device/detail/device_config_helper.hpp | 80 ++ .../device/detail/device_reduce_by_key.hpp | 42 +- .../detail/device_run_length_encode.hpp | 877 ++++++++++++++++++ .../rocprim/device/device_reduce_by_key.hpp | 3 +- .../device/device_run_length_encode.hpp | 422 +++++---- .../device_run_length_encode_config.hpp | 125 ++- .../include/rocprim/thread/thread_reduce.hpp | 52 +- rocprim/include/rocprim/type_traits.hpp | 64 ++ scripts/autotune/create_optimization.py | 8 + .../run_length_encode_config_template | 28 + .../rocprim/test_device_run_length_encode.cpp | 100 +- 20 files changed, 2558 insertions(+), 471 deletions(-) create mode 100644 benchmark/benchmark_device_run_length_encode.parallel.cpp.in create mode 100644 benchmark/benchmark_device_run_length_encode.parallel.hpp create mode 100644 rocprim/include/rocprim/device/detail/config/device_run_length_encode_non_trivial.hpp create mode 100644 rocprim/include/rocprim/device/detail/device_run_length_encode.hpp create mode 100644 scripts/autotune/templates/run_length_encode_config_template diff --git a/CHANGELOG.md b/CHANGELOG.md index f503db5d2..5896dedaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,8 @@ Full documentation for rocPRIM is available at [https://rocm.docs.amd.com/projec ## rocPRIM 3.5.0 for ROCm 6.5.0 ### Added -* gfx950 support + +* Added gfx950 support. ## rocPRIM 3.4.0 for ROCm 6.4.0 @@ -26,6 +27,8 @@ Full documentation for rocPRIM is available at [https://rocm.docs.amd.com/projec * Added the parallel `search` and `find_end` device functions similar to `std::search` and `std::find_end`, these functions search for the first and last occurrence of the sequence respectively. * Added a parallel device-level function, `rocprim::search_n`, similar to the C++ Standard Library `std::search_n` algorithm. * Added new constructors and a `base` function, and added `constexpr` specifier to all functions in `rocprim::reverse_iterator` to improve parity with the C++17 `std::reverse_iterator`. +* Added hipGraph support to device run-length-encode for non trivial runs (`rocprim::run_length_encode_non_trivial_runs`). +* Added configuration autotuning to device run-length-encode for non trivial runs (`rocprim::run_length_encode_non_trivial_runs`) for improved performance on selected architectures. ### Changed diff --git a/benchmark/ConfigAutotuneSettings.cmake b/benchmark/ConfigAutotuneSettings.cmake index d0d11c62b..f5555df6a 100644 --- a/benchmark/ConfigAutotuneSettings.cmake +++ b/benchmark/ConfigAutotuneSettings.cmake @@ -123,5 +123,9 @@ DataType;BlockSize;" PARENT_SCOPE) set(list_across_names "KeyType;ValueType;BlockSize" PARENT_SCOPE) set(list_across "${TUNING_TYPES};rocprim::empty_type ${LIMITED_TUNING_TYPES};32 64 128 256 512 1024" PARENT_SCOPE) set(output_pattern_suffix "@KeyType@_@ValueType@_@BlockSize@" PARENT_SCOPE) + elseif(file STREQUAL "benchmark_device_run_length_encode") + set(list_across_names "KeyType;BlockSize;BlockLoadMethod" PARENT_SCOPE) + set(list_across "${TUNING_TYPES};64 128 256 512 1024;block_load_vectorize block_load_warp_transpose" PARENT_SCOPE) + set(output_pattern_suffix "@KeyType@_@BlockSize@_@BlockLoadMethod@" PARENT_SCOPE) endif() endfunction() diff --git a/benchmark/benchmark_device_run_length_encode.cpp b/benchmark/benchmark_device_run_length_encode.cpp index 345708983..73d855650 100644 --- a/benchmark/benchmark_device_run_length_encode.cpp +++ b/benchmark/benchmark_device_run_length_encode.cpp @@ -20,7 +20,9 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +#include "benchmark_device_run_length_encode.parallel.hpp" #include "benchmark_utils.hpp" + // CmdParser #include "cmdparser.hpp" @@ -33,9 +35,7 @@ // rocPRIM #include -#include -#include -#include +#include #include #include @@ -52,7 +52,7 @@ void run_encode_benchmark(benchmark::State& state, const managed_seed& seed, hipStream_t stream) { - using key_type = T; + using key_type = T; using count_type = unsigned int; const size_t size = bytes / sizeof(T); @@ -60,7 +60,7 @@ void run_encode_benchmark(benchmark::State& state, // Generate data std::vector input(size); - unsigned int runs_count = 0; + unsigned int runs_count = 0; const auto random_range = limit_random_range(1, max_length); std::vector key_counts = get_random_data(100000, random_range.first, random_range.second, seed.get_0()); @@ -68,7 +68,7 @@ void run_encode_benchmark(benchmark::State& state, while(offset < size) { const size_t key_count = key_counts[runs_count % key_counts.size()]; - const size_t end = std::min(size, offset + key_count); + const size_t end = std::min(size, offset + key_count); for(size_t i = offset; i < end; i++) { input[i] = runs_count; @@ -78,34 +78,30 @@ void run_encode_benchmark(benchmark::State& state, offset += key_count; } - key_type * d_input; + key_type* d_input; HIP_CHECK(hipMalloc(reinterpret_cast(&d_input), size * sizeof(key_type))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - size * sizeof(key_type), - hipMemcpyHostToDevice - ) - ); - - key_type * d_unique_output; - count_type * d_counts_output; - count_type * d_runs_count_output; + HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(key_type), hipMemcpyHostToDevice)); + + key_type* d_unique_output; + count_type* d_counts_output; + count_type* d_runs_count_output; HIP_CHECK(hipMalloc(reinterpret_cast(&d_unique_output), runs_count * sizeof(key_type))); - HIP_CHECK(hipMalloc(reinterpret_cast(&d_counts_output), runs_count * sizeof(count_type))); + HIP_CHECK( + hipMalloc(reinterpret_cast(&d_counts_output), runs_count * sizeof(count_type))); HIP_CHECK(hipMalloc(reinterpret_cast(&d_runs_count_output), sizeof(count_type))); - void * d_temporary_storage = nullptr; + void* d_temporary_storage = nullptr; size_t temporary_storage_bytes = 0; - HIP_CHECK( - rp::run_length_encode( - nullptr, temporary_storage_bytes, - d_input, size, - d_unique_output, d_counts_output, d_runs_count_output, - stream, false - ) - ); + HIP_CHECK(rp::run_length_encode(nullptr, + temporary_storage_bytes, + d_input, + size, + d_unique_output, + d_counts_output, + d_runs_count_output, + stream, + false)); HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); @@ -113,14 +109,15 @@ void run_encode_benchmark(benchmark::State& state, // Warm-up for(size_t i = 0; i < 10; i++) { - HIP_CHECK( - rp::run_length_encode( - d_temporary_storage, temporary_storage_bytes, - d_input, size, - d_unique_output, d_counts_output, d_runs_count_output, - stream, false - ) - ); + HIP_CHECK(rp::run_length_encode(d_temporary_storage, + temporary_storage_bytes, + d_input, + size, + d_unique_output, + d_counts_output, + d_runs_count_output, + stream, + false)); } HIP_CHECK(hipDeviceSynchronize()); @@ -130,7 +127,7 @@ void run_encode_benchmark(benchmark::State& state, HIP_CHECK(hipEventCreate(&stop)); const unsigned int batch_size = 10; - for (auto _ : state) + for(auto _ : state) { // Record start event HIP_CHECK(hipEventRecord(start, stream)); @@ -171,152 +168,25 @@ void run_encode_benchmark(benchmark::State& state, HIP_CHECK(hipFree(d_runs_count_output)); } -template -void run_non_trivial_runs_benchmark(benchmark::State& state, - size_t max_length, - size_t bytes, - const managed_seed& seed, - hipStream_t stream) -{ - using key_type = T; - using offset_type = unsigned int; - using count_type = unsigned int; - - const size_t size = bytes / sizeof(T); - - // Generate data - std::vector input(size); - - unsigned int runs_count = 0; - const auto random_range = limit_random_range(1, max_length); - std::vector key_counts - = get_random_data(100000, random_range.first, random_range.second, seed.get_0()); - size_t offset = 0; - while(offset < size) - { - const size_t key_count = key_counts[runs_count % key_counts.size()]; - const size_t end = std::min(size, offset + key_count); - for(size_t i = offset; i < end; i++) - { - input[i] = runs_count; - } - - runs_count++; - offset += key_count; - } - - key_type * d_input; - HIP_CHECK(hipMalloc(reinterpret_cast(&d_input), size * sizeof(key_type))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - size * sizeof(key_type), - hipMemcpyHostToDevice - ) - ); - - offset_type * d_offsets_output; - count_type * d_counts_output; - count_type * d_runs_count_output; - HIP_CHECK(hipMalloc(reinterpret_cast(&d_offsets_output), runs_count * sizeof(offset_type))); - HIP_CHECK(hipMalloc(reinterpret_cast(&d_counts_output), runs_count * sizeof(count_type))); - HIP_CHECK(hipMalloc(reinterpret_cast(&d_runs_count_output), sizeof(count_type))); - - void * d_temporary_storage = nullptr; - size_t temporary_storage_bytes = 0; - - HIP_CHECK( - rp::run_length_encode_non_trivial_runs( - nullptr, temporary_storage_bytes, - d_input, size, - d_offsets_output, d_counts_output, d_runs_count_output, - stream, false - ) - ); - - HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < 10; i++) - { - HIP_CHECK( - rp::run_length_encode_non_trivial_runs( - d_temporary_storage, temporary_storage_bytes, - d_input, size, - d_offsets_output, d_counts_output, d_runs_count_output, - stream, false - ) - ); - } - HIP_CHECK(hipDeviceSynchronize()); - - // HIP events creation - hipEvent_t start, stop; - HIP_CHECK(hipEventCreate(&start)); - HIP_CHECK(hipEventCreate(&stop)); - - const unsigned int batch_size = 10; - for (auto _ : state) - { - // Record start event - HIP_CHECK(hipEventRecord(start, stream)); - - for(size_t i = 0; i < batch_size; i++) - { - HIP_CHECK(rp::run_length_encode_non_trivial_runs(d_temporary_storage, - temporary_storage_bytes, - d_input, - size, - d_offsets_output, - d_counts_output, - d_runs_count_output, - stream, - false)); - } - - // Record stop event and wait until it completes - HIP_CHECK(hipEventRecord(stop, stream)); - HIP_CHECK(hipEventSynchronize(stop)); - - float elapsed_mseconds; - HIP_CHECK(hipEventElapsedTime(&elapsed_mseconds, start, stop)); - state.SetIterationTime(elapsed_mseconds / 1000); - } - - // Destroy HIP events - HIP_CHECK(hipEventDestroy(start)); - HIP_CHECK(hipEventDestroy(stop)); - - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(key_type)); - state.SetItemsProcessed(state.iterations() * batch_size * size); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_offsets_output)); - HIP_CHECK(hipFree(d_counts_output)); - HIP_CHECK(hipFree(d_runs_count_output)); -} - #define CREATE_ENCODE_BENCHMARK(T) \ benchmark::RegisterBenchmark( \ bench_naming::format_name( \ "{lvl:device,algo:run_length_encode,subalgo:trivial,key_type:" #T ",keys_max_length:" \ - + std::to_string(max_length) + ",cfg:default_config}") \ + + std::to_string(MaxLength) + ",cfg:default_config}") \ .c_str(), \ run_encode_benchmark, \ - max_length, \ + MaxLength, \ size, \ seed, \ stream) -void add_encode_benchmarks(size_t max_length, - std::vector& benchmarks, +template +void add_encode_benchmarks(std::vector& benchmarks, size_t size, const managed_seed& seed, hipStream_t stream) { - using custom_float2 = custom_type; + using custom_float2 = custom_type; using custom_double2 = custom_type; std::vector bs = { @@ -336,45 +206,35 @@ void add_encode_benchmarks(size_t max_len benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); } -#define CREATE_NON_TRIVIAL_RUNS_BENCHMARK(T) \ - benchmark::RegisterBenchmark( \ - bench_naming::format_name( \ - "{lvl:device,algo:run_length_encode,subalgo:non_trivial,key_type:" #T \ - ",keys_max_length:" \ - + std::to_string(max_length) + ",cfg:default_config}") \ - .c_str(), \ - run_non_trivial_runs_benchmark, \ - max_length, \ - size, \ - seed, \ - stream) +// CHANGE +#define CREATE_NON_TRIVIAL_RUNS_BENCHMARK(T, ML) \ + { \ + const device_non_trivial_runs_benchmark instance; \ + REGISTER_BENCHMARK(benchmarks, size, seed, stream, instance); \ + } -void add_non_trivial_runs_benchmarks(size_t max_length, - std::vector& benchmarks, +template +void add_non_trivial_runs_benchmarks(std::vector& benchmarks, size_t size, const managed_seed& seed, hipStream_t stream) { - using custom_float2 = custom_type; + using custom_float2 = custom_type; using custom_double2 = custom_type; - std::vector bs = - { - CREATE_NON_TRIVIAL_RUNS_BENCHMARK(int), - CREATE_NON_TRIVIAL_RUNS_BENCHMARK(long long), - - CREATE_NON_TRIVIAL_RUNS_BENCHMARK(int8_t), - CREATE_NON_TRIVIAL_RUNS_BENCHMARK(uint8_t), - CREATE_NON_TRIVIAL_RUNS_BENCHMARK(rocprim::half), - - CREATE_NON_TRIVIAL_RUNS_BENCHMARK(custom_float2), - CREATE_NON_TRIVIAL_RUNS_BENCHMARK(custom_double2), - }; + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(int8_t, MaxLength); + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(int16_t, MaxLength); + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(int32_t, MaxLength); + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(int64_t, MaxLength); + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(rocprim::half, MaxLength); + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(float, MaxLength); + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(double, MaxLength); - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(custom_float2, MaxLength); + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(custom_double2, MaxLength); } -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { cli::Parser parser(argc, argv); parser.set_optional("size", "size", DEFAULT_BYTES, "number of bytes"); @@ -384,12 +244,24 @@ int main(int argc, char *argv[]) "human", "either: json,human,txt"); parser.set_optional("seed", "seed", "random", get_seed_message()); +#ifdef BENCHMARK_CONFIG_TUNING + // optionally run an evenly split subset of benchmarks, when making multiple program invocations + parser.set_optional("parallel_instance", + "parallel_instance", + 0, + "parallel instance index"); + parser.set_optional("parallel_instances", + "parallel_instances", + 1, + "total parallel instances"); +#endif + parser.run_and_exit_if_error(); // Parse argv benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); + const size_t size = parser.get("size"); + const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); const managed_seed seed(seed_type); @@ -402,12 +274,26 @@ int main(int argc, char *argv[]) benchmark::AddCustomContext("size", std::to_string(size)); benchmark::AddCustomContext("seed", seed_type); - // Add benchmarks std::vector benchmarks; - add_encode_benchmarks(1000, benchmarks, size, seed, stream); - add_encode_benchmarks(10, benchmarks, size, seed, stream); - add_non_trivial_runs_benchmarks(1000, benchmarks, size, seed, stream); - add_non_trivial_runs_benchmarks(10, benchmarks, size, seed, stream); + + // Add benchmarks +#ifdef BENCHMARK_CONFIG_TUNING + const int parallel_instance = parser.get("parallel_instance"); + const int parallel_instances = parser.get("parallel_instances"); + config_autotune_register::register_benchmark_subset(benchmarks, + parallel_instance, + parallel_instances, + size, + seed, + stream); +#else + add_encode_benchmarks<1000>(benchmarks, size, seed, stream); + add_encode_benchmarks<10>(benchmarks, size, seed, stream); + add_non_trivial_runs_benchmarks<16>(benchmarks, size, seed, stream); + add_non_trivial_runs_benchmarks<256>(benchmarks, size, seed, stream); + add_non_trivial_runs_benchmarks<4096>(benchmarks, size, seed, stream); + +#endif // Use manual timing for(auto& b : benchmarks) diff --git a/benchmark/benchmark_device_run_length_encode.parallel.cpp.in b/benchmark/benchmark_device_run_length_encode.parallel.cpp.in new file mode 100644 index 000000000..a09a4f261 --- /dev/null +++ b/benchmark/benchmark_device_run_length_encode.parallel.cpp.in @@ -0,0 +1,34 @@ +// MIT License +// +// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include + +#include "benchmark_device_run_length_encode.parallel.hpp" +#include "benchmark_utils.hpp" + +namespace { + auto benchmarks = config_autotune_register::create_bulk( + device_non_trivial_runs_benchmark_generator< + @KeyType@, + @BlockSize@, + rocprim::block_load_method::@BlockLoadMethod@>::create); +} // namespace diff --git a/benchmark/benchmark_device_run_length_encode.parallel.hpp b/benchmark/benchmark_device_run_length_encode.parallel.hpp new file mode 100644 index 000000000..ee7817e76 --- /dev/null +++ b/benchmark/benchmark_device_run_length_encode.parallel.hpp @@ -0,0 +1,259 @@ +// MIT License +// +// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#ifndef ROCPRIM_BENCHMARK_DEVICE_RUN_LENGTH_ENCODE_PARALLEL_HPP_ +#define ROCPRIM_BENCHMARK_DEVICE_RUN_LENGTH_ENCODE_PARALLEL_HPP_ + +#include "benchmark_utils.hpp" + +// Google Benchmark +#include + +// HIP API +#include + +// rocPRIM +#include +#include + +#include +#include +#include +#include +#include +#ifdef BENCHMARK_CONFIG_TUNING + #include +#endif + +template +std::string non_trivial_runs_config_name() +{ + const rocprim::detail::non_trivial_runs_config_params config = Config(); + return "{bs:" + std::to_string(config.kernel_config.block_size) + + ",ipt:" + std::to_string(config.kernel_config.items_per_thread) + + ",load_method:" + get_block_load_method_name(config.load_input_method) + "}"; +} + +template<> +inline std::string non_trivial_runs_config_name() +{ + return "default_config"; +} + +template +struct device_non_trivial_runs_benchmark : public config_autotune_interface +{ + std::string name() const override + { + using namespace std::string_literals; + return bench_naming::format_name( + "{lvl:device,algo:run_length_encode,subalgo:non_trivial,key_type:" + + std::string(Traits::name()) + ",keys_max_length:" + std::to_string(MaxLength) + + ",cfg:" + non_trivial_runs_config_name() + "}"); + } + + void run(benchmark::State& state, + size_t bytes, + const managed_seed& seed, + hipStream_t stream) const override + { + using offset_type = unsigned int; + using count_type = unsigned int; + + constexpr int batch_size = 10; + constexpr int warmup_size = 5; + constexpr std::array tuning_max_segment_lengths = {10, 1000}; + constexpr int num_input_arrays = is_tuning ? tuning_max_segment_lengths.size() : 1; + + constexpr size_t item_size = sizeof(T) + sizeof(offset_type) + sizeof(count_type); + + const size_t size = bytes / item_size; + + // Generate data + std::vector input[num_input_arrays]; + if(is_tuning) + { + for(size_t i = 0; i < tuning_max_segment_lengths.size(); ++i) + { + input[i] = get_random_segments_iota(size, + tuning_max_segment_lengths[i], + seed.get_0()); + } + } + else + { + input[0] = get_random_segments_iota(size, MaxLength, seed.get_0()); + } + + T* d_input[num_input_arrays]; + for(int i = 0; i < num_input_arrays; ++i) + { + HIP_CHECK(hipMalloc(&d_input[i], size * sizeof(*d_input[i]))); + HIP_CHECK(hipMemcpy(d_input[i], + input[i].data(), + size * sizeof(*d_input[i]), + hipMemcpyHostToDevice)); + } + + offset_type* d_offsets_output; + HIP_CHECK(hipMalloc(&d_offsets_output, size * sizeof(*d_offsets_output))); + count_type* d_counts_output; + HIP_CHECK(hipMalloc(&d_counts_output, size * sizeof(*d_counts_output))); + count_type* d_runs_count_output; + HIP_CHECK(hipMalloc(&d_runs_count_output, sizeof(*d_runs_count_output))); + + const auto dispatch = [&](void* d_temporary_storage, size_t& temporary_storage_bytes) + { + const auto dispatch_input = [&](T* d_input) + { + HIP_CHECK( + rocprim::run_length_encode_non_trivial_runs(d_temporary_storage, + temporary_storage_bytes, + d_input, + size, + d_offsets_output, + d_counts_output, + d_runs_count_output, + stream, + false)); + }; + + for(int i = 0; i < num_input_arrays; ++i) + { + dispatch_input(d_input[i]); + } + }; + + // Allocate temporary storage memory + size_t temporary_storage_bytes = 0; + dispatch(nullptr, temporary_storage_bytes); + void* d_temporary_storage; + HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); + HIP_CHECK(hipDeviceSynchronize()); + + // Warm-up + for(int i = 0; i < warmup_size; ++i) + { + dispatch(d_temporary_storage, temporary_storage_bytes); + } + HIP_CHECK(hipDeviceSynchronize()); + + // HIP events creation + hipEvent_t start, stop; + HIP_CHECK(hipEventCreate(&start)); + HIP_CHECK(hipEventCreate(&stop)); + + for(auto _ : state) + { + // Record start event + HIP_CHECK(hipEventRecord(start, stream)); + + for(int i = 0; i < batch_size; ++i) + { + dispatch(d_temporary_storage, temporary_storage_bytes); + } + HIP_CHECK(hipStreamSynchronize(stream)); + + // Record stop event and wait until it completes + HIP_CHECK(hipEventRecord(stop, stream)); + HIP_CHECK(hipEventSynchronize(stop)); + + HIP_CHECK(hipGetLastError()); + HIP_CHECK(hipDeviceSynchronize()); + + float elapsed_mseconds{}; + HIP_CHECK(hipEventElapsedTime(&elapsed_mseconds, start, stop)); + state.SetIterationTime(elapsed_mseconds / 1000); + } + + // Destroy HIP events + HIP_CHECK(hipEventDestroy(start)); + HIP_CHECK(hipEventDestroy(stop)); + + state.SetBytesProcessed(state.iterations() * batch_size * size * item_size); + state.SetItemsProcessed(state.iterations() * batch_size * size); + + HIP_CHECK(hipFree(d_temporary_storage)); + for(int i = 0; i < num_input_arrays; ++i) + { + HIP_CHECK(hipFree(d_input[i])); + } + HIP_CHECK(hipFree(d_offsets_output)); + HIP_CHECK(hipFree(d_counts_output)); + HIP_CHECK(hipFree(d_runs_count_output)); + } + static constexpr bool is_tuning = !std::is_same::value; +}; + +#ifdef BENCHMARK_CONFIG_TUNING + +template +struct device_non_trivial_runs_benchmark_generator +{ + using OffsetCountPairT = ::rocprim::tuple; + + static constexpr unsigned int max_shared_memory = TUNING_SHARED_MEMORY_MAX; + static constexpr unsigned int max_size_per_element + = std::max(sizeof(T), sizeof(OffsetCountPairT)); + static constexpr unsigned int max_items_per_thread + = max_shared_memory / (BlockSize * max_size_per_element); + static constexpr unsigned int max_items_per_thread_exponent + = rocprim::Log2::VALUE - 1; + static constexpr unsigned int min_items_per_thread_exponent = 3u; + + static constexpr bool is_load_warp_transpose + = BlockLoadMethod == ::rocprim::block_load_method::block_load_warp_transpose; + static constexpr bool is_warp_load_supp + = is_load_warp_transpose && BlockSize == ROCPRIM_WARP_SIZE_64; + + template + struct create_ipt + { + void operator()(std::vector>& storage) + { + if(!is_load_warp_transpose || is_warp_load_supp) + { + using config = rocprim::non_trivial_runs_config< + BlockSize, + items_per_thread, + BlockLoadMethod, + rocprim::block_scan_algorithm::using_warp_scan>; + storage.emplace_back( + std::make_unique>()); + } + } + + private: + static constexpr unsigned int items_per_thread = 1u << ItemsPerThreadExp; + }; + + static void create(std::vector>& storage) + { + static_for_each< + make_index_range, + create_ipt>(storage); + } +}; + +#endif // BENCHMARK_CONFIG_TUNING + +#endif // ROCPRIM_BENCHMARK_DEVICE_RUN_LENGTH_ENCODE_PARALLEL_HPP_ diff --git a/benchmark/benchmark_device_scan.parallel.hpp b/benchmark/benchmark_device_scan.parallel.hpp index 8fb58b83e..e4ba20f2f 100644 --- a/benchmark/benchmark_device_scan.parallel.hpp +++ b/benchmark/benchmark_device_scan.parallel.hpp @@ -45,8 +45,8 @@ std::string config_name() { const rocprim::detail::scan_config_params config = Config(); return "{bs:" + std::to_string(config.kernel_config.block_size) - + ",ipt:" + std::to_string(config.kernel_config.items_per_thread) - + ",method:" + std::string(get_block_scan_method_name(config.block_scan_method)) + "}"; + + ",ipt:" + std::to_string(config.kernel_config.items_per_thread) + ",method:" + + std::string(get_block_scan_algorithm_name(config.block_scan_method)) + "}"; } template<> diff --git a/benchmark/benchmark_device_scan_by_key.parallel.hpp b/benchmark/benchmark_device_scan_by_key.parallel.hpp index b0c80d842..7d81fd8a0 100644 --- a/benchmark/benchmark_device_scan_by_key.parallel.hpp +++ b/benchmark/benchmark_device_scan_by_key.parallel.hpp @@ -45,8 +45,8 @@ std::string config_name() { const rocprim::detail::scan_by_key_config_params config = Config(); return "{bs:" + std::to_string(config.kernel_config.block_size) - + ",ipt:" + std::to_string(config.kernel_config.items_per_thread) - + ",method:" + std::string(get_block_scan_method_name(config.block_scan_method)) + "}"; + + ",ipt:" + std::to_string(config.kernel_config.items_per_thread) + ",method:" + + std::string(get_block_scan_algorithm_name(config.block_scan_method)) + "}"; } template<> diff --git a/benchmark/benchmark_utils.hpp b/benchmark/benchmark_utils.hpp index 6b5c11cd4..5b7b4dc0b 100644 --- a/benchmark/benchmark_utils.hpp +++ b/benchmark/benchmark_utils.hpp @@ -24,6 +24,7 @@ #include // rocPRIM +#include #include #include #include // partition_config_params @@ -1132,7 +1133,7 @@ inline void add_common_benchmark_info() num("hdp_arch_has_dynamic_parallelism", arch.hasDynamicParallelism); } -inline const char* get_block_scan_method_name(rocprim::block_scan_algorithm alg) +inline const char* get_block_scan_algorithm_name(rocprim::block_scan_algorithm alg) { switch(alg) { @@ -1142,7 +1143,25 @@ inline const char* get_block_scan_method_name(rocprim::block_scan_algorithm alg) return "block_scan_algorithm::reduce_then_scan"; // Not using `default: ...` because it kills effectiveness of -Wswitch } - return "unknown_algorithm"; + return "default_algorithm"; +} + +inline const char* get_block_load_method_name(rocprim::block_load_method method) +{ + switch(method) + { + case rocprim::block_load_method::block_load_direct: + return "block_load_method::block_load_direct"; + case rocprim::block_load_method::block_load_striped: + return "block_load_method::block_load_striped"; + case rocprim::block_load_method::block_load_vectorize: + return "block_load_method::block_load_vectorize"; + case rocprim::block_load_method::block_load_transpose: + return "block_load_method::block_load_transpose"; + case rocprim::block_load_method::block_load_warp_transpose: + return "block_load_method::block_load_warp_transpose"; + } + return "default_method"; } template diff --git a/rocprim/include/rocprim/device/detail/config/device_run_length_encode_non_trivial.hpp b/rocprim/include/rocprim/device/detail/config/device_run_length_encode_non_trivial.hpp new file mode 100644 index 000000000..8345dd2cd --- /dev/null +++ b/rocprim/include/rocprim/device/detail/config/device_run_length_encode_non_trivial.hpp @@ -0,0 +1,599 @@ +// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RUN_LENGTH_ENCODE_HPP_ +#define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RUN_LENGTH_ENCODE_HPP_ + +#include "../../../type_traits.hpp" +#include "../device_config_helper.hpp" + +#include + +/* DO NOT EDIT THIS FILE + * This file is automatically generated by `/scripts/autotune/create_optimization.py`. + * so most likely you want to edit rocprim/device/device_(algo)_config.hpp + */ + +/// \addtogroup primitivesmodule_deviceconfigs +/// @{ + +BEGIN_ROCPRIM_NAMESPACE + +namespace detail +{ + +template +struct default_non_trivial_runs_config : default_non_trivial_runs_config_base::type +{}; + +// Based on key_type = double +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx1030), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = float +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx1030), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::half +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx1030), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 2))>> + : non_trivial_runs_config<128, + 32, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int64_t +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx1030), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx1030), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = short +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx1030), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1))>> + : non_trivial_runs_config<128, + 32, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int8_t +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx1030), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 1))>> + : non_trivial_runs_config<128, + 32, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = double +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx1100), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = float +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx1100), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : non_trivial_runs_config<128, + 32, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::half +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx1100), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 2))>> + : non_trivial_runs_config<128, + 32, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int64_t +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx1100), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx1100), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : non_trivial_runs_config<128, + 32, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = short +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx1100), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1))>> + : non_trivial_runs_config<128, + 32, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int8_t +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx1100), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 1))>> + : non_trivial_runs_config<128, + 32, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = double +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx906), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : non_trivial_runs_config<64, + 16, + ::rocprim::block_load_method::block_load_warp_transpose, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = float +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx906), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : non_trivial_runs_config<128, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::half +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx906), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 2))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int64_t +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx906), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : non_trivial_runs_config<64, + 16, + ::rocprim::block_load_method::block_load_warp_transpose, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx906), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : non_trivial_runs_config<128, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = short +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx906), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1))>> + : non_trivial_runs_config<128, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int8_t +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx906), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 1))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = double +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx908), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : non_trivial_runs_config<64, + 16, + ::rocprim::block_load_method::block_load_warp_transpose, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = float +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx908), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::half +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx908), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 2))>> + : non_trivial_runs_config<64, + 64, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int64_t +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx908), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : non_trivial_runs_config<64, + 16, + ::rocprim::block_load_method::block_load_warp_transpose, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx908), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = short +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx908), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int8_t +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx908), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 1))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = double +template +struct default_non_trivial_runs_config< + static_cast(target_arch::unknown), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : non_trivial_runs_config<64, + 16, + ::rocprim::block_load_method::block_load_warp_transpose, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = float +template +struct default_non_trivial_runs_config< + static_cast(target_arch::unknown), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::half +template +struct default_non_trivial_runs_config< + static_cast(target_arch::unknown), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 2))>> + : non_trivial_runs_config<64, + 64, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int64_t +template +struct default_non_trivial_runs_config< + static_cast(target_arch::unknown), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : non_trivial_runs_config<64, + 16, + ::rocprim::block_load_method::block_load_warp_transpose, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int +template +struct default_non_trivial_runs_config< + static_cast(target_arch::unknown), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = short +template +struct default_non_trivial_runs_config< + static_cast(target_arch::unknown), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int8_t +template +struct default_non_trivial_runs_config< + static_cast(target_arch::unknown), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 1))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = double +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx90a), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : non_trivial_runs_config<64, + 16, + ::rocprim::block_load_method::block_load_warp_transpose, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = float +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx90a), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::half +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx90a), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 2))>> + : non_trivial_runs_config<64, + 64, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int64_t +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx90a), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : non_trivial_runs_config<64, + 16, + ::rocprim::block_load_method::block_load_warp_transpose, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx90a), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = short +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx90a), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int8_t +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx90a), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 1))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +} // end namespace detail + +END_ROCPRIM_NAMESPACE + +/// @} +// end of group primitivesmodule_deviceconfigs + +#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RUN_LENGTH_ENCODE_HPP_ \ No newline at end of file diff --git a/rocprim/include/rocprim/device/detail/device_config_helper.hpp b/rocprim/include/rocprim/device/detail/device_config_helper.hpp index 87146056f..50db4283b 100644 --- a/rocprim/include/rocprim/device/detail/device_config_helper.hpp +++ b/rocprim/include/rocprim/device/detail/device_config_helper.hpp @@ -1087,6 +1087,86 @@ struct nth_element_config : public detail::nth_element_config_params namespace detail { +struct non_trivial_runs_config_tag +{}; + +struct non_trivial_runs_config_params +{ + kernel_config_params kernel_config; + block_load_method load_input_method; + block_scan_algorithm scan_algorithm; +}; + +} // namespace detail + +/// \brief Configuration of device-level run length encode (non-trivial runs) operation. +/// +/// \tparam BlockSize - number of threads in a block. +/// \tparam ItemsPerThread - number of items processed by each thread. +/// \tparam LoadInputMethod - method for loading inputs. +/// \tparam BlockScanMethod - algorithm for block scan. +template +struct non_trivial_runs_config : public detail::non_trivial_runs_config_params +{ + /// \brief Identifies the algorithm associated to the config. + using tag = detail::non_trivial_runs_config_tag; +#ifndef DOXYGEN_DOCUMENTATION_BUILD + /// \brief Number of threads in a block. + static constexpr unsigned int block_size = BlockSize; + /// \brief Number of items processed by each thread. + static constexpr unsigned int items_per_thread = ItemsPerThread; + /// \brief Method for loading inputs. + static constexpr block_load_method load_input_method = LoadInputMethod; + /// \brief Algorithm for block scan. + static constexpr block_scan_algorithm scan_algorithm = BlockScanAlgorithm; + + constexpr non_trivial_runs_config() + : detail::non_trivial_runs_config_params{ + {BlockSize, ItemsPerThread}, + LoadInputMethod, BlockScanAlgorithm + } {}; +#endif // DOXYGEN_DOCUMENTATION_BUILD +}; + +namespace detail +{ + +template +struct default_non_trivial_runs_config_base +{ + static constexpr unsigned int items_per_thread = 16; + using small_config = non_trivial_runs_config<256U, + items_per_thread, + block_load_method::block_load_vectorize, + block_scan_algorithm::reduce_then_scan>; + + using OffsetCountPairT = ::rocprim::tuple; + + static constexpr unsigned int size_memory_per_item + = std::max(sizeof(InputT), sizeof(OffsetCountPairT)); + + // Additional shared memory is required by the lookback scan state. + static constexpr unsigned int shared_mem_offset + = sizeof(typename offset_lookback_scan_prefix_op< + OffsetCountPairT, + lookback_scan_state>::storage_type); + + using big_config + = non_trivial_runs_config::value, + items_per_thread, + block_load_method::block_load_warp_transpose, + block_scan_algorithm::reduce_then_scan>; + + using type = std::conditional_t; +}; struct find_first_of_config_params { diff --git a/rocprim/include/rocprim/device/detail/device_reduce_by_key.hpp b/rocprim/include/rocprim/device/detail/device_reduce_by_key.hpp index 3ec5c1427..a4e37d2db 100644 --- a/rocprim/include/rocprim/device/detail/device_reduce_by_key.hpp +++ b/rocprim/include/rocprim/device/detail/device_reduce_by_key.hpp @@ -34,6 +34,7 @@ #include "../../thread/thread_operators.hpp" #include "../../config.hpp" +#include "../../type_traits.hpp" #include #include @@ -47,12 +48,10 @@ namespace detail namespace reduce_by_key { -template -using value_type_t = typename std::iterator_traits::value_type; - template using accumulator_type_t = - typename invoke_result_binary_op, BinaryOp>::type; + typename invoke_result_binary_op<::rocprim::detail::value_type_t, + BinaryOp>::type; template using wrapped_type_t = rocprim::tuple; @@ -61,36 +60,6 @@ template using lookback_scan_state_t = detail::lookback_scan_state, UseSleep>; -template -struct guarded_inequality_wrapper -{ - /// Wrapped equality operator - EqualityOp op; - - /// Out-of-bounds limit - size_t guard; - - /// Constructor - ROCPRIM_HOST_DEVICE inline guarded_inequality_wrapper(EqualityOp op, size_t guard) - : op(op), guard(guard) - {} - - /// \brief Guarded boolean inequality operator. - /// - /// \tparam T Type of the operands compared by the equality operator - /// \param a Left hand-side operand - /// \param b Right hand-side operand - /// \param idx Index of the thread calling to this operator. This is used to determine which - /// operations are out-of-bounds - /// \returns !op(a, b) for a certain equality operator \p op when in-bounds. - template - ROCPRIM_HOST_DEVICE inline bool operator()(const T& a, const T& b, size_t idx) const - { - // In-bounds return operation result, out-of-bounds return false. - return (idx < guard) ? !op(a, b) : 0; - } -}; - template(compare, remaining); + = ::rocprim::detail::guarded_inequality_wrapper(compare, + remaining); if(!is_global_first_tile) { @@ -545,7 +515,7 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE auto static constexpr block_scan_algorithm scan_algorithm = params.scan_algorithm; static constexpr unsigned int items_per_tile = block_size * items_per_thread; - using key_type = reduce_by_key::value_type_t; + using key_type = ::rocprim::detail::value_type_t; using tile_processor = tile_helper +using offset_count_pair_type_t = ::rocprim::tuple; + +template +struct load_helper +{ + using block_load_input = block_load; + union storage_type + { + typename block_load_input::storage_type input; + }; + + template + ROCPRIM_DEVICE + void load_input_values(InputIterator block_input, + const bool is_last_block, + const unsigned int valid_in_last_block, + InputType (&input)[ItemsPerThread], + storage_type& storage) + { + if(!is_last_block) + { + block_load_input{}.load(block_input, input, storage.input); + } + else + { + block_load_input{}.load(block_input, input, valid_in_last_block, storage.input); + } + ::rocprim::syncthreads(); + } +}; + +template +struct discontinuity_helper +{ + using block_discontinuity_type = block_discontinuity; + using storage_type = typename block_discontinuity_type::storage_type; + + template + ROCPRIM_DEVICE + void flag_heads_and_tails(InputIterator block_input, + const InputType (&input)[ItemsPerThread], + unsigned int (&head_flags)[ItemsPerThread], + unsigned int (&tail_flags)[ItemsPerThread], + const bool is_first_block, + const bool is_last_block, + const size_t valid_in_last_block, + storage_type& storage) + { + if(is_last_block) + { + // If it's the last block, the out-of-bound items should not be flagged. + auto guarded_not_equal + = ::rocprim::detail::guarded_inequality_wrapper( + CompareFunction(), + valid_in_last_block); + + if(is_first_block) + { + block_discontinuity_type{}.flag_heads_and_tails(head_flags, + tail_flags, + input, + guarded_not_equal, + storage); + } + else + { + const InputType block_predecessor = block_input[-1]; + block_discontinuity_type{}.flag_heads_and_tails(head_flags, + block_predecessor, + tail_flags, + input, + guarded_not_equal, + storage); + } + } + else + { + auto not_equal + = ::rocprim::detail::inequality_wrapper(CompareFunction()); + + constexpr unsigned int block_size = BlockSize * ItemsPerThread; + const InputType block_successor = block_input[block_size]; + + if(is_first_block) + { + block_discontinuity_type{}.flag_heads_and_tails(head_flags, + tail_flags, + block_successor, + input, + not_equal, + storage); + } + else + { + const InputType block_predecessor = block_input[-1]; + block_discontinuity_type{}.flag_heads_and_tails(head_flags, + block_predecessor, + tail_flags, + block_successor, + input, + not_equal, + storage); + } + } + } +}; + +/// Custom warp_exchange class with extra check in scatter_to_striped for out-of-bound accesses. +template +class custom_warp_exchange +{ + static_assert(::rocprim::detail::is_power_of_two(WarpSize), + "Logical warp size must be a power of two."); + ROCPRIM_DETAIL_DEVICE_STATIC_ASSERT( + WarpSize <= ::rocprim::device_warp_size(), + "Logical warp size cannot be larger than physical warp size."); + + static constexpr unsigned int warp_items = WarpSize * ItemsPerThread; + + // Struct used for creating a raw_storage object for this primitive's temporary storage. + struct storage_type_ + { + uninitialized_array buffer; + }; + +public: + /// \brief Struct used to allocate a temporary memory that is required for thread + /// communication during operations provided by the related parallel primitive. + /// + /// Depending on the implementation the operations exposed by parallel primitive may + /// require a temporary storage for thread communication. The storage should be allocated + /// using keywords __shared__. It can be aliased to + /// an externally allocated memory, or be a part of a union type with other storage types + /// to increase shared memory reusability. + using storage_type = storage_type_; // only for Doxygen + + /// \brief Orders \p input values according to ranks using temporary storage, + /// then writes the values to \p output in a striped manner. + /// Values in \p ranks that exceed \p WarpSize*ItemsPerThread-1 are not orderer nor written + /// to output. + /// \tparam U - [inferred] the output type. + /// \tparam OffsetT - [inferred] the offset type. + /// + /// \param [in] input - array that data is loaded from. + /// \param [out] output - array that data is loaded to. + /// \param [in] ranks - array containing the positions. + /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// + /// \par Storage reuse + /// Synchronization barrier should be placed before \p storage is reused + /// or repurposed: \p __syncthreads() or \p rocprim::syncthreads(). + template + ROCPRIM_DEVICE ROCPRIM_INLINE + void scatter_to_striped(const T (&input)[ItemsPerThread], + U (&output)[ItemsPerThread], + const OffsetT (&ranks)[ItemsPerThread], + storage_type& storage) + { + const unsigned int flat_id = ::rocprim::detail::logical_lane_id(); + + ROCPRIM_UNROLL + for(unsigned int i = 0; i < ItemsPerThread; i++) + { + if(ranks[i] < warp_items) + { + storage.buffer.emplace(ranks[i], input[i]); + } + } + ::rocprim::syncthreads(); + const auto& storage_buffer = storage.buffer.get_unsafe_array(); + + ROCPRIM_UNROLL + for(unsigned int i = 0; i < ItemsPerThread; i++) + { + unsigned int item_offset = (i * WarpSize) + flat_id; + output[i] = storage_buffer[item_offset]; + } + } +}; + +template +struct scatter_helper +{ + using offsets_storage_type = typename WarpExchangeOffsetType::storage_type; + using counts_storage_type = typename WarpExchangeCountType::storage_type; + + template + ROCPRIM_DEVICE + void scatter(const OffsetCountPairType (&offsets_and_counts)[ItemsPerThread], + OffsetsOutputIterator offsets_output, + CountsOutputIterator counts_output, + size_t block_num_runs_aggregate, + size_t block_num_runs_exclusive_in_global, + size_t warp_num_runs_aggregate, + size_t warp_num_runs_exclusive_in_block, + const size_t (&thread_num_runs_exclusive_in_warp)[ItemsPerThread], + offsets_storage_type& offsets_storage, + counts_storage_type& counts_storage) + { + // Direct scatter + if((ItemsPerThread == 1) || (block_num_runs_aggregate < BlockSize)) + { + // If the warp has any non-trivial run start, scatter + if(warp_num_runs_aggregate) + { + for(unsigned int i = 0; i < ItemsPerThread; ++i) + { + if(thread_num_runs_exclusive_in_warp[i] < warp_num_runs_aggregate) + { + size_t item_offset = block_num_runs_exclusive_in_global + + warp_num_runs_exclusive_in_block + + thread_num_runs_exclusive_in_warp[i]; + + // Scatter offset + offsets_output[item_offset] = ::rocprim::get<0>(offsets_and_counts[i]); + + // Scatter count if not the first (global) length + if(item_offset > 0) + { + counts_output[item_offset - 1] + = ::rocprim::get<1>(offsets_and_counts[i]); + } + } + } + } + } + else // Two-phase scatter + { + using offset_type = unsigned int; + using count_type = unsigned int; + + unsigned int lane_id = ::rocprim::detail::logical_lane_id(); + + // Unzip + offset_type run_offsets[ItemsPerThread]; + count_type run_counts[ItemsPerThread]; + + for(unsigned int i = 0; i < ItemsPerThread; ++i) + { + run_offsets[i] = ::rocprim::get<0>(offsets_and_counts[i]); + run_counts[i] = ::rocprim::get<1>(offsets_and_counts[i]); + } + + WarpExchangeOffsetType().scatter_to_striped(run_offsets, + run_offsets, + thread_num_runs_exclusive_in_warp, + offsets_storage); + + ::rocprim::syncthreads(); + + WarpExchangeCountType().scatter_to_striped(run_counts, + run_counts, + thread_num_runs_exclusive_in_warp, + counts_storage); + + ::rocprim::syncthreads(); + + // Each thread t in the warp scatters the valid runs with index (i * warp_size + t), for + // i in [0, ItemsPerThread-1]. That is, consecutive threads scatter consecutive non-trivial + // runs output values. + for(unsigned int i = 0; i < ItemsPerThread; ++i) + { + if((i * WarpSize) + lane_id < warp_num_runs_aggregate) + { + size_t item_offset = block_num_runs_exclusive_in_global + + warp_num_runs_exclusive_in_block + (i * WarpSize) + + lane_id; + + // Scatter offset + offsets_output[item_offset] = run_offsets[i]; + + // Scatter length if the scattered offset above was not for the first + // (global) non-trivial run + if((i != 0) || (item_offset > 0)) + { + counts_output[item_offset - 1] = run_counts[i]; + } + } + } + } + } +}; + +template +struct scan_helper +{ + using warp_scan_pairs_type = ::rocprim::warp_scan; + using warp_scan_storage_type = typename warp_scan_pairs_type::storage_type; + + template + ROCPRIM_DEVICE + void scan(OffsetCountPairType (&offsets_and_run_items)[ItemsPerThread], + OffsetCountPairType& block_aggregate, + OffsetCountPairType& warp_aggregate, + OffsetCountPairType& warp_exclusive_in_block, + OffsetCountPairType& thread_exclusive_in_warp, + ScanOp scan_op, + warp_scan_storage_type (&warp_scan_storage)[WarpsNo], + OffsetCountPairType* warp_aggregates_storage) + { + const unsigned int warp_id = ::rocprim::warp_id(); + const unsigned int lane_id = ::rocprim::detail::logical_lane_id(); + + OffsetCountPairType init; + ::rocprim::get<0>(init) = 0; + ::rocprim::get<1>(init) = 0; + + // [0]: number of non-trivial run starts in this and previous threads of the warp + // [1]: number of items in the last non-trivial run of this and previous threads of the warp + OffsetCountPairType thread_inclusive; + + // [0]: number of non-trivial run starts in this thread + // [1]: number of items in the last non-trivial run of this thread + OffsetCountPairType thread_aggregate + = ::rocprim::thread_reduce(&offsets_and_run_items[0], scan_op); + + // Warp scan results for thread i: + // - thread_inclusive = scan_op(thread_aggregate_0, thread_aggregate_1, ..., + // thread_aggregate_i) + // - thread_exclusive = scan_op(init, thread_aggregate_0, thread_aggregate_1, ... + // thread_aggregate_{i-1}) + warp_scan_pairs_type().scan(thread_aggregate, + thread_inclusive, + thread_exclusive_in_warp, + init, + warp_scan_storage[warp_id], + scan_op); + + // Last thread of the warp sets the warp-aggregate. + if(lane_id == WarpSize - 1) + { + warp_aggregates_storage[warp_id] = thread_inclusive; + } + + ::rocprim::syncthreads(); + + warp_exclusive_in_block = init; + warp_aggregate = warp_aggregates_storage[warp_id]; + + block_aggregate = warp_aggregates_storage[0]; + for(unsigned int i = 1; i < WarpsNo; ++i) + { + // The aggregate from previous warps is the partial value of block_aggregate. + if(warp_id == i) + { + warp_exclusive_in_block = block_aggregate; + } + // Update block_aggregate by adding up the warp_aggregate of warp i. + block_aggregate = scan_op(block_aggregate, warp_aggregates_storage[i]); + } + + // Ensure all threads have read warp aggregates before the storage is repurposed in the + // subsequent scatter stage. + ::rocprim::syncthreads(); + } +}; + +template +class block_helper +{ +private: + using equal_op = ::rocprim::equal_to; + using prefix_op_factory = detail::offset_lookback_scan_factory; + + // Helper class for loading input values. + using load_type + = run_length_encode::load_helper; + // Helper class for flagging the heads and tails of the input values. + using discontinuity_type + = run_length_encode::discontinuity_helper; + + // Warp size. + static constexpr unsigned int warp_size + = detail::get_min_warp_size(BlockSize, ::rocprim::device_warp_size()); + // Number of warps in block. + static constexpr unsigned int warps_no = (BlockSize + warp_size - 1) / warp_size; + + // warp_scan primitive that will be used to perform warp-level exclusive scans + // on pairs. + using warp_scan_pairs_type = ::rocprim::warp_scan; + // Helper class for warp-wide scanning of pairs. + using warp_scan_type = run_length_encode::scan_helper; + + // warp_exchange primitives that will be used to perform warp-level scatter_to_striped + // on offsets and counts. + using warp_exchange_offsets_type + = custom_warp_exchange; + using warp_exchange_counts_type + = custom_warp_exchange; + // Helper class for scattering offsets and counts. + using warp_scatter_type = run_length_encode::scatter_helper; + +public: + union storage_type + { + typename load_type::storage_type load; + + OffsetCountPairType block_exclusive; + + struct + { + typename discontinuity_type::storage_type flags; + typename prefix_op_factory::storage_type prefix; + typename warp_scan_pairs_type::storage_type warp_scan[warps_no]; + ROCPRIM_DETAIL_SUPPRESS_DEPRECATION_WITH_PUSH + typename detail::raw_storage warp_aggregates; + ROCPRIM_DETAIL_SUPPRESS_DEPRECATION_POP + } scan; + + union ScatterStorage + { + typename warp_exchange_offsets_type::storage_type scatter_offsets[warps_no]; + typename warp_exchange_counts_type::storage_type scatter_counts[warps_no]; + } scatter; + }; + + ROCPRIM_DETAIL_SUPPRESS_DEPRECATION_WITH_PUSH + using storage_type_ = detail::raw_storage; + ROCPRIM_DETAIL_SUPPRESS_DEPRECATION_POP + + template + ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE + OffsetCountPairType process_block(const InputIterator block_input, + OffsetsOutputIterator offsets_output, + CountsOutputIterator counts_output, + LookbackScanState scan_state, + const unsigned int block_id, + const std::size_t grid_size, + const std::size_t size, + storage_type_& storage_) + { + storage_type& storage = storage_.get(); + + static constexpr unsigned int items_per_block = BlockSize * ItemsPerThread; + const std::size_t block_offset = block_id * items_per_block; + + // First and last blocks + const bool is_first_block = block_id == 0; + const bool is_last_block = block_id == grid_size - 1; + + // Input items remaining in last block + const unsigned int valid_in_last_block + = static_cast(size - ((grid_size - 1) * items_per_block)); + + const unsigned int flat_thread_id = ::rocprim::detail::block_thread_id<0>(); + unsigned int warp_id = ::rocprim::warp_id(); + + InputType input[ItemsPerThread]; + + // Load items. + load_type{}.load_input_values(block_input, + is_last_block, + valid_in_last_block, + input, + storage.load); + + // Flag items. + unsigned int head_flags[ItemsPerThread]; + unsigned int tail_flags[ItemsPerThread]; + discontinuity_type{}.flag_heads_and_tails(block_input, + input, + head_flags, + tail_flags, + is_first_block, + is_last_block, + valid_in_last_block, + storage.scan.flags); + + // Heads and tails are flagged, so we can identify which runs are non-trivial: + // input [1, 1, 1, 2, 10, 10, 10, 88] + // heads [1, 0, 0, 1, 1, 0, 0, 1] + // tails [0, 0, 1, 1, 0, 0, 1, 1] + // offset [1, 0, 0, 0, 1, 0, 0, 0] = head && !tail (first items of non-trivial runs) + // count [1, 1, 1, 0, 1, 1, 1, 0] = !head || !tail (items of non-trivial runs) + OffsetCountPairType offsets_and_run_items[ItemsPerThread]; + for(unsigned int i = 0; i < ItemsPerThread; ++i) + { + ::rocprim::get<0>(offsets_and_run_items[i]) = head_flags[i] && !tail_flags[i]; + ::rocprim::get<1>(offsets_and_run_items[i]) = !head_flags[i] || !tail_flags[i]; + } + + // Exclusive scan of offsets and counts + + // [0]: number of non-trivial run starts in this block + // [1]: number of items in the last non-trivial run of this block + OffsetCountPairType block_aggregate; + + // [0]: number of non-trivial run starts in this warp + // [1]: number of items in the last non-trivial run of this warp + OffsetCountPairType warp_aggregate; + + // [0]: number of non-trivial run starts in previous warps from this block + // [1]: number of items in the last non-trivial run in previous warps from this block + OffsetCountPairType warp_exclusive_in_block; + + // [0]: number of non-trivial run starts in previous threads from this warp + // [1]: number of items in the last non-trivial run in previous threads from this warp + OffsetCountPairType thread_exclusive_in_warp; + + // Scan function: + // - always adds the number of non-trivial-run starts (first component) and + // - always keeps updated the number of items in the last non-trivial run found. Two + // cases can happen + // 1. when the first component of rhs is 0, it is either a trivial run or a non-head + // item of the latest non-trivial run found. In this case, we always add the + // second component of rhs to the one of lhs because for the former it is 0 + // (so nothing changes) and for the latter it is 1 so we add 1 to the length of + // the latest non-trivial run found). + // 2. when the first component of rhs is 1, a new non-trivial run start + // has been found, so we start counting from 1, which is the second component + // of rhs. + auto scan_op = [&](const OffsetCountPairType& lhs, const OffsetCountPairType& rhs) + { + return OffsetCountPairType{rocprim::get<0>(lhs) + rocprim::get<0>(rhs), + rocprim::get<0>(rhs) == 0 + ? rocprim::get<1>(lhs) + rocprim::get<1>(rhs) + : rocprim::get<1>(rhs)}; + }; + + // Warp scan. + warp_scan_type{}.scan(offsets_and_run_items, + block_aggregate, + warp_aggregate, + warp_exclusive_in_block, + thread_exclusive_in_warp, + scan_op, + storage.scan.warp_scan, + storage.scan.warp_aggregates.get()); + + // At this point, we have computed: + // - the pairs (offsets_and_run_items) with the heads and elements of non-trivial runs + // - the pair (thread_exclusive_in_warp) with + // 1. the number of non-trivial runs starts in previous threads of the warp + // 2. the length of the last non-trivial run in previous threads of the warp + // - the pair (warp_exclusive_in_block) with + // 1. the number of non-trivial runs starts in previous warps of the block + // 2. the length of the last non-trivial run in previous warps of the block + // - the pair (warp_aggregate) with + // 1. the number of non-trivial runs starts in this warp + // 2. the length of the last non-trivial run in this warp + // - the pair (block_aggregate) with + // 1. the number of non-trivial runs starts in this block + // 2. the length of the last non-trivial run in this block + + OffsetCountPairType offsets_and_run_items_output[ItemsPerThread]; + OffsetCountPairType offsets_and_counts[ItemsPerThread]; + OffsetCountPairType reduction; + + // Number of non-trivial run starts in previous threads from this warp + size_t thread_num_runs_exclusive_in_warp[ItemsPerThread]; + + size_t block_num_runs_aggregate; + size_t block_num_runs_exclusive_in_global; + size_t warp_num_runs_aggregate; + size_t warp_num_runs_exclusive_in_block; + + if(is_first_block) + { + // Update block status if this is not the last (and only) block + if(!is_last_block && (flat_thread_id == 0)) + { + scan_state.set_complete(0, block_aggregate); + } + + // If there are no non-trivial runs starts in the previous warp threads, then + // `thread_exclusive_in_warp<1>` denotes the number of items in the last + // non-trivial run of the previous block threads + if(::rocprim::get<0>(thread_exclusive_in_warp) == 0) + { + ::rocprim::get<1>(thread_exclusive_in_warp) + += ::rocprim::get<1>(warp_exclusive_in_block); + } + + // Update offsets_and_run_items with the non-trivial runs from previous threads + // of the warp + thread_scan_exclusive(offsets_and_run_items, + offsets_and_run_items_output, + scan_op, + thread_exclusive_in_warp /*initial_value*/); + + // Compute the offsets of the non-trivial runs. + // If there was a run that started in a previous block, there will be no run start + // flagged for it. Instead, either its length will be scattered when scattering the + // offset of the next non-trivial run or it is the last non-trivial run and its + // length will be stored when finishing processing the last block. + for(unsigned int i = 0; i < ItemsPerThread; ++i) + { + ::rocprim::get<0>(offsets_and_counts[i]) + = block_offset + (flat_thread_id * ItemsPerThread) + i; // offset of run + ::rocprim::get<1>(offsets_and_counts[i]) + = ::rocprim::get<1>(offsets_and_run_items_output[i]); // length of run + thread_num_runs_exclusive_in_warp[i] + = (::rocprim::get<0>( + offsets_and_run_items + [i])) // if there was a non-trivial run start keep offset + ? ::rocprim::get<0>(offsets_and_run_items_output[i]) + : warp_size * ItemsPerThread; // else, discard offset + } + + block_num_runs_aggregate = ::rocprim::get<0>(block_aggregate); + block_num_runs_exclusive_in_global = 0; + warp_num_runs_aggregate = ::rocprim::get<0>(warp_aggregate); + warp_num_runs_exclusive_in_block = ::rocprim::get<0>(warp_exclusive_in_block); + + // Return running total inclusive of this block + reduction = block_aggregate; + } + else + { + auto lookback_op = detail::lookback_scan_prefix_op{block_id, + scan_op, + scan_state}; + + auto offset_lookback_op = prefix_op_factory::create(lookback_op, storage.scan.prefix); + + // First warp of the block computes the block prefix in lane 0 + if(warp_id == 0) + { + // 1. Set block_aggregate as partial prefix for next block + // 2. Get prefix from previous block + // 3. Set scan_op(prefix, block_aggregate) as complete (inclusive) prefix for next block + // 4. Store block_reduction (block_aggregate) and prefix (exclusive prefix) + offset_lookback_op(block_aggregate); + + if(flat_thread_id == 0) + { + storage.block_exclusive = prefix_op_factory::get_prefix(storage.scan.prefix); + } + } + + ::rocprim::syncthreads(); + + OffsetCountPairType block_exclusive_in_global = storage.block_exclusive; + OffsetCountPairType thread_exclusive + = scan_op(block_exclusive_in_global, warp_exclusive_in_block); + + // If there are no non-trivial runs starts in the previous warp threads, then + // `thread_exclusive_in_warp<1>` denotes the number of items in the last + // non-trivial run of the previous grids threads + if(::rocprim::get<0>(thread_exclusive_in_warp) == 0) + { + ::rocprim::get<1>(thread_exclusive_in_warp) += ::rocprim::get<1>(thread_exclusive); + } + + thread_scan_exclusive(offsets_and_run_items, + offsets_and_run_items_output, + scan_op, + thread_exclusive_in_warp /*initial_value*/); + + // Compute the offsets of the non-trivial runs. + for(unsigned int i = 0; i < ItemsPerThread; ++i) + { + ::rocprim::get<0>(offsets_and_counts[i]) + = block_offset + (flat_thread_id * ItemsPerThread) + i; // offset of run + ::rocprim::get<1>(offsets_and_counts[i]) + = ::rocprim::get<1>(offsets_and_run_items_output[i]); // length of last run + thread_num_runs_exclusive_in_warp[i] + = (::rocprim::get<0>( + offsets_and_run_items[i])) // if there was a non-trivial run start + ? ::rocprim::get<0>(offsets_and_run_items_output[i]) // keep offset + : warp_size * ItemsPerThread; // else, discard offset + } + + block_num_runs_aggregate = ::rocprim::get<0>(block_aggregate); + block_num_runs_exclusive_in_global = ::rocprim::get<0>(block_exclusive_in_global); + warp_num_runs_aggregate = ::rocprim::get<0>(warp_aggregate); + warp_num_runs_exclusive_in_block = ::rocprim::get<0>(warp_exclusive_in_block); + + // Return running total (inclusive of this block) + reduction = scan_state.get_complete_value(block_id); + } + + // Scatter + warp_scatter_type{}.scatter(offsets_and_counts, + offsets_output, + counts_output, + block_num_runs_aggregate, + block_num_runs_exclusive_in_global, + warp_num_runs_aggregate, + warp_num_runs_exclusive_in_block, + thread_num_runs_exclusive_in_warp, + storage.scatter.scatter_offsets[warp_id], + storage.scatter.scatter_counts[warp_id]); + + // Return running total (inclusive of this block) + return reduction; + } +}; + +template +ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE auto + non_trivial_kernel_impl(InputIterator, + const OffsetsOutputIterator, + const CountsOutputIterator, + const RunsCountOutputIterator, + const LookbackScanState, + OrderedBlockIdType, + const size_t, + const size_t) + -> std::enable_if_t()> +{ + // No need to build the kernel with sleep on a device that does not require it +} + +template +ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE auto + non_trivial_kernel_impl(InputIterator input, + const OffsetsOutputIterator offsets_output, + const CountsOutputIterator counts_output, + const RunsCountOutputIterator runs_count_output, + const LookbackScanState scan_state, + OrderedBlockIdType ordered_block_id, + const size_t grid_size, + const size_t size) + -> std::enable_if_t()> +{ + static constexpr non_trivial_runs_config_params params = device_params(); + static constexpr unsigned int block_size = params.kernel_config.block_size; + static constexpr unsigned int items_per_thread = params.kernel_config.items_per_thread; + static constexpr block_load_method load_input_method = params.load_input_method; + static constexpr block_scan_algorithm scan_algorithm = params.scan_algorithm; + static constexpr unsigned int items_per_block = block_size * items_per_thread; + + using input_type = ::rocprim::detail::value_type_t; + using offset_type = unsigned int; + using count_type = unsigned int; + + using block_processor = block_helper; + + ROCPRIM_SHARED_MEMORY union + { + typename OrderedBlockIdType::storage_type block_id; + typename block_processor::storage_type_ block; + } storage; + + const size_t block_id = ordered_block_id.get(threadIdx.x, storage.block_id); + + const size_t block_offset = block_id * items_per_block; + const InputIterator block_input = input + block_offset; + + const size_t valid_in_last_block + = static_cast(size - (size_t{grid_size - 1} * items_per_block)); + + if(block_id < grid_size - 1) + { + block_processor{}.process_block(block_input, + offsets_output, + counts_output, + scan_state, + block_id, + grid_size, + size, + storage.block); + } + else if(valid_in_last_block > 0) + { + OffsetCountPairType total = block_processor{}.process_block(block_input, + offsets_output, + counts_output, + scan_state, + block_id, + grid_size, + size, + storage.block); + // First thread of last block sets the total number of non-trivial runs found and updates + // the counts with the last run's length if necessary. + if(threadIdx.x == 0) + { + *runs_count_output = ::rocprim::get<0>(total); + + if(::rocprim::get<0>(total) > 0) + { + counts_output[::rocprim::get<0>(total) - 1] = ::rocprim::get<1>(total); + } + } + } +} + +} // namespace run_length_encode +} // namespace detail + +END_ROCPRIM_NAMESPACE + +#endif // ROCPRIM_DEVICE_DETAIL_DEVICE_RUN_LENGTH_ENCODE_HPP_ diff --git a/rocprim/include/rocprim/device/device_reduce_by_key.hpp b/rocprim/include/rocprim/device/device_reduce_by_key.hpp index 64b577b48..74cdde683 100644 --- a/rocprim/include/rocprim/device/device_reduce_by_key.hpp +++ b/rocprim/include/rocprim/device/device_reduce_by_key.hpp @@ -37,6 +37,7 @@ #include "../functional.hpp" #include "../intrinsics/thread.hpp" #include "../iterator/constant_iterator.hpp" +#include "../type_traits.hpp" #include #include @@ -160,7 +161,7 @@ hipError_t reduce_by_key_impl(void* temporary_storage, const hipStream_t stream, const bool debug_synchronous) { - using key_type = reduce_by_key::value_type_t; + using key_type = ::rocprim::detail::value_type_t; using accumulator_type = reduce_by_key::accumulator_type_t; using config = wrapped_reduce_by_key_config; diff --git a/rocprim/include/rocprim/device/device_run_length_encode.hpp b/rocprim/include/rocprim/device/device_run_length_encode.hpp index 2accf5a26..1f53d1d7e 100644 --- a/rocprim/include/rocprim/device/device_run_length_encode.hpp +++ b/rocprim/include/rocprim/device/device_run_length_encode.hpp @@ -21,29 +21,218 @@ #ifndef ROCPRIM_DEVICE_DEVICE_RUN_LENGTH_ENCODE_HPP_ #define ROCPRIM_DEVICE_DEVICE_RUN_LENGTH_ENCODE_HPP_ -#include -#include -#include +#include "device_reduce_by_key.hpp" +#include "device_run_length_encode_config.hpp" + +#include "detail/device_config_helper.hpp" +#include "detail/device_run_length_encode.hpp" #include "../config.hpp" #include "../common.hpp" #include "../detail/various.hpp" - #include "../iterator/constant_iterator.hpp" -#include "../iterator/counting_iterator.hpp" -#include "../iterator/discard_iterator.hpp" -#include "../iterator/zip_iterator.hpp" +#include "../type_traits.hpp" -#include "detail/device_config_helper.hpp" -#include "device_reduce_by_key.hpp" -#include "device_run_length_encode_config.hpp" -#include "device_select.hpp" +#include +#include +#include +#include +#include +#include BEGIN_ROCPRIM_NAMESPACE /// \addtogroup devicemodule /// @{ +namespace detail +{ + +namespace run_length_encode +{ + +template +ROCPRIM_KERNEL + __launch_bounds__(device_params().kernel_config.block_size) +void non_trivial_kernel(const InputIterator input, + const OffsetsOutputIterator offsets_output, + const CountsOutputIterator counts_output, + const RunsCountOutputIterator runs_count_output, + const LookbackScanState scan_state, + const OrderedBlockIdType ordered_block_id, + const std::size_t grid_size, + const std::size_t size) +{ + run_length_encode::non_trivial_kernel_impl(input, + offsets_output, + counts_output, + runs_count_output, + scan_state, + ordered_block_id, + grid_size, + size); +} + +template +hipError_t run_length_encode_non_trivial_runs_impl(void* temporary_storage, + size_t& storage_size, + InputIterator input, + const size_t size, + OffsetsOutputIterator offsets_output, + CountsOutputIterator counts_output, + RunsCountOutputIterator runs_count_output, + const hipStream_t stream, + const bool debug_synchronous) +{ + using input_type = ::rocprim::detail::value_type_t; + using offset_type = unsigned int; + using count_type = unsigned int; + using offset_count_pair_type + = run_length_encode::offset_count_pair_type_t; // accumulator_type + + using config = rocprim::detail::wrapped_non_trivial_runs_config; + + using scan_state_type + = ::rocprim::detail::lookback_scan_state; + using scan_state_with_sleep_type + = ::rocprim::detail::lookback_scan_state; + + using ordered_block_id_type = detail::ordered_block_id; + + detail::target_arch target_arch; + ROCPRIM_RETURN_ON_ERROR(host_target_arch(stream, target_arch)); + + const non_trivial_runs_config_params params = dispatch_target_arch(target_arch); + const unsigned int block_size = params.kernel_config.block_size; + const unsigned int items_per_block = block_size * params.kernel_config.items_per_thread; + const std::size_t grid_size = detail::ceiling_div(size, items_per_block); + + // Calculate required temporary storage + void* scan_state_storage; + ordered_block_id_type::id_type* ordered_bid_storage; + + detail::temp_storage::layout layout{}; + ROCPRIM_RETURN_ON_ERROR(scan_state_type::get_temp_storage_layout(grid_size, stream, layout)); + + hipError_t result = detail::temp_storage::partition( + temporary_storage, + storage_size, + detail::temp_storage::make_linear_partition( + // This is valid even with scan_state_with_sleep_type + detail::temp_storage::make_partition(&scan_state_storage, layout), + detail::temp_storage::make_partition( + &ordered_bid_storage, + ordered_block_id_type::get_temp_storage_layout()))); + if(result != hipSuccess || temporary_storage == nullptr) + { + return result; + } + + bool use_sleep; + ROCPRIM_RETURN_ON_ERROR(detail::is_sleep_scan_state_used(stream, use_sleep)); + + scan_state_type scan_state{}; + scan_state_with_sleep_type scan_state_with_sleep{}; + ROCPRIM_RETURN_ON_ERROR(scan_state_type::create(scan_state, scan_state_storage, grid_size, stream)); + ROCPRIM_RETURN_ON_ERROR(scan_state_with_sleep_type::create(scan_state_with_sleep, + scan_state_storage, + grid_size, + stream)); + + auto with_scan_state + = [use_sleep, scan_state, scan_state_with_sleep](auto&& func) mutable -> decltype(auto) + { + if(use_sleep) + { + return func(scan_state_with_sleep); + } + else + { + return func(scan_state); + } + }; + + auto ordered_bid = ordered_block_id_type::create(ordered_bid_storage); + + if(size == 0) + { + // Fill out runs_count_output with zero + return rocprim::transform(rocprim::constant_iterator(0), + runs_count_output, + 1, + rocprim::identity{}, + stream, + debug_synchronous); + } + + // Start point for time measurements + std::chrono::steady_clock::time_point start; + if(debug_synchronous) + { + std::cout << "size: " << size << '\n'; + std::cout << "block_size: " << block_size << '\n'; + std::cout << "grid_size: " << grid_size << '\n'; + std::cout << "items_per_block: " << items_per_block << '\n'; + start = std::chrono::steady_clock::now(); + } + + with_scan_state( + [&](const auto scan_state) + { + const unsigned int init_block_size = ROCPRIM_DEFAULT_MAX_BLOCK_SIZE; + const std::size_t init_grid_size = detail::ceiling_div(grid_size, init_block_size); + hipLaunchKernelGGL(init_lookback_scan_state_kernel, + dim3(init_grid_size), + dim3(init_block_size), + 0, + stream, + scan_state, + grid_size, + ordered_bid); + }); + ROCPRIM_DETAIL_HIP_SYNC_AND_RETURN_ON_ERROR("init_lookback_scan_state_kernel", + grid_size, + start); + + with_scan_state( + [&](const auto scan_state) + { + hipLaunchKernelGGL( + HIP_KERNEL_NAME( + run_length_encode::non_trivial_kernel), + dim3(grid_size), + dim3(block_size), + 0, + stream, + input + 0, + offsets_output, + counts_output, + runs_count_output, + scan_state, + ordered_bid, + grid_size, + size); + }); + ROCPRIM_DETAIL_HIP_SYNC_AND_RETURN_ON_ERROR("run_length_encode::non_trivial_kernel", + size, + start); + + return hipSuccess; +} +} // namespace run_length_encode +} // namespace detail + /// \brief Parallel run-length encoding for device level. /// /// run_length_encode function performs a device-wide run-length encoding of runs (groups) @@ -123,23 +312,20 @@ BEGIN_ROCPRIM_NAMESPACE /// // runs_count_output: [4] /// \endcode /// \endparblock -template< - class Config = default_config, - class InputIterator, - class UniqueOutputIterator, - class CountsOutputIterator, - class RunsCountOutputIterator -> -inline -hipError_t run_length_encode(void * temporary_storage, - size_t& storage_size, - InputIterator input, - unsigned int size, - UniqueOutputIterator unique_output, - CountsOutputIterator counts_output, - RunsCountOutputIterator runs_count_output, - hipStream_t stream = 0, - bool debug_synchronous = false) +template +inline hipError_t run_length_encode(void* temporary_storage, + size_t& storage_size, + InputIterator input, + unsigned int size, + UniqueOutputIterator unique_output, + CountsOutputIterator counts_output, + RunsCountOutputIterator runs_count_output, + hipStream_t stream = 0, + bool debug_synchronous = false) { using input_type = typename std::iterator_traits::value_type; using count_type = unsigned int; @@ -153,12 +339,18 @@ hipError_t run_length_encode(void * temporary_storage, default_config>>; return ::rocprim::reduce_by_key( - temporary_storage, storage_size, - input, make_constant_iterator(1), size, - unique_output, counts_output, runs_count_output, - ::rocprim::plus(), ::rocprim::equal_to(), - stream, debug_synchronous - ); + temporary_storage, + storage_size, + input, + make_constant_iterator(1), + size, + unique_output, + counts_output, + runs_count_output, + ::rocprim::plus(), + ::rocprim::equal_to(), + stream, + debug_synchronous); } /// \brief Parallel run-length encoding of non-trivial runs for device level. @@ -241,153 +433,33 @@ hipError_t run_length_encode(void * temporary_storage, /// // runs_count_output: [2] /// \endcode /// \endparblock -template< - class Config = default_config, - class InputIterator, - class OffsetsOutputIterator, - class CountsOutputIterator, - class RunsCountOutputIterator -> -inline -hipError_t run_length_encode_non_trivial_runs(void * temporary_storage, - size_t& storage_size, - InputIterator input, - unsigned int size, - OffsetsOutputIterator offsets_output, - CountsOutputIterator counts_output, - RunsCountOutputIterator runs_count_output, - hipStream_t stream = 0, - bool debug_synchronous = false) +template +inline hipError_t run_length_encode_non_trivial_runs(void* temporary_storage, + size_t& storage_size, + InputIterator input, + unsigned int size, + OffsetsOutputIterator offsets_output, + CountsOutputIterator counts_output, + RunsCountOutputIterator runs_count_output, + hipStream_t stream = 0, + bool debug_synchronous = false) { - using input_type = typename std::iterator_traits::value_type; - using offset_type = unsigned int; - using count_type = unsigned int; - using offset_count_pair = typename ::rocprim::tuple; - - using config = detail::default_or_custom_config< - Config, - detail::default_run_length_encode_config>>; - - hipError_t error; - - auto reduce_op = [] __device__ (const offset_count_pair& a, const offset_count_pair& b) - { - return offset_count_pair( - ::rocprim::get<0>(a), // Always use offset of the first item of the run - ::rocprim::get<1>(a) + ::rocprim::get<1>(b) // Number of items in the run - ); - }; - auto non_trivial_runs_select_op = [] __device__ (const offset_count_pair& a) - { - return ::rocprim::get<1>(a) > 1; - }; - - offset_type * offsets_tmp = nullptr; - count_type * counts_tmp = nullptr; - count_type * all_runs_count_tmp = nullptr; - - // Calculate size of temporary storage for reduce_by_key operation - size_t reduce_by_key_bytes; - error = ::rocprim::reduce_by_key( - nullptr, reduce_by_key_bytes, - input, - ::rocprim::make_zip_iterator( - ::rocprim::make_tuple( - ::rocprim::make_counting_iterator(0), - ::rocprim::make_constant_iterator(1) - ) - ), - size, - ::rocprim::make_discard_iterator(), - ::rocprim::make_zip_iterator(::rocprim::make_tuple(offsets_tmp, counts_tmp)), - all_runs_count_tmp, - reduce_op, ::rocprim::equal_to(), - stream, debug_synchronous - ); - if(error != hipSuccess) return error; - reduce_by_key_bytes = ::rocprim::detail::align_size(reduce_by_key_bytes); - - // Calculate size of temporary storage for select operation - size_t select_bytes; - error = ::rocprim::select( - nullptr, select_bytes, - ::rocprim::make_zip_iterator(::rocprim::make_tuple(offsets_tmp, counts_tmp)), - ::rocprim::make_zip_iterator(::rocprim::make_tuple(offsets_output, counts_output)), - runs_count_output, - size, - non_trivial_runs_select_op, - stream, debug_synchronous - ); - if(error != hipSuccess) return error; - select_bytes = ::rocprim::detail::align_size(select_bytes); - - const size_t offsets_tmp_bytes = ::rocprim::detail::align_size(size * sizeof(offset_type)); - const size_t counts_tmp_bytes = ::rocprim::detail::align_size(size * sizeof(count_type)); - const size_t all_runs_count_tmp_bytes = sizeof(count_type); - if(temporary_storage == nullptr) - { - storage_size = ::rocprim::max(reduce_by_key_bytes, select_bytes) + - offsets_tmp_bytes + counts_tmp_bytes + all_runs_count_tmp_bytes; - return hipSuccess; - } - - char * ptr = reinterpret_cast(temporary_storage); - ptr += ::rocprim::max(reduce_by_key_bytes, select_bytes); - offsets_tmp = reinterpret_cast(ptr); - ptr += offsets_tmp_bytes; - counts_tmp = reinterpret_cast(ptr); - ptr += counts_tmp_bytes; - all_runs_count_tmp = reinterpret_cast(ptr); - - std::chrono::steady_clock::time_point start; - - if(debug_synchronous) start = std::chrono::steady_clock::now(); - error = ::rocprim::reduce_by_key( - temporary_storage, reduce_by_key_bytes, + return detail::run_length_encode::run_length_encode_non_trivial_runs_impl( + temporary_storage, + storage_size, input, - ::rocprim::make_zip_iterator( - ::rocprim::make_tuple( - ::rocprim::make_counting_iterator(0), - ::rocprim::make_constant_iterator(1) - ) - ), size, - ::rocprim::make_discard_iterator(), // Ignore unique output - ::rocprim::make_zip_iterator(rocprim::make_tuple(offsets_tmp, counts_tmp)), - all_runs_count_tmp, - reduce_op, ::rocprim::equal_to(), - stream, debug_synchronous - ); - ROCPRIM_DETAIL_HIP_SYNC_AND_RETURN_ON_ERROR("rocprim::reduce_by_key", size, start); - - // Read count of all runs (including trivial runs) - count_type all_runs_count; - error = detail::memcpy_and_sync(&all_runs_count, - all_runs_count_tmp, - sizeof(count_type), - hipMemcpyDeviceToHost, - stream); - if(error != hipSuccess) - return error; - - // Select non-trivial runs - if(debug_synchronous) start = std::chrono::steady_clock::now(); - error = ::rocprim::select( - temporary_storage, select_bytes, - ::rocprim::make_zip_iterator(::rocprim::make_tuple(offsets_tmp, counts_tmp)), - ::rocprim::make_zip_iterator(::rocprim::make_tuple(offsets_output, counts_output)), + offsets_output, + counts_output, runs_count_output, - all_runs_count, - non_trivial_runs_select_op, - stream, debug_synchronous - ); - ROCPRIM_DETAIL_HIP_SYNC_AND_RETURN_ON_ERROR("rocprim::select", all_runs_count, start); - - return hipSuccess; + stream, + debug_synchronous); } - - /// @} // end of group devicemodule diff --git a/rocprim/include/rocprim/device/device_run_length_encode_config.hpp b/rocprim/include/rocprim/device/device_run_length_encode_config.hpp index cdcbf17b0..99a798682 100644 --- a/rocprim/include/rocprim/device/device_run_length_encode_config.hpp +++ b/rocprim/include/rocprim/device/device_run_length_encode_config.hpp @@ -21,13 +21,15 @@ #ifndef ROCPRIM_DEVICE_DEVICE_RUN_LENGTH_ENCODE_CONFIG_HPP_ #define ROCPRIM_DEVICE_DEVICE_RUN_LENGTH_ENCODE_CONFIG_HPP_ -#include +#include "config_types.hpp" +#include "detail/device_config_helper.hpp" #include "../config.hpp" #include "../detail/various.hpp" -#include "detail/device_config_helper.hpp" +#include "../type_traits.hpp" +#include "detail/config/device_run_length_encode_non_trivial.hpp" -#include "config_types.hpp" +#include /// \addtogroup primitivesmodule_deviceconfigs /// @{ @@ -55,12 +57,117 @@ struct run_length_encode_config namespace detail { -// The problem size for the selection is the number of runs, which may be much smaller than the -// total problem size. The tuning assumption of reasonably large inputs does therefore not hold. -template -using default_run_length_encode_config - = run_length_encode_config::type>; +// Wrap around run_length_encode_config and the newly added non_trivial_runs_config for the +// run_length_encode_non_trivial_runs algorithm. Three cases are considered for selecting +// the appropriate config: +// +// - When a run_length_encode_config struct is passed as argument, an specialization of +// this struct takes care of mapping the parameters of that config to the newly added +// non_trivial_runs_config. +// +// - When a default config is passed, another specialization takes care of using the +// default set up of non_trivial_runs_config. +// +// - When a non_trivial_runs_config is passed, the params are set from this config. +// +template +struct wrapped_non_trivial_runs_config +{ + static_assert(std::is_same::value, + "Config must be a specialization of struct template non_trivial_runs_config"); + + template + struct architecture_config + { + static constexpr non_trivial_runs_config_params params = RLENonTrivialRunsConfig{}; + }; +}; + +template +struct wrapped_non_trivial_runs_config< + rocprim::run_length_encode_config, + InputType> +{ + template + struct architecture_config + { + // Mapping to non_trivial_runs_config. + // Beware that this mapping may impact performance of executions of + // run_length_encode_non_trivial_runs with the former run_length_encode_config, + // as it may not be the best for all cases. + static constexpr unsigned int block_size = ReduceByKeyConfig::block_size; + static constexpr unsigned int items_per_thread = ReduceByKeyConfig::items_per_thread; + + static constexpr block_load_method load_input_method = ReduceByKeyConfig::load_keys_method; + static constexpr block_scan_algorithm scan_algorithm = ReduceByKeyConfig::scan_algorithm; + + static constexpr non_trivial_runs_config_params params + = non_trivial_runs_config{}; + }; +}; + +// Generic for default config: instantiate base config. +template +struct wrapped_non_trivial_runs_impl +{ + template + struct architecture_config + { + static constexpr non_trivial_runs_config_params params = + typename default_non_trivial_runs_config_base::type{}; + }; +}; + +// Specialization for default config if types are arithmetic or half/bfloat16-precision +// floating point types: instantiate the tuned config. +template +struct wrapped_non_trivial_runs_impl::value>> +{ + template + struct architecture_config + { + static constexpr non_trivial_runs_config_params params + = default_non_trivial_runs_config(Arch), InputType>{}; + }; +}; + +// Specialization for default config. +template +struct wrapped_non_trivial_runs_config + : wrapped_non_trivial_runs_impl +{}; + +#ifndef DOXYGEN_DOCUMENTATION_BUILD + +template +template +constexpr non_trivial_runs_config_params + wrapped_non_trivial_runs_config::architecture_config::params; + +template +template +constexpr non_trivial_runs_config_params wrapped_non_trivial_runs_config< + rocprim::run_length_encode_config, + InputType>::architecture_config::params; + +template +template +constexpr non_trivial_runs_config_params + wrapped_non_trivial_runs_impl::architecture_config::params; + +template +template +constexpr non_trivial_runs_config_params wrapped_non_trivial_runs_impl< + InputType, + std::enable_if_t::value>>::architecture_config::params; + +#endif // DOXYGEN_DOCUMENTATION_BUILD } // end namespace detail diff --git a/rocprim/include/rocprim/thread/thread_reduce.hpp b/rocprim/include/rocprim/thread/thread_reduce.hpp index 40a891b02..97aa1d55a 100644 --- a/rocprim/include/rocprim/thread/thread_reduce.hpp +++ b/rocprim/include/rocprim/thread/thread_reduce.hpp @@ -30,8 +30,11 @@ #ifndef ROCPRIM_THREAD_THREAD_REDUCE_HPP_ #define ROCPRIM_THREAD_THREAD_REDUCE_HPP_ - #include "../config.hpp" +#include "../functional.hpp" +#include "../type_traits.hpp" + +#include BEGIN_ROCPRIM_NAMESPACE @@ -50,28 +53,55 @@ BEGIN_ROCPRIM_NAMESPACE /// \param reduction_op [in] Instance of the reduction operator functor /// \param prefix [in] Value to be used as prefix, if NoPrefix is false /// \return Value obtained from reduction of input array -template < - int LENGTH, - typename T, - typename ReductionOp, - bool NoPrefix = false> -ROCPRIM_DEVICE ROCPRIM_INLINE T thread_reduce( - T* input, - ReductionOp reduction_op, - T prefix = T(0)) +template +ROCPRIM_DEVICE ROCPRIM_INLINE +auto thread_reduce(T* input, ReductionOp reduction_op, T prefix = T(0)) + -> std::enable_if_t::value, T> +{ + T retval; + if(NoPrefix) + { + retval = input[0]; + } + else + { + retval = prefix; + } + + ROCPRIM_UNROLL + for(int i = 0 + NoPrefix; i < LENGTH; ++i) + { + retval = reduction_op(retval, input[i]); + } + + return retval; +} + +/// \cond thread_reduce_specialization +template +ROCPRIM_DEVICE ROCPRIM_INLINE +auto thread_reduce(T* input, ReductionOp reduction_op, T prefix = T{}) + -> std::enable_if_t::value, T> { T retval; if(NoPrefix) + { retval = input[0]; + } else + { retval = prefix; + } ROCPRIM_UNROLL - for (int i = 0 + NoPrefix; i < LENGTH; ++i) + for(int i = 0 + NoPrefix; i < LENGTH; ++i) + { retval = reduction_op(retval, input[i]); + } return retval; } +/// \endcond /// \brief Carry out a reduction on an array of elements in one thread /// \tparam LENGTH Length of the array to be reduced diff --git a/rocprim/include/rocprim/type_traits.hpp b/rocprim/include/rocprim/type_traits.hpp index f8368a4ba..623e7feef 100644 --- a/rocprim/include/rocprim/type_traits.hpp +++ b/rocprim/include/rocprim/type_traits.hpp @@ -369,6 +369,35 @@ struct invoke_result_impl(), std::declval(), std::declval()...)); }; +template +struct is_tuple +{ +public: + static constexpr bool value = false; +}; + +template +struct is_tuple<::rocprim::tuple> +{ +private: + template + ROCPRIM_HOST_DEVICE + static constexpr bool is_tuple_impl() + { + return is_tuple_impl(); + } + + template<> + ROCPRIM_HOST_DEVICE + static constexpr bool is_tuple_impl() + { + return true; + } + +public: + static constexpr bool value = is_tuple_impl<0>(); +}; + template struct is_tuple_of_references { @@ -439,6 +468,41 @@ struct float_bit_mask template using void_t = void; +template +using value_type_t = typename std::iterator_traits::value_type; + +template +struct guarded_inequality_wrapper +{ + /// Wrapped equality operator + EqualityOp op; + + /// Out-of-bounds limit + size_t guard; + + /// Constructor + ROCPRIM_HOST_DEVICE inline guarded_inequality_wrapper(EqualityOp op, size_t guard) + : op(op), guard(guard) + {} + + /// \brief Guarded boolean inequality operator. + /// + /// \tparam T Type of the operands compared by the equality operator + /// \param a Left hand-side operand + /// \param b Right hand-side operand + /// \param idx Index of the thread calling to this operator. This is used to determine which + /// operations are out-of-bounds + /// \returns !op(a, b) for a certain equality operator \p op when in-bounds. + template + ROCPRIM_HOST_DEVICE + inline bool + operator()(const T& a, const T& b, size_t idx) const + { + // In-bounds return operation result, out-of-bounds return ret. + return (idx < guard) ? !op(a, b) : Ret; + } +}; + } // end namespace detail /// \brief Behaves like ``std::invoke_result``, but allows the use of invoke_result diff --git a/scripts/autotune/create_optimization.py b/scripts/autotune/create_optimization.py index 093f7b79b..018ce843e 100755 --- a/scripts/autotune/create_optimization.py +++ b/scripts/autotune/create_optimization.py @@ -653,6 +653,12 @@ class AlgorithmDeviceMerge(Algorithm): config_selection_params = [ SelectionType(name="key_type", is_optional=False, select_on_size_only=False), SelectionType(name="value_type", is_optional=True, select_on_size_only=True)] + +class AlgorithmDeviceRunLengthEncodeNonTrivial(Algorithm): + algorithm_name = 'device_run_length_encode_non_trivial' + cpp_configuration_template_name = 'run_length_encode_config_template' + config_selection_params = [SelectionType(name='key_type', is_optional=False, select_on_size_only=False)] + def __init__(self, fallback_entries): Algorithm.__init__(self, fallback_entries) @@ -721,6 +727,8 @@ def create_algorithm(algorithm_name: str, fallback_entries: List[FallbackCase]): return AlgorithmDeviceFindFirstOf(fallback_entries) elif algorithm_name == 'device_merge': return AlgorithmDeviceMerge(fallback_entries) + elif algorithm_name == 'device_run_length_encode_non_trivial': + return AlgorithmDeviceRunLengthEncodeNonTrivial(fallback_entries) else: raise(NotSupportedError(f'Algorithm "{algorithm_name}" is not supported (yet)')) diff --git a/scripts/autotune/templates/run_length_encode_config_template b/scripts/autotune/templates/run_length_encode_config_template new file mode 100644 index 000000000..5c2824099 --- /dev/null +++ b/scripts/autotune/templates/run_length_encode_config_template @@ -0,0 +1,28 @@ +{% extends "config_template" %} + +{% macro get_header_guard() %} +ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RUN_LENGTH_ENCODE_HPP_ +{%- endmacro %} + +{% macro kernel_configuration(measurement) -%} +non_trivial_runs_config< + {{ measurement['cfg']['bs'] }}, + {{ measurement['cfg']['ipt'] }}, + ::rocprim::{{ measurement['cfg']['load_method'] }}, + ::rocprim::block_scan_algorithm::using_warp_scan> { }; +{%- endmacro %} + +{% macro general_case() -%} +template +struct default_non_trivial_runs_config : default_non_trivial_runs_config_base::type +{}; +{%- endmacro %} + +{% macro configuration_fallback(benchmark_of_architecture, based_on_type, fallback_selection_criteria) -%} +// Based on {{ based_on_type }} +template +struct default_non_trivial_runs_config< + static_cast({{ benchmark_of_architecture.name }}), + key_type, + {{ fallback_selection_criteria }}> : +{%- endmacro %} \ No newline at end of file diff --git a/test/rocprim/test_device_run_length_encode.cpp b/test/rocprim/test_device_run_length_encode.cpp index 6d837e047..a2d07e268 100644 --- a/test/rocprim/test_device_run_length_encode.cpp +++ b/test/rocprim/test_device_run_length_encode.cpp @@ -26,6 +26,9 @@ #include // required test headers +#include "rocprim/block/block_load.hpp" +#include "rocprim/block/block_scan.hpp" +#include "rocprim/device/detail/lookback_scan_state.hpp" #include "rocprim/types.hpp" #include "test_utils_types.hpp" @@ -35,12 +38,14 @@ template + class Config = rocprim::default_config, + class NonTrivialConfig = rocprim::default_config> struct params { using key_type = Key; using count_type = Count; using config = Config; + using non_trivial_config = NonTrivialConfig; static constexpr unsigned int min_segment_length = MinSegmentLength; static constexpr unsigned int max_segment_length = MaxSegmentLength; static constexpr bool use_identity_iterator = UseIdentityIterator; @@ -56,35 +61,43 @@ using custom_int2 = test_utils::custom_test_type; using custom_double2 = test_utils::custom_test_type; typedef ::testing::Types< - params, - params, - params, + // Tests with default configuration + params, + params, + params, + params, + params, + params, + params, params, - params, params, - params, - rocprim::select_config<64, 3>>>, + params, + params, params, - params, - params, - params, - params, params, // half should be supported, but is missing some key operators. // we should uncomment these, as soon as these are implemented and the tests compile and work as intended. //params, - params, - params, params, params, - params, - params, - params> + // Tests for custom types + params, + params, + params, + // Tests for supported config structs + params, + rocprim::select_config<64, 3>>, + // RLE non-trivial config + rocprim::run_length_encode_config, + rocprim::select_config<256, 13>>>, + // Tests for when output's value_type is void + params> Params; TYPED_TEST_SUITE(RocprimDeviceRunLengthEncode, Params); @@ -130,6 +143,9 @@ TYPED_TEST(RocprimDeviceRunLengthEncode, Encode) hipStream_t stream = 0; // default + // Default stream does not support hipGraph stream capture, so create one + HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); + // Generate data and calculate expected results std::vector unique_expected; std::vector counts_expected; @@ -198,6 +214,10 @@ TYPED_TEST(RocprimDeviceRunLengthEncode, Encode) void * d_temporary_storage; HIP_CHECK(test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + HIP_CHECK(hipDeviceSynchronize()); + + test_utils::GraphHelper gHelper; + gHelper.startStreamCapture(stream); HIP_CHECK(rocprim::run_length_encode( d_temporary_storage, @@ -210,6 +230,8 @@ TYPED_TEST(RocprimDeviceRunLengthEncode, Encode) stream, debug_synchronous)); + gHelper.createAndLaunchGraph(stream); + HIP_CHECK(hipDeviceSynchronize()); HIP_CHECK(hipFree(d_temporary_storage)); @@ -217,6 +239,7 @@ TYPED_TEST(RocprimDeviceRunLengthEncode, Encode) std::vector unique_output(runs_count_expected); std::vector counts_output(runs_count_expected); std::vector runs_count_output(1); + HIP_CHECK( hipMemcpy( unique_output.data(), d_unique_output, @@ -244,6 +267,9 @@ TYPED_TEST(RocprimDeviceRunLengthEncode, Encode) HIP_CHECK(hipFree(d_counts_output)); HIP_CHECK(hipFree(d_runs_count_output)); + gHelper.cleanupGraphHelper(); + HIP_CHECK(hipStreamDestroy(stream)); + // Validating results std::vector runs_count_expected_2; @@ -266,9 +292,10 @@ TYPED_TEST(RocprimDeviceRunLengthEncode, NonTrivialRuns) using key_type = typename TestFixture::params::key_type; using count_type = typename TestFixture::params::count_type; using offset_type = typename TestFixture::params::count_type; - using config = typename TestFixture::params::config; + using config = typename TestFixture::params::non_trivial_config; constexpr bool use_identity_iterator = TestFixture::params::use_identity_iterator; + const bool debug_synchronous = false; const unsigned int seed = 123; @@ -286,6 +313,9 @@ TYPED_TEST(RocprimDeviceRunLengthEncode, NonTrivialRuns) hipStream_t stream = 0; // default + // Default stream does not support hipGraph stream capture, so create one + HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); + // Generate data and calculate expected results std::vector offsets_expected; std::vector counts_expected; @@ -297,7 +327,6 @@ TYPED_TEST(RocprimDeviceRunLengthEncode, NonTrivialRuns) TestFixture::params::max_segment_length ); std::bernoulli_distribution is_trivial_dis(0.1); - std::vector values_input = test_utils::get_random_data(size, 0, 100, seed_value); size_t offset = 0; key_type current_key = get_random_value_no_duplicate(key_type(0), random_keys, size); @@ -367,6 +396,10 @@ TYPED_TEST(RocprimDeviceRunLengthEncode, NonTrivialRuns) void * d_temporary_storage; HIP_CHECK(test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + HIP_CHECK(hipDeviceSynchronize()); + + test_utils::GraphHelper gHelper; + gHelper.startStreamCapture(stream); HIP_CHECK(rocprim::run_length_encode_non_trivial_runs( d_temporary_storage, @@ -379,11 +412,16 @@ TYPED_TEST(RocprimDeviceRunLengthEncode, NonTrivialRuns) stream, debug_synchronous)); + gHelper.createAndLaunchGraph(stream); + + HIP_CHECK(hipDeviceSynchronize()); + HIP_CHECK(hipFree(d_temporary_storage)); std::vector offsets_output(runs_count_expected); std::vector counts_output(runs_count_expected); std::vector runs_count_output(1); + if(runs_count_expected > 0) { HIP_CHECK( @@ -414,14 +452,22 @@ TYPED_TEST(RocprimDeviceRunLengthEncode, NonTrivialRuns) HIP_CHECK(hipFree(d_counts_output)); HIP_CHECK(hipFree(d_runs_count_output)); + gHelper.cleanupGraphHelper(); + HIP_CHECK(hipStreamDestroy(stream)); + // Validating results std::vector runs_count_expected_2; runs_count_expected_2.push_back(static_cast(runs_count_expected)); - test_utils::assert_eq(runs_count_output, runs_count_expected_2, 1); - - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(offsets_output, offsets_expected, runs_count_expected)); - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(counts_output, counts_expected, runs_count_expected)); + SCOPED_TRACE(testing::Message() << "runs_count_output"); + ASSERT_NO_FATAL_FAILURE( + test_utils::assert_eq(runs_count_output, runs_count_expected_2, 1)); + SCOPED_TRACE(testing::Message() << "offsets_output"); + ASSERT_NO_FATAL_FAILURE( + test_utils::assert_eq(offsets_output, offsets_expected, runs_count_expected)); + SCOPED_TRACE(testing::Message() << "counts_output"); + ASSERT_NO_FATAL_FAILURE( + test_utils::assert_eq(counts_output, counts_expected, runs_count_expected)); } } From 29970bf25ea2af033fe54eef37ef9bc1a1f9828b Mon Sep 17 00:00:00 2001 From: Beatriz Navidad Vilches Date: Thu, 21 Nov 2024 12:58:59 +0100 Subject: [PATCH 04/84] Resolve "Investigate RLE (non trivial runs) failure" --- .../rocprim/device/detail/device_run_length_encode.hpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/rocprim/include/rocprim/device/detail/device_run_length_encode.hpp b/rocprim/include/rocprim/device/detail/device_run_length_encode.hpp index 8be9efe19..bab041509 100644 --- a/rocprim/include/rocprim/device/detail/device_run_length_encode.hpp +++ b/rocprim/include/rocprim/device/detail/device_run_length_encode.hpp @@ -216,7 +216,7 @@ class custom_warp_exchange storage.buffer.emplace(ranks[i], input[i]); } } - ::rocprim::syncthreads(); + ::rocprim::wave_barrier(); const auto& storage_buffer = storage.buffer.get_unsafe_array(); ROCPRIM_UNROLL @@ -297,20 +297,19 @@ struct scatter_helper run_counts[i] = ::rocprim::get<1>(offsets_and_counts[i]); } + // Force synchronization point + ::rocprim::syncthreads(); + WarpExchangeOffsetType().scatter_to_striped(run_offsets, run_offsets, thread_num_runs_exclusive_in_warp, offsets_storage); - ::rocprim::syncthreads(); - WarpExchangeCountType().scatter_to_striped(run_counts, run_counts, thread_num_runs_exclusive_in_warp, counts_storage); - ::rocprim::syncthreads(); - // Each thread t in the warp scatters the valid runs with index (i * warp_size + t), for // i in [0, ItemsPerThread-1]. That is, consecutive threads scatter consecutive non-trivial // runs output values. From cfeb4a4b7e872172363c041af5f37b17ba04754d Mon Sep 17 00:00:00 2001 From: Yung-sheng Tu Date: Thu, 21 Nov 2024 12:59:03 +0100 Subject: [PATCH 05/84] Resolve "Config tuning and dynamic dispatch for device run length encode" --- CHANGELOG.md | 1 + benchmark/CMakeLists.txt | 1 + benchmark/ConfigAutotuneSettings.cmake | 12 +- .../benchmark_device_run_length_encode.cpp | 197 +---- ...k_device_run_length_encode.parallel.cpp.in | 4 +- ...mark_device_run_length_encode.parallel.hpp | 213 +++-- ...ice_run_length_encode_non_trivial_runs.cpp | 151 ++++ ...th_encode_non_trivial_runs.parallel.cpp.in | 34 + ...ength_encode_non_trivial_runs.parallel.hpp | 258 +++++++ docs/concepts/tuning.rst | 2 +- .../config/device_run_length_encode.hpp | 725 ++++++++++++++++++ .../device_run_length_encode_non_trivial.hpp | 6 +- .../rocprim/device/device_reduce_by_key.hpp | 152 ++-- .../device/device_run_length_encode.hpp | 113 ++- .../device_run_length_encode_config.hpp | 74 +- scripts/autotune/create_optimization.py | 28 +- .../run_length_encode_config_template | 27 +- ...th_encode_non_trivial_runs_config_template | 28 + 18 files changed, 1585 insertions(+), 441 deletions(-) create mode 100644 benchmark/benchmark_device_run_length_encode_non_trivial_runs.cpp create mode 100644 benchmark/benchmark_device_run_length_encode_non_trivial_runs.parallel.cpp.in create mode 100644 benchmark/benchmark_device_run_length_encode_non_trivial_runs.parallel.hpp create mode 100644 rocprim/include/rocprim/device/detail/config/device_run_length_encode.hpp create mode 100644 scripts/autotune/templates/run_length_encode_non_trivial_runs_config_template diff --git a/CHANGELOG.md b/CHANGELOG.md index 5896dedaf..28b2f5872 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ Full documentation for rocPRIM is available at [https://rocm.docs.amd.com/projec * Added new constructors and a `base` function, and added `constexpr` specifier to all functions in `rocprim::reverse_iterator` to improve parity with the C++17 `std::reverse_iterator`. * Added hipGraph support to device run-length-encode for non trivial runs (`rocprim::run_length_encode_non_trivial_runs`). * Added configuration autotuning to device run-length-encode for non trivial runs (`rocprim::run_length_encode_non_trivial_runs`) for improved performance on selected architectures. +* Added configuration autotuning to device run-length-encode for trivial runs (`rocprim::run_length_encode`) for improved performance on selected architectures. ### Changed diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 94eeb1e76..fd8ee7804 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -161,6 +161,7 @@ add_rocprim_benchmark(benchmark_device_reduce_by_key.cpp) add_rocprim_benchmark(benchmark_device_reduce_by_key_deterministic.cpp) add_rocprim_benchmark(benchmark_device_reduce.cpp) add_rocprim_benchmark(benchmark_device_run_length_encode.cpp) +add_rocprim_benchmark(benchmark_device_run_length_encode_non_trivial_runs.cpp) add_rocprim_benchmark(benchmark_device_scan.cpp) add_rocprim_benchmark(benchmark_device_scan_deterministic.cpp) add_rocprim_benchmark(benchmark_device_scan_by_key.cpp) diff --git a/benchmark/ConfigAutotuneSettings.cmake b/benchmark/ConfigAutotuneSettings.cmake index f5555df6a..b4c25e53c 100644 --- a/benchmark/ConfigAutotuneSettings.cmake +++ b/benchmark/ConfigAutotuneSettings.cmake @@ -119,13 +119,17 @@ DataType;BlockSize;" PARENT_SCOPE) set(list_across_names "DataType;BlockSize" PARENT_SCOPE) set(list_across "${LIMITED_TUNING_TYPES};32 64 128 256 512 1024" PARENT_SCOPE) set(output_pattern_suffix "@DataType@_@BlockSize@" PARENT_SCOPE) - elseif(file STREQUAL "benchmark_device_merge") - set(list_across_names "KeyType;ValueType;BlockSize" PARENT_SCOPE) - set(list_across "${TUNING_TYPES};rocprim::empty_type ${LIMITED_TUNING_TYPES};32 64 128 256 512 1024" PARENT_SCOPE) - set(output_pattern_suffix "@KeyType@_@ValueType@_@BlockSize@" PARENT_SCOPE) elseif(file STREQUAL "benchmark_device_run_length_encode") + set(list_across_names "KeyType;BlockSize;TilesPerBlock" PARENT_SCOPE) + set(list_across "${TUNING_TYPES};128 192 256 384 512;1 2" PARENT_SCOPE) + set(output_pattern_suffix "@KeyType@_@BlockSize@_@TilesPerBlock@" PARENT_SCOPE) + elseif(file STREQUAL "benchmark_device_run_length_encode_non_trivial_runs") set(list_across_names "KeyType;BlockSize;BlockLoadMethod" PARENT_SCOPE) set(list_across "${TUNING_TYPES};64 128 256 512 1024;block_load_vectorize block_load_warp_transpose" PARENT_SCOPE) set(output_pattern_suffix "@KeyType@_@BlockSize@_@BlockLoadMethod@" PARENT_SCOPE) + elseif(file STREQUAL "benchmark_device_merge") + set(list_across_names "KeyType;ValueType;BlockSize" PARENT_SCOPE) + set(list_across "${TUNING_TYPES};rocprim::empty_type ${LIMITED_TUNING_TYPES};32 64 128 256 512 1024" PARENT_SCOPE) + set(output_pattern_suffix "@KeyType@_@ValueType@_@BlockSize@" PARENT_SCOPE) endif() endfunction() diff --git a/benchmark/benchmark_device_run_length_encode.cpp b/benchmark/benchmark_device_run_length_encode.cpp index 73d855650..80abe2d9b 100644 --- a/benchmark/benchmark_device_run_length_encode.cpp +++ b/benchmark/benchmark_device_run_length_encode.cpp @@ -43,143 +43,12 @@ constexpr size_t DEFAULT_BYTES = size_t{2} << 30; // 2 GiB #endif -namespace rp = rocprim; - -template -void run_encode_benchmark(benchmark::State& state, - size_t max_length, - size_t bytes, - const managed_seed& seed, - hipStream_t stream) -{ - using key_type = T; - using count_type = unsigned int; - - const size_t size = bytes / sizeof(T); - - // Generate data - std::vector input(size); - - unsigned int runs_count = 0; - const auto random_range = limit_random_range(1, max_length); - std::vector key_counts - = get_random_data(100000, random_range.first, random_range.second, seed.get_0()); - size_t offset = 0; - while(offset < size) - { - const size_t key_count = key_counts[runs_count % key_counts.size()]; - const size_t end = std::min(size, offset + key_count); - for(size_t i = offset; i < end; i++) - { - input[i] = runs_count; - } - - runs_count++; - offset += key_count; - } - - key_type* d_input; - HIP_CHECK(hipMalloc(reinterpret_cast(&d_input), size * sizeof(key_type))); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(key_type), hipMemcpyHostToDevice)); - - key_type* d_unique_output; - count_type* d_counts_output; - count_type* d_runs_count_output; - HIP_CHECK(hipMalloc(reinterpret_cast(&d_unique_output), runs_count * sizeof(key_type))); - HIP_CHECK( - hipMalloc(reinterpret_cast(&d_counts_output), runs_count * sizeof(count_type))); - HIP_CHECK(hipMalloc(reinterpret_cast(&d_runs_count_output), sizeof(count_type))); - - void* d_temporary_storage = nullptr; - size_t temporary_storage_bytes = 0; - - HIP_CHECK(rp::run_length_encode(nullptr, - temporary_storage_bytes, - d_input, - size, - d_unique_output, - d_counts_output, - d_runs_count_output, - stream, - false)); - - HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < 10; i++) - { - HIP_CHECK(rp::run_length_encode(d_temporary_storage, - temporary_storage_bytes, - d_input, - size, - d_unique_output, - d_counts_output, - d_runs_count_output, - stream, - false)); - } - HIP_CHECK(hipDeviceSynchronize()); - - // HIP events creation - hipEvent_t start, stop; - HIP_CHECK(hipEventCreate(&start)); - HIP_CHECK(hipEventCreate(&stop)); - - const unsigned int batch_size = 10; - for(auto _ : state) - { - // Record start event - HIP_CHECK(hipEventRecord(start, stream)); - - for(size_t i = 0; i < batch_size; i++) - { - HIP_CHECK(rp::run_length_encode(d_temporary_storage, - temporary_storage_bytes, - d_input, - size, - d_unique_output, - d_counts_output, - d_runs_count_output, - stream, - false)); - } - - // Record stop event and wait until it completes - HIP_CHECK(hipEventRecord(stop, stream)); - HIP_CHECK(hipEventSynchronize(stop)); - - float elapsed_mseconds; - HIP_CHECK(hipEventElapsedTime(&elapsed_mseconds, start, stop)); - state.SetIterationTime(elapsed_mseconds / 1000); +#define CREATE_ENCODE_BENCHMARK(T, ML) \ + { \ + const device_run_length_encode_benchmark instance; \ + REGISTER_BENCHMARK(benchmarks, size, seed, stream, instance); \ } - // Destroy HIP events - HIP_CHECK(hipEventDestroy(start)); - HIP_CHECK(hipEventDestroy(stop)); - - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(key_type)); - state.SetItemsProcessed(state.iterations() * batch_size * size); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_unique_output)); - HIP_CHECK(hipFree(d_counts_output)); - HIP_CHECK(hipFree(d_runs_count_output)); -} - -#define CREATE_ENCODE_BENCHMARK(T) \ - benchmark::RegisterBenchmark( \ - bench_naming::format_name( \ - "{lvl:device,algo:run_length_encode,subalgo:trivial,key_type:" #T ",keys_max_length:" \ - + std::to_string(MaxLength) + ",cfg:default_config}") \ - .c_str(), \ - run_encode_benchmark, \ - MaxLength, \ - size, \ - seed, \ - stream) - template void add_encode_benchmarks(std::vector& benchmarks, size_t size, @@ -189,49 +58,17 @@ void add_encode_benchmarks(std::vector& benchma using custom_float2 = custom_type; using custom_double2 = custom_type; - std::vector bs = { - // all tuned types - CREATE_ENCODE_BENCHMARK(int8_t), - CREATE_ENCODE_BENCHMARK(int16_t), - CREATE_ENCODE_BENCHMARK(int32_t), - CREATE_ENCODE_BENCHMARK(int64_t), - CREATE_ENCODE_BENCHMARK(rocprim::half), - CREATE_ENCODE_BENCHMARK(float), - CREATE_ENCODE_BENCHMARK(double), - // custom types - CREATE_ENCODE_BENCHMARK(custom_float2), - CREATE_ENCODE_BENCHMARK(custom_double2), - }; - - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); -} - -// CHANGE -#define CREATE_NON_TRIVIAL_RUNS_BENCHMARK(T, ML) \ - { \ - const device_non_trivial_runs_benchmark instance; \ - REGISTER_BENCHMARK(benchmarks, size, seed, stream, instance); \ - } - -template -void add_non_trivial_runs_benchmarks(std::vector& benchmarks, - size_t size, - const managed_seed& seed, - hipStream_t stream) -{ - using custom_float2 = custom_type; - using custom_double2 = custom_type; - - CREATE_NON_TRIVIAL_RUNS_BENCHMARK(int8_t, MaxLength); - CREATE_NON_TRIVIAL_RUNS_BENCHMARK(int16_t, MaxLength); - CREATE_NON_TRIVIAL_RUNS_BENCHMARK(int32_t, MaxLength); - CREATE_NON_TRIVIAL_RUNS_BENCHMARK(int64_t, MaxLength); - CREATE_NON_TRIVIAL_RUNS_BENCHMARK(rocprim::half, MaxLength); - CREATE_NON_TRIVIAL_RUNS_BENCHMARK(float, MaxLength); - CREATE_NON_TRIVIAL_RUNS_BENCHMARK(double, MaxLength); - - CREATE_NON_TRIVIAL_RUNS_BENCHMARK(custom_float2, MaxLength); - CREATE_NON_TRIVIAL_RUNS_BENCHMARK(custom_double2, MaxLength); + // all tuned types + CREATE_ENCODE_BENCHMARK(int8_t, MaxLength); + CREATE_ENCODE_BENCHMARK(int16_t, MaxLength); + CREATE_ENCODE_BENCHMARK(int32_t, MaxLength); + CREATE_ENCODE_BENCHMARK(int64_t, MaxLength); + CREATE_ENCODE_BENCHMARK(rocprim::half, MaxLength); + CREATE_ENCODE_BENCHMARK(float, MaxLength); + CREATE_ENCODE_BENCHMARK(double, MaxLength); + // custom types + CREATE_ENCODE_BENCHMARK(custom_float2, MaxLength); + CREATE_ENCODE_BENCHMARK(custom_double2, MaxLength); } int main(int argc, char* argv[]) @@ -289,10 +126,6 @@ int main(int argc, char* argv[]) #else add_encode_benchmarks<1000>(benchmarks, size, seed, stream); add_encode_benchmarks<10>(benchmarks, size, seed, stream); - add_non_trivial_runs_benchmarks<16>(benchmarks, size, seed, stream); - add_non_trivial_runs_benchmarks<256>(benchmarks, size, seed, stream); - add_non_trivial_runs_benchmarks<4096>(benchmarks, size, seed, stream); - #endif // Use manual timing diff --git a/benchmark/benchmark_device_run_length_encode.parallel.cpp.in b/benchmark/benchmark_device_run_length_encode.parallel.cpp.in index a09a4f261..b1588898b 100644 --- a/benchmark/benchmark_device_run_length_encode.parallel.cpp.in +++ b/benchmark/benchmark_device_run_length_encode.parallel.cpp.in @@ -27,8 +27,8 @@ namespace { auto benchmarks = config_autotune_register::create_bulk( - device_non_trivial_runs_benchmark_generator< + device_run_length_encode_benchmark_generator< @KeyType@, @BlockSize@, - rocprim::block_load_method::@BlockLoadMethod@>::create); + @TilesPerBlock@>::create); } // namespace diff --git a/benchmark/benchmark_device_run_length_encode.parallel.hpp b/benchmark/benchmark_device_run_length_encode.parallel.hpp index ee7817e76..d02003400 100644 --- a/benchmark/benchmark_device_run_length_encode.parallel.hpp +++ b/benchmark/benchmark_device_run_length_encode.parallel.hpp @@ -36,7 +36,6 @@ #include #include -#include #include #include #include @@ -45,115 +44,104 @@ #endif template -std::string non_trivial_runs_config_name() +std::string run_length_encode_config_name() { - const rocprim::detail::non_trivial_runs_config_params config = Config(); + const rocprim::detail::reduce_by_key_config_params config = Config(); return "{bs:" + std::to_string(config.kernel_config.block_size) + ",ipt:" + std::to_string(config.kernel_config.items_per_thread) - + ",load_method:" + get_block_load_method_name(config.load_input_method) + "}"; + + ",tpb:" + std::to_string(config.tiles_per_block) + "}"; } template<> -inline std::string non_trivial_runs_config_name() +inline std::string run_length_encode_config_name() { return "default_config"; } -template -struct device_non_trivial_runs_benchmark : public config_autotune_interface +template +struct device_run_length_encode_benchmark : public config_autotune_interface { std::string name() const override { - using namespace std::string_literals; - return bench_naming::format_name( - "{lvl:device,algo:run_length_encode,subalgo:non_trivial,key_type:" - + std::string(Traits::name()) + ",keys_max_length:" + std::to_string(MaxLength) - + ",cfg:" + non_trivial_runs_config_name() + "}"); + return bench_naming::format_name("{lvl:device,algo:run_length_encode,key_type:" + + std::string(Traits::name()) + + ",keys_max_length:" + std::to_string(MaxLength) + + ",cfg:" + run_length_encode_config_name() + "}"); } - void run(benchmark::State& state, size_t bytes, const managed_seed& seed, hipStream_t stream) const override { - using offset_type = unsigned int; - using count_type = unsigned int; - - constexpr int batch_size = 10; - constexpr int warmup_size = 5; - constexpr std::array tuning_max_segment_lengths = {10, 1000}; - constexpr int num_input_arrays = is_tuning ? tuning_max_segment_lengths.size() : 1; - - constexpr size_t item_size = sizeof(T) + sizeof(offset_type) + sizeof(count_type); + using key_type = T; + using count_type = unsigned int; - const size_t size = bytes / item_size; + const size_t size = bytes / sizeof(T); // Generate data - std::vector input[num_input_arrays]; - if(is_tuning) + std::vector input(size); + + unsigned int runs_count = 0; + const auto random_range = limit_random_range(1, MaxLength); + std::vector key_counts = get_random_data(100000, + random_range.first, + random_range.second, + seed.get_0()); + size_t offset = 0; + while(offset < size) { - for(size_t i = 0; i < tuning_max_segment_lengths.size(); ++i) + const size_t key_count = key_counts[runs_count % key_counts.size()]; + const size_t end = std::min(size, offset + key_count); + for(size_t i = offset; i < end; ++i) { - input[i] = get_random_segments_iota(size, - tuning_max_segment_lengths[i], - seed.get_0()); + input[i] = runs_count; } - } - else - { - input[0] = get_random_segments_iota(size, MaxLength, seed.get_0()); - } - T* d_input[num_input_arrays]; - for(int i = 0; i < num_input_arrays; ++i) - { - HIP_CHECK(hipMalloc(&d_input[i], size * sizeof(*d_input[i]))); - HIP_CHECK(hipMemcpy(d_input[i], - input[i].data(), - size * sizeof(*d_input[i]), - hipMemcpyHostToDevice)); + ++runs_count; + offset += key_count; } - offset_type* d_offsets_output; - HIP_CHECK(hipMalloc(&d_offsets_output, size * sizeof(*d_offsets_output))); + key_type* d_input; + HIP_CHECK(hipMalloc(reinterpret_cast(&d_input), size * sizeof(key_type))); + HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(key_type), hipMemcpyHostToDevice)); + + key_type* d_unique_output; count_type* d_counts_output; - HIP_CHECK(hipMalloc(&d_counts_output, size * sizeof(*d_counts_output))); count_type* d_runs_count_output; - HIP_CHECK(hipMalloc(&d_runs_count_output, sizeof(*d_runs_count_output))); + HIP_CHECK( + hipMalloc(reinterpret_cast(&d_unique_output), runs_count * sizeof(key_type))); + HIP_CHECK( + hipMalloc(reinterpret_cast(&d_counts_output), runs_count * sizeof(count_type))); + HIP_CHECK(hipMalloc(reinterpret_cast(&d_runs_count_output), sizeof(count_type))); - const auto dispatch = [&](void* d_temporary_storage, size_t& temporary_storage_bytes) - { - const auto dispatch_input = [&](T* d_input) - { - HIP_CHECK( - rocprim::run_length_encode_non_trivial_runs(d_temporary_storage, - temporary_storage_bytes, - d_input, - size, - d_offsets_output, - d_counts_output, - d_runs_count_output, - stream, - false)); - }; - - for(int i = 0; i < num_input_arrays; ++i) - { - dispatch_input(d_input[i]); - } - }; - - // Allocate temporary storage memory + void* d_temporary_storage = nullptr; size_t temporary_storage_bytes = 0; - dispatch(nullptr, temporary_storage_bytes); - void* d_temporary_storage; + + HIP_CHECK(rocprim::run_length_encode(nullptr, + temporary_storage_bytes, + d_input, + size, + d_unique_output, + d_counts_output, + d_runs_count_output, + stream, + false)); + HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); // Warm-up - for(int i = 0; i < warmup_size; ++i) + for(size_t i = 0; i < 10; ++i) { - dispatch(d_temporary_storage, temporary_storage_bytes); + HIP_CHECK(rocprim::run_length_encode(d_temporary_storage, + temporary_storage_bytes, + d_input, + size, + d_unique_output, + d_counts_output, + d_runs_count_output, + stream, + false)); } HIP_CHECK(hipDeviceSynchronize()); @@ -162,25 +150,30 @@ struct device_non_trivial_runs_benchmark : public config_autotune_interface HIP_CHECK(hipEventCreate(&start)); HIP_CHECK(hipEventCreate(&stop)); + const unsigned int batch_size = 10; for(auto _ : state) { // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(int i = 0; i < batch_size; ++i) + for(size_t i = 0; i < batch_size; ++i) { - dispatch(d_temporary_storage, temporary_storage_bytes); + HIP_CHECK(rocprim::run_length_encode(d_temporary_storage, + temporary_storage_bytes, + d_input, + size, + d_unique_output, + d_counts_output, + d_runs_count_output, + stream, + false)); } - HIP_CHECK(hipStreamSynchronize(stream)); // Record stop event and wait until it completes HIP_CHECK(hipEventRecord(stop, stream)); HIP_CHECK(hipEventSynchronize(stop)); - HIP_CHECK(hipGetLastError()); - HIP_CHECK(hipDeviceSynchronize()); - - float elapsed_mseconds{}; + float elapsed_mseconds; HIP_CHECK(hipEventElapsedTime(&elapsed_mseconds, start, stop)); state.SetIterationTime(elapsed_mseconds / 1000); } @@ -189,68 +182,44 @@ struct device_non_trivial_runs_benchmark : public config_autotune_interface HIP_CHECK(hipEventDestroy(start)); HIP_CHECK(hipEventDestroy(stop)); - state.SetBytesProcessed(state.iterations() * batch_size * size * item_size); + state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(key_type)); state.SetItemsProcessed(state.iterations() * batch_size * size); HIP_CHECK(hipFree(d_temporary_storage)); - for(int i = 0; i < num_input_arrays; ++i) - { - HIP_CHECK(hipFree(d_input[i])); - } - HIP_CHECK(hipFree(d_offsets_output)); + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_unique_output)); HIP_CHECK(hipFree(d_counts_output)); HIP_CHECK(hipFree(d_runs_count_output)); } - static constexpr bool is_tuning = !std::is_same::value; }; #ifdef BENCHMARK_CONFIG_TUNING -template -struct device_non_trivial_runs_benchmark_generator +template +struct device_run_length_encode_benchmark_generator { - using OffsetCountPairT = ::rocprim::tuple; - - static constexpr unsigned int max_shared_memory = TUNING_SHARED_MEMORY_MAX; - static constexpr unsigned int max_size_per_element - = std::max(sizeof(T), sizeof(OffsetCountPairT)); - static constexpr unsigned int max_items_per_thread - = max_shared_memory / (BlockSize * max_size_per_element); - static constexpr unsigned int max_items_per_thread_exponent - = rocprim::Log2::VALUE - 1; - static constexpr unsigned int min_items_per_thread_exponent = 3u; - - static constexpr bool is_load_warp_transpose - = BlockLoadMethod == ::rocprim::block_load_method::block_load_warp_transpose; - static constexpr bool is_warp_load_supp - = is_load_warp_transpose && BlockSize == ROCPRIM_WARP_SIZE_64; - - template + template struct create_ipt { void operator()(std::vector>& storage) { - if(!is_load_warp_transpose || is_warp_load_supp) - { - using config = rocprim::non_trivial_runs_config< - BlockSize, - items_per_thread, - BlockLoadMethod, - rocprim::block_scan_algorithm::using_warp_scan>; - storage.emplace_back( - std::make_unique>()); - } + using config + = rocprim::reduce_by_key_config; + storage.emplace_back( + std::make_unique>()); + storage.emplace_back( + std::make_unique>()); } - - private: - static constexpr unsigned int items_per_thread = 1u << ItemsPerThreadExp; }; static void create(std::vector>& storage) { - static_for_each< - make_index_range, - create_ipt>(storage); + static_for_each, create_ipt>(storage); } }; diff --git a/benchmark/benchmark_device_run_length_encode_non_trivial_runs.cpp b/benchmark/benchmark_device_run_length_encode_non_trivial_runs.cpp new file mode 100644 index 000000000..56cb78777 --- /dev/null +++ b/benchmark/benchmark_device_run_length_encode_non_trivial_runs.cpp @@ -0,0 +1,151 @@ +// MIT License +// +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "benchmark_device_run_length_encode_non_trivial_runs.parallel.hpp" +#include "benchmark_utils.hpp" + +// CmdParser +#include "cmdparser.hpp" + +// Google Benchmark +#include + +// HIP API +#include + +// rocPRIM +#include + +#include +#include +#include + +#ifndef DEFAULT_BYTES +constexpr size_t DEFAULT_BYTES = size_t{2} << 30; // 2 GiB +#endif + +// CHANGE +#define CREATE_NON_TRIVIAL_RUNS_BENCHMARK(T, ML) \ + { \ + const device_non_trivial_runs_benchmark instance; \ + REGISTER_BENCHMARK(benchmarks, size, seed, stream, instance); \ + } + +template +void add_non_trivial_runs_benchmarks(std::vector& benchmarks, + size_t size, + const managed_seed& seed, + hipStream_t stream) +{ + using custom_float2 = custom_type; + using custom_double2 = custom_type; + + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(int8_t, MaxLength); + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(int16_t, MaxLength); + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(int32_t, MaxLength); + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(int64_t, MaxLength); + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(rocprim::half, MaxLength); + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(float, MaxLength); + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(double, MaxLength); + + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(custom_float2, MaxLength); + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(custom_double2, MaxLength); +} + +int main(int argc, char* argv[]) +{ + cli::Parser parser(argc, argv); + parser.set_optional("size", "size", DEFAULT_BYTES, "number of bytes"); + parser.set_optional("trials", "trials", -1, "number of iterations"); + parser.set_optional("name_format", + "name_format", + "human", + "either: json,human,txt"); + parser.set_optional("seed", "seed", "random", get_seed_message()); +#ifdef BENCHMARK_CONFIG_TUNING + // optionally run an evenly split subset of benchmarks, when making multiple program invocations + parser.set_optional("parallel_instance", + "parallel_instance", + 0, + "parallel instance index"); + parser.set_optional("parallel_instances", + "parallel_instances", + 1, + "total parallel instances"); +#endif + + parser.run_and_exit_if_error(); + + // Parse argv + benchmark::Initialize(&argc, argv); + const size_t size = parser.get("size"); + const int trials = parser.get("trials"); + bench_naming::set_format(parser.get("name_format")); + const std::string seed_type = parser.get("seed"); + const managed_seed seed(seed_type); + + // HIP + hipStream_t stream = 0; // default + + // Benchmark info + add_common_benchmark_info(); + benchmark::AddCustomContext("size", std::to_string(size)); + benchmark::AddCustomContext("seed", seed_type); + + std::vector benchmarks; + + // Add benchmarks +#ifdef BENCHMARK_CONFIG_TUNING + const int parallel_instance = parser.get("parallel_instance"); + const int parallel_instances = parser.get("parallel_instances"); + config_autotune_register::register_benchmark_subset(benchmarks, + parallel_instance, + parallel_instances, + size, + seed, + stream); +#else + add_non_trivial_runs_benchmarks<16>(benchmarks, size, seed, stream); + add_non_trivial_runs_benchmarks<256>(benchmarks, size, seed, stream); + add_non_trivial_runs_benchmarks<4096>(benchmarks, size, seed, stream); +#endif + + // Use manual timing + for(auto& b : benchmarks) + { + b->UseManualTime(); + b->Unit(benchmark::kMillisecond); + } + + // Force number of iterations + if(trials > 0) + { + for(auto& b : benchmarks) + { + b->Iterations(trials); + } + } + + // Run benchmarks + benchmark::RunSpecifiedBenchmarks(); + return 0; +} diff --git a/benchmark/benchmark_device_run_length_encode_non_trivial_runs.parallel.cpp.in b/benchmark/benchmark_device_run_length_encode_non_trivial_runs.parallel.cpp.in new file mode 100644 index 000000000..55981f60d --- /dev/null +++ b/benchmark/benchmark_device_run_length_encode_non_trivial_runs.parallel.cpp.in @@ -0,0 +1,34 @@ +// MIT License +// +// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include + +#include "benchmark_device_run_length_encode_non_trivial_runs.parallel.hpp" +#include "benchmark_utils.hpp" + +namespace { + auto benchmarks = config_autotune_register::create_bulk( + device_non_trivial_runs_benchmark_generator< + @KeyType@, + @BlockSize@, + rocprim::block_load_method::@BlockLoadMethod@>::create); +} // namespace diff --git a/benchmark/benchmark_device_run_length_encode_non_trivial_runs.parallel.hpp b/benchmark/benchmark_device_run_length_encode_non_trivial_runs.parallel.hpp new file mode 100644 index 000000000..b057d7b6f --- /dev/null +++ b/benchmark/benchmark_device_run_length_encode_non_trivial_runs.parallel.hpp @@ -0,0 +1,258 @@ +// MIT License +// +// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#ifndef ROCPRIM_BENCHMARK_DEVICE_RUN_LENGTH_ENCODE_NON_TRIVIAL_RUNS_PARALLEL_HPP_ +#define ROCPRIM_BENCHMARK_DEVICE_RUN_LENGTH_ENCODE_NON_TRIVIAL_RUNS_PARALLEL_HPP_ + +#include "benchmark_utils.hpp" + +// Google Benchmark +#include + +// HIP API +#include + +// rocPRIM +#include +#include + +#include +#include +#include +#include +#include +#ifdef BENCHMARK_CONFIG_TUNING + #include +#endif + +template +std::string non_trivial_runs_config_name() +{ + const rocprim::detail::non_trivial_runs_config_params config = Config(); + return "{bs:" + std::to_string(config.kernel_config.block_size) + + ",ipt:" + std::to_string(config.kernel_config.items_per_thread) + + ",load_method:" + get_block_load_method_name(config.load_input_method) + "}"; +} + +template<> +inline std::string non_trivial_runs_config_name() +{ + return "default_config"; +} + +template +struct device_non_trivial_runs_benchmark : public config_autotune_interface +{ + std::string name() const override + { + return bench_naming::format_name( + "{lvl:device,algo:run_length_encode,subalgo:non_trivial,key_type:" + + std::string(Traits::name()) + ",keys_max_length:" + std::to_string(MaxLength) + + ",cfg:" + non_trivial_runs_config_name() + "}"); + } + + void run(benchmark::State& state, + size_t bytes, + const managed_seed& seed, + hipStream_t stream) const override + { + using offset_type = unsigned int; + using count_type = unsigned int; + + constexpr int batch_size = 10; + constexpr int warmup_size = 5; + constexpr std::array tuning_max_segment_lengths = {10, 1000}; + constexpr int num_input_arrays = is_tuning ? tuning_max_segment_lengths.size() : 1; + + constexpr size_t item_size = sizeof(T) + sizeof(offset_type) + sizeof(count_type); + + const size_t size = bytes / item_size; + + // Generate data + std::vector input[num_input_arrays]; + if(is_tuning) + { + for(size_t i = 0; i < tuning_max_segment_lengths.size(); ++i) + { + input[i] = get_random_segments_iota(size, + tuning_max_segment_lengths[i], + seed.get_0()); + } + } + else + { + input[0] = get_random_segments_iota(size, MaxLength, seed.get_0()); + } + + T* d_input[num_input_arrays]; + for(int i = 0; i < num_input_arrays; ++i) + { + HIP_CHECK(hipMalloc(&d_input[i], size * sizeof(*d_input[i]))); + HIP_CHECK(hipMemcpy(d_input[i], + input[i].data(), + size * sizeof(*d_input[i]), + hipMemcpyHostToDevice)); + } + + offset_type* d_offsets_output; + HIP_CHECK(hipMalloc(&d_offsets_output, size * sizeof(*d_offsets_output))); + count_type* d_counts_output; + HIP_CHECK(hipMalloc(&d_counts_output, size * sizeof(*d_counts_output))); + count_type* d_runs_count_output; + HIP_CHECK(hipMalloc(&d_runs_count_output, sizeof(*d_runs_count_output))); + + const auto dispatch = [&](void* d_temporary_storage, size_t& temporary_storage_bytes) + { + const auto dispatch_input = [&](T* d_input) + { + HIP_CHECK( + rocprim::run_length_encode_non_trivial_runs(d_temporary_storage, + temporary_storage_bytes, + d_input, + size, + d_offsets_output, + d_counts_output, + d_runs_count_output, + stream, + false)); + }; + + for(int i = 0; i < num_input_arrays; ++i) + { + dispatch_input(d_input[i]); + } + }; + + // Allocate temporary storage memory + size_t temporary_storage_bytes = 0; + dispatch(nullptr, temporary_storage_bytes); + void* d_temporary_storage; + HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); + HIP_CHECK(hipDeviceSynchronize()); + + // Warm-up + for(int i = 0; i < warmup_size; ++i) + { + dispatch(d_temporary_storage, temporary_storage_bytes); + } + HIP_CHECK(hipDeviceSynchronize()); + + // HIP events creation + hipEvent_t start, stop; + HIP_CHECK(hipEventCreate(&start)); + HIP_CHECK(hipEventCreate(&stop)); + + for(auto _ : state) + { + // Record start event + HIP_CHECK(hipEventRecord(start, stream)); + + for(int i = 0; i < batch_size; ++i) + { + dispatch(d_temporary_storage, temporary_storage_bytes); + } + HIP_CHECK(hipStreamSynchronize(stream)); + + // Record stop event and wait until it completes + HIP_CHECK(hipEventRecord(stop, stream)); + HIP_CHECK(hipEventSynchronize(stop)); + + HIP_CHECK(hipGetLastError()); + HIP_CHECK(hipDeviceSynchronize()); + + float elapsed_mseconds{}; + HIP_CHECK(hipEventElapsedTime(&elapsed_mseconds, start, stop)); + state.SetIterationTime(elapsed_mseconds / 1000); + } + + // Destroy HIP events + HIP_CHECK(hipEventDestroy(start)); + HIP_CHECK(hipEventDestroy(stop)); + + state.SetBytesProcessed(state.iterations() * batch_size * size * item_size); + state.SetItemsProcessed(state.iterations() * batch_size * size); + + HIP_CHECK(hipFree(d_temporary_storage)); + for(int i = 0; i < num_input_arrays; ++i) + { + HIP_CHECK(hipFree(d_input[i])); + } + HIP_CHECK(hipFree(d_offsets_output)); + HIP_CHECK(hipFree(d_counts_output)); + HIP_CHECK(hipFree(d_runs_count_output)); + } + static constexpr bool is_tuning = !std::is_same::value; +}; + +#ifdef BENCHMARK_CONFIG_TUNING + +template +struct device_non_trivial_runs_benchmark_generator +{ + using OffsetCountPairT = ::rocprim::tuple; + + static constexpr unsigned int max_shared_memory = TUNING_SHARED_MEMORY_MAX; + static constexpr unsigned int max_size_per_element + = std::max(sizeof(T), sizeof(OffsetCountPairT)); + static constexpr unsigned int max_items_per_thread + = max_shared_memory / (BlockSize * max_size_per_element); + static constexpr unsigned int max_items_per_thread_exponent + = rocprim::Log2::VALUE - 1; + static constexpr unsigned int min_items_per_thread_exponent = 3u; + + static constexpr bool is_load_warp_transpose + = BlockLoadMethod == ::rocprim::block_load_method::block_load_warp_transpose; + static constexpr bool is_warp_load_supp + = is_load_warp_transpose && BlockSize == ROCPRIM_WARP_SIZE_64; + + template + struct create_ipt + { + void operator()(std::vector>& storage) + { + if(!is_load_warp_transpose || is_warp_load_supp) + { + using config = rocprim::non_trivial_runs_config< + BlockSize, + items_per_thread, + BlockLoadMethod, + rocprim::block_scan_algorithm::using_warp_scan>; + storage.emplace_back( + std::make_unique>()); + } + } + + private: + static constexpr unsigned int items_per_thread = 1u << ItemsPerThreadExp; + }; + + static void create(std::vector>& storage) + { + static_for_each< + make_index_range, + create_ipt>(storage); + } +}; + +#endif // BENCHMARK_CONFIG_TUNING + +#endif // ROCPRIM_BENCHMARK_DEVICE_RUN_LENGTH_ENCODE_NON_TRIVIAL_RUNS_PARALLEL_HPP_ diff --git a/docs/concepts/tuning.rst b/docs/concepts/tuning.rst index 393d44969..673214146 100644 --- a/docs/concepts/tuning.rst +++ b/docs/concepts/tuning.rst @@ -35,4 +35,4 @@ Due to the modularity of rocPRIM, some device-level algorithms depend on other d * ``segmented_radix_sort`` depends on ``partition`` and ``partition_three_way`` but does not use the tuned configurations. * ``segmented_reduce`` does not depend on ``reduce`` but uses the same tuned configurations. * ``segmented_scan`` does not depend on ``scan`` but uses the same tuned configurations. -* ``run_length_encode`` depends on ``reduce_by_key``, but does not use its tuned configurations. ``run_length_encode_non_trivial_runs`` depends on ``reduce_by_key`` and ``select``, but does not use the tuned configurations from ``reduce_by_key`` and ``select``. +* ``run_length_encode`` depends on ``reduce_by_key``, but uses separate tuned configurations. ``run_length_encode_non_trivial_runs`` depends on ``reduce_by_key`` and ``select``, but does not use the tuned configurations from ``reduce_by_key`` and ``select``. diff --git a/rocprim/include/rocprim/device/detail/config/device_run_length_encode.hpp b/rocprim/include/rocprim/device/detail/config/device_run_length_encode.hpp new file mode 100644 index 000000000..d41b7acef --- /dev/null +++ b/rocprim/include/rocprim/device/detail/config/device_run_length_encode.hpp @@ -0,0 +1,725 @@ +// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RUN_LENGTH_ENCODE_HPP_ +#define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RUN_LENGTH_ENCODE_HPP_ + +#include "../../../type_traits.hpp" +#include "../device_config_helper.hpp" + +#include + +/* DO NOT EDIT THIS FILE + * This file is automatically generated by `/scripts/autotune/create_optimization.py`. + * so most likely you want to edit rocprim/device/device_(algo)_config.hpp + */ + +/// \addtogroup primitivesmodule_deviceconfigs +/// @{ + +BEGIN_ROCPRIM_NAMESPACE + +namespace detail +{ + +template +struct default_trivial_runs_config : default_reduce_by_key_config_base::type +{}; + +// Based on key_type = double +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_by_key_config<256, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = float +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_by_key_config<512, + 14, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = rocprim::half +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 2))>> + : reduce_by_key_config<384, + 14, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int64_t +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_by_key_config<256, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_by_key_config<512, + 14, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = short +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1))>> + : reduce_by_key_config<384, + 14, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int8_t +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 1))>> + : reduce_by_key_config<384, + 13, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = double +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 2> +{}; + +// Based on key_type = float +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = rocprim::half +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 2))>> + : reduce_by_key_config<512, + 14, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int64_t +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 2> +{}; + +// Based on key_type = int +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = short +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1))>> + : reduce_by_key_config<512, + 14, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int8_t +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 1))>> + : reduce_by_key_config<384, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = double +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_by_key_config<256, + 10, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = float +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_by_key_config<256, + 14, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = rocprim::half +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 2))>> + : reduce_by_key_config<256, + 14, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int64_t +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_by_key_config<256, + 10, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_by_key_config<256, + 14, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = short +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1))>> + : reduce_by_key_config<256, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int8_t +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 1))>> + : reduce_by_key_config<192, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = double +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_by_key_config<256, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = float +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_by_key_config<256, + 14, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = rocprim::half +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 2))>> + : reduce_by_key_config<512, + 10, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int64_t +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_by_key_config<256, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_by_key_config<256, + 14, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = short +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1))>> + : reduce_by_key_config<512, + 10, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int8_t +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 1))>> + : reduce_by_key_config<192, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = double +template +struct default_trivial_runs_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_by_key_config<256, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = float +template +struct default_trivial_runs_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_by_key_config<256, + 14, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = rocprim::half +template +struct default_trivial_runs_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 2))>> + : reduce_by_key_config<512, + 10, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int64_t +template +struct default_trivial_runs_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_by_key_config<256, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int +template +struct default_trivial_runs_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_by_key_config<256, + 14, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = short +template +struct default_trivial_runs_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1))>> + : reduce_by_key_config<512, + 10, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int8_t +template +struct default_trivial_runs_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 1))>> + : reduce_by_key_config<192, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = double +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_by_key_config<256, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = float +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_by_key_config<256, + 14, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = rocprim::half +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 2))>> + : reduce_by_key_config<512, + 10, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int64_t +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_by_key_config<256, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_by_key_config<256, + 14, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = short +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1))>> + : reduce_by_key_config<512, + 10, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int8_t +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 1))>> + : reduce_by_key_config<192, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +} // end namespace detail + +END_ROCPRIM_NAMESPACE + +/// @} +// end of group primitivesmodule_deviceconfigs + +#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RUN_LENGTH_ENCODE_HPP_ diff --git a/rocprim/include/rocprim/device/detail/config/device_run_length_encode_non_trivial.hpp b/rocprim/include/rocprim/device/detail/config/device_run_length_encode_non_trivial.hpp index 8345dd2cd..78ed92ca0 100644 --- a/rocprim/include/rocprim/device/detail/config/device_run_length_encode_non_trivial.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_run_length_encode_non_trivial.hpp @@ -18,8 +18,8 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -#ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RUN_LENGTH_ENCODE_HPP_ -#define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RUN_LENGTH_ENCODE_HPP_ +#ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RUN_LENGTH_ENCODE_NON_TRIVIAL_RUNS_HPP_ +#define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RUN_LENGTH_ENCODE_NON_TRIVIAL_RUNS_HPP_ #include "../../../type_traits.hpp" #include "../device_config_helper.hpp" @@ -596,4 +596,4 @@ END_ROCPRIM_NAMESPACE /// @} // end of group primitivesmodule_deviceconfigs -#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RUN_LENGTH_ENCODE_HPP_ \ No newline at end of file +#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RUN_LENGTH_ENCODE_NON_TRIVIAL_RUNS_HPP_ diff --git a/rocprim/include/rocprim/device/device_reduce_by_key.hpp b/rocprim/include/rocprim/device/device_reduce_by_key.hpp index 74cdde683..a5caa68fe 100644 --- a/rocprim/include/rocprim/device/device_reduce_by_key.hpp +++ b/rocprim/include/rocprim/device/device_reduce_by_key.hpp @@ -30,8 +30,8 @@ #include "detail/device_scan_common.hpp" #include "detail/lookback_scan_state.hpp" -#include "../config.hpp" #include "../common.hpp" +#include "../config.hpp" #include "../detail/temp_storage.hpp" #include "../detail/various.hpp" #include "../functional.hpp" @@ -104,23 +104,22 @@ template -ROCPRIM_KERNEL - __launch_bounds__(device_params().kernel_config.block_size) void reduce_by_key_kernel( - const KeyIterator keys_input, - const ValueIterator values_input, - const UniqueIterator unique_keys, - const ReductionIterator reductions, - const UniqueCountIterator unique_count, - const BinaryOp reduce_op, - const CompareFunction compare, - const LookbackScanState scan_state, - const ordered_block_id ordered_tile_id, - const std::size_t starting_tile, - const std::size_t total_number_of_tiles, - const std::size_t size, - const std::size_t* const global_head_count, - const AccumulatorType* const previous_accumulated, - const std::size_t number_of_tiles_launch) +ROCPRIM_KERNEL __launch_bounds__(device_params().kernel_config.block_size) +void reduce_by_key_kernel(const KeyIterator keys_input, + const ValueIterator values_input, + const UniqueIterator unique_keys, + const ReductionIterator reductions, + const UniqueCountIterator unique_count, + const BinaryOp reduce_op, + const CompareFunction compare, + const LookbackScanState scan_state, + const ordered_block_id ordered_tile_id, + const std::size_t starting_tile, + const std::size_t total_number_of_tiles, + const std::size_t size, + const std::size_t* const global_head_count, + const AccumulatorType* const previous_accumulated, + const std::size_t number_of_tiles_launch) { reduce_by_key::kernel_impl(keys_input, values_input, @@ -140,32 +139,28 @@ ROCPRIM_KERNEL } template -hipError_t reduce_by_key_impl(void* temporary_storage, - size_t& storage_size, - KeysInputIterator keys_input, - ValuesInputIterator values_input, - const size_t size, - UniqueOutputIterator unique_output, - AggregatesOutputIterator aggregates_output, - UniqueCountOutputIterator unique_count_output, - BinaryFunction reduce_op, - KeyCompareFunction key_compare_op, - const hipStream_t stream, - const bool debug_synchronous) + typename config, + typename KeysInputIterator, + typename ValuesInputIterator, + typename UniqueOutputIterator, + typename AggregatesOutputIterator, + typename UniqueCountOutputIterator, + typename BinaryFunction, + typename KeyCompareFunction> +hipError_t reduce_by_key_impl_wrapped_config(void* temporary_storage, + size_t& storage_size, + KeysInputIterator keys_input, + ValuesInputIterator values_input, + const size_t size, + UniqueOutputIterator unique_output, + AggregatesOutputIterator aggregates_output, + UniqueCountOutputIterator unique_count_output, + BinaryFunction reduce_op, + KeyCompareFunction key_compare_op, + const hipStream_t stream, + const bool debug_synchronous) { - using key_type = ::rocprim::detail::value_type_t; using accumulator_type = reduce_by_key::accumulator_type_t; - - using config = wrapped_reduce_by_key_config; - detail::target_arch target_arch; hipError_t result = host_target_arch(stream, target_arch); if(result != hipSuccess) @@ -293,7 +288,7 @@ hipError_t reduce_by_key_impl(void* temporary_storage, std::cout << "items_per_tile: " << items_per_tile << '\n'; } - for(size_t i = 0, offset = 0; i < number_of_launch; i++, offset += limited_size) + for(size_t i = 0, offset = 0; i < number_of_launch; ++i, offset += limited_size) { const std::size_t current_size = std::min(size - offset, limited_size); const std::size_t number_of_tiles_launch = ceiling_div(current_size, items_per_tile); @@ -359,7 +354,46 @@ hipError_t reduce_by_key_impl(void* temporary_storage, return hipSuccess; } +template +hipError_t reduce_by_key_impl(void* temporary_storage, + size_t& storage_size, + KeysInputIterator keys_input, + ValuesInputIterator values_input, + const size_t size, + UniqueOutputIterator unique_output, + AggregatesOutputIterator aggregates_output, + UniqueCountOutputIterator unique_count_output, + BinaryFunction reduce_op, + KeyCompareFunction key_compare_op, + const hipStream_t stream, + const bool debug_synchronous) +{ + using key_type = ::rocprim::detail::value_type_t; + using accumulator_type = reduce_by_key::accumulator_type_t; + + using config = wrapped_reduce_by_key_config; + return detail::reduce_by_key_impl_wrapped_config(temporary_storage, + storage_size, + keys_input, + values_input, + size, + unique_output, + aggregates_output, + unique_count_output, + reduce_op, + key_compare_op, + stream, + debug_synchronous); +} } // namespace detail @@ -470,15 +504,15 @@ hipError_t reduce_by_key_impl(void* temporary_storage, /// // unique_count_output: [4] /// \endcode /// \endparblock -template::value_type>, - class KeyCompareFunction + typename KeyCompareFunction = ::rocprim::equal_to::value_type>> inline hipError_t reduce_by_key(void* temporary_storage, size_t& storage_size, @@ -515,15 +549,15 @@ inline hipError_t reduce_by_key(void* temporary_storage, /// non-associative scan operators like floating point arithmetic operations. /// Refer to the documentation for \link reduce_by_key() rocprim::reduce_by_key \endlink /// for a detailed description of this function. -template::value_type>, - class KeyCompareFunction + typename KeyCompareFunction = ::rocprim::equal_to::value_type>> inline hipError_t deterministic_reduce_by_key(void* temporary_storage, size_t& storage_size, diff --git a/rocprim/include/rocprim/device/device_run_length_encode.hpp b/rocprim/include/rocprim/device/device_run_length_encode.hpp index 1f53d1d7e..09b535639 100644 --- a/rocprim/include/rocprim/device/device_run_length_encode.hpp +++ b/rocprim/include/rocprim/device/device_run_length_encode.hpp @@ -27,8 +27,8 @@ #include "detail/device_config_helper.hpp" #include "detail/device_run_length_encode.hpp" -#include "../config.hpp" #include "../common.hpp" +#include "../config.hpp" #include "../detail/various.hpp" #include "../iterator/constant_iterator.hpp" #include "../type_traits.hpp" @@ -51,6 +51,49 @@ namespace detail namespace run_length_encode { +template +hipError_t run_length_encode_impl(void* temporary_storage, + size_t& storage_size, + KeysInputIterator keys_input, + ValuesInputIterator values_input, + const size_t size, + UniqueOutputIterator unique_output, + AggregatesOutputIterator aggregates_output, + UniqueCountOutputIterator unique_count_output, + BinaryFunction reduce_op, + KeyCompareFunction key_compare_op, + const hipStream_t stream, + const bool debug_synchronous) +{ + using key_type = ::rocprim::detail::value_type_t; + using accumulator_type = reduce_by_key::accumulator_type_t; + + using config = wrapped_trivial_runs_config; + + return detail::reduce_by_key_impl_wrapped_config< + detail::lookback_scan_determinism::default_determinism, + config>(temporary_storage, + storage_size, + keys_input, + values_input, + size, + unique_output, + aggregates_output, + unique_count_output, + reduce_op, + key_compare_op, + stream, + debug_synchronous); +} + template -ROCPRIM_KERNEL - __launch_bounds__(device_params().kernel_config.block_size) +ROCPRIM_KERNEL __launch_bounds__(device_params().kernel_config.block_size) void non_trivial_kernel(const InputIterator input, const OffsetsOutputIterator offsets_output, const CountsOutputIterator counts_output, @@ -80,11 +122,11 @@ void non_trivial_kernel(const InputIterator input, size); } -template +template hipError_t run_length_encode_non_trivial_runs_impl(void* temporary_storage, size_t& storage_size, InputIterator input, @@ -312,11 +354,11 @@ hipError_t run_length_encode_non_trivial_runs_impl(void* tempo /// // runs_count_output: [4] /// \endcode /// \endparblock -template +template inline hipError_t run_length_encode(void* temporary_storage, size_t& storage_size, InputIterator input, @@ -330,27 +372,20 @@ inline hipError_t run_length_encode(void* temporary_storage, using input_type = typename std::iterator_traits::value_type; using count_type = unsigned int; - // run_length_encode does not use a tuned reduce by key config, as using a constant iterator - // instead of a device array has different performance characteristics - using config = detail::default_or_custom_config< - Config, - run_length_encode_config< - typename detail::default_reduce_by_key_config_base::type, - default_config>>; - - return ::rocprim::reduce_by_key( - temporary_storage, - storage_size, - input, - make_constant_iterator(1), - size, - unique_output, - counts_output, - runs_count_output, - ::rocprim::plus(), - ::rocprim::equal_to(), - stream, - debug_synchronous); + return detail::run_length_encode:: + run_length_encode_impl( + temporary_storage, + storage_size, + input, + make_constant_iterator(1), + size, + unique_output, + counts_output, + runs_count_output, + ::rocprim::plus(), + ::rocprim::equal_to(), + stream, + debug_synchronous); } /// \brief Parallel run-length encoding of non-trivial runs for device level. @@ -433,11 +468,11 @@ inline hipError_t run_length_encode(void* temporary_storage, /// // runs_count_output: [2] /// \endcode /// \endparblock -template +template inline hipError_t run_length_encode_non_trivial_runs(void* temporary_storage, size_t& storage_size, InputIterator input, diff --git a/rocprim/include/rocprim/device/device_run_length_encode_config.hpp b/rocprim/include/rocprim/device/device_run_length_encode_config.hpp index 99a798682..53b7ee8ff 100644 --- a/rocprim/include/rocprim/device/device_run_length_encode_config.hpp +++ b/rocprim/include/rocprim/device/device_run_length_encode_config.hpp @@ -22,11 +22,12 @@ #define ROCPRIM_DEVICE_DEVICE_RUN_LENGTH_ENCODE_CONFIG_HPP_ #include "config_types.hpp" -#include "detail/device_config_helper.hpp" +#include "device_reduce_by_key_config.hpp" #include "../config.hpp" #include "../detail/various.hpp" #include "../type_traits.hpp" +#include "detail/config/device_run_length_encode.hpp" #include "detail/config/device_run_length_encode_non_trivial.hpp" #include @@ -42,10 +43,7 @@ BEGIN_ROCPRIM_NAMESPACE /// Must be \p reduce_by_key_config or \p default_config. /// \tparam SelectConfig - configuration of device-level select operation. /// Must be \p select_config or \p default_config. -template< - class ReduceByKeyConfig, - class SelectConfig = default_config -> +template struct run_length_encode_config { /// \brief Configuration of device-level reduce-by-key operation. @@ -57,6 +55,58 @@ struct run_length_encode_config namespace detail { +template +struct wrapped_trivial_runs_config + : wrapped_reduce_by_key_config +{}; + +template +struct wrapped_trivial_runs_config< + rocprim::run_length_encode_config, + KeyType, + AccumulatorType, + BinaryFunction> + : wrapped_reduce_by_key_config +{}; + +template +struct wrapped_trivial_runs_impl + : wrapped_reduce_by_key_impl +{}; + +template +struct wrapped_trivial_runs_impl< + KeyType, + AccumulatorType, + BinaryFunction, + std::enable_if_t::value && is_arithmetic::value + && is_binary_functional::value>> +{ + template + struct architecture_config + { + static constexpr reduce_by_key_config_params params + = default_trivial_runs_config(Arch), + KeyType, + AccumulatorType>{}; + }; +}; + +template +struct wrapped_trivial_runs_config + : wrapped_trivial_runs_impl +{}; + // Wrap around run_length_encode_config and the newly added non_trivial_runs_config for the // run_length_encode_non_trivial_runs algorithm. Three cases are considered for selecting // the appropriate config: @@ -84,7 +134,7 @@ struct wrapped_non_trivial_runs_config }; }; -template +template struct wrapped_non_trivial_runs_config< rocprim::run_length_encode_config, InputType> @@ -144,13 +194,23 @@ struct wrapped_non_trivial_runs_config #ifndef DOXYGEN_DOCUMENTATION_BUILD +template +template +constexpr reduce_by_key_config_params wrapped_trivial_runs_impl< + KeyType, + AccumulatorType, + BinaryFunction, + std::enable_if_t::value && is_arithmetic::value + && is_binary_functional::value>>::architecture_config:: + params; + template template constexpr non_trivial_runs_config_params wrapped_non_trivial_runs_config::architecture_config::params; -template +template template constexpr non_trivial_runs_config_params wrapped_non_trivial_runs_config< rocprim::run_length_encode_config, diff --git a/scripts/autotune/create_optimization.py b/scripts/autotune/create_optimization.py index 018ce843e..f29e00f5a 100755 --- a/scripts/autotune/create_optimization.py +++ b/scripts/autotune/create_optimization.py @@ -647,21 +647,27 @@ class AlgorithmDeviceFindFirstOf(Algorithm): def __init__(self, fallback_entries): Algorithm.__init__(self, fallback_entries) -class AlgorithmDeviceMerge(Algorithm): - algorithm_name = "device_merge" - cpp_configuration_template_name = "merge_config_template" - config_selection_params = [ - SelectionType(name="key_type", is_optional=False, select_on_size_only=False), - SelectionType(name="value_type", is_optional=True, select_on_size_only=True)] +class AlgorithmDeviceRunLengthEncode(Algorithm): + algorithm_name = 'device_run_length_encode' + cpp_configuration_template_name = 'run_length_encode_config_template' + config_selection_params = [SelectionType(name='key_type', is_optional=False, select_on_size_only=False)] + def __init__(self, fallback_entries): + Algorithm.__init__(self, fallback_entries) class AlgorithmDeviceRunLengthEncodeNonTrivial(Algorithm): algorithm_name = 'device_run_length_encode_non_trivial' - cpp_configuration_template_name = 'run_length_encode_config_template' + cpp_configuration_template_name = 'run_length_encode_non_trivial_runs_config_template' config_selection_params = [SelectionType(name='key_type', is_optional=False, select_on_size_only=False)] - def __init__(self, fallback_entries): Algorithm.__init__(self, fallback_entries) +class AlgorithmDeviceMerge(Algorithm): + algorithm_name = "device_merge" + cpp_configuration_template_name = "merge_config_template" + config_selection_params = [ + SelectionType(name="key_type", is_optional=False, select_on_size_only=False), + SelectionType(name="value_type", is_optional=True, select_on_size_only=True)] + def filt_algo_regex(e: FallbackCase, algorithm_name): if e.algo_regex: return re.match(e.algo_regex, algorithm_name) is not None @@ -725,10 +731,12 @@ def create_algorithm(algorithm_name: str, fallback_entries: List[FallbackCase]): return AlgorithmDeviceReduceByKey(fallback_entries) elif algorithm_name == 'device_find_first_of': return AlgorithmDeviceFindFirstOf(fallback_entries) - elif algorithm_name == 'device_merge': - return AlgorithmDeviceMerge(fallback_entries) + elif algorithm_name == 'device_run_length_encode': + return AlgorithmDeviceRunLengthEncode(fallback_entries) elif algorithm_name == 'device_run_length_encode_non_trivial': return AlgorithmDeviceRunLengthEncodeNonTrivial(fallback_entries) + elif algorithm_name == 'device_merge': + return AlgorithmDeviceMerge(fallback_entries) else: raise(NotSupportedError(f'Algorithm "{algorithm_name}" is not supported (yet)')) diff --git a/scripts/autotune/templates/run_length_encode_config_template b/scripts/autotune/templates/run_length_encode_config_template index 5c2824099..1800f9909 100644 --- a/scripts/autotune/templates/run_length_encode_config_template +++ b/scripts/autotune/templates/run_length_encode_config_template @@ -5,24 +5,27 @@ ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RUN_LENGTH_ENCODE_HPP_ {%- endmacro %} {% macro kernel_configuration(measurement) -%} -non_trivial_runs_config< - {{ measurement['cfg']['bs'] }}, - {{ measurement['cfg']['ipt'] }}, - ::rocprim::{{ measurement['cfg']['load_method'] }}, - ::rocprim::block_scan_algorithm::using_warp_scan> { }; +reduce_by_key_config< + {{ measurement['cfg']['bs'] }}, + {{ measurement['cfg']['ipt'] }}, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + {{ measurement['cfg']['tpb'] }}> { }; {%- endmacro %} {% macro general_case() -%} -template -struct default_non_trivial_runs_config : default_non_trivial_runs_config_base::type +template +struct default_trivial_runs_config : default_reduce_by_key_config_base::type {}; {%- endmacro %} {% macro configuration_fallback(benchmark_of_architecture, based_on_type, fallback_selection_criteria) -%} // Based on {{ based_on_type }} -template -struct default_non_trivial_runs_config< - static_cast({{ benchmark_of_architecture.name }}), - key_type, +template +struct default_trivial_runs_config< + static_cast({{ benchmark_of_architecture.name }}), + key_type, + value_type, {{ fallback_selection_criteria }}> : -{%- endmacro %} \ No newline at end of file +{%- endmacro %} diff --git a/scripts/autotune/templates/run_length_encode_non_trivial_runs_config_template b/scripts/autotune/templates/run_length_encode_non_trivial_runs_config_template new file mode 100644 index 000000000..49ce27a78 --- /dev/null +++ b/scripts/autotune/templates/run_length_encode_non_trivial_runs_config_template @@ -0,0 +1,28 @@ +{% extends "config_template" %} + +{% macro get_header_guard() %} +ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RUN_LENGTH_ENCODE_NON_TRIVIAL_RUNS_HPP_ +{%- endmacro %} + +{% macro kernel_configuration(measurement) -%} +non_trivial_runs_config< + {{ measurement['cfg']['bs'] }}, + {{ measurement['cfg']['ipt'] }}, + ::rocprim::{{ measurement['cfg']['load_method'] }}, + ::rocprim::block_scan_algorithm::using_warp_scan> { }; +{%- endmacro %} + +{% macro general_case() -%} +template +struct default_non_trivial_runs_config : default_non_trivial_runs_config_base::type +{}; +{%- endmacro %} + +{% macro configuration_fallback(benchmark_of_architecture, based_on_type, fallback_selection_criteria) -%} +// Based on {{ based_on_type }} +template +struct default_non_trivial_runs_config< + static_cast({{ benchmark_of_architecture.name }}), + key_type, + {{ fallback_selection_criteria }}> : +{%- endmacro %} From d3aa223185efd1a292ba4a6a06cf425a0c0a27b2 Mon Sep 17 00:00:00 2001 From: Cenxuan Tian Date: Thu, 21 Nov 2024 12:59:11 +0100 Subject: [PATCH 06/84] Resolve "Implement type traits interface for rocPRIM's compiler-extended types and third-party's types" --- CHANGELOG.md | 1 + docs/concepts/concepts.rst | 1 + docs/concepts/developer.rst | 2 +- docs/concepts/type_traits.rst | 51 ++ docs/index.rst | 1 + docs/reference/reference.rst | 1 + docs/sphinx/_toc.yml.in | 1 + example/CMakeLists.txt | 1 + example/example_type_traits_interface.cpp | 125 +++ .../device/detail/device_radix_sort.hpp | 10 +- .../rocprim/thread/radix_key_codec.hpp | 6 +- rocprim/include/rocprim/type_traits.hpp | 130 +-- .../include/rocprim/type_traits_interface.hpp | 777 ++++++++++++++++++ test/rocprim/CMakeLists.txt | 1 + test/rocprim/test_device_radix_sort.cpp.in | 1 + test/rocprim/test_device_radix_sort.hpp | 4 +- test/rocprim/test_type_traits_interface.cpp | 332 ++++++++ .../test_utils_custom_float_traits_type.hpp | 129 +++ test/rocprim/test_utils_sort_comparator.hpp | 16 +- 19 files changed, 1451 insertions(+), 139 deletions(-) create mode 100644 docs/concepts/type_traits.rst create mode 100644 example/example_type_traits_interface.cpp create mode 100644 rocprim/include/rocprim/type_traits_interface.hpp create mode 100644 test/rocprim/test_type_traits_interface.cpp create mode 100644 test/rocprim/test_utils_custom_float_traits_type.hpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 28b2f5872..cfff549e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ Full documentation for rocPRIM is available at [https://rocm.docs.amd.com/projec * Added hipGraph support to device run-length-encode for non trivial runs (`rocprim::run_length_encode_non_trivial_runs`). * Added configuration autotuning to device run-length-encode for non trivial runs (`rocprim::run_length_encode_non_trivial_runs`) for improved performance on selected architectures. * Added configuration autotuning to device run-length-encode for trivial runs (`rocprim::run_length_encode`) for improved performance on selected architectures. +* Added a new type traits interface to enable users to provide additional type trait information to rocPRIM, facilitating better compatibility with custom types. ### Changed diff --git a/docs/concepts/concepts.rst b/docs/concepts/concepts.rst index b0508744c..e5e5bcc6b 100644 --- a/docs/concepts/concepts.rst +++ b/docs/concepts/concepts.rst @@ -12,3 +12,4 @@ * :ref:`glossary` * :ref:`developer` * :ref:`tuning` +* :ref:`type_traits` diff --git a/docs/concepts/developer.rst b/docs/concepts/developer.rst index 780f842cd..20a842b75 100644 --- a/docs/concepts/developer.rst +++ b/docs/concepts/developer.rst @@ -116,7 +116,7 @@ Selecting the default configuration is done based on the target architecture ``t struct default_ALGO_config< static_cast(target_arch::gfx1030), Type, - std::enable_if_t::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2)>> + std::enable_if_t().is_floating_point()) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2)>> : ALGO_config<256, 16> {}; diff --git a/docs/concepts/type_traits.rst b/docs/concepts/type_traits.rst new file mode 100644 index 000000000..836c8761a --- /dev/null +++ b/docs/concepts/type_traits.rst @@ -0,0 +1,51 @@ +.. meta:: + :description: rocPRIM documentation and API reference library + :keywords: rocPRIM, ROCm, API, documentation + +.. _type_traits: + +******************************************************************** + Implement traits for custom types +******************************************************************** + +Overview +======== + +This interface is designed to enable users to provide additional type trait information to rocPRIM, facilitating better compatibility with custom types. + +Custom types that implement arithmetic operators can behave like built-in arithmetic types; however, algorithms in rocPRIM may still interpret them as generic `struct` or `class` types. For some algorithms, processing methods differ between floating-point and integral types. Accurately describing custom types, therefore, is essential not only for performance optimization but also for ensuring correctness. The rocPRIM type traits interface allows users to supply custom trait information for their types, greatly enhancing compatibility between these types and rocPRIM algorithms. + +This interface is strict but flexible, similar to operator overloading. Users should only implement traits as required by specific algorithms, and some traits cannot be defined if they can be inferred from others. Adhering to these rules is crucial for successful compilation. Fortunately, the interface provides clear error messages to guide users when issues arise. + + +Interface +========= + +.. doxygengroup:: type_traits_interfaces + :content-only: + :members: + + +Available traits +================ + +.. doxygengroup:: available_traits + :content-only: + :members: + +Type traits wrappers +==================== +.. doxygengroup:: rocprim_type_traits_wrapper + :content-only: + :no-link: + +Types with predefined traits +============================ + +.. doxygengroup:: rocprim_pre_defined_traits + :content-only: + :members: + :outline: + :no-link: + + \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 51e1f50fd..7d1015c2c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -33,6 +33,7 @@ The rocPRIM documentation is structured as follows: * :ref:`thread-index` * :ref:`iterators` * :ref:`intrinsics` + * :ref:`type_traits` To contribute to the documentation refer to `Contributing to ROCm `_. diff --git a/docs/reference/reference.rst b/docs/reference/reference.rst index 803797a97..1aee5deb6 100644 --- a/docs/reference/reference.rst +++ b/docs/reference/reference.rst @@ -16,4 +16,5 @@ * :ref:`thread-index` * :ref:`iterators` * :ref:`intrinsics` +* :ref:`type_traits` * :ref:`types` diff --git a/docs/sphinx/_toc.yml.in b/docs/sphinx/_toc.yml.in index 584671f4d..7c52b9a55 100644 --- a/docs/sphinx/_toc.yml.in +++ b/docs/sphinx/_toc.yml.in @@ -80,6 +80,7 @@ subtrees: - file: thread_ops/thread_store.rst - file: reference/iterators.rst - file: reference/intrinsics.rst + - file: concepts/type_traits.rst - file: reference/types.rst - file: reference/acknowledge.rst - file: license.rst diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index b4020ae5f..7239a7df4 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -68,3 +68,4 @@ endfunction() # **************************************************************************** add_rocprim_example(example_temporary_storage.cpp) +add_rocprim_example(example_type_traits_interface.cpp) diff --git a/example/example_type_traits_interface.cpp b/example/example_type_traits_interface.cpp new file mode 100644 index 000000000..d07cbf9a1 --- /dev/null +++ b/example/example_type_traits_interface.cpp @@ -0,0 +1,125 @@ +// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +#include + +#include +#include +#include + +template().is_floating_point()>* = nullptr> +void some_algo() +{ + constexpr auto input_traits = rocprim::traits::get(); + + std::stringstream ss; + + ss << "root_type:\t[" << typeid(InputType).name() << "] this type is floating point\n"; + ss << "is_arithmetic:\t" << input_traits.is_arithmetic() << '\n'; + ss << "is_fundamental:\t" << input_traits.is_fundamental() << '\n'; + ss << "is_scalar:\t" << input_traits.is_scalar() << '\n'; + constexpr auto mask = input_traits.float_bit_mask(); + ss << "sign_bit:\t" << mask.sign_bit << '\n'; + ss << "exponent:\t" << mask.exponent << '\n'; + ss << "mantissa:\t" << mask.mantissa << '\n'; + std::cout << ss.str() << "\n"; +} + +template().is_integral()>* = nullptr> +void some_algo() +{ + constexpr auto input_traits = rocprim::traits::get(); + + std::stringstream ss; + ss << "root_type:\t[" << typeid(InputType).name() << "] this type is integral\n"; + ss << "is_arithmetic:\t" << input_traits.is_arithmetic() << '\n'; + ss << "is_fundamental:\t" << input_traits.is_fundamental() << '\n'; + ss << "is_scalar:\t" << input_traits.is_scalar() << '\n'; + ss << "is_signed:\t" << input_traits.is_signed() << '\n'; + ss << "is_unsigned:\t" << input_traits.is_unsigned() << '\n'; + + std::cout << ss.str() << "\n"; +} + +template().is_integral() + && !rocprim::traits::get().is_floating_point()>* = nullptr> +void some_algo() +{ + constexpr auto input_traits = rocprim::traits::get(); + std::stringstream ss; + ss << "root_type:\t[" << typeid(InputType).name() + << "] this type is neither integral nor floating point\n"; + ss << "is_arithmetic:\t" << input_traits.is_arithmetic() << '\n'; + ss << "is_fundamental:\t" << input_traits.is_fundamental() << '\n'; + ss << "is_signed:\t" << input_traits.is_signed() << '\n'; + ss << "is_unsigned:\t" << input_traits.is_unsigned() << '\n'; + ss << "is_scalar:\t" << input_traits.is_scalar() << '\n'; + + std::cout << ss.str() << "\n"; +} + +// Your type definition +struct custom_float_type +{}; + +// you should add this struct specialization for the type to implement the traits +template<> +struct rocprim::traits::define +{ + using is_arithmetic = rocprim::traits::is_arithmetic::values; + using number_format + = rocprim::traits::number_format::values; + using float_bit_mask = rocprim::traits::float_bit_mask::values; +}; + +// Your type definition +struct custom_int_type +{}; + +// you should add this struct specialization for the type to implement the traits +template<> +struct rocprim::traits::define +{ + using is_arithmetic = rocprim::traits::is_arithmetic::values; + using number_format + = rocprim::traits::number_format::values; + using integral_sign + = rocprim::traits::integral_sign::values; +}; + +int main() +{ + // C++ types + some_algo(); + some_algo(); + some_algo(); + + // rocprim type + some_algo(); + some_algo(); + some_algo(); + some_algo(); + + // other types + some_algo(); + some_algo(); +} diff --git a/rocprim/include/rocprim/device/detail/device_radix_sort.hpp b/rocprim/include/rocprim/device/detail/device_radix_sort.hpp index fe53de847..30650d196 100644 --- a/rocprim/include/rocprim/device/detail/device_radix_sort.hpp +++ b/rocprim/include/rocprim/device/detail/device_radix_sort.hpp @@ -373,9 +373,9 @@ struct radix_sort_and_scatter_helper class ValuesInputIterator, class ValuesOutputIterator> ROCPRIM_DEVICE ROCPRIM_INLINE - void sort_and_scatter(KeysInputIterator keys_input, - KeysOutputIterator keys_output, - ValuesInputIterator values_input, + void sort_and_scatter(KeysInputIterator keys_input, + KeysOutputIterator keys_output, + ValuesInputIterator values_input, ValuesOutputIterator values_output, Offset begin_offset, Offset end_offset, @@ -621,8 +621,8 @@ auto compare_nan_sensitive(const T& a, const T& b) // Always check benchmark_device_segmented_radix_sort and benchmark_device_radix_sort // when making changes to this function. - using bit_key_type = typename float_bit_mask::bit_type; - static constexpr auto sign_bit = float_bit_mask::sign_bit; + static constexpr auto sign_bit = ::rocprim::traits::get().float_bit_mask().sign_bit; + using bit_key_type = decltype(sign_bit); auto a_bits = ::rocprim::detail::bit_cast(a); auto b_bits = ::rocprim::detail::bit_cast(b); diff --git a/rocprim/include/rocprim/thread/radix_key_codec.hpp b/rocprim/include/rocprim/thread/radix_key_codec.hpp index a8b6026cb..81ebe0758 100644 --- a/rocprim/include/rocprim/thread/radix_key_codec.hpp +++ b/rocprim/include/rocprim/thread/radix_key_codec.hpp @@ -113,9 +113,11 @@ struct radix_key_codec_floating { using bit_key_type = BitKey; - static constexpr bit_key_type sign_bit = ::rocprim::detail::float_bit_mask::sign_bit; + static constexpr bit_key_type sign_bit + = ::rocprim::traits::get().float_bit_mask().sign_bit; - ROCPRIM_HOST_DEVICE ROCPRIM_INLINE static bit_key_type encode(Key key) + ROCPRIM_HOST_DEVICE ROCPRIM_INLINE + static bit_key_type encode(Key key) { bit_key_type bit_key = ::rocprim::detail::bit_cast(key); bit_key ^= (sign_bit & bit_key) == 0 ? sign_bit : bit_key_type(-1); diff --git a/rocprim/include/rocprim/type_traits.hpp b/rocprim/include/rocprim/type_traits.hpp index 623e7feef..2bed27199 100644 --- a/rocprim/include/rocprim/type_traits.hpp +++ b/rocprim/include/rocprim/type_traits.hpp @@ -23,12 +23,12 @@ #include "config.hpp" #include "functional.hpp" -#include "types.hpp" + +#include "type_traits_interface.hpp" #include "types/tuple.hpp" #include -#include #include /// \addtogroup utilsmodule_typetraits @@ -36,82 +36,6 @@ BEGIN_ROCPRIM_NAMESPACE -/// \brief Extension of `std::is_floating_point`, which includes support for \ref rocprim::half and \ref rocprim::bfloat16. -template -struct is_floating_point - : std::integral_constant< - bool, - std::is_floating_point::value || - std::is_same<::rocprim::half, typename std::remove_cv::type>::value || - std::is_same<::rocprim::bfloat16, typename std::remove_cv::type>::value - > {}; - -/// \brief Extension of `std::is_integral`, which includes support for 128-bit integers. -template -struct is_integral - : std::integral_constant< - bool, - std::is_integral::value - || std::is_same<::rocprim::int128_t, typename std::remove_cv::type>::value - || std::is_same<::rocprim::uint128_t, typename std::remove_cv::type>::value> -{}; - -/// \brief Extension of `std::is_arithmetic`, which includes support for \ref rocprim::half , \ref rocprim::bfloat16 and 128-bit integers. -template -struct is_arithmetic - : std::integral_constant< - bool, - std::is_arithmetic::value - || std::is_same<::rocprim::half, typename std::remove_cv::type>::value - || std::is_same<::rocprim::bfloat16, typename std::remove_cv::type>::value - || std::is_same<::rocprim::int128_t, typename std::remove_cv::type>::value - || std::is_same<::rocprim::uint128_t, typename std::remove_cv::type>::value> -{}; - -/// \brief Extension of `std::is_fundamental`, which includes support for \ref rocprim::half , \ref rocprim::bfloat16 and 128-bit integers. -template -struct is_fundamental - : std::integral_constant< - bool, - std::is_fundamental::value - || std::is_same<::rocprim::half, typename std::remove_cv::type>::value - || std::is_same<::rocprim::bfloat16, typename std::remove_cv::type>::value - || std::is_same<::rocprim::int128_t, typename std::remove_cv::type>::value - || std::is_same<::rocprim::uint128_t, typename std::remove_cv::type>::value> -{}; - -/// \brief Extension of `std::is_unsigned`, which includes support for 128-bit integers. -template -struct is_unsigned - : std::integral_constant< - bool, - std::is_unsigned::value - || std::is_same<::rocprim::uint128_t, typename std::remove_cv::type>::value> -{}; - -/// \brief Extension of `std::is_signed`, which includes support for \ref rocprim::half , \ref rocprim::bfloat16 and 128-bit integers. -template -struct is_signed - : std::integral_constant< - bool, - std::is_signed::value - || std::is_same<::rocprim::half, typename std::remove_cv::type>::value - || std::is_same<::rocprim::bfloat16, typename std::remove_cv::type>::value - || std::is_same<::rocprim::int128_t, typename std::remove_cv::type>::value> -{}; - -/// \brief Extension of `std::is_scalar`, which includes support for \ref rocprim::half , \ref rocprim::bfloat16 and 128-bit integers. -template -struct is_scalar - : std::integral_constant< - bool, - std::is_scalar::value - || std::is_same<::rocprim::half, typename std::remove_cv::type>::value - || std::is_same<::rocprim::bfloat16, typename std::remove_cv::type>::value - || std::is_same<::rocprim::int128_t, typename std::remove_cv::type>::value - || std::is_same<::rocprim::uint128_t, typename std::remove_cv::type>::value> -{}; - /// \brief Extension of `std::make_unsigned`, which includes support for 128-bit integers. template struct make_unsigned : std::make_unsigned @@ -134,14 +58,6 @@ struct make_unsigned<::rocprim::uint128_t> static_assert(std::is_same::type, ::rocprim::uint128_t>::value, "'rocprim::int128_t' needs to implement 'make_unsigned' trait."); -/// \brief Extension of `std::is_compound`, which includes support for \ref rocprim::half , \ref rocprim::bfloat16 and 128-bit integers. -template -struct is_compound - : std::integral_constant< - bool, - !is_fundamental::value - > {}; - /// \brief Extension of `std::numeric_limits`, which includes support for 128-bit integers. template struct numeric_limits : std::numeric_limits @@ -426,48 +342,6 @@ struct is_tuple_of_references<::rocprim::tuple> static constexpr bool value = is_tuple_of_references_impl<0>(); }; -template -struct float_bit_mask; - -template<> -struct float_bit_mask -{ - static constexpr uint32_t sign_bit = 0x80000000; - static constexpr uint32_t exponent = 0x7F800000; - static constexpr uint32_t mantissa = 0x007FFFFF; - using bit_type = uint32_t; -}; - -template<> -struct float_bit_mask -{ - static constexpr uint64_t sign_bit = 0x8000000000000000; - static constexpr uint64_t exponent = 0x7FF0000000000000; - static constexpr uint64_t mantissa = 0x000FFFFFFFFFFFFF; - using bit_type = uint64_t; -}; - -template<> -struct float_bit_mask -{ - static constexpr uint16_t sign_bit = 0x8000; - static constexpr uint16_t exponent = 0x7F80; - static constexpr uint16_t mantissa = 0x007F; - using bit_type = uint16_t; -}; - -template<> -struct float_bit_mask -{ - static constexpr uint16_t sign_bit = 0x8000; - static constexpr uint16_t exponent = 0x7C00; - static constexpr uint16_t mantissa = 0x03FF; - using bit_type = uint16_t; -}; - -template -using void_t = void; - template using value_type_t = typename std::iterator_traits::value_type; diff --git a/rocprim/include/rocprim/type_traits_interface.hpp b/rocprim/include/rocprim/type_traits_interface.hpp new file mode 100644 index 000000000..2efcc5372 --- /dev/null +++ b/rocprim/include/rocprim/type_traits_interface.hpp @@ -0,0 +1,777 @@ +// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef ROCPRIM_TYPE_TRAITS_INTERFACE_HPP_ +#define ROCPRIM_TYPE_TRAITS_INTERFACE_HPP_ + +#include "types.hpp" + +#include + +// common macros + +/// \brief A reverse version of static_assert aims to increase code readability +#ifndef ROCPRIM_DO_NOT_COMPILE_IF + #define ROCPRIM_DO_NOT_COMPILE_IF(condition, msg) static_assert(!(condition), msg) +#endif +#ifndef ROCPRIM_COMPILE_ERROR + #define ROCPRIM_COMPILE_ERROR(msg) static_assert(false, msg) +#endif +/// \brief Wrapper macro for std::enable_if aims to increase code readability +#ifndef ROCPRIM_REQUIRES + #define ROCPRIM_REQUIRES(...) typename std::enable_if<(__VA_ARGS__)>::type* = nullptr +#endif +#ifndef DOXYGEN_DOCUMENTATION_BUILD + /// \brief Since every definable traits need to use `is_defined`, this macro reduce the amount of code + #define ROCPRIM_TRAITS_GENERATE_IS_DEFINE(traits_name) \ + template \ + static constexpr bool is_defined = false; \ + template \ + static constexpr bool \ + is_defined::traits_name>> \ + = true +#endif + +BEGIN_ROCPRIM_NAMESPACE + +namespace detail +{ + +template +using void_t = void; + +template +struct [[deprecated]] float_bit_mask; + +} // namespace detail + +namespace traits +{ +/// \defgroup type_traits_interfaces Interfaces for defining and obtaining the traits +/// \addtogroup type_traits_interfaces +/// @{ + +/// \par Overview +/// This template struct provides an interface for downstream libraries to implement type traits for +/// their custom types. Users can utilize this template struct to define traits for these types. Users +/// should only implement traits as required by specific algorithms, and some traits cannot be defined +/// if they can be inferred from others. This API is not static because of ODR. +/// \tparam T The type for which you want to define traits. +/// +/// \par Example +/// \parblock +/// The example below demonstrates how to implement traits for a custom floating-point type. +/// \code{.cpp} +/// // Your type definition +/// struct custom_float_type +/// {}; +/// // Implement the traits +/// template<> +/// struct rocprim::traits::define +/// { +/// using is_arithmetic = rocprim::traits::is_arithmetic::values; +/// using number_format = rocprim::traits::number_format::values; +/// using float_bit_mask = rocprim::traits::float_bit_mask::values; +/// }; +/// \endcode +/// The example below demonstrates how to implement traits for a custom integral type. +/// \code{.cpp} +/// // Your type definition +/// struct custom_int_type +/// {}; +/// // Implement the traits +/// template<> +/// struct rocprim::traits::define +/// { +/// using is_arithmetic = rocprim::traits::is_arithmetic::values; +/// using number_format = rocprim::traits::number_format::values; +/// using integral_sign = rocprim::traits::integral_sign::values; +/// }; +/// \endcode +/// \endparblock +template +struct define +{}; + +/// @} + +/// predef +template +struct get; + +/// \defgroup available_traits Traits that can be used +/// \addtogroup available_traits +/// @{ + +/// \par Definability +/// * **Undefinable**: For types with `predefined traits`. +/// * **Optional**: For other types. +/// \par How to define +/// \parblock +/// \code{.cpp} +/// using is_arithmetic = rocprim::traits::is_arithmetic::values; +/// \endcode +/// \endparblock +/// \par How to use +/// \parblock +/// \code{.cpp} +/// rocprim::traits::get().is_arithmetic(); +/// \endcode +/// \endparblock +struct is_arithmetic +{ + /// \brief Value of this trait + template + struct values + { + /// \brief This indicates if the `InputType` is arithmetic. + static constexpr auto value = Val; + }; + +#ifndef DOXYGEN_DOCUMENTATION_BUILD + + ROCPRIM_TRAITS_GENERATE_IS_DEFINE(is_arithmetic); + + // For c++ arithmetic types, return true, but will throw compile error when user try to define this trait for them + template::value)> + static constexpr auto get() + { + ROCPRIM_DO_NOT_COMPILE_IF(is_defined, + "Do not define trait `is_arithmetic` for c++ arithmetic types"); + return values{}; + } + + // For third party types, if trait `is_arithmetic` not defined, will return default value `false` + template::value && !is_defined)> + static constexpr auto get() + { + return values{}; + } + + // For third party types, if trait `is_arithmetic` is defined, then should return its value + template::value && is_defined)> + static constexpr auto get() + { + return typename define::is_arithmetic{}; + } +#endif +}; + +/// \brief Arithmetic types, pointers, member pointers, and null pointers are considered scalar types. +/// \par Definability +/// * **Undefinable**: For types with `predefined traits`. +/// * **Optional**: For other types. If both `is_arithmetic` and `is_scalar` are defined, their values +/// must be consistent; otherwise, a compile-time error will occur. +/// \par How to define +/// \parblock +/// \code{.cpp} +/// using is_scalar = rocprim::traits::is_scalar::values; +/// \endcode +/// \endparblock +/// \par How to use +/// \parblock +/// \code{.cpp} +/// rocprim::traits::get().is_scalar(); +/// \endcode +/// \endparblock +struct is_scalar +{ + /// \brief Value of this trait + template + struct values + { + /// \brief This indicates if the `InputType` is scalar. + static constexpr auto value = Val; + }; + +#ifndef DOXYGEN_DOCUMENTATION_BUILD + + ROCPRIM_TRAITS_GENERATE_IS_DEFINE(is_scalar); + + // For c++ scalar types, return true, but will throw compile error when user try to define this trait for them + template::value)> + static constexpr auto get() + { + ROCPRIM_DO_NOT_COMPILE_IF(is_defined, + "Do not define trait `is_scalar` for c++ scalar types"); + return values{}; + } + + // For third party types, if trait `is_scalar` is not defined, will return default value `false` + // For rocprim or third party types that defined trait `is_arithmetic` as true the result should be `true` + template::value && !is_defined)> + static constexpr auto get() + { + return values().value>{}; + } + // For third party types and rocprim types, if trait `is_scalar` is defined, will return the value + // check if the `is_scalar` equals to `is_arithmetic`, or throw a compile error + template::value && is_defined)> + static constexpr auto get() + { + ROCPRIM_DO_NOT_COMPILE_IF( + is_arithmetic::get().value != typename define::is_scalar{}.value, + "Trait `is_arithmetic` and trait `is_scalar` should have the same value"); + return typename define::is_scalar{}; + } +#endif +}; + +/// \par Definability +/// * **Undefinable**: For types with `predefined traits` and non-arithmetic types. +/// * **Required**: If you define `is_arithmetic` as `true`, you must also define this trait; otherwise, a +/// compile-time error will occur. +/// \par How to define +/// \parblock +/// \code{.cpp} +/// using number_format = rocprim::traits::number_format::values; +/// \endcode +/// \endparblock +/// \par How to use +/// \parblock +/// \code{.cpp} +/// rocprim::traits::get().is_integral(); +/// rocprim::traits::get().is_floating_point(); +/// \endcode +/// \endparblock +struct number_format +{ + /// \brief The kind enum that indecates the values avaliable for this trait + enum class kind + { + unknown_type = 0, + floating_point_type = 1, + integral_type = 2 + }; + + /// \brief Value of this trait + template + struct values + { + /// \brief This indicates if the `InputType` is floating_point_type or integral_type or unknown_type. + static constexpr auto value = Val; + }; + +#ifndef DOXYGEN_DOCUMENTATION_BUILD + + ROCPRIM_TRAITS_GENERATE_IS_DEFINE(number_format); + + // For c++ arithmetic types + template::value)> + static constexpr auto get() + { // C++ build-in arithmetic types are either floating point or integral + return values < std::is_floating_point::value ? kind::floating_point_type + : kind::integral_type > {}; + } + + // For rocprim arithmetic types + template::value + && is_arithmetic::get().value)> + static constexpr auto get() + { + ROCPRIM_DO_NOT_COMPILE_IF(!is_defined, + "You must define trait `number_format` for arithmetic types"); + return typename define::number_format{}; + } + + // For other types + template::value + && !is_arithmetic::get().value)> + static constexpr auto get() + { + ROCPRIM_DO_NOT_COMPILE_IF( + is_defined, + "You cannot define trait `number_format` for non-arithmetic types"); + return values{}; + } +#endif +}; + +/// \par Definability +/// * **Undefinable**: For types with `predefined traits`, non-arithmetic types and floating-point types. +/// * **Required**: If you define `number_format` as `number_format::kind::floating_point_type`, you must also define this trait; otherwise, a +/// compile-time error will occur. +/// \par How to define +/// \parblock +/// \code{.cpp} +/// using integral_sign = rocprim::traits::integral_sign::values; +/// \endcode +/// \endparblock +/// \par How to use +/// \parblock +/// \code{.cpp} +/// rocprim::traits::get().is_signed(); +/// rocprim::traits::get().is_unsigned(); +/// \endcode +/// \endparblock +struct integral_sign +{ + /// \brief The kind enum that indecates the values avaliable for this trait + enum class kind + { + unknown_type = 0, + signed_type = 1, + unsigned_type = 2 + }; + + /// \brief Value of this trait + template + struct values + { + /// \brief This indicates if the `InputType` is signed_type or unsigned_type or unknown_type. + static constexpr auto value = Val; + }; + +#ifndef DOXYGEN_DOCUMENTATION_BUILD + + ROCPRIM_TRAITS_GENERATE_IS_DEFINE(integral_sign); + + // For c++ arithmetic types + template::value)> + static constexpr auto get() + { // cpp arithmetic types are either signed point or unsignned + return values < std::is_signed::value ? kind::signed_type + : kind::unsigned_type > {}; + } + + // For rocprim arithmetic integral + template::value && is_arithmetic::get().value + && number_format::get().value == number_format::kind::integral_type)> + static constexpr auto get() + { + ROCPRIM_DO_NOT_COMPILE_IF(!is_defined, + "Trait `integral_sign` is required for arithmetic " + "integral types, please define"); + return typename define::integral_sign{}; + } + + // For rocprim arithmetic non-integral + template::value && is_arithmetic::get().value + && number_format::get().value != number_format::kind::integral_type)> + static constexpr auto get() + { + ROCPRIM_DO_NOT_COMPILE_IF( + is_defined, + "You cannot define trait `integral_sign` for arithmetic non-integral types"); + return values{}; + } + + // For other types + template::value + && !is_arithmetic::get().value)> + static constexpr auto get() + { // For other types, trait is_floating_point is a must + ROCPRIM_DO_NOT_COMPILE_IF( + is_defined, + "You cannot define trait `integral_sign` for non-arithmetic types"); + return values{}; + } +#endif +}; + +/// \warning For some types, if this trait is not implemented in their traits definition, it will +/// link to `rocprim::detail::float_bit_mask` to maintain compatibility with downstream libraries. +/// However, this linkage will be removed in the next major release. Please ensure that these types +/// are updated to the latest interface. +/// \par Definability +/// * **Undefinable**: For types with `predefined traits`, non-arithmetic types and integral types. +/// * **Required**: If you define `number_format` as `number_format::kind::unknown_type`, you must also define this trait; otherwise, a +/// compile-time error will occur. +/// \par How to define +/// \parblock +/// \code{.cpp} +/// using float_bit_mask = rocprim::traits::float_bit_mask::values; +/// \endcode +/// \endparblock +/// \par How to use +/// \parblock +/// \code{.cpp} +/// rocprim::traits::get().float_bit_mask(); +/// \endcode +/// \endparblock +struct float_bit_mask +{ + /// \brief Value of this trait + template + struct values + { + ROCPRIM_DO_NOT_COMPILE_IF(number_format::get().value + != number_format::kind::integral_type, + "BitType should be integral"); + /// \brief Trait sign_bit for the `InputType`. + static constexpr BitType sign_bit = SignBit; + /// \brief Trait exponent for the `InputType`. + static constexpr BitType exponent = Exponent; + /// \brief Trait mantissa for the `InputType`. + static constexpr BitType mantissa = Mantissa; + }; + +#ifndef DOXYGEN_DOCUMENTATION_BUILD + + ROCPRIM_TRAITS_GENERATE_IS_DEFINE(float_bit_mask); + + template + static constexpr bool has_old_float_bit_mask = false; + template + static constexpr bool has_old_float_bit_mask< + InputType, + detail::void_t{})>> + = true; + + // If this trait is defined, then use the new interface + template)> + static constexpr auto get() + { + ROCPRIM_DO_NOT_COMPILE_IF( + number_format::get().value != number_format::kind::floating_point_type, + "You cannot use trait `float_bit_mask` for `non-floating_point` types"); + return typename define::float_bit_mask{}; + } + + // This function acts as a bridge for old interface. Will be removed in certain version + // "`rocprim::detail::float_bit_mask` will be deprecated on next main release," + // "`please use rocprim::trait::define` to define tratis for types." + template && has_old_float_bit_mask)> + static constexpr auto get() + { + using mask = typename ::rocprim::detail::float_bit_mask; + return values{}; + } + + // For types that don't have a trait `float_bit_mask` defined neither a rocprim::detail::float_bit_mask specialization + template && !has_old_float_bit_mask)> + static constexpr auto get() + { + ROCPRIM_DO_NOT_COMPILE_IF( + number_format::get().value != number_format::kind::floating_point_type, + "You cannot use trait `float_bit_mask` for `non-floating_point` types"); + ROCPRIM_DO_NOT_COMPILE_IF(number_format::get().value + == number_format::kind::floating_point_type, + "Trait `float_bit_mask` is required for `floating_point` types"); + return values{}; + } +#endif +}; + +/// \brief The trait `is_fundamental` is undefinable, as it is the union of `std::is_fundamental` +/// and `rocprim::traits::is_arithmetic`. +/// \par Definability +/// * **Undefinable**: If you attempt to define this trait in any form, a compile-time error will occur. +/// \par How to use +/// \parblock +/// \code{.cpp} +/// rocprim::traits::get().is_fundamental(); +/// rocprim::traits::get().is_compound(); +/// \endcode +/// \endparblock +struct is_fundamental +{ + + /// \brief Value of this trait + template + struct values + { + /// \brief This indicates if the `InputType` is fundamental. + static constexpr auto value = Val; + }; + +#ifndef DOXYGEN_DOCUMENTATION_BUILD + + ROCPRIM_TRAITS_GENERATE_IS_DEFINE(is_fundamental); + + // For all types + template + static constexpr auto get() + { + ROCPRIM_DO_NOT_COMPILE_IF(is_defined, "Trait `is_fundamental` is undefinable"); + return values < std::is_fundamental::value + || is_arithmetic::get().value > {}; + } +#endif +}; + +/// @} + +/// \addtogroup type_traits_interfaces +/// @{ + +/// \par Overview +/// This template struct is designed to allow rocPRIM algorithms to retrieve trait information from C++ +/// build-in arithmetic types, rocPRIM types, and custom types. This API is not static because of ODR. +/// * All member functions are `compiled only when invoked`. +/// * Different algorithms require different traits. +/// \tparam T The type from which you want to retrieve the traits. +/// \par Example +/// \parblock +/// The following code demonstrates how to retrieve the traits of type `T`. +/// \code{.cpp} +/// // Get the trait in a template parameter +/// template().is_integral()>::type* = nullptr> +/// void get_traits_in_template_parameter(){} +/// // Get the trait in a function body +/// template +/// void get_traits_in_function_body(){ +/// constexpr auto input_traits = rocprim::traits::get(); +/// // Then you can use the member functinos +/// constexpr bool is_arithmetic = input_traits.is_arithmetic(); +/// } +/// \endcode +/// \endparblock +template +struct get +{ + /// \brief Get the value of trait `is_arithmetic`. + /// \returns `true` if `std::is_arithmetic_v` is `true`, or if type `T` is a rocPRIM arithmetic + /// type, or if the `is_arithmetic` trait has been defined as `true`; otherwise, returns `false`. + constexpr bool is_arithmetic() const + { + return rocprim::traits::is_arithmetic{}.get().value; + }; + + /// \brief Get trait `is_fundamental`. + /// \returns `true` if `T` is a fundamental type (that is, rocPRIM arithmetic type, void, or nullptr_t); + /// otherwise, returns `false`. + constexpr bool is_fundamental() const + { + return rocprim::traits::is_fundamental{}.get().value; + }; + + /// \brief If `T` is fundamental type, then returns `false`. + /// \returns `false` if `T` is a fundamental type (that is, rocPRIM arithmetic type, void, or nullptr_t); + /// otherwise, returns `true`. + constexpr bool is_compound() const + { + return !rocprim::traits::is_fundamental{}.get().value; + } + + /// \brief To check if `T` is floating-point type. + /// \warning You cannot call this function when `is_arithmetic()` returns `false`; + /// doing so will result in a compile-time error. + constexpr bool is_floating_point() const + { + return rocprim::traits::number_format{}.get().value + == number_format::kind::floating_point_type; + }; + + /// \brief To check if `T` is integral type. + /// \warning You cannot call this function when `is_arithmetic()` returns `false`; + /// doing so will result in a compile-time error. + constexpr bool is_integral() const + { + return rocprim::traits::number_format{}.get().value + == number_format::kind::integral_type; + } + + /// \brief To check if `T` is signed integral type. + /// \warning You cannot call this function when `is_integral()` returns `false`; + /// doing so will result in a compile-time error. + constexpr bool is_signed() const + { + return rocprim::traits::integral_sign{}.get().value == integral_sign::kind::signed_type; + } + + /// \brief To check if `T` is unsigned integral type. + /// \warning You cannot call this function when `is_integral()` returns `false`; + /// doing so will result in a compile-time error. + constexpr bool is_unsigned() const + { + return rocprim::traits::integral_sign{}.get().value + == integral_sign::kind::unsigned_type; + } + + /// \brief Get trait `is_scalar`. + /// \returns `true` if `std::is_scalar_v` is `true`, or if type `T` is a rocPRIM arithmetic + /// type, or if the `is_scalar` trait has been defined as `true`; otherwise, returns `false`. + constexpr bool is_scalar() const + { + return rocprim::traits::is_scalar{}.get().value; + } + + /// \brief Get trait `float_bit_mask`. + /// \warning You cannot call this function when `is_floating_point()` returns `false`; + /// doing so will result in a compile-time error. + /// \returns A constexpr instance of the specialization of `rocprim::traits::float_bit_mask::values` + /// as provided in the traits definition of type T. If the `float_bit_mask trait` is not defined, it + /// returns the rocprim::detail::float_bit_mask values, provided a specialization of + /// `rocprim::detail::float_bit_mask` exists. + constexpr auto float_bit_mask() const + { + return rocprim::traits::float_bit_mask{}.get(); + }; +}; + +/// @} + +} // namespace traits + +/// \defgroup rocprim_pre_defined_traits Trait definitions for rocPRIM arithmetic types and additional traits for +/// C++ build-in arithmetic types. +/// \addtogroup rocprim_pre_defined_traits +/// @{ + +/// \brief This is the definition of traits of `float` +/// C++ build-in type +template<> +struct traits::define +{ + /// \brief Trait `float_bit_mask` for this type + using float_bit_mask + = traits::float_bit_mask::values; +}; + +/// \brief This is the definition of traits of `double` +/// C++ build-in type +template<> +struct traits::define +{ + /// \brief Trait `float_bit_mask` for this type + using float_bit_mask = traits::float_bit_mask:: + values; +}; + +/// \brief This is the definition of traits of `rocprim::bfloat16` +/// rocPRIM arithmetic type +template<> +struct traits::define +{ + /// \brief Trait `is_arithmetic` for this type + using is_arithmetic = traits::is_arithmetic::values; + /// \brief Trait `number_format` for this type + using number_format + = traits::number_format::values; + /// \brief Trait `float_bit_mask` for this type + using float_bit_mask = traits::float_bit_mask::values; +}; + +/// \brief This is the definition of traits of `rocprim::half` +/// rocPRIM arithmetic type +template<> +struct traits::define +{ + /// \brief Trait `is_arithmetic` for this type + using is_arithmetic = traits::is_arithmetic::values; + /// \brief Trait `number_format` for this type + using number_format + = traits::number_format::values; + /// \brief Trait `float_bit_mask` for this type + using float_bit_mask = traits::float_bit_mask::values; +}; + +/// \brief This is the definition of traits of `rocprim::int128_t` +/// rocPRIM arithmetic type +template<> +struct traits::define +{ + /// \brief Trait `is_arithmetic` for this type + using is_arithmetic = traits::is_arithmetic::values; + /// \brief Trait `number_format` for this type + using number_format = traits::number_format::values; + /// \brief Trait `integral_sign` for this type + using integral_sign = traits::integral_sign::values; +}; + +/// \brief This is the definition of traits of `rocprim::uint128_t` +/// rocPRIM arithmetic type +template<> +struct traits::define +{ + /// \brief Trait `is_arithmetic` for this type + using is_arithmetic = traits::is_arithmetic::values; + /// \brief Trait `number_format` for this type + using number_format = traits::number_format::values; + /// \brief Trait `integral_sign` for this type + using integral_sign = traits::integral_sign::values; +}; +/// @} + +/// \defgroup rocprim_type_traits_wrapper Handy wrappers for obtaining type traits +/// \addtogroup rocprim_type_traits_wrapper +/// @{ + +/// \brief An extension of `std::is_floating_point` that supports additional arithmetic types, +/// including `rocprim::half`, `rocprim::bfloat16`, and any types with trait +/// `rocprim::traits::number_format::values` implemented. +template +struct is_floating_point + : std::integral_constant().is_floating_point()> +{}; + +/// \brief An extension of `std::is_integral` that supports additional arithmetic types, +/// including `rocprim::int128_t`, `rocprim::uint128_t`, and any types with trait +/// `rocprim::traits::number_format::values` implemented. +template +struct is_integral : std::integral_constant().is_integral()> +{}; + +/// \brief An extension of `std::is_arithmetic` that supports additional arithmetic types, +/// including any types with trait `rocprim::traits::is_arithmetic::values` implemented. +template +struct is_arithmetic : std::integral_constant().is_arithmetic()> +{}; + +/// \brief An extension of `std::is_fundamental` that supports additional arithmetic types, +/// including any types with trait `rocprim::traits::is_arithmetic::values` implemented. +template +struct is_fundamental : std::integral_constant().is_fundamental()> +{}; + +/// \brief An extension of `std::is_unsigned` that supports additional arithmetic types, +/// including `rocprim::uint128_t`, and any types with trait +/// `rocprim::traits::integral_sign::values` implemented. +template +struct is_unsigned : std::integral_constant().is_unsigned()> +{}; + +/// \brief An extension of `std::is_signed` that supports additional arithmetic types, +/// including `rocprim::int128_t`, and any types with trait +/// `rocprim::traits::integral_sign::values` implemented. +template +struct is_signed : std::integral_constant().is_signed()> +{}; + +/// \brief An extension of `std::is_scalar` that supports additional arithmetic types, +/// including any types with trait `rocprim::traits::is_scalar::values` implemented. +template +struct is_scalar : std::integral_constant().is_scalar()> +{}; + +/// \brief An extension of `std::is_scalar` that supports additional non-arithmetic types. +template +struct is_compound : std::integral_constant().is_compound()> +{}; + +/// @} + +END_ROCPRIM_NAMESPACE + +#endif diff --git a/test/rocprim/CMakeLists.txt b/test/rocprim/CMakeLists.txt index 24e87ebeb..4d210c203 100644 --- a/test/rocprim/CMakeLists.txt +++ b/test/rocprim/CMakeLists.txt @@ -294,6 +294,7 @@ add_rocprim_test("rocprim.texture_cache_iterator" test_texture_cache_iterator.cp add_rocprim_test("rocprim.thread" test_thread.cpp) add_rocprim_test("rocprim.thread_algos" test_thread_algos.cpp) add_rocprim_test("rocprim.transform_iterator" test_transform_iterator.cpp) +add_rocprim_test("rocprim.type_traits_interface" test_type_traits_interface.cpp) add_rocprim_test("rocprim.no_half_operators" test_no_half_operators.cpp) add_rocprim_test("rocprim.intrinsics" test_intrinsics.cpp) add_rocprim_test("rocprim.invoke_result" test_invoke_result.cpp) diff --git a/test/rocprim/test_device_radix_sort.cpp.in b/test/rocprim/test_device_radix_sort.cpp.in index a49138ada..31addc9f4 100644 --- a/test/rocprim/test_device_radix_sort.cpp.in +++ b/test/rocprim/test_device_radix_sort.cpp.in @@ -72,6 +72,7 @@ INSTANTIATE(params) INSTANTIATE(params>) INSTANTIATE(params) + INSTANTIATE(params) INSTANTIATE(params>) // start_bit and end_bit diff --git a/test/rocprim/test_device_radix_sort.hpp b/test/rocprim/test_device_radix_sort.hpp index 2a109e58f..e41308b10 100644 --- a/test/rocprim/test_device_radix_sort.hpp +++ b/test/rocprim/test_device_radix_sort.hpp @@ -30,6 +30,7 @@ // required test headers #include "test_seed.hpp" +#include "test_utils_custom_float_traits_type.hpp" #include "test_utils_custom_float_type.hpp" #include "test_utils_custom_test_types.hpp" #include "test_utils_sort_comparator.hpp" @@ -96,7 +97,8 @@ auto generate_key_input(KeyIter keys_input, size_t size, engine_type& rng_engine template constexpr bool is_custom_not_float_test_type = test_utils::is_custom_test_type::value - && !std::is_same::value; + && !std::is_same::value + && !std::is_same::value; template auto invoke_sort_keys(void* d_temporary_storage, diff --git a/test/rocprim/test_type_traits_interface.cpp b/test/rocprim/test_type_traits_interface.cpp new file mode 100644 index 000000000..a3c5d550e --- /dev/null +++ b/test/rocprim/test_type_traits_interface.cpp @@ -0,0 +1,332 @@ +// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "../common_test_header.hpp" + +#include "test_utils_custom_test_types.hpp" + +#include + +#include +#include + +#include + +#define ROCPRIM_STATIC_ASSERT(cond) static_assert((cond), "Rocprim Traits Assertion failed!") +#define ROCPRIM_STATIC_ASSERT_TRUE(cond) ROCPRIM_STATIC_ASSERT((cond)) +#define ROCPRIM_STATIC_ASSERT_FALSE(cond) ROCPRIM_STATIC_ASSERT(!(cond)) +#define ROCPRIM_STATIC_ASSERT_EQ(val1, val2) ROCPRIM_STATIC_ASSERT((val1) == (val2)) +#define ROCPRIM_STATIC_ASSERT_NE(val1, val2) ROCPRIM_STATIC_ASSERT((val1) != (val2)) + +namespace type_traits_test +{ +// Custom type to model types like Eigen::half or Eigen::bfloat16, that wrap around floating point +// types. +struct custom_float_type +{ + float x; + + // Constructor for the data generation utilities, simply ignore the second number + ROCPRIM_HOST_DEVICE custom_float_type(float val, float /*ignored*/) : x{val} + {} + + ROCPRIM_HOST_DEVICE custom_float_type(float val) : x{val} {} + + custom_float_type() = default; + + ROCPRIM_HOST_DEVICE + custom_float_type + operator+(const custom_float_type& other) const + { + return custom_float_type(x + other.x); + } + + ROCPRIM_HOST_DEVICE + custom_float_type + operator-(const custom_float_type& other) const + { + return custom_float_type(x - other.x); + } + + ROCPRIM_HOST_DEVICE + bool operator<(const custom_float_type& other) const + { + return x < other.x; + } + + ROCPRIM_HOST_DEVICE + bool operator>(const custom_float_type& other) const + { + return x > other.x; + } + + ROCPRIM_HOST_DEVICE + bool operator==(const custom_float_type& other) const + { + return x == other.x; + } + + ROCPRIM_HOST_DEVICE + bool operator!=(const custom_float_type& other) const + { + return !(*this == other); + } +}; + +inline bool signbit(const custom_float_type& val) +{ + return std::signbit(val.x); +} + +inline std::ostream& operator<<(std::ostream& stream, const custom_float_type& value) +{ + stream << "[" << value.x << "]"; + return stream; +} + +struct float_bit_masked_type +{}; + +// Custom type to model types like Eigen::half or Eigen::bfloat16, that wrap around floating point +// types. +struct custom_int_type +{ + int x; + + // Constructor for the data generation utilities, simply ignore the second number + ROCPRIM_HOST_DEVICE custom_int_type(int val, int /*ignored*/) : x{val} + {} + + ROCPRIM_HOST_DEVICE custom_int_type(int val) : x{val} {} + + custom_int_type() = default; + + ROCPRIM_HOST_DEVICE + custom_int_type + operator+(const custom_int_type& other) const + { + return custom_int_type(x + other.x); + } + + ROCPRIM_HOST_DEVICE + custom_int_type + operator-(const custom_int_type& other) const + { + return custom_int_type(x - other.x); + } + + ROCPRIM_HOST_DEVICE + bool operator<(const custom_int_type& other) const + { + return x < other.x; + } + + ROCPRIM_HOST_DEVICE + bool operator>(const custom_int_type& other) const + { + return x > other.x; + } + + ROCPRIM_HOST_DEVICE + bool operator==(const custom_int_type& other) const + { + return x == other.x; + } + + ROCPRIM_HOST_DEVICE + bool operator!=(const custom_int_type& other) const + { + return !(*this == other); + } +}; + +inline std::ostream& operator<<(std::ostream& stream, const custom_int_type& value) +{ + stream << "[" << value.x << "]"; + return stream; +} + +} // namespace type_traits_test + +template<> +struct rocprim::traits::define +{ + using is_arithmetic = rocprim::traits::is_arithmetic::values; + using number_format + = rocprim::traits::number_format::values; + using float_bit_mask = rocprim::traits::float_bit_mask::values; +}; + +template<> +struct rocprim::traits::define +{ + using is_arithmetic = rocprim::traits::is_arithmetic::values; + using number_format + = rocprim::traits::number_format::values; + using integral_sign + = rocprim::traits::integral_sign::values; +}; + +template<> +struct rocprim::detail::float_bit_mask +{ + static constexpr uint32_t sign_bit = 0x80000000; + static constexpr uint32_t exponent = 0x7F800000; + static constexpr uint32_t mantissa = 0x007FFFFF; + using bit_type = uint32_t; +}; + +template +class RocprimFloatingPointTests : public ::testing::Test +{ +public: + using input_type = Params; +}; +using FloatingPointTypeTestParams = ::testing:: + Types; + +template +class RocprimIntegralTests : public ::testing::Test +{ +public: + using input_type = Params; +}; +using IntegralTypeTestParams = ::testing::Types; + +TYPED_TEST_SUITE(RocprimFloatingPointTests, FloatingPointTypeTestParams); + +TYPED_TEST_SUITE(RocprimIntegralTests, IntegralTypeTestParams); + +TYPED_TEST(RocprimFloatingPointTests, FloatingPoint) +{ + using input_type = typename TestFixture::input_type; + constexpr auto input_traits = rocprim::traits::get(); + + ROCPRIM_STATIC_ASSERT_TRUE(input_traits.is_arithmetic()); + ROCPRIM_STATIC_ASSERT_TRUE(input_traits.is_fundamental()); + ROCPRIM_STATIC_ASSERT_FALSE(input_traits.is_compound()); + ROCPRIM_STATIC_ASSERT_TRUE(input_traits.is_scalar()); + ROCPRIM_STATIC_ASSERT_TRUE(input_traits.is_floating_point()); + ROCPRIM_STATIC_ASSERT_FALSE(input_traits.is_integral()); + + ROCPRIM_STATIC_ASSERT_EQ(input_traits.is_integral(), rocprim::is_integral::value); + + // cannot do static_assert because under c++ 14 there is no if constexpr + if ROCPRIM_IF_CONSTEXPR(rocprim::is_arithmetic::value) + { // for c++ arithmetic types + ASSERT_EQ(input_traits.is_compound(), rocprim::is_compound::value); + ASSERT_EQ(input_traits.is_scalar(), rocprim::is_scalar::value); + ASSERT_EQ(input_traits.is_fundamental(), rocprim::is_fundamental::value); + ASSERT_EQ(input_traits.is_arithmetic(), rocprim::is_arithmetic::value); + ASSERT_EQ(input_traits.is_floating_point(), rocprim::is_floating_point::value); + } + else + { // for custom_types + ASSERT_NE(input_traits.is_compound(), rocprim::is_compound::value); + ASSERT_NE(input_traits.is_scalar(), rocprim::is_scalar::value); + ASSERT_NE(input_traits.is_fundamental(), rocprim::is_fundamental::value); + ASSERT_NE(input_traits.is_arithmetic(), rocprim::is_arithmetic::value); + ASSERT_NE(input_traits.is_floating_point(), rocprim::is_floating_point::value); + } + + [[maybe_unused]] constexpr auto float_bit_mask = input_traits.float_bit_mask(); +} + +TYPED_TEST(RocprimIntegralTests, Integral) +{ + using input_type = typename TestFixture::input_type; + + constexpr auto input_traits = rocprim::traits::get(); + + ROCPRIM_STATIC_ASSERT_TRUE(input_traits.is_arithmetic()); + ROCPRIM_STATIC_ASSERT_TRUE(input_traits.is_fundamental()); + ROCPRIM_STATIC_ASSERT_FALSE(input_traits.is_compound()); + ROCPRIM_STATIC_ASSERT_TRUE(input_traits.is_scalar()); + ROCPRIM_STATIC_ASSERT_FALSE(input_traits.is_floating_point()); + ROCPRIM_STATIC_ASSERT_TRUE(input_traits.is_integral()); + + ROCPRIM_STATIC_ASSERT_EQ(input_traits.is_floating_point(), + rocprim::is_floating_point::value); + ROCPRIM_STATIC_ASSERT_NE(input_traits.is_signed(), input_traits.is_unsigned()); + + if ROCPRIM_IF_CONSTEXPR(rocprim::is_arithmetic::value) + { // for c++ arithmetic types + ASSERT_EQ(input_traits.is_compound(), rocprim::is_compound::value); + ASSERT_EQ(input_traits.is_scalar(), rocprim::is_scalar::value); + ASSERT_EQ(input_traits.is_fundamental(), rocprim::is_fundamental::value); + ASSERT_EQ(input_traits.is_arithmetic(), rocprim::is_arithmetic::value); + ASSERT_EQ(input_traits.is_integral(), rocprim::is_integral::value); + ASSERT_EQ(input_traits.is_signed(), rocprim::is_signed::value); + ASSERT_EQ(input_traits.is_unsigned(), rocprim::is_unsigned::value); + } + else + { // for custom_types + ASSERT_NE(input_traits.is_compound(), rocprim::is_compound::value); + ASSERT_NE(input_traits.is_scalar(), rocprim::is_scalar::value); + ASSERT_NE(input_traits.is_fundamental(), rocprim::is_fundamental::value); + ASSERT_NE(input_traits.is_arithmetic(), rocprim::is_arithmetic::value); + ASSERT_NE(input_traits.is_integral(), rocprim::is_integral::value); + } +} + +TEST(TraitsInterface, OldType) +{ + using input_type = type_traits_test::float_bit_masked_type; + using bit_mask = rocprim::detail::float_bit_mask; + + constexpr auto input_traits = rocprim::traits::get(); + ROCPRIM_STATIC_ASSERT_FALSE(input_traits.is_arithmetic()); + ROCPRIM_STATIC_ASSERT_FALSE(input_traits.is_fundamental()); + ROCPRIM_STATIC_ASSERT_TRUE(input_traits.is_compound()); + + ROCPRIM_STATIC_ASSERT_TRUE(input_traits.is_compound()); + ROCPRIM_STATIC_ASSERT_FALSE(input_traits.is_scalar()); + ROCPRIM_STATIC_ASSERT_FALSE(input_traits.is_floating_point()); + ROCPRIM_STATIC_ASSERT_FALSE(input_traits.is_integral()); + + constexpr auto float_bit_mask = input_traits.float_bit_mask(); + + ROCPRIM_STATIC_ASSERT_EQ(float_bit_mask.sign_bit, bit_mask::sign_bit); + ROCPRIM_STATIC_ASSERT_EQ(float_bit_mask.exponent, bit_mask::exponent); + ROCPRIM_STATIC_ASSERT_EQ(float_bit_mask.mantissa, bit_mask::mantissa); +} + +TEST(TraitsInterface, OtherType) +{ + struct TestT + {}; + constexpr auto input_traits = rocprim::traits::get(); + ROCPRIM_STATIC_ASSERT_FALSE(input_traits.is_arithmetic()); + ROCPRIM_STATIC_ASSERT_FALSE(input_traits.is_fundamental()); + ROCPRIM_STATIC_ASSERT_TRUE(input_traits.is_compound()); + ROCPRIM_STATIC_ASSERT_FALSE(input_traits.is_scalar()); + ROCPRIM_STATIC_ASSERT_FALSE(input_traits.is_floating_point()); + ROCPRIM_STATIC_ASSERT_FALSE(input_traits.is_integral()); +} diff --git a/test/rocprim/test_utils_custom_float_traits_type.hpp b/test/rocprim/test_utils_custom_float_traits_type.hpp new file mode 100644 index 000000000..1f11f4091 --- /dev/null +++ b/test/rocprim/test_utils_custom_float_traits_type.hpp @@ -0,0 +1,129 @@ +// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef ROCPRIM_TEST_UTILS_CUSTOM_FLOAT_TRAITS_TYPE_HPP_ +#define ROCPRIM_TEST_UTILS_CUSTOM_FLOAT_TRAITS_TYPE_HPP_ + +#include "test_utils_custom_test_types.hpp" + +// For radix_key_codec +#include + +#include +#include + +#include + +namespace test_utils +{ +// Custom type to model types like Eigen::half or Eigen::bfloat16, that wrap around floating point +// types. +struct custom_float_traits_type +{ + using value_type = float; + float x; + + // Constructor for the data generation utilities, simply ignore the second number + ROCPRIM_HOST_DEVICE custom_float_traits_type(float val, float /*ignored*/) : x{val} + {} + + ROCPRIM_HOST_DEVICE custom_float_traits_type(float val) : x{val} {} + + custom_float_traits_type() = default; + + ROCPRIM_HOST_DEVICE + custom_float_traits_type + operator+(const custom_float_traits_type& other) const + { + return custom_float_traits_type(x + other.x); + } + + ROCPRIM_HOST_DEVICE + custom_float_traits_type + operator-(const custom_float_traits_type& other) const + { + return custom_float_traits_type(x - other.x); + } + + ROCPRIM_HOST_DEVICE + bool operator<(const custom_float_traits_type& other) const + { + return x < other.x; + } + + ROCPRIM_HOST_DEVICE + bool operator>(const custom_float_traits_type& other) const + { + return x > other.x; + } + + ROCPRIM_HOST_DEVICE + bool operator==(const custom_float_traits_type& other) const + { + return x == other.x; + } + + ROCPRIM_HOST_DEVICE + bool operator!=(const custom_float_traits_type& other) const + { + return !(*this == other); + } +}; + +inline bool signbit(const custom_float_traits_type& val) +{ + return std::signbit(val.x); +} + +inline std::ostream& operator<<(std::ostream& stream, const custom_float_traits_type& value) +{ + stream << "[" << value.x << "]"; + return stream; +} + +template<> +struct is_custom_test_type : std::true_type +{}; + +template<> +struct inner_type +{ + using type = custom_float_traits_type::value_type; +}; + +} // namespace test_utils + +template<> +struct ::rocprim::traits::define +{ + using is_arithmetic = ::rocprim::traits::is_arithmetic::values; + using number_format = ::rocprim::traits::number_format::values< + ::rocprim::traits::number_format::kind::floating_point_type>; + using float_bit_mask + = ::rocprim::traits::float_bit_mask::values; +}; + +template<> +struct ::rocprim::detail::radix_key_codec_base + : ::rocprim::detail::radix_key_codec_floating +{}; + +#endif //ROCPRIM_TEST_UTILS_CUSTOM_FLOAT_TYPE_HPP_ diff --git a/test/rocprim/test_utils_sort_comparator.hpp b/test/rocprim/test_utils_sort_comparator.hpp index 5f26eb00e..f9e26e373 100644 --- a/test/rocprim/test_utils_sort_comparator.hpp +++ b/test/rocprim/test_utils_sort_comparator.hpp @@ -26,6 +26,7 @@ #include #include "test_utils_bfloat16.hpp" +#include "test_utils_custom_float_traits_type.hpp" #include "test_utils_custom_float_type.hpp" #include "test_utils_custom_test_types.hpp" #include "test_utils_half.hpp" @@ -89,7 +90,8 @@ template // that we must counter here. - && !std::is_same::value, + && !std::is_same::value + && !std::is_same::value, int> = 0> auto to_bits(const Key key) -> typename rocprim::get_unsigned_bits_type::unsigned_type @@ -127,7 +129,8 @@ template // that we must counter here. - && !std::is_same::value, + && !std::is_same::value + && !std::is_same::value, int> = 0> auto to_bits(const Key& key) -> typename rocprim::get_unsigned_bits_type::unsigned_type @@ -163,6 +166,15 @@ auto to_bits(const Key key) -> typename rocprim::get_unsigned_bits_type::un return to_bits(key.x); } +template::value, int> = 0> +auto to_bits(const Key key) -> typename rocprim::get_unsigned_bits_type::unsigned_type +{ + return to_bits(key.x); +} + } // namespace detail template From b9c85021d5fde4b031bbd62ad62b0a38a7658369 Mon Sep 17 00:00:00 2001 From: Robin Voetter Date: Thu, 21 Nov 2024 12:59:15 +0100 Subject: [PATCH 07/84] remove ordered block id from run length encode A missing call to __syncthreads() caused this algorithm to occasionally produce an invalid index and cause a crash. Instead of just adding that call, we can remove the use of ordered block id by the regular block id, which is already ordered per HSA guarantees. There still remains an issue with a hang, but this seems a runtime issue and will be investigated separately. --- .../detail/device_run_length_encode.hpp | 21 +++------- .../device/device_run_length_encode.hpp | 38 +++++++------------ 2 files changed, 19 insertions(+), 40 deletions(-) diff --git a/rocprim/include/rocprim/device/detail/device_run_length_encode.hpp b/rocprim/include/rocprim/device/detail/device_run_length_encode.hpp index bab041509..8571ab2db 100644 --- a/rocprim/include/rocprim/device/detail/device_run_length_encode.hpp +++ b/rocprim/include/rocprim/device/detail/device_run_length_encode.hpp @@ -23,7 +23,6 @@ #include "device_partition.hpp" #include "lookback_scan_state.hpp" -#include "ordered_block_id.hpp" #include "../../detail/binary_op_wrappers.hpp" #include "../../detail/various.hpp" @@ -764,15 +763,13 @@ template + typename LookbackScanState> ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE auto non_trivial_kernel_impl(InputIterator, const OffsetsOutputIterator, const CountsOutputIterator, const RunsCountOutputIterator, const LookbackScanState, - OrderedBlockIdType, const size_t, const size_t) -> std::enable_if_t()> @@ -786,15 +783,13 @@ template + typename LookbackScanState> ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE auto non_trivial_kernel_impl(InputIterator input, const OffsetsOutputIterator offsets_output, const CountsOutputIterator counts_output, const RunsCountOutputIterator runs_count_output, const LookbackScanState scan_state, - OrderedBlockIdType ordered_block_id, const size_t grid_size, const size_t size) -> std::enable_if_t()> @@ -819,13 +814,9 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE auto load_input_method, scan_algorithm>; - ROCPRIM_SHARED_MEMORY union - { - typename OrderedBlockIdType::storage_type block_id; - typename block_processor::storage_type_ block; - } storage; + ROCPRIM_SHARED_MEMORY typename block_processor::storage_type_ storage; - const size_t block_id = ordered_block_id.get(threadIdx.x, storage.block_id); + const size_t block_id = flat_block_id(); const size_t block_offset = block_id * items_per_block; const InputIterator block_input = input + block_offset; @@ -842,7 +833,7 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE auto block_id, grid_size, size, - storage.block); + storage); } else if(valid_in_last_block > 0) { @@ -853,7 +844,7 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE auto block_id, grid_size, size, - storage.block); + storage); // First thread of last block sets the total number of non-trivial runs found and updates // the counts with the last run's length if necessary. if(threadIdx.x == 0) diff --git a/rocprim/include/rocprim/device/device_run_length_encode.hpp b/rocprim/include/rocprim/device/device_run_length_encode.hpp index 09b535639..95abaad31 100644 --- a/rocprim/include/rocprim/device/device_run_length_encode.hpp +++ b/rocprim/include/rocprim/device/device_run_length_encode.hpp @@ -100,24 +100,21 @@ template -ROCPRIM_KERNEL __launch_bounds__(device_params().kernel_config.block_size) -void non_trivial_kernel(const InputIterator input, - const OffsetsOutputIterator offsets_output, - const CountsOutputIterator counts_output, - const RunsCountOutputIterator runs_count_output, - const LookbackScanState scan_state, - const OrderedBlockIdType ordered_block_id, - const std::size_t grid_size, - const std::size_t size) + typename LookbackScanState> +ROCPRIM_KERNEL __launch_bounds__(device_params().kernel_config.block_size) void + non_trivial_kernel(const InputIterator input, + const OffsetsOutputIterator offsets_output, + const CountsOutputIterator counts_output, + const RunsCountOutputIterator runs_count_output, + const LookbackScanState scan_state, + const std::size_t grid_size, + const std::size_t size) { run_length_encode::non_trivial_kernel_impl(input, offsets_output, counts_output, runs_count_output, scan_state, - ordered_block_id, grid_size, size); } @@ -150,8 +147,6 @@ hipError_t run_length_encode_non_trivial_runs_impl(void* tempo using scan_state_with_sleep_type = ::rocprim::detail::lookback_scan_state; - using ordered_block_id_type = detail::ordered_block_id; - detail::target_arch target_arch; ROCPRIM_RETURN_ON_ERROR(host_target_arch(stream, target_arch)); @@ -161,8 +156,7 @@ hipError_t run_length_encode_non_trivial_runs_impl(void* tempo const std::size_t grid_size = detail::ceiling_div(size, items_per_block); // Calculate required temporary storage - void* scan_state_storage; - ordered_block_id_type::id_type* ordered_bid_storage; + void* scan_state_storage; detail::temp_storage::layout layout{}; ROCPRIM_RETURN_ON_ERROR(scan_state_type::get_temp_storage_layout(grid_size, stream, layout)); @@ -172,10 +166,8 @@ hipError_t run_length_encode_non_trivial_runs_impl(void* tempo storage_size, detail::temp_storage::make_linear_partition( // This is valid even with scan_state_with_sleep_type - detail::temp_storage::make_partition(&scan_state_storage, layout), - detail::temp_storage::make_partition( - &ordered_bid_storage, - ordered_block_id_type::get_temp_storage_layout()))); + detail::temp_storage::make_partition(&scan_state_storage, layout))); + if(result != hipSuccess || temporary_storage == nullptr) { return result; @@ -205,8 +197,6 @@ hipError_t run_length_encode_non_trivial_runs_impl(void* tempo } }; - auto ordered_bid = ordered_block_id_type::create(ordered_bid_storage); - if(size == 0) { // Fill out runs_count_output with zero @@ -240,8 +230,7 @@ hipError_t run_length_encode_non_trivial_runs_impl(void* tempo 0, stream, scan_state, - grid_size, - ordered_bid); + grid_size); }); ROCPRIM_DETAIL_HIP_SYNC_AND_RETURN_ON_ERROR("init_lookback_scan_state_kernel", grid_size, @@ -262,7 +251,6 @@ hipError_t run_length_encode_non_trivial_runs_impl(void* tempo counts_output, runs_count_output, scan_state, - ordered_bid, grid_size, size); }); From 2e8622f0a07b8299f8ba4d048f899f92581fe430 Mon Sep 17 00:00:00 2001 From: Robin Voetter Date: Tue, 17 Dec 2024 15:09:08 +0000 Subject: [PATCH 08/84] MI200 & MI300 tuning --- .../config/device_adjacent_difference.hpp | 82 ++- .../device_adjacent_difference_inplace.hpp | 80 ++- .../detail/config/device_binary_search.hpp | 367 +++++++++++- .../detail/config/device_find_first_of.hpp | 43 +- .../device/detail/config/device_histogram.hpp | 501 +++++++++++++++- .../detail/config/device_lower_bound.hpp | 369 +++++++++++- .../device/detail/config/device_merge.hpp | 435 +++++++++++++- .../config/device_merge_sort_block_merge.hpp | 546 ++++++++++++++++- .../config/device_merge_sort_block_sort.hpp | 2 +- .../detail/config/device_partition_flag.hpp | 80 ++- .../config/device_partition_predicate.hpp | 80 ++- .../config/device_partition_three_way.hpp | 80 ++- .../config/device_partition_two_way_flag.hpp | 80 ++- .../device_partition_two_way_predicate.hpp | 78 ++- .../config/device_radix_sort_block_sort.hpp | 139 +++-- .../config/device_radix_sort_onesweep.hpp | 160 ++--- .../device/detail/config/device_reduce.hpp | 20 +- .../detail/config/device_reduce_by_key.hpp | 559 ++++++++++++++++-- .../config/device_run_length_encode.hpp | 126 +++- .../device_run_length_encode_non_trivial.hpp | 113 +++- .../device/detail/config/device_scan.hpp | 122 +++- .../detail/config/device_scan_by_key.hpp | 525 ++++++++++++++-- .../detail/config/device_select_flag.hpp | 78 ++- .../detail/config/device_select_predicate.hpp | 78 ++- .../config/device_select_predicated_flag.hpp | 368 +++++++++++- .../detail/config/device_select_unique.hpp | 80 ++- .../config/device_select_unique_by_key.hpp | 362 +++++++++++- .../device/detail/config/device_transform.hpp | 74 ++- .../detail/config/device_upper_bound.hpp | 363 +++++++++++- .../device/detail/device_config_helper.hpp | 12 +- .../mergesort_block_sort_config_template | 4 +- 31 files changed, 5449 insertions(+), 557 deletions(-) diff --git a/rocprim/include/rocprim/device/detail/config/device_adjacent_difference.hpp b/rocprim/include/rocprim/device/detail/config/device_adjacent_difference.hpp index e1e2cc2c9..0f8cbc51b 100644 --- a/rocprim/include/rocprim/device/detail/config/device_adjacent_difference.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_adjacent_difference.hpp @@ -390,7 +390,7 @@ struct default_adjacent_difference_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : adjacent_difference_config<128, 1> + : adjacent_difference_config<256, 2> {}; // Based on value_type = float @@ -400,7 +400,7 @@ struct default_adjacent_difference_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<128, 3> + : adjacent_difference_config<512, 5> {}; // Based on value_type = rocprim::half @@ -409,7 +409,7 @@ struct default_adjacent_difference_config< static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2))>> : adjacent_difference_config<64, 7> + && (sizeof(value_type) <= 2))>> : adjacent_difference_config<256, 11> {}; // Based on value_type = int64_t @@ -419,7 +419,7 @@ struct default_adjacent_difference_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : adjacent_difference_config<128, 1> + : adjacent_difference_config<256, 2> {}; // Based on value_type = int @@ -429,7 +429,7 @@ struct default_adjacent_difference_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<128, 2> + : adjacent_difference_config<512, 5> {}; // Based on value_type = short @@ -439,7 +439,7 @@ struct default_adjacent_difference_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : adjacent_difference_config<64, 7> + : adjacent_difference_config<64, 17> {}; // Based on value_type = int8_t @@ -451,6 +451,74 @@ struct default_adjacent_difference_config< && (sizeof(value_type) <= 1))>> : adjacent_difference_config<64, 19> {}; +// Based on value_type = double +template +struct default_adjacent_difference_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : adjacent_difference_config<256, 7> +{}; + +// Based on value_type = float +template +struct default_adjacent_difference_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : adjacent_difference_config<256, 11> +{}; + +// Based on value_type = rocprim::half +template +struct default_adjacent_difference_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2))>> : adjacent_difference_config<64, 19> +{}; + +// Based on value_type = int64_t +template +struct default_adjacent_difference_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : adjacent_difference_config<1024, 2> +{}; + +// Based on value_type = int +template +struct default_adjacent_difference_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : adjacent_difference_config<1024, 5> +{}; + +// Based on value_type = short +template +struct default_adjacent_difference_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : adjacent_difference_config<128, 17> +{}; + +// Based on value_type = int8_t +template +struct default_adjacent_difference_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1))>> : adjacent_difference_config<64, 23> +{}; + } // end namespace detail END_ROCPRIM_NAMESPACE @@ -458,4 +526,4 @@ END_ROCPRIM_NAMESPACE /// @} // end of group primitivesmodule_deviceconfigs -#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_ADJACENT_DIFFERENCE_HPP_ \ No newline at end of file +#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_ADJACENT_DIFFERENCE_HPP_ diff --git a/rocprim/include/rocprim/device/detail/config/device_adjacent_difference_inplace.hpp b/rocprim/include/rocprim/device/detail/config/device_adjacent_difference_inplace.hpp index b4f4473b0..f6c9d3cf8 100644 --- a/rocprim/include/rocprim/device/detail/config/device_adjacent_difference_inplace.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_adjacent_difference_inplace.hpp @@ -400,7 +400,7 @@ struct default_adjacent_difference_inplace_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<1024, 5> + : adjacent_difference_config<512, 5> {}; // Based on value_type = rocprim::half @@ -409,7 +409,7 @@ struct default_adjacent_difference_inplace_config< static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2))>> : adjacent_difference_config<32, 23> + && (sizeof(value_type) <= 2))>> : adjacent_difference_config<256, 11> {}; // Based on value_type = int64_t @@ -429,7 +429,7 @@ struct default_adjacent_difference_inplace_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<1024, 5> + : adjacent_difference_config<512, 5> {}; // Based on value_type = short @@ -439,7 +439,7 @@ struct default_adjacent_difference_inplace_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : adjacent_difference_config<32, 23> + : adjacent_difference_config<64, 17> {}; // Based on value_type = int8_t @@ -448,7 +448,75 @@ struct default_adjacent_difference_inplace_config< static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1))>> : adjacent_difference_config<64, 19> + && (sizeof(value_type) <= 1))>> : adjacent_difference_config<128, 19> +{}; + +// Based on value_type = double +template +struct default_adjacent_difference_inplace_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : adjacent_difference_config<128, 11> +{}; + +// Based on value_type = float +template +struct default_adjacent_difference_inplace_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : adjacent_difference_config<512, 7> +{}; + +// Based on value_type = rocprim::half +template +struct default_adjacent_difference_inplace_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2))>> : adjacent_difference_config<128, 17> +{}; + +// Based on value_type = int64_t +template +struct default_adjacent_difference_inplace_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : adjacent_difference_config<128, 11> +{}; + +// Based on value_type = int +template +struct default_adjacent_difference_inplace_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : adjacent_difference_config<512, 7> +{}; + +// Based on value_type = short +template +struct default_adjacent_difference_inplace_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : adjacent_difference_config<128, 17> +{}; + +// Based on value_type = int8_t +template +struct default_adjacent_difference_inplace_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1))>> : adjacent_difference_config<128, 19> {}; } // end namespace detail @@ -458,4 +526,4 @@ END_ROCPRIM_NAMESPACE /// @} // end of group primitivesmodule_deviceconfigs -#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_ADJACENT_DIFFERENCE_INPLACE_HPP_ \ No newline at end of file +#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_ADJACENT_DIFFERENCE_INPLACE_HPP_ diff --git a/rocprim/include/rocprim/device/detail/config/device_binary_search.hpp b/rocprim/include/rocprim/device/detail/config/device_binary_search.hpp index 09b42bf2c..86ff91c82 100644 --- a/rocprim/include/rocprim/device/detail/config/device_binary_search.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_binary_search.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2023-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -1989,7 +1989,7 @@ struct default_binary_search_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : binary_search_config<256, 1> + : binary_search_config<64, 2> {}; // Based on value_type = double, output_type = int @@ -2001,7 +2001,7 @@ struct default_binary_search_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : binary_search_config<256, 1> + : binary_search_config<64, 2> {}; // Based on value_type = double, output_type = short @@ -2013,7 +2013,7 @@ struct default_binary_search_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<256, 16> + : binary_search_config<64, 2> {}; // Based on value_type = double, output_type = int8_t @@ -2024,7 +2024,7 @@ struct default_binary_search_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : binary_search_config<128, 1> + && (sizeof(output_type) <= 1))>> : binary_search_config<64, 2> {}; // Based on value_type = float, output_type = int64_t @@ -2060,7 +2060,7 @@ struct default_binary_search_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<128, 1> + : binary_search_config<256, 1> {}; // Based on value_type = float, output_type = int8_t @@ -2082,7 +2082,7 @@ struct default_binary_search_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : binary_search_config<256, 4> + && (sizeof(output_type) > 4))>> : binary_search_config<128, 4> {}; // Based on value_type = rocprim::half, output_type = int @@ -2093,7 +2093,7 @@ struct default_binary_search_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : binary_search_config<64, 4> + && (sizeof(output_type) > 2))>> : binary_search_config<64, 2> {}; // Based on value_type = rocprim::half, output_type = short @@ -2104,7 +2104,7 @@ struct default_binary_search_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : binary_search_config<256, 4> + && (sizeof(output_type) > 1))>> : binary_search_config<64, 2> {}; // Based on value_type = rocprim::half, output_type = int8_t @@ -2115,7 +2115,7 @@ struct default_binary_search_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> - : binary_search_config<256, 4> + : binary_search_config<64, 4> {}; // Based on value_type = int64_t, output_type = int64_t @@ -2127,7 +2127,7 @@ struct default_binary_search_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : binary_search_config<256, 1> + : binary_search_config<64, 2> {}; // Based on value_type = int64_t, output_type = int @@ -2139,7 +2139,7 @@ struct default_binary_search_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : binary_search_config<256, 1> + : binary_search_config<256, 2> {}; // Based on value_type = int64_t, output_type = short @@ -2151,7 +2151,7 @@ struct default_binary_search_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<256, 16> + : binary_search_config<256, 2> {}; // Based on value_type = int64_t, output_type = int8_t @@ -2162,7 +2162,7 @@ struct default_binary_search_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> + && (sizeof(output_type) <= 1))>> : binary_search_config<64, 2> {}; // Based on value_type = int, output_type = int64_t @@ -2198,7 +2198,7 @@ struct default_binary_search_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<256, 8> + : binary_search_config<256, 1> {}; // Based on value_type = int, output_type = int8_t @@ -2221,7 +2221,7 @@ struct default_binary_search_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : binary_search_config<128, 1> + : binary_search_config<256, 1> {}; // Based on value_type = short, output_type = int @@ -2233,7 +2233,7 @@ struct default_binary_search_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : binary_search_config<128, 1> + : binary_search_config<256, 1> {}; // Based on value_type = short, output_type = short @@ -2245,7 +2245,7 @@ struct default_binary_search_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<64, 4> + : binary_search_config<256, 1> {}; // Based on value_type = short, output_type = int8_t @@ -2256,7 +2256,7 @@ struct default_binary_search_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 1))>> : binary_search_config<128, 1> + && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> {}; // Based on value_type = int8_t, output_type = int64_t @@ -2267,7 +2267,7 @@ struct default_binary_search_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : binary_search_config<128, 2> + && (sizeof(output_type) > 4))>> : binary_search_config<256, 1> {}; // Based on value_type = int8_t, output_type = int @@ -2278,7 +2278,7 @@ struct default_binary_search_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : binary_search_config<128, 1> + && (sizeof(output_type) > 2))>> : binary_search_config<256, 1> {}; // Based on value_type = int8_t, output_type = short @@ -2289,7 +2289,7 @@ struct default_binary_search_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : binary_search_config<256, 4> + && (sizeof(output_type) > 1))>> : binary_search_config<256, 1> {}; // Based on value_type = int8_t, output_type = int8_t @@ -2300,9 +2300,332 @@ struct default_binary_search_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> + : binary_search_config<256, 1> +{}; + +// Based on value_type = double, output_type = int64_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : binary_search_config<256, 1> +{}; + +// Based on value_type = double, output_type = int +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : binary_search_config<64, 16> +{}; + +// Based on value_type = double, output_type = short +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : binary_search_config<64, 16> +{}; + +// Based on value_type = double, output_type = int8_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 1))>> : binary_search_config<64, 16> +{}; + +// Based on value_type = float, output_type = int64_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : binary_search_config<256, 1> +{}; + +// Based on value_type = float, output_type = int +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : binary_search_config<256, 1> +{}; + +// Based on value_type = float, output_type = short +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : binary_search_config<256, 1> +{}; + +// Based on value_type = float, output_type = int8_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> +{}; + +// Based on value_type = rocprim::half, output_type = int64_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) + && (sizeof(output_type) > 4))>> : binary_search_config<256, 8> +{}; + +// Based on value_type = rocprim::half, output_type = int +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) + && (sizeof(output_type) > 2))>> : binary_search_config<256, 4> +{}; + +// Based on value_type = rocprim::half, output_type = short +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) + && (sizeof(output_type) > 1))>> : binary_search_config<256, 1> +{}; + +// Based on value_type = rocprim::half, output_type = int8_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> : binary_search_config<256, 4> {}; +// Based on value_type = int64_t, output_type = int64_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : binary_search_config<256, 1> +{}; + +// Based on value_type = int64_t, output_type = int +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : binary_search_config<256, 1> +{}; + +// Based on value_type = int64_t, output_type = short +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : binary_search_config<256, 1> +{}; + +// Based on value_type = int64_t, output_type = int8_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> +{}; + +// Based on value_type = int, output_type = int64_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : binary_search_config<256, 1> +{}; + +// Based on value_type = int, output_type = int +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : binary_search_config<256, 1> +{}; + +// Based on value_type = int, output_type = short +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : binary_search_config<256, 1> +{}; + +// Based on value_type = int, output_type = int8_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> +{}; + +// Based on value_type = short, output_type = int64_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : binary_search_config<256, 1> +{}; + +// Based on value_type = short, output_type = int +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : binary_search_config<256, 1> +{}; + +// Based on value_type = short, output_type = short +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : binary_search_config<256, 1> +{}; + +// Based on value_type = short, output_type = int8_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> +{}; + +// Based on value_type = int8_t, output_type = int64_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) + && (sizeof(output_type) > 4))>> : binary_search_config<256, 16> +{}; + +// Based on value_type = int8_t, output_type = int +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) + && (sizeof(output_type) > 2))>> : binary_search_config<128, 4> +{}; + +// Based on value_type = int8_t, output_type = short +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) + && (sizeof(output_type) > 1))>> : binary_search_config<64, 16> +{}; + +// Based on value_type = int8_t, output_type = int8_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> + : binary_search_config<64, 16> +{}; + } // end namespace detail END_ROCPRIM_NAMESPACE diff --git a/rocprim/include/rocprim/device/detail/config/device_find_first_of.hpp b/rocprim/include/rocprim/device/detail/config/device_find_first_of.hpp index ce06277a8..4886d50af 100644 --- a/rocprim/include/rocprim/device/detail/config/device_find_first_of.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_find_first_of.hpp @@ -224,7 +224,7 @@ struct default_find_first_of_config< static_cast(target_arch::gfx90a), value_type, std::enable_if_t<((sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : find_first_of_config<256, 8> + : find_first_of_config<256, 6> {}; // Based on value_type = int @@ -233,7 +233,7 @@ struct default_find_first_of_config< static_cast(target_arch::gfx90a), value_type, std::enable_if_t<((sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : find_first_of_config<256, 10> + : find_first_of_config<128, 9> {}; // Based on value_type = short @@ -242,7 +242,7 @@ struct default_find_first_of_config< static_cast(target_arch::gfx90a), value_type, std::enable_if_t<((sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : find_first_of_config<256, 11> + : find_first_of_config<256, 15> {}; // Based on value_type = int8_t @@ -250,7 +250,42 @@ template struct default_find_first_of_config(target_arch::gfx90a), value_type, std::enable_if_t<((sizeof(value_type) <= 1))>> - : find_first_of_config<256, 10> + : find_first_of_config<512, 1> +{}; + +// Based on value_type = int64_t +template +struct default_find_first_of_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<((sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : find_first_of_config<1024, 7> +{}; + +// Based on value_type = int +template +struct default_find_first_of_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<((sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : find_first_of_config<1024, 7> +{}; + +// Based on value_type = short +template +struct default_find_first_of_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<((sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : find_first_of_config<1024, 9> +{}; + +// Based on value_type = int8_t +template +struct default_find_first_of_config(target_arch::gfx942), + value_type, + std::enable_if_t<((sizeof(value_type) <= 1))>> + : find_first_of_config<1024, 12> {}; } // end namespace detail diff --git a/rocprim/include/rocprim/device/detail/config/device_histogram.hpp b/rocprim/include/rocprim/device/detail/config/device_histogram.hpp index cfd3dc72a..33c8269ae 100644 --- a/rocprim/include/rocprim/device/detail/config/device_histogram.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_histogram.hpp @@ -2727,7 +2727,7 @@ struct default_histogram_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = double, channels = 2, active_channels = 2 @@ -2740,7 +2740,7 @@ struct default_histogram_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 3> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = double, channels = 3, active_channels = 3 @@ -2753,7 +2753,7 @@ struct default_histogram_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = double, channels = 4, active_channels = 3 @@ -2766,7 +2766,7 @@ struct default_histogram_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = double, channels = 4, active_channels = 4 @@ -2779,7 +2779,7 @@ struct default_histogram_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = float, channels = 1, active_channels = 1 @@ -2792,7 +2792,7 @@ struct default_histogram_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = float, channels = 2, active_channels = 2 @@ -2805,7 +2805,7 @@ struct default_histogram_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = float, channels = 3, active_channels = 3 @@ -2818,7 +2818,7 @@ struct default_histogram_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = float, channels = 4, active_channels = 3 @@ -2844,7 +2844,7 @@ struct default_histogram_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 1, active_channels = 1 @@ -2856,7 +2856,7 @@ struct default_histogram_config< active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 2, active_channels = 2 @@ -2880,7 +2880,7 @@ struct default_histogram_config< active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 4, active_channels = 3 @@ -2892,7 +2892,7 @@ struct default_histogram_config< active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 4, active_channels = 4 @@ -2904,7 +2904,7 @@ struct default_histogram_config< active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int64_t, channels = 1, active_channels = 1 @@ -2917,7 +2917,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = int64_t, channels = 2, active_channels = 2 @@ -2930,7 +2930,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 2> + : histogram_config, 2048, 2048, 2> {}; // Based on value_type = int64_t, channels = 3, active_channels = 3 @@ -2943,7 +2943,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 2> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int64_t, channels = 4, active_channels = 3 @@ -2956,7 +2956,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 2> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = int64_t, channels = 4, active_channels = 4 @@ -2969,7 +2969,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 2> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int, channels = 1, active_channels = 1 @@ -2982,7 +2982,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int, channels = 2, active_channels = 2 @@ -2995,7 +2995,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int, channels = 3, active_channels = 3 @@ -3008,7 +3008,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int, channels = 4, active_channels = 3 @@ -3047,7 +3047,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = short, channels = 2, active_channels = 2 @@ -3060,7 +3060,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = short, channels = 3, active_channels = 3 @@ -3073,7 +3073,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = short, channels = 4, active_channels = 3 @@ -3086,7 +3086,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = short, channels = 4, active_channels = 4 @@ -3099,7 +3099,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int8_t, channels = 1, active_channels = 1 @@ -3111,7 +3111,7 @@ struct default_histogram_config< active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int8_t, channels = 2, active_channels = 2 @@ -3162,6 +3162,451 @@ struct default_histogram_config< : histogram_config, 2048, 2048, 4> {}; +// Based on value_type = double, channels = 1, active_channels = 1 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 1) + && (active_channels == 1))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = double, channels = 2, active_channels = 2 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 2) + && (active_channels == 2))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = double, channels = 3, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 3) + && (active_channels == 3))>> + : histogram_config, 2048, 2048, 3> +{}; + +// Based on value_type = double, channels = 4, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) + && (active_channels == 3))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = double, channels = 4, active_channels = 4 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) + && (active_channels == 4))>> + : histogram_config, 2048, 2048, 3> +{}; + +// Based on value_type = float, channels = 1, active_channels = 1 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 1) + && (active_channels == 1))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = float, channels = 2, active_channels = 2 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 2) + && (active_channels == 2))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = float, channels = 3, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 3) + && (active_channels == 3))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = float, channels = 4, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 4) + && (active_channels == 3))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = float, channels = 4, active_channels = 4 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 4) + && (active_channels == 4))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = rocprim::half, channels = 1, active_channels = 1 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (channels == 1) && (active_channels == 1))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = rocprim::half, channels = 2, active_channels = 2 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (channels == 2) && (active_channels == 2))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = rocprim::half, channels = 3, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (channels == 3) && (active_channels == 3))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = rocprim::half, channels = 4, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (channels == 4) && (active_channels == 3))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = rocprim::half, channels = 4, active_channels = 4 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (channels == 4) && (active_channels == 4))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = int64_t, channels = 1, active_channels = 1 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 1) + && (active_channels == 1))>> + : histogram_config, 2048, 2048, 3> +{}; + +// Based on value_type = int64_t, channels = 2, active_channels = 2 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 2) + && (active_channels == 2))>> + : histogram_config, 2048, 2048, 3> +{}; + +// Based on value_type = int64_t, channels = 3, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 3) + && (active_channels == 3))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = int64_t, channels = 4, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) + && (active_channels == 3))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = int64_t, channels = 4, active_channels = 4 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) + && (active_channels == 4))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = int, channels = 1, active_channels = 1 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 1) + && (active_channels == 1))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = int, channels = 2, active_channels = 2 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 2) + && (active_channels == 2))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = int, channels = 3, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 3) + && (active_channels == 3))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = int, channels = 4, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 4) + && (active_channels == 3))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = int, channels = 4, active_channels = 4 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 4) + && (active_channels == 4))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = short, channels = 1, active_channels = 1 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 1) + && (active_channels == 1))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = short, channels = 2, active_channels = 2 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 2) + && (active_channels == 2))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = short, channels = 3, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 3) + && (active_channels == 3))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = short, channels = 4, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 4) + && (active_channels == 3))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = short, channels = 4, active_channels = 4 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 4) + && (active_channels == 4))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = int8_t, channels = 1, active_channels = 1 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (channels == 1) && (active_channels == 1))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = int8_t, channels = 2, active_channels = 2 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (channels == 2) && (active_channels == 2))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = int8_t, channels = 3, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (channels == 3) && (active_channels == 3))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = int8_t, channels = 4, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (channels == 4) && (active_channels == 3))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = int8_t, channels = 4, active_channels = 4 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (channels == 4) && (active_channels == 4))>> + : histogram_config, 2048, 2048, 4> +{}; + } // end namespace detail END_ROCPRIM_NAMESPACE @@ -3169,4 +3614,4 @@ END_ROCPRIM_NAMESPACE /// @} // end of group primitivesmodule_deviceconfigs -#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_HISTOGRAM_HPP_ \ No newline at end of file +#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_HISTOGRAM_HPP_ diff --git a/rocprim/include/rocprim/device/detail/config/device_lower_bound.hpp b/rocprim/include/rocprim/device/detail/config/device_lower_bound.hpp index 076bc48fb..91599aeac 100644 --- a/rocprim/include/rocprim/device/detail/config/device_lower_bound.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_lower_bound.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2023-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -1989,7 +1989,7 @@ struct default_lower_bound_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : lower_bound_config<64, 8> + : lower_bound_config<64, 4> {}; // Based on value_type = double, output_type = int @@ -2001,7 +2001,7 @@ struct default_lower_bound_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : lower_bound_config<64, 8> + : lower_bound_config<64, 4> {}; // Based on value_type = double, output_type = short @@ -2013,7 +2013,7 @@ struct default_lower_bound_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : lower_bound_config<256, 16> + : lower_bound_config<256, 4> {}; // Based on value_type = double, output_type = int8_t @@ -2024,7 +2024,7 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> + && (sizeof(output_type) <= 1))>> : lower_bound_config<64, 4> {}; // Based on value_type = float, output_type = int64_t @@ -2082,7 +2082,7 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : lower_bound_config<64, 4> + && (sizeof(output_type) > 4))>> : lower_bound_config<128, 4> {}; // Based on value_type = rocprim::half, output_type = int @@ -2093,7 +2093,7 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : lower_bound_config<64, 4> + && (sizeof(output_type) > 2))>> : lower_bound_config<64, 2> {}; // Based on value_type = rocprim::half, output_type = short @@ -2104,7 +2104,7 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : lower_bound_config<64, 4> + && (sizeof(output_type) > 1))>> : lower_bound_config<64, 2> {}; // Based on value_type = rocprim::half, output_type = int8_t @@ -2115,7 +2115,7 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> - : lower_bound_config<64, 4> + : lower_bound_config<64, 2> {}; // Based on value_type = int64_t, output_type = int64_t @@ -2127,7 +2127,7 @@ struct default_lower_bound_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : lower_bound_config<128, 1> + : lower_bound_config<128, 4> {}; // Based on value_type = int64_t, output_type = int @@ -2139,7 +2139,7 @@ struct default_lower_bound_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : lower_bound_config<64, 8> + : lower_bound_config<64, 4> {}; // Based on value_type = int64_t, output_type = short @@ -2151,7 +2151,7 @@ struct default_lower_bound_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : lower_bound_config<256, 16> + : lower_bound_config<128, 4> {}; // Based on value_type = int64_t, output_type = int8_t @@ -2162,7 +2162,7 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 2> + && (sizeof(output_type) <= 1))>> : lower_bound_config<64, 4> {}; // Based on value_type = int, output_type = int64_t @@ -2174,7 +2174,7 @@ struct default_lower_bound_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : lower_bound_config<128, 1> + : lower_bound_config<256, 1> {}; // Based on value_type = int, output_type = int @@ -2186,7 +2186,7 @@ struct default_lower_bound_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : lower_bound_config<128, 1> + : lower_bound_config<256, 1> {}; // Based on value_type = int, output_type = short @@ -2221,7 +2221,7 @@ struct default_lower_bound_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : lower_bound_config<128, 1> + : lower_bound_config<256, 1> {}; // Based on value_type = short, output_type = int @@ -2233,7 +2233,7 @@ struct default_lower_bound_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : lower_bound_config<128, 1> + : lower_bound_config<256, 1> {}; // Based on value_type = short, output_type = short @@ -2245,7 +2245,7 @@ struct default_lower_bound_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : lower_bound_config<256, 4> + : lower_bound_config<256, 1> {}; // Based on value_type = short, output_type = int8_t @@ -2256,7 +2256,7 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 1))>> : lower_bound_config<64, 4> + && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> {}; // Based on value_type = int8_t, output_type = int64_t @@ -2267,7 +2267,7 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : lower_bound_config<128, 1> + && (sizeof(output_type) > 4))>> : lower_bound_config<128, 2> {}; // Based on value_type = int8_t, output_type = int @@ -2278,7 +2278,7 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : lower_bound_config<128, 1> + && (sizeof(output_type) > 2))>> : lower_bound_config<256, 2> {}; // Based on value_type = int8_t, output_type = short @@ -2289,7 +2289,7 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : lower_bound_config<128, 1> + && (sizeof(output_type) > 1))>> : lower_bound_config<256, 2> {}; // Based on value_type = int8_t, output_type = int8_t @@ -2300,7 +2300,330 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> - : lower_bound_config<64, 4> + : lower_bound_config<128, 2> +{}; + +// Based on value_type = double, output_type = int64_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : lower_bound_config<64, 16> +{}; + +// Based on value_type = double, output_type = int +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : lower_bound_config<256, 1> +{}; + +// Based on value_type = double, output_type = short +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : lower_bound_config<256, 1> +{}; + +// Based on value_type = double, output_type = int8_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 1))>> : lower_bound_config<128, 1> +{}; + +// Based on value_type = float, output_type = int64_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : lower_bound_config<256, 1> +{}; + +// Based on value_type = float, output_type = int +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : lower_bound_config<256, 1> +{}; + +// Based on value_type = float, output_type = short +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : lower_bound_config<256, 1> +{}; + +// Based on value_type = float, output_type = int8_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> +{}; + +// Based on value_type = rocprim::half, output_type = int64_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) + && (sizeof(output_type) > 4))>> : lower_bound_config<256, 8> +{}; + +// Based on value_type = rocprim::half, output_type = int +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) + && (sizeof(output_type) > 2))>> : lower_bound_config<256, 8> +{}; + +// Based on value_type = rocprim::half, output_type = short +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) + && (sizeof(output_type) > 1))>> : lower_bound_config<256, 4> +{}; + +// Based on value_type = rocprim::half, output_type = int8_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> + : lower_bound_config<256, 4> +{}; + +// Based on value_type = int64_t, output_type = int64_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : lower_bound_config<256, 1> +{}; + +// Based on value_type = int64_t, output_type = int +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : lower_bound_config<128, 1> +{}; + +// Based on value_type = int64_t, output_type = short +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : lower_bound_config<256, 1> +{}; + +// Based on value_type = int64_t, output_type = int8_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> +{}; + +// Based on value_type = int, output_type = int64_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : lower_bound_config<256, 1> +{}; + +// Based on value_type = int, output_type = int +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : lower_bound_config<256, 1> +{}; + +// Based on value_type = int, output_type = short +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : lower_bound_config<256, 1> +{}; + +// Based on value_type = int, output_type = int8_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> +{}; + +// Based on value_type = short, output_type = int64_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : lower_bound_config<256, 1> +{}; + +// Based on value_type = short, output_type = int +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : lower_bound_config<256, 1> +{}; + +// Based on value_type = short, output_type = short +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : lower_bound_config<256, 1> +{}; + +// Based on value_type = short, output_type = int8_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> +{}; + +// Based on value_type = int8_t, output_type = int64_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) + && (sizeof(output_type) > 4))>> : lower_bound_config<128, 4> +{}; + +// Based on value_type = int8_t, output_type = int +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) + && (sizeof(output_type) > 2))>> : lower_bound_config<128, 4> +{}; + +// Based on value_type = int8_t, output_type = short +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) + && (sizeof(output_type) > 1))>> : lower_bound_config<64, 16> +{}; + +// Based on value_type = int8_t, output_type = int8_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> + : lower_bound_config<64, 16> {}; } // end namespace detail diff --git a/rocprim/include/rocprim/device/detail/config/device_merge.hpp b/rocprim/include/rocprim/device/detail/config/device_merge.hpp index 83c2367ab..ec7e922b8 100644 --- a/rocprim/include/rocprim/device/detail/config/device_merge.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_merge.hpp @@ -2036,7 +2036,7 @@ struct default_merge_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : merge_config<1024, 2> + && (sizeof(value_type) > 4))>> : merge_config<512, 2> {}; // Based on key_type = double, value_type = int @@ -2047,7 +2047,7 @@ struct default_merge_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : merge_config<512, 2> + && (sizeof(value_type) > 2))>> : merge_config<256, 5> {}; // Based on key_type = double, value_type = short @@ -2082,7 +2082,7 @@ struct default_merge_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_config<256, 7> + : merge_config<256, 4> {}; // Based on key_type = float, value_type = int64_t @@ -2104,7 +2104,7 @@ struct default_merge_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : merge_config<512, 2> + && (sizeof(value_type) > 2))>> : merge_config<512, 4> {}; // Based on key_type = float, value_type = short @@ -2139,7 +2139,7 @@ struct default_merge_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : merge_config<512, 4> + : merge_config<256, 10> {}; // Based on key_type = rocprim::half, value_type = int64_t @@ -2161,7 +2161,7 @@ struct default_merge_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_config<1024, 2> + : merge_config<512, 2> {}; // Based on key_type = rocprim::half, value_type = short @@ -2172,7 +2172,7 @@ struct default_merge_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_config<256, 4> + : merge_config<512, 4> {}; // Based on key_type = rocprim::half, value_type = int8_t @@ -2184,7 +2184,7 @@ struct default_merge_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_config<256, 4> + : merge_config<256, 10> {}; // Based on key_type = rocprim::half, value_type = empty_type @@ -2195,7 +2195,7 @@ struct default_merge_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (std::is_same::value))>> - : merge_config<256, 8> + : merge_config<256, 11> {}; // Based on key_type = int64_t, value_type = int64_t @@ -2206,7 +2206,7 @@ struct default_merge_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : merge_config<1024, 2> + && (sizeof(value_type) > 4))>> : merge_config<512, 2> {}; // Based on key_type = int64_t, value_type = int @@ -2217,7 +2217,7 @@ struct default_merge_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : merge_config<512, 2> + && (sizeof(value_type) > 2))>> : merge_config<256, 5> {}; // Based on key_type = int64_t, value_type = short @@ -2252,7 +2252,7 @@ struct default_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_config<256, 7> + : merge_config<256, 4> {}; // Based on key_type = int, value_type = int64_t @@ -2274,7 +2274,7 @@ struct default_merge_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : merge_config<512, 2> + && (sizeof(value_type) > 2))>> : merge_config<1024, 2> {}; // Based on key_type = int, value_type = short @@ -2320,7 +2320,7 @@ struct default_merge_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : merge_config<1024, 2> + && (sizeof(value_type) > 4))>> : merge_config<512, 2> {}; // Based on key_type = short, value_type = int @@ -2331,7 +2331,7 @@ struct default_merge_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : merge_config<512, 2> + && (sizeof(value_type) > 2))>> : merge_config<512, 4> {}; // Based on key_type = short, value_type = short @@ -2354,7 +2354,7 @@ struct default_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_config<512, 4> + : merge_config<256, 10> {}; // Based on key_type = short, value_type = empty_type @@ -2366,7 +2366,7 @@ struct default_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (std::is_same::value))>> - : merge_config<512, 8> + : merge_config<256, 16> {}; // Based on key_type = int8_t, value_type = int64_t @@ -2411,7 +2411,7 @@ struct default_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_config<512, 4> + : merge_config<256, 10> {}; // Based on key_type = int8_t, value_type = empty_type @@ -2422,9 +2422,406 @@ struct default_merge_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (std::is_same::value))>> + : merge_config<256, 16> +{}; + +// Based on key_type = double, value_type = int64_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_config<256, 4> +{}; + +// Based on key_type = double, value_type = int +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_config<256, 4> +{}; + +// Based on key_type = double, value_type = short +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_config<256, 5> +{}; + +// Based on key_type = double, value_type = int8_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_config<256, 5> +{}; + +// Based on key_type = double, value_type = empty_type +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) + && (std::is_same::value))>> + : merge_config<256, 7> +{}; + +// Based on key_type = float, value_type = int64_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_config<256, 4> +{}; + +// Based on key_type = float, value_type = int +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_config<512, 4> +{}; + +// Based on key_type = float, value_type = short +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_config<256, 8> +{}; + +// Based on key_type = float, value_type = int8_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_config<256, 8> +{}; + +// Based on key_type = float, value_type = empty_type +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) + && (std::is_same::value))>> + : merge_config<256, 10> +{}; + +// Based on key_type = rocprim::half, value_type = int64_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : merge_config<512, 2> +{}; + +// Based on key_type = rocprim::half, value_type = int +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : merge_config<512, 4> +{}; + +// Based on key_type = rocprim::half, value_type = short +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : merge_config<512, 4> +{}; + +// Based on key_type = rocprim::half, value_type = int8_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_config<256, 10> +{}; + +// Based on key_type = rocprim::half, value_type = empty_type +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (std::is_same::value))>> + : merge_config<256, 16> +{}; + +// Based on key_type = int64_t, value_type = int64_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_config<256, 4> +{}; + +// Based on key_type = int64_t, value_type = int +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_config<256, 4> +{}; + +// Based on key_type = int64_t, value_type = short +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_config<256, 5> +{}; + +// Based on key_type = int64_t, value_type = int8_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_config<256, 5> +{}; + +// Based on key_type = int64_t, value_type = empty_type +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) + && (std::is_same::value))>> + : merge_config<256, 7> +{}; + +// Based on key_type = int, value_type = int64_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_config<256, 4> +{}; + +// Based on key_type = int, value_type = int +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_config<512, 4> +{}; + +// Based on key_type = int, value_type = short +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_config<256, 8> +{}; + +// Based on key_type = int, value_type = int8_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_config<256, 8> +{}; + +// Based on key_type = int, value_type = empty_type +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) + && (std::is_same::value))>> + : merge_config<256, 10> +{}; + +// Based on key_type = short, value_type = int64_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_config<256, 4> +{}; + +// Based on key_type = short, value_type = int +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_config<512, 4> +{}; + +// Based on key_type = short, value_type = short +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_config<256, 10> +{}; + +// Based on key_type = short, value_type = int8_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_config<256, 10> +{}; + +// Based on key_type = short, value_type = empty_type +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) + && (std::is_same::value))>> + : merge_config<256, 16> +{}; + +// Based on key_type = int8_t, value_type = int64_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : merge_config<1024, 2> +{}; + +// Based on key_type = int8_t, value_type = int +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : merge_config<512, 4> +{}; + +// Based on key_type = int8_t, value_type = short +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : merge_config<512, 4> +{}; + +// Based on key_type = int8_t, value_type = int8_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> : merge_config<512, 8> {}; +// Based on key_type = int8_t, value_type = empty_type +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (std::is_same::value))>> + : merge_config<512, 16> +{}; + } // end namespace detail END_ROCPRIM_NAMESPACE @@ -2432,4 +2829,4 @@ END_ROCPRIM_NAMESPACE /// @} // end of group primitivesmodule_deviceconfigs -#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_MERGE_HPP_ \ No newline at end of file +#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_MERGE_HPP_ diff --git a/rocprim/include/rocprim/device/detail/config/device_merge_sort_block_merge.hpp b/rocprim/include/rocprim/device/detail/config/device_merge_sort_block_merge.hpp index 885f1e942..39c44610d 100644 --- a/rocprim/include/rocprim/device/detail/config/device_merge_sort_block_merge.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_merge_sort_block_merge.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -2031,7 +2031,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> {}; // Based on key_type = double, value_type = int64_t @@ -2043,7 +2043,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; // Based on key_type = double, value_type = int @@ -2055,7 +2055,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = double, value_type = short @@ -2067,7 +2067,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = double, value_type = int8_t @@ -2079,7 +2079,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = double, value_type = empty_type @@ -2103,7 +2103,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; // Based on key_type = float, value_type = int64_t @@ -2115,7 +2115,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = float, value_type = int @@ -2127,7 +2127,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = float, value_type = short @@ -2139,7 +2139,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; // Based on key_type = float, value_type = int8_t @@ -2185,7 +2185,7 @@ struct default_merge_sort_block_merge_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = rocprim::half, value_type = int @@ -2196,7 +2196,7 @@ struct default_merge_sort_block_merge_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; // Based on key_type = rocprim::half, value_type = short @@ -2207,7 +2207,7 @@ struct default_merge_sort_block_merge_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; // Based on key_type = rocprim::half, value_type = int8_t @@ -2242,7 +2242,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> {}; // Based on key_type = int64_t, value_type = int64_t @@ -2254,7 +2254,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; // Based on key_type = int64_t, value_type = int @@ -2266,7 +2266,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = int64_t, value_type = short @@ -2278,7 +2278,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = int64_t, value_type = int8_t @@ -2290,7 +2290,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = int64_t, value_type = empty_type @@ -2302,7 +2302,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = int, value_type = custom_type @@ -2326,7 +2326,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = int, value_type = int @@ -2350,7 +2350,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; // Based on key_type = int, value_type = int8_t @@ -2398,7 +2398,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = short, value_type = int @@ -2410,7 +2410,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; // Based on key_type = short, value_type = short @@ -2468,7 +2468,7 @@ struct default_merge_sort_block_merge_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = int8_t, value_type = int @@ -2479,7 +2479,7 @@ struct default_merge_sort_block_merge_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; // Based on key_type = int8_t, value_type = short @@ -2677,6 +2677,500 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; +// Based on key_type = double, value_type = custom_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> +{}; + +// Based on key_type = double, value_type = int64_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> +{}; + +// Based on key_type = double, value_type = int +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 4> +{}; + +// Based on key_type = double, value_type = short +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = double, value_type = int8_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> +{}; + +// Based on key_type = double, value_type = empty_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) + && (std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> +{}; + +// Based on key_type = float, value_type = custom_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = float, value_type = int64_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> +{}; + +// Based on key_type = float, value_type = int +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = float, value_type = short +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = float, value_type = int8_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = float, value_type = empty_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) + && (std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = rocprim::half, value_type = custom_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = rocprim::half, value_type = int64_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = rocprim::half, value_type = int +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = rocprim::half, value_type = short +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = rocprim::half, value_type = int8_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = rocprim::half, value_type = empty_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = int64_t, value_type = custom_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> +{}; + +// Based on key_type = int64_t, value_type = int64_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = int64_t, value_type = int +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = int64_t, value_type = short +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = int64_t, value_type = int8_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = int64_t, value_type = empty_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) + && (std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> +{}; + +// Based on key_type = int, value_type = custom_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = int, value_type = int64_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = int, value_type = int +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = int, value_type = short +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = int, value_type = int8_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = int, value_type = empty_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) + && (std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = short, value_type = custom_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = short, value_type = int64_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = short, value_type = int +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = short, value_type = short +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = short, value_type = int8_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = short, value_type = empty_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) + && (std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = int8_t, value_type = custom_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = int8_t, value_type = int64_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> +{}; + +// Based on key_type = int8_t, value_type = int +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = int8_t, value_type = short +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = int8_t, value_type = int8_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = int8_t, value_type = empty_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + } // end namespace detail END_ROCPRIM_NAMESPACE @@ -2684,4 +3178,4 @@ END_ROCPRIM_NAMESPACE /// @} // end of group primitivesmodule_deviceconfigs -#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_MERGE_SORT_BLOCK_MERGE_HPP_ \ No newline at end of file +#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_MERGE_SORT_BLOCK_MERGE_HPP_ diff --git a/rocprim/include/rocprim/device/detail/config/device_merge_sort_block_sort.hpp b/rocprim/include/rocprim/device/detail/config/device_merge_sort_block_sort.hpp index 6e557104d..3343376cc 100644 --- a/rocprim/include/rocprim/device/detail/config/device_merge_sort_block_sort.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_merge_sort_block_sort.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal diff --git a/rocprim/include/rocprim/device/detail/config/device_partition_flag.hpp b/rocprim/include/rocprim/device/detail/config/device_partition_flag.hpp index 93f5f7cbd..56f548238 100644 --- a/rocprim/include/rocprim/device/detail/config/device_partition_flag.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_partition_flag.hpp @@ -379,7 +379,7 @@ struct default_partition_flag_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<128, 7> + && (sizeof(data_type) > 4))>> : select_config<192, 4> {}; // Based on data_type = float @@ -388,7 +388,7 @@ struct default_partition_flag_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<128, 12> + && (sizeof(data_type) > 2))>> : select_config<256, 10> {}; // Based on data_type = rocprim::half @@ -397,7 +397,7 @@ struct default_partition_flag_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 28> + && (sizeof(data_type) <= 2))>> : select_config<256, 20> {}; // Based on data_type = int64_t @@ -407,7 +407,7 @@ struct default_partition_flag_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<128, 7> + : select_config<192, 4> {}; // Based on data_type = int @@ -417,7 +417,7 @@ struct default_partition_flag_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<128, 12> + : select_config<256, 10> {}; // Based on data_type = short @@ -427,7 +427,7 @@ struct default_partition_flag_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 28> + : select_config<256, 20> {}; // Based on data_type = int8_t @@ -436,7 +436,73 @@ struct default_partition_flag_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 1))>> : select_config<256, 28> + && (sizeof(data_type) <= 1))>> : select_config<256, 24> +{}; + +// Based on data_type = double +template +struct default_partition_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) + && (sizeof(data_type) > 4))>> : select_config<512, 5> +{}; + +// Based on data_type = float +template +struct default_partition_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) + && (sizeof(data_type) > 2))>> : select_config<512, 15> +{}; + +// Based on data_type = rocprim::half +template +struct default_partition_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2))>> : select_config<512, 28> +{}; + +// Based on data_type = int64_t +template +struct default_partition_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> + : select_config<512, 5> +{}; + +// Based on data_type = int +template +struct default_partition_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> + : select_config<512, 15> +{}; + +// Based on data_type = short +template +struct default_partition_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> + : select_config<512, 30> +{}; + +// Based on data_type = int8_t +template +struct default_partition_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 1))>> : select_config<512, 24> {}; } // end namespace detail diff --git a/rocprim/include/rocprim/device/detail/config/device_partition_predicate.hpp b/rocprim/include/rocprim/device/detail/config/device_partition_predicate.hpp index 6f92494b3..b77fb0d67 100644 --- a/rocprim/include/rocprim/device/detail/config/device_partition_predicate.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_partition_predicate.hpp @@ -379,7 +379,7 @@ struct default_partition_predicate_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<128, 7> + && (sizeof(data_type) > 4))>> : select_config<192, 4> {}; // Based on data_type = float @@ -388,7 +388,7 @@ struct default_partition_predicate_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<256, 15> + && (sizeof(data_type) > 2))>> : select_config<256, 10> {}; // Based on data_type = rocprim::half @@ -397,7 +397,7 @@ struct default_partition_predicate_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 24> + && (sizeof(data_type) <= 2))>> : select_config<256, 20> {}; // Based on data_type = int64_t @@ -407,7 +407,7 @@ struct default_partition_predicate_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<192, 7> + : select_config<192, 4> {}; // Based on data_type = int @@ -417,7 +417,7 @@ struct default_partition_predicate_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<256, 15> + : select_config<256, 10> {}; // Based on data_type = short @@ -427,7 +427,7 @@ struct default_partition_predicate_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 24> + : select_config<256, 20> {}; // Based on data_type = int8_t @@ -436,7 +436,73 @@ struct default_partition_predicate_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 1))>> : select_config<256, 24> + && (sizeof(data_type) <= 1))>> : select_config<512, 24> +{}; + +// Based on data_type = double +template +struct default_partition_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) + && (sizeof(data_type) > 4))>> : select_config<512, 5> +{}; + +// Based on data_type = float +template +struct default_partition_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) + && (sizeof(data_type) > 2))>> : select_config<512, 15> +{}; + +// Based on data_type = rocprim::half +template +struct default_partition_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2))>> : select_config<512, 30> +{}; + +// Based on data_type = int64_t +template +struct default_partition_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> + : select_config<512, 5> +{}; + +// Based on data_type = int +template +struct default_partition_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> + : select_config<512, 15> +{}; + +// Based on data_type = short +template +struct default_partition_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> + : select_config<512, 30> +{}; + +// Based on data_type = int8_t +template +struct default_partition_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 1))>> : select_config<512, 23> {}; } // end namespace detail diff --git a/rocprim/include/rocprim/device/detail/config/device_partition_three_way.hpp b/rocprim/include/rocprim/device/detail/config/device_partition_three_way.hpp index 75b74d843..7538d8c87 100644 --- a/rocprim/include/rocprim/device/detail/config/device_partition_three_way.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_partition_three_way.hpp @@ -379,7 +379,7 @@ struct default_partition_three_way_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<192, 6> + && (sizeof(data_type) > 4))>> : select_config<256, 6> {}; // Based on data_type = float @@ -388,7 +388,7 @@ struct default_partition_three_way_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<256, 15> + && (sizeof(data_type) > 2))>> : select_config<256, 11> {}; // Based on data_type = rocprim::half @@ -397,7 +397,7 @@ struct default_partition_three_way_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 14> + && (sizeof(data_type) <= 2))>> : select_config<256, 16> {}; // Based on data_type = int64_t @@ -407,7 +407,7 @@ struct default_partition_three_way_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<256, 7> + : select_config<256, 6> {}; // Based on data_type = int @@ -417,7 +417,7 @@ struct default_partition_three_way_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<256, 15> + : select_config<256, 11> {}; // Based on data_type = short @@ -427,7 +427,7 @@ struct default_partition_three_way_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 14> + : select_config<256, 16> {}; // Based on data_type = int8_t @@ -436,7 +436,73 @@ struct default_partition_three_way_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 1))>> : select_config<256, 15> + && (sizeof(data_type) <= 1))>> : select_config<256, 17> +{}; + +// Based on data_type = double +template +struct default_partition_three_way_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) + && (sizeof(data_type) > 4))>> : select_config<512, 5> +{}; + +// Based on data_type = float +template +struct default_partition_three_way_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) + && (sizeof(data_type) > 2))>> : select_config<512, 15> +{}; + +// Based on data_type = rocprim::half +template +struct default_partition_three_way_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2))>> : select_config<256, 26> +{}; + +// Based on data_type = int64_t +template +struct default_partition_three_way_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> + : select_config<512, 5> +{}; + +// Based on data_type = int +template +struct default_partition_three_way_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> + : select_config<512, 15> +{}; + +// Based on data_type = short +template +struct default_partition_three_way_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> + : select_config<512, 31> +{}; + +// Based on data_type = int8_t +template +struct default_partition_three_way_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 1))>> : select_config<512, 26> {}; } // end namespace detail diff --git a/rocprim/include/rocprim/device/detail/config/device_partition_two_way_flag.hpp b/rocprim/include/rocprim/device/detail/config/device_partition_two_way_flag.hpp index d8c114d96..309213176 100644 --- a/rocprim/include/rocprim/device/detail/config/device_partition_two_way_flag.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_partition_two_way_flag.hpp @@ -379,7 +379,7 @@ struct default_partition_two_way_flag_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<128, 7> + && (sizeof(data_type) > 4))>> : select_config<192, 4> {}; // Based on data_type = float @@ -388,7 +388,7 @@ struct default_partition_two_way_flag_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<256, 13> + && (sizeof(data_type) > 2))>> : select_config<256, 10> {}; // Based on data_type = rocprim::half @@ -397,7 +397,7 @@ struct default_partition_two_way_flag_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 28> + && (sizeof(data_type) <= 2))>> : select_config<256, 20> {}; // Based on data_type = int64_t @@ -407,7 +407,7 @@ struct default_partition_two_way_flag_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<128, 7> + : select_config<192, 4> {}; // Based on data_type = int @@ -417,7 +417,7 @@ struct default_partition_two_way_flag_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<256, 12> + : select_config<256, 7> {}; // Based on data_type = short @@ -427,7 +427,7 @@ struct default_partition_two_way_flag_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 28> + : select_config<256, 20> {}; // Based on data_type = int8_t @@ -436,7 +436,73 @@ struct default_partition_two_way_flag_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 1))>> : select_config<256, 28> + && (sizeof(data_type) <= 1))>> : select_config<256, 24> +{}; + +// Based on data_type = double +template +struct default_partition_two_way_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) + && (sizeof(data_type) > 4))>> : select_config<512, 5> +{}; + +// Based on data_type = float +template +struct default_partition_two_way_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) + && (sizeof(data_type) > 2))>> : select_config<512, 15> +{}; + +// Based on data_type = rocprim::half +template +struct default_partition_two_way_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2))>> : select_config<512, 28> +{}; + +// Based on data_type = int64_t +template +struct default_partition_two_way_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> + : select_config<512, 5> +{}; + +// Based on data_type = int +template +struct default_partition_two_way_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> + : select_config<512, 15> +{}; + +// Based on data_type = short +template +struct default_partition_two_way_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> + : select_config<512, 28> +{}; + +// Based on data_type = int8_t +template +struct default_partition_two_way_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 1))>> : select_config<512, 24> {}; } // end namespace detail diff --git a/rocprim/include/rocprim/device/detail/config/device_partition_two_way_predicate.hpp b/rocprim/include/rocprim/device/detail/config/device_partition_two_way_predicate.hpp index 554a79c09..246fe23c7 100644 --- a/rocprim/include/rocprim/device/detail/config/device_partition_two_way_predicate.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_partition_two_way_predicate.hpp @@ -380,7 +380,7 @@ struct default_partition_two_way_predicate_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<128, 7> + && (sizeof(data_type) > 4))>> : select_config<128, 6> {}; // Based on data_type = float @@ -389,7 +389,7 @@ struct default_partition_two_way_predicate_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<256, 15> + && (sizeof(data_type) > 2))>> : select_config<256, 10> {}; // Based on data_type = rocprim::half @@ -398,7 +398,7 @@ struct default_partition_two_way_predicate_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 24> + && (sizeof(data_type) <= 2))>> : select_config<256, 20> {}; // Based on data_type = int64_t @@ -408,7 +408,7 @@ struct default_partition_two_way_predicate_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<128, 7> + : select_config<192, 4> {}; // Based on data_type = int @@ -418,7 +418,7 @@ struct default_partition_two_way_predicate_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<256, 13> + : select_config<256, 10> {}; // Based on data_type = short @@ -428,7 +428,7 @@ struct default_partition_two_way_predicate_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 24> + : select_config<256, 28> {}; // Based on data_type = int8_t @@ -440,6 +440,72 @@ struct default_partition_two_way_predicate_config< && (sizeof(data_type) <= 1))>> : select_config<256, 24> {}; +// Based on data_type = double +template +struct default_partition_two_way_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) + && (sizeof(data_type) > 4))>> : select_config<512, 7> +{}; + +// Based on data_type = float +template +struct default_partition_two_way_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) + && (sizeof(data_type) > 2))>> : select_config<512, 15> +{}; + +// Based on data_type = rocprim::half +template +struct default_partition_two_way_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2))>> : select_config<512, 30> +{}; + +// Based on data_type = int64_t +template +struct default_partition_two_way_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> + : select_config<512, 5> +{}; + +// Based on data_type = int +template +struct default_partition_two_way_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> + : select_config<512, 15> +{}; + +// Based on data_type = short +template +struct default_partition_two_way_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> + : select_config<512, 30> +{}; + +// Based on data_type = int8_t +template +struct default_partition_two_way_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 1))>> : select_config<512, 24> +{}; + } // end namespace detail END_ROCPRIM_NAMESPACE diff --git a/rocprim/include/rocprim/device/detail/config/device_radix_sort_block_sort.hpp b/rocprim/include/rocprim/device/detail/config/device_radix_sort_block_sort.hpp index f3e77d427..02897f37f 100644 --- a/rocprim/include/rocprim/device/detail/config/device_radix_sort_block_sort.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_radix_sort_block_sort.hpp @@ -2039,7 +2039,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 12> + && (sizeof(value_type) > 4))>> : kernel_config<256, 15> {}; // Based on key_type = double, value_type = int @@ -2050,7 +2050,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<256, 12> + && (sizeof(value_type) > 2))>> : kernel_config<256, 15> {}; // Based on key_type = double, value_type = short @@ -2061,7 +2061,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<256, 13> + && (sizeof(value_type) > 1))>> : kernel_config<256, 15> {}; // Based on key_type = double, value_type = int8_t @@ -2073,7 +2073,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<256, 12> + : kernel_config<256, 15> {}; // Based on key_type = double, value_type = empty_type @@ -2085,7 +2085,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : kernel_config<256, 16> + : kernel_config<256, 15> {}; // Based on key_type = float, value_type = int64_t @@ -2096,7 +2096,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 12> + && (sizeof(value_type) > 4))>> : kernel_config<256, 15> {}; // Based on key_type = float, value_type = int @@ -2107,7 +2107,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<512, 31> + && (sizeof(value_type) > 2))>> : kernel_config<256, 21> {}; // Based on key_type = float, value_type = short @@ -2118,7 +2118,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<512, 31> + && (sizeof(value_type) > 1))>> : kernel_config<512, 15> {}; // Based on key_type = float, value_type = int8_t @@ -2130,7 +2130,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<512, 31> + : kernel_config<256, 21> {}; // Based on key_type = float, value_type = empty_type @@ -2142,7 +2142,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : kernel_config<512, 31> + : kernel_config<256, 21> {}; // Based on key_type = rocprim::half, value_type = int64_t @@ -2153,7 +2153,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : kernel_config<256, 12> + : kernel_config<256, 10> {}; // Based on key_type = rocprim::half, value_type = int @@ -2164,7 +2164,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : kernel_config<512, 31> + : kernel_config<512, 15> {}; // Based on key_type = rocprim::half, value_type = short @@ -2175,7 +2175,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : kernel_config<512, 32> + : kernel_config<512, 16> {}; // Based on key_type = rocprim::half, value_type = int8_t @@ -2187,7 +2187,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<512, 31> + : kernel_config<256, 32> {}; // Based on key_type = rocprim::half, value_type = empty_type @@ -2198,7 +2198,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (std::is_same::value))>> - : kernel_config<1024, 23> + : kernel_config<256, 32> {}; // Based on key_type = int64_t, value_type = int64_t @@ -2209,7 +2209,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 12> + && (sizeof(value_type) > 4))>> : kernel_config<256, 15> {}; // Based on key_type = int64_t, value_type = int @@ -2220,7 +2220,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<256, 12> + && (sizeof(value_type) > 2))>> : kernel_config<256, 15> {}; // Based on key_type = int64_t, value_type = short @@ -2231,7 +2231,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<256, 8> + && (sizeof(value_type) > 1))>> : kernel_config<256, 15> {}; // Based on key_type = int64_t, value_type = int8_t @@ -2243,7 +2243,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<256, 12> + : kernel_config<512, 15> {}; // Based on key_type = int64_t, value_type = empty_type @@ -2255,7 +2255,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : kernel_config<256, 16> + : kernel_config<256, 15> {}; // Based on key_type = int, value_type = int64_t @@ -2266,7 +2266,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 12> + && (sizeof(value_type) > 4))>> : kernel_config<256, 15> {}; // Based on key_type = int, value_type = int @@ -2277,7 +2277,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<512, 31> + && (sizeof(value_type) > 2))>> : kernel_config<256, 21> {}; // Based on key_type = int, value_type = short @@ -2288,7 +2288,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<512, 31> + && (sizeof(value_type) > 1))>> : kernel_config<512, 15> {}; // Based on key_type = int, value_type = int8_t @@ -2300,7 +2300,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<512, 31> + : kernel_config<256, 21> {}; // Based on key_type = int, value_type = empty_type @@ -2312,7 +2312,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : kernel_config<512, 31> + : kernel_config<256, 21> {}; // Based on key_type = short, value_type = int64_t @@ -2323,7 +2323,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 12> + && (sizeof(value_type) > 4))>> : kernel_config<256, 10> {}; // Based on key_type = short, value_type = int @@ -2334,7 +2334,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<512, 31> + && (sizeof(value_type) > 2))>> : kernel_config<512, 15> {}; // Based on key_type = short, value_type = short @@ -2345,7 +2345,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<512, 32> + && (sizeof(value_type) > 1))>> : kernel_config<256, 32> {}; // Based on key_type = short, value_type = int8_t @@ -2357,7 +2357,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<512, 32> + : kernel_config<256, 32> {}; // Based on key_type = short, value_type = empty_type @@ -2369,7 +2369,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (std::is_same::value))>> - : kernel_config<1024, 23> + : kernel_config<256, 32> {}; // Based on key_type = int8_t, value_type = int64_t @@ -2380,7 +2380,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : kernel_config<256, 13> + : kernel_config<512, 5> {}; // Based on key_type = int8_t, value_type = int @@ -2391,7 +2391,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : kernel_config<512, 31> + : kernel_config<512, 13> {}; // Based on key_type = int8_t, value_type = short @@ -2402,7 +2402,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : kernel_config<512, 32> + : kernel_config<512, 16> {}; // Based on key_type = int8_t, value_type = int8_t @@ -2414,7 +2414,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<512, 32> + : kernel_config<256, 32> {}; // Based on key_type = int8_t, value_type = empty_type @@ -2425,7 +2425,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (std::is_same::value))>> - : kernel_config<1024, 23> + : kernel_config<512, 32> {}; // Based on key_type = double, value_type = int64_t @@ -2436,7 +2436,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 16> + && (sizeof(value_type) > 4))>> : kernel_config<256, 15> {}; // Based on key_type = double, value_type = int @@ -2447,7 +2447,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<256, 16> + && (sizeof(value_type) > 2))>> : kernel_config<512, 15> {}; // Based on key_type = double, value_type = short @@ -2458,7 +2458,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<256, 16> + && (sizeof(value_type) > 1))>> : kernel_config<512, 14> {}; // Based on key_type = double, value_type = int8_t @@ -2470,7 +2470,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<256, 16> + : kernel_config<512, 15> {}; // Based on key_type = double, value_type = empty_type @@ -2482,7 +2482,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : kernel_config<256, 16> + : kernel_config<256, 15> {}; // Based on key_type = float, value_type = int64_t @@ -2493,7 +2493,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 8> + && (sizeof(value_type) > 4))>> : kernel_config<256, 15> {}; // Based on key_type = float, value_type = int @@ -2504,7 +2504,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<512, 16> + && (sizeof(value_type) > 2))>> : kernel_config<256, 19> {}; // Based on key_type = float, value_type = short @@ -2515,7 +2515,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<256, 32> + && (sizeof(value_type) > 1))>> : kernel_config<256, 16> {}; // Based on key_type = float, value_type = int8_t @@ -2527,7 +2527,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<256, 32> + : kernel_config<512, 31> {}; // Based on key_type = float, value_type = empty_type @@ -2539,7 +2539,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : kernel_config<256, 32> + : kernel_config<512, 31> {}; // Based on key_type = rocprim::half, value_type = int64_t @@ -2550,7 +2550,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : kernel_config<512, 8> + : kernel_config<256, 15> {}; // Based on key_type = rocprim::half, value_type = int @@ -2561,7 +2561,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : kernel_config<256, 32> + : kernel_config<512, 15> {}; // Based on key_type = rocprim::half, value_type = short @@ -2572,7 +2572,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : kernel_config<512, 18> + : kernel_config<512, 16> {}; // Based on key_type = rocprim::half, value_type = int8_t @@ -2595,7 +2595,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (std::is_same::value))>> - : kernel_config<512, 21> + : kernel_config<256, 32> {}; // Based on key_type = int64_t, value_type = int64_t @@ -2606,7 +2606,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 16> + && (sizeof(value_type) > 4))>> : kernel_config<128, 21> {}; // Based on key_type = int64_t, value_type = int @@ -2617,7 +2617,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<256, 16> + && (sizeof(value_type) > 2))>> : kernel_config<512, 15> {}; // Based on key_type = int64_t, value_type = short @@ -2628,7 +2628,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<256, 16> + && (sizeof(value_type) > 1))>> : kernel_config<512, 14> {}; // Based on key_type = int64_t, value_type = int8_t @@ -2640,7 +2640,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<256, 16> + : kernel_config<512, 15> {}; // Based on key_type = int64_t, value_type = empty_type @@ -2652,7 +2652,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : kernel_config<256, 16> + : kernel_config<256, 15> {}; // Based on key_type = int, value_type = int64_t @@ -2663,7 +2663,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 10> + && (sizeof(value_type) > 4))>> : kernel_config<256, 15> {}; // Based on key_type = int, value_type = int @@ -2674,7 +2674,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<256, 32> + && (sizeof(value_type) > 2))>> : kernel_config<256, 21> {}; // Based on key_type = int, value_type = short @@ -2685,7 +2685,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<256, 32> + && (sizeof(value_type) > 1))>> : kernel_config<512, 31> {}; // Based on key_type = int, value_type = int8_t @@ -2697,7 +2697,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<256, 32> + : kernel_config<512, 31> {}; // Based on key_type = int, value_type = empty_type @@ -2709,7 +2709,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : kernel_config<256, 32> + : kernel_config<256, 31> {}; // Based on key_type = short, value_type = int64_t @@ -2720,7 +2720,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 16> + && (sizeof(value_type) > 4))>> : kernel_config<256, 15> {}; // Based on key_type = short, value_type = int @@ -2731,7 +2731,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<256, 11> + && (sizeof(value_type) > 2))>> : kernel_config<256, 30> {}; // Based on key_type = short, value_type = short @@ -2742,7 +2742,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<512, 18> + && (sizeof(value_type) > 1))>> : kernel_config<512, 16> {}; // Based on key_type = short, value_type = int8_t @@ -2766,7 +2766,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (std::is_same::value))>> - : kernel_config<512, 23> + : kernel_config<256, 32> {}; // Based on key_type = int8_t, value_type = int64_t @@ -2777,7 +2777,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : kernel_config<256, 16> + : kernel_config<512, 5> {}; // Based on key_type = int8_t, value_type = int @@ -2788,7 +2788,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : kernel_config<256, 21> + : kernel_config<512, 13> {}; // Based on key_type = int8_t, value_type = short @@ -2799,7 +2799,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : kernel_config<256, 21> + : kernel_config<512, 16> {}; // Based on key_type = int8_t, value_type = int8_t @@ -2811,7 +2811,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<256, 23> + : kernel_config<1024, 20> {}; // Based on key_type = int8_t, value_type = empty_type @@ -2822,9 +2822,8 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (std::is_same::value))>> - : kernel_config<512, 21> + : kernel_config<512, 32> {}; - } // end namespace detail END_ROCPRIM_NAMESPACE @@ -2832,4 +2831,4 @@ END_ROCPRIM_NAMESPACE /// @} // end of group primitivesmodule_deviceconfigs -#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RADIX_SORT_BLOCK_SORT_HPP_ \ No newline at end of file +#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RADIX_SORT_BLOCK_SORT_HPP_ diff --git a/rocprim/include/rocprim/device/detail/config/device_radix_sort_onesweep.hpp b/rocprim/include/rocprim/device/detail/config/device_radix_sort_onesweep.hpp index 64c6acb6d..fa9d53ac0 100644 --- a/rocprim/include/rocprim/device/detail/config/device_radix_sort_onesweep.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_radix_sort_onesweep.hpp @@ -2168,8 +2168,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<512, 6>, + : radix_sort_onesweep_config, + kernel_config<256, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -2183,8 +2183,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<512, 6>, + : radix_sort_onesweep_config, + kernel_config<256, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -2198,8 +2198,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, - kernel_config<1024, 6>, + : radix_sort_onesweep_config, + kernel_config<512, 6>, 8, block_radix_rank_algorithm::match> {}; @@ -2213,8 +2213,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, - kernel_config<256, 22>, + : radix_sort_onesweep_config, + kernel_config<512, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -2228,8 +2228,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<256, 22>, + : radix_sort_onesweep_config, + kernel_config<1024, 6>, 8, block_radix_rank_algorithm::match> {}; @@ -2243,8 +2243,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 22>, + : radix_sort_onesweep_config, + kernel_config<1024, 6>, 8, block_radix_rank_algorithm::match> {}; @@ -2258,8 +2258,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 22>, + : radix_sort_onesweep_config, + kernel_config<512, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -2286,8 +2286,8 @@ struct default_radix_sort_onesweep_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<512, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -2300,8 +2300,8 @@ struct default_radix_sort_onesweep_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<512, 22>, 8, block_radix_rank_algorithm::match> {}; @@ -2315,8 +2315,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<512, 16>, 8, block_radix_rank_algorithm::match> {}; @@ -2329,8 +2329,8 @@ struct default_radix_sort_onesweep_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<512, 16>, 8, block_radix_rank_algorithm::match> {}; @@ -2346,7 +2346,7 @@ struct default_radix_sort_onesweep_config< && (sizeof(value_type) > 4))>> : radix_sort_onesweep_config, kernel_config<1024, 6>, - 6, + 8, block_radix_rank_algorithm::match> {}; @@ -2359,8 +2359,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, - kernel_config<256, 12>, + : radix_sort_onesweep_config, + kernel_config<1024, 6>, 8, block_radix_rank_algorithm::match> {}; @@ -2389,8 +2389,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<512, 6>, + : radix_sort_onesweep_config, + kernel_config<256, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -2419,8 +2419,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, - kernel_config<256, 12>, + : radix_sort_onesweep_config, + kernel_config<512, 6>, 8, block_radix_rank_algorithm::match> {}; @@ -2434,9 +2434,9 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, - kernel_config<512, 12>, - 5, + : radix_sort_onesweep_config, + kernel_config<1024, 6>, + 8, block_radix_rank_algorithm::match> {}; @@ -2449,8 +2449,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<512, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -2464,8 +2464,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 22>, + : radix_sort_onesweep_config, + kernel_config<1024, 6>, 8, block_radix_rank_algorithm::match> {}; @@ -2479,9 +2479,9 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 22>, - 7, + : radix_sort_onesweep_config, + kernel_config<512, 12>, + 8, block_radix_rank_algorithm::match> {}; @@ -2494,8 +2494,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, - kernel_config<256, 18>, + : radix_sort_onesweep_config, + kernel_config<1024, 6>, 8, block_radix_rank_algorithm::match> {}; @@ -2509,9 +2509,9 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, - kernel_config<256, 18>, - 8, + : radix_sort_onesweep_config, + kernel_config<1024, 6>, + 6, block_radix_rank_algorithm::match> {}; @@ -2524,9 +2524,9 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<1024, 22>, - 6, + : radix_sort_onesweep_config, + kernel_config<512, 22>, + 8, block_radix_rank_algorithm::match> {}; @@ -2539,8 +2539,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 18>, + : radix_sort_onesweep_config, + kernel_config<512, 22>, 8, block_radix_rank_algorithm::match> {}; @@ -2554,8 +2554,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 22>, + : radix_sort_onesweep_config, + kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -2582,8 +2582,8 @@ struct default_radix_sort_onesweep_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, - kernel_config<256, 22>, + : radix_sort_onesweep_config, + kernel_config<512, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -2596,8 +2596,8 @@ struct default_radix_sort_onesweep_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<512, 22>, 8, block_radix_rank_algorithm::match> {}; @@ -2611,10 +2611,10 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<1024, 22>, - 4, - block_radix_rank_algorithm::basic> + : radix_sort_onesweep_config, + kernel_config<1024, 12>, + 8, + block_radix_rank_algorithm::match> {}; // Based on key_type = int8_t, value_type = empty_type @@ -2625,10 +2625,10 @@ struct default_radix_sort_onesweep_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, - 8, - block_radix_rank_algorithm::match> + : radix_sort_onesweep_config, + kernel_config<512, 22>, + 4, + block_radix_rank_algorithm::basic> {}; // Based on key_type = double, value_type = int64_t @@ -3363,8 +3363,8 @@ struct default_radix_sort_onesweep_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<512, 22>, + : radix_sort_onesweep_config, + kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -3423,8 +3423,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<512, 12>, + : radix_sort_onesweep_config, + kernel_config<1024, 6>, 8, block_radix_rank_algorithm::match> {}; @@ -3528,8 +3528,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, - kernel_config<1024, 6>, + : radix_sort_onesweep_config, + kernel_config<512, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -3558,8 +3558,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<512, 22>, + : radix_sort_onesweep_config, + kernel_config<1024, 22>, 8, block_radix_rank_algorithm::match> {}; @@ -3573,8 +3573,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<1024, 12>, + : radix_sort_onesweep_config, + kernel_config<1024, 22>, 8, block_radix_rank_algorithm::match> {}; @@ -3588,8 +3588,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<512, 22>, + : radix_sort_onesweep_config, + kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -3630,8 +3630,8 @@ struct default_radix_sort_onesweep_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<512, 22>, + : radix_sort_onesweep_config, + kernel_config<1024, 22>, 8, block_radix_rank_algorithm::match> {}; @@ -3645,8 +3645,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<1024, 12>, + : radix_sort_onesweep_config, + kernel_config<1024, 22>, 8, block_radix_rank_algorithm::match> {}; @@ -3672,4 +3672,4 @@ END_ROCPRIM_NAMESPACE /// @} // end of group primitivesmodule_deviceconfigs -#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RADIX_SORT_ONESWEEP_HPP_ \ No newline at end of file +#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RADIX_SORT_ONESWEEP_HPP_ diff --git a/rocprim/include/rocprim/device/detail/config/device_reduce.hpp b/rocprim/include/rocprim/device/detail/config/device_reduce.hpp index 3dcbf8576..b4d506d21 100644 --- a/rocprim/include/rocprim/device/detail/config/device_reduce.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_reduce.hpp @@ -467,7 +467,7 @@ struct default_reduce_config< key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2))>> - : reduce_config<256, 4, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<128, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = rocprim::half @@ -476,7 +476,7 @@ struct default_reduce_config(target_arch::gfx90a), key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2))>> - : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<64, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = int64_t @@ -486,7 +486,7 @@ struct default_reduce_config< key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4))>> - : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<256, 4, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = int @@ -496,7 +496,7 @@ struct default_reduce_config< key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2))>> - : reduce_config<256, 4, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<128, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = short @@ -506,7 +506,7 @@ struct default_reduce_config< key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1))>> - : reduce_config<128, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<64, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = int8_t @@ -515,7 +515,7 @@ struct default_reduce_config(target_arch::gfx90a), key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1))>> - : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<128, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = double @@ -525,7 +525,7 @@ struct default_reduce_config< key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4))>> - : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<256, 2, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = float @@ -535,7 +535,7 @@ struct default_reduce_config< key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2))>> - : reduce_config<256, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<256, 4, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = rocprim::half @@ -574,7 +574,7 @@ struct default_reduce_config< key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1))>> - : reduce_config<128, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<256, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = int8_t @@ -593,4 +593,4 @@ END_ROCPRIM_NAMESPACE /// @} // end of group primitivesmodule_deviceconfigs -#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_REDUCE_HPP_ \ No newline at end of file +#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_REDUCE_HPP_ diff --git a/rocprim/include/rocprim/device/detail/config/device_reduce_by_key.hpp b/rocprim/include/rocprim/device/detail/config/device_reduce_by_key.hpp index 892b42987..30ab7deb4 100644 --- a/rocprim/include/rocprim/device/detail/config/device_reduce_by_key.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_reduce_by_key.hpp @@ -2417,8 +2417,8 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : reduce_by_key_config<256, - 15, + : reduce_by_key_config<192, + 7, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -2451,8 +2451,8 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2))>> - : reduce_by_key_config<256, - 10, + : reduce_by_key_config<192, + 7, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -2468,8 +2468,8 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && !bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : reduce_by_key_config<256, - 15, + : reduce_by_key_config<192, + 7, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -2485,8 +2485,8 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && !bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : reduce_by_key_config<256, - 10, + : reduce_by_key_config<192, + 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -2502,8 +2502,8 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && !bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : reduce_by_key_config<256, - 10, + : reduce_by_key_config<192, + 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -2536,8 +2536,8 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : reduce_by_key_config<256, - 10, + : reduce_by_key_config<192, + 7, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -2553,8 +2553,8 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : reduce_by_key_config<256, - 15, + : reduce_by_key_config<192, + 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -2570,8 +2570,8 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2))>> - : reduce_by_key_config<256, - 15, + : reduce_by_key_config<192, + 12, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -2587,8 +2587,8 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && !bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : reduce_by_key_config<256, - 10, + : reduce_by_key_config<192, + 7, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -2604,7 +2604,7 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && !bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : reduce_by_key_config<256, + : reduce_by_key_config<192, 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, @@ -2622,7 +2622,7 @@ struct default_reduce_by_key_config< && !bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : reduce_by_key_config<192, - 15, + 12, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -2639,7 +2639,7 @@ struct default_reduce_by_key_config< && !bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1))>> : reduce_by_key_config<192, - 15, + 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -2655,12 +2655,12 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : reduce_by_key_config<256, - 14, + : reduce_by_key_config<128, + 9, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, - 1> + 2> {}; // Based on key_type = short, value_type = float @@ -2672,7 +2672,7 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : reduce_by_key_config<256, + : reduce_by_key_config<192, 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, @@ -2689,8 +2689,8 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2))>> - : reduce_by_key_config<256, - 15, + : reduce_by_key_config<192, + 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -2706,7 +2706,7 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && !bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : reduce_by_key_config<256, + : reduce_by_key_config<192, 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, @@ -2723,7 +2723,7 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && !bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : reduce_by_key_config<256, + : reduce_by_key_config<192, 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, @@ -2740,8 +2740,8 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && !bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : reduce_by_key_config<256, - 15, + : reduce_by_key_config<192, + 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -2757,8 +2757,8 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && !bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1))>> - : reduce_by_key_config<512, - 10, + : reduce_by_key_config<192, + 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -2773,8 +2773,8 @@ struct default_reduce_by_key_config< value_type, std::enable_if_t<((sizeof(key_type) <= 1) && bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : reduce_by_key_config<256, - 15, + : reduce_by_key_config<192, + 7, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -2789,8 +2789,8 @@ struct default_reduce_by_key_config< value_type, std::enable_if_t<((sizeof(key_type) <= 1) && bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : reduce_by_key_config<256, - 15, + : reduce_by_key_config<192, + 12, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -2805,8 +2805,8 @@ struct default_reduce_by_key_config< value_type, std::enable_if_t<((sizeof(key_type) <= 1) && bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2))>> - : reduce_by_key_config<256, - 15, + : reduce_by_key_config<192, + 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -2822,8 +2822,8 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 1) && !bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : reduce_by_key_config<256, - 15, + : reduce_by_key_config<192, + 7, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -2840,7 +2840,7 @@ struct default_reduce_by_key_config< && !bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : reduce_by_key_config<192, - 15, + 12, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -2857,7 +2857,7 @@ struct default_reduce_by_key_config< && !bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : reduce_by_key_config<192, - 15, + 13, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -2874,6 +2874,479 @@ struct default_reduce_by_key_config< && !bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1))>> : reduce_by_key_config<192, + 13, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int64_t, value_type = double +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 8) && (sizeof(key_type) > 4) + && bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int64_t, value_type = float +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 8) && (sizeof(key_type) > 4) + && bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int64_t, value_type = rocprim::half +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 8) && (sizeof(key_type) > 4) + && bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int64_t, value_type = int64_t +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 8) && (sizeof(key_type) > 4) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int64_t, value_type = int +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 8) && (sizeof(key_type) > 4) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int64_t, value_type = short +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 8) && (sizeof(key_type) > 4) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int64_t, value_type = int8_t +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 8) && (sizeof(key_type) > 4) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int, value_type = double +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 4) && (sizeof(key_type) > 2) + && bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int, value_type = float +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 4) && (sizeof(key_type) > 2) + && bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int, value_type = rocprim::half +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 4) && (sizeof(key_type) > 2) + && bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int, value_type = int64_t +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 4) && (sizeof(key_type) > 2) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int, value_type = int +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 4) && (sizeof(key_type) > 2) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int, value_type = short +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 4) && (sizeof(key_type) > 2) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int, value_type = int8_t +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 4) && (sizeof(key_type) > 2) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = short, value_type = double +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 2) && (sizeof(key_type) > 1) + && bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = short, value_type = float +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 2) && (sizeof(key_type) > 1) + && bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = short, value_type = rocprim::half +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 2) && (sizeof(key_type) > 1) + && bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = short, value_type = int64_t +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 2) && (sizeof(key_type) > 1) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = short, value_type = int +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 2) && (sizeof(key_type) > 1) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = short, value_type = short +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 2) && (sizeof(key_type) > 1) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = short, value_type = int8_t +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 2) && (sizeof(key_type) > 1) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int8_t, value_type = double +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 1) && bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int8_t, value_type = float +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 1) && bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int8_t, value_type = rocprim::half +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 1) && bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int8_t, value_type = int64_t +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 1) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int8_t, value_type = int +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 1) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int8_t, value_type = short +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 1) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int8_t, value_type = int8_t +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 1) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1))>> + : reduce_by_key_config<512, 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, diff --git a/rocprim/include/rocprim/device/detail/config/device_run_length_encode.hpp b/rocprim/include/rocprim/device/detail/config/device_run_length_encode.hpp index d41b7acef..1b473943f 100644 --- a/rocprim/include/rocprim/device/detail/config/device_run_length_encode.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_run_length_encode.hpp @@ -628,7 +628,7 @@ struct default_trivial_runs_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2))>> : reduce_by_key_config<256, - 14, + 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -643,8 +643,8 @@ struct default_trivial_runs_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2))>> - : reduce_by_key_config<512, - 10, + : reduce_by_key_config<192, + 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -676,7 +676,7 @@ struct default_trivial_runs_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2))>> : reduce_by_key_config<256, - 14, + 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -691,8 +691,8 @@ struct default_trivial_runs_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1))>> - : reduce_by_key_config<512, - 10, + : reduce_by_key_config<192, + 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, block_scan_algorithm::using_warp_scan, @@ -707,7 +707,119 @@ struct default_trivial_runs_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1))>> - : reduce_by_key_config<192, + : reduce_by_key_config<256, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = double +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = float +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = rocprim::half +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 2))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int64_t +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = short +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1))>> + : reduce_by_key_config<512, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan, + 1> +{}; + +// Based on key_type = int8_t +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 1))>> + : reduce_by_key_config<512, 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, diff --git a/rocprim/include/rocprim/device/detail/config/device_run_length_encode_non_trivial.hpp b/rocprim/include/rocprim/device/detail/config/device_run_length_encode_non_trivial.hpp index 78ed92ca0..a8b6c105f 100644 --- a/rocprim/include/rocprim/device/detail/config/device_run_length_encode_non_trivial.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_run_length_encode_non_trivial.hpp @@ -499,7 +499,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = double -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx90a), key_type, @@ -507,12 +507,12 @@ struct default_non_trivial_runs_config< && (sizeof(key_type) > 4))>> : non_trivial_runs_config<64, 16, - ::rocprim::block_load_method::block_load_warp_transpose, + ::rocprim::block_load_method::block_load_vectorize, ::rocprim::block_scan_algorithm::using_warp_scan> {}; // Based on key_type = float -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx90a), key_type, @@ -525,20 +525,20 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = rocprim::half -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx90a), key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2))>> - : non_trivial_runs_config<64, - 64, + : non_trivial_runs_config<256, + 16, ::rocprim::block_load_method::block_load_vectorize, ::rocprim::block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx90a), key_type, @@ -546,12 +546,12 @@ struct default_non_trivial_runs_config< && (sizeof(key_type) > 4))>> : non_trivial_runs_config<64, 16, - ::rocprim::block_load_method::block_load_warp_transpose, + ::rocprim::block_load_method::block_load_vectorize, ::rocprim::block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx90a), key_type, @@ -564,7 +564,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = short -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx90a), key_type, @@ -577,7 +577,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = int8_t -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx90a), key_type, @@ -589,6 +589,97 @@ struct default_non_trivial_runs_config< ::rocprim::block_scan_algorithm::using_warp_scan> {}; +// Based on key_type = double +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx942), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = float +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx942), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::half +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx942), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 2))>> + : non_trivial_runs_config<128, + 32, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int64_t +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx942), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx942), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = short +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx942), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int8_t +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx942), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 1))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + } // end namespace detail END_ROCPRIM_NAMESPACE diff --git a/rocprim/include/rocprim/device/detail/config/device_scan.hpp b/rocprim/include/rocprim/device/detail/config/device_scan.hpp index c402b2178..44d65fce0 100644 --- a/rocprim/include/rocprim/device/detail/config/device_scan.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_scan.hpp @@ -529,11 +529,11 @@ struct default_scan_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_config<256, - 6, + : scan_config<128, + 10, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on value_type = float @@ -543,7 +543,7 @@ struct default_scan_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_config<256, + : scan_config<128, 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, @@ -556,8 +556,8 @@ struct default_scan_config(target_arch::gfx90a), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2))>> - : scan_config<256, - 18, + : scan_config<64, + 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -570,8 +570,8 @@ struct default_scan_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_config<256, - 6, + : scan_config<64, + 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -584,8 +584,104 @@ struct default_scan_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : scan_config<128, + 10, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on value_type = short +template +struct default_scan_config< + static_cast(target_arch::gfx90a), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_config<256, - 14, + 10, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on value_type = int8_t +template +struct default_scan_config(target_arch::gfx90a), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1))>> + : scan_config<256, + 24, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on value_type = double +template +struct default_scan_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : scan_config<256, + 24, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on value_type = float +template +struct default_scan_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : scan_config<256, + 21, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on value_type = rocprim::half +template +struct default_scan_config(target_arch::gfx942), + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2))>> + : scan_config<256, + 24, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on value_type = int64_t +template +struct default_scan_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : scan_config<256, + 24, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on value_type = int +template +struct default_scan_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : scan_config<256, + 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -594,12 +690,12 @@ struct default_scan_config< // Based on value_type = short template struct default_scan_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::gfx942), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_config<128, - 14, + : scan_config<256, + 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -607,7 +703,7 @@ struct default_scan_config< // Based on value_type = int8_t template -struct default_scan_config(target_arch::gfx90a), +struct default_scan_config(target_arch::gfx942), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1))>> diff --git a/rocprim/include/rocprim/device/detail/config/device_scan_by_key.hpp b/rocprim/include/rocprim/device/detail/config/device_scan_by_key.hpp index f5b10bce7..037a224f0 100644 --- a/rocprim/include/rocprim/device/detail/config/device_scan_by_key.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_scan_by_key.hpp @@ -2226,8 +2226,8 @@ struct default_scan_by_key_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_by_key_config<256, - 14, + : scan_by_key_config<64, + 19, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -2246,7 +2246,7 @@ struct default_scan_by_key_config< 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = double, value_type = short @@ -2259,10 +2259,10 @@ struct default_scan_by_key_config< && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<64, - 14, + 19, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = double, value_type = int8_t @@ -2273,7 +2273,7 @@ struct default_scan_by_key_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1))>> - : scan_by_key_config<64, + : scan_by_key_config<256, 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, @@ -2289,8 +2289,8 @@ struct default_scan_by_key_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_by_key_config<128, - 10, + : scan_by_key_config<64, + 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -2305,11 +2305,11 @@ struct default_scan_by_key_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_by_key_config<256, - 12, + : scan_by_key_config<128, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = float, value_type = short @@ -2321,8 +2321,8 @@ struct default_scan_by_key_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_by_key_config<256, - 14, + : scan_by_key_config<64, + 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -2340,7 +2340,7 @@ struct default_scan_by_key_config< 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = rocprim::half, value_type = int64_t @@ -2352,7 +2352,7 @@ struct default_scan_by_key_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_by_key_config<64, - 18, + 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -2367,10 +2367,10 @@ struct default_scan_by_key_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<256, - 18, + 9, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = rocprim::half, value_type = short @@ -2382,7 +2382,7 @@ struct default_scan_by_key_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<256, - 20, + 22, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -2397,10 +2397,10 @@ struct default_scan_by_key_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1))>> : scan_by_key_config<256, - 24, + 18, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = int64_t, value_type = int64_t @@ -2412,11 +2412,11 @@ struct default_scan_by_key_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_by_key_config<256, - 14, + : scan_by_key_config<64, + 19, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = int @@ -2432,7 +2432,7 @@ struct default_scan_by_key_config< 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = short @@ -2445,7 +2445,7 @@ struct default_scan_by_key_config< && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<64, - 14, + 19, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -2463,7 +2463,7 @@ struct default_scan_by_key_config< 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = int, value_type = int64_t @@ -2475,11 +2475,11 @@ struct default_scan_by_key_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_by_key_config<128, - 10, + : scan_by_key_config<64, + 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = int, value_type = int @@ -2492,10 +2492,10 @@ struct default_scan_by_key_config< && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<256, - 12, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = short @@ -2507,8 +2507,8 @@ struct default_scan_by_key_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_by_key_config<256, - 14, + : scan_by_key_config<64, + 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -2526,7 +2526,7 @@ struct default_scan_by_key_config< 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = short, value_type = int64_t @@ -2539,10 +2539,10 @@ struct default_scan_by_key_config< && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_by_key_config<64, - 18, + 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int @@ -2555,10 +2555,10 @@ struct default_scan_by_key_config< && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<256, - 18, + 9, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = short, value_type = short @@ -2571,7 +2571,7 @@ struct default_scan_by_key_config< && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<256, - 20, + 22, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -2586,10 +2586,10 @@ struct default_scan_by_key_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1))>> : scan_by_key_config<256, - 24, + 18, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = int8_t, value_type = int64_t @@ -2601,10 +2601,10 @@ struct default_scan_by_key_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_by_key_config<64, - 18, + 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = int8_t, value_type = int @@ -2616,7 +2616,7 @@ struct default_scan_by_key_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<128, - 18, + 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -2631,7 +2631,7 @@ struct default_scan_by_key_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<256, - 20, + 18, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -2652,6 +2652,441 @@ struct default_scan_by_key_config< block_scan_algorithm::reduce_then_scan> {}; +// Based on key_type = double, value_type = int64_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : scan_by_key_config<256, + 13, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = double, value_type = int +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : scan_by_key_config<256, + 21, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = double, value_type = short +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : scan_by_key_config<256, + 24, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = double, value_type = int8_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1))>> + : scan_by_key_config<256, + 14, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = float, value_type = int64_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : scan_by_key_config<256, + 21, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = float, value_type = int +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : scan_by_key_config<256, + 19, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = float, value_type = short +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : scan_by_key_config<256, + 19, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = float, value_type = int8_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1))>> + : scan_by_key_config<256, + 19, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::half, value_type = int64_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : scan_by_key_config<256, + 24, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = rocprim::half, value_type = int +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : scan_by_key_config<256, + 21, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::half, value_type = short +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : scan_by_key_config<256, + 22, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::half, value_type = int8_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 1))>> + : scan_by_key_config<256, + 23, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = int64_t, value_type = int64_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : scan_by_key_config<256, + 13, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int64_t, value_type = int +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : scan_by_key_config<256, + 21, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int64_t, value_type = short +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : scan_by_key_config<256, + 24, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int64_t, value_type = int8_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1))>> + : scan_by_key_config<256, + 14, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int, value_type = int64_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : scan_by_key_config<256, + 21, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = int, value_type = int +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : scan_by_key_config<256, + 19, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int, value_type = short +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : scan_by_key_config<256, + 19, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int, value_type = int8_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1))>> + : scan_by_key_config<256, + 19, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = short, value_type = int64_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : scan_by_key_config<256, + 24, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = short, value_type = int +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : scan_by_key_config<256, + 21, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = short, value_type = short +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : scan_by_key_config<256, + 22, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = short, value_type = int8_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1))>> + : scan_by_key_config<256, + 23, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = int8_t, value_type = int64_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : scan_by_key_config<256, + 24, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = int8_t, value_type = int +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : scan_by_key_config<256, + 21, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int8_t, value_type = short +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : scan_by_key_config<256, + 24, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int8_t, value_type = int8_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 1))>> + : scan_by_key_config<256, + 24, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + } // end namespace detail END_ROCPRIM_NAMESPACE diff --git a/rocprim/include/rocprim/device/detail/config/device_select_flag.hpp b/rocprim/include/rocprim/device/detail/config/device_select_flag.hpp index e4a97e417..3ffd1217c 100644 --- a/rocprim/include/rocprim/device/detail/config/device_select_flag.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_select_flag.hpp @@ -379,7 +379,7 @@ struct default_select_flag_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<128, 7> + && (sizeof(data_type) > 4))>> : select_config<192, 4> {}; // Based on data_type = float @@ -388,7 +388,7 @@ struct default_select_flag_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<512, 12> + && (sizeof(data_type) > 2))>> : select_config<256, 7> {}; // Based on data_type = rocprim::half @@ -397,7 +397,7 @@ struct default_select_flag_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 24> + && (sizeof(data_type) <= 2))>> : select_config<256, 20> {}; // Based on data_type = int64_t @@ -407,7 +407,7 @@ struct default_select_flag_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<128, 7> + : select_config<192, 4> {}; // Based on data_type = int @@ -417,7 +417,7 @@ struct default_select_flag_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<512, 12> + : select_config<256, 7> {}; // Based on data_type = short @@ -427,7 +427,7 @@ struct default_select_flag_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 24> + : select_config<256, 14> {}; // Based on data_type = int8_t @@ -439,6 +439,72 @@ struct default_select_flag_config< && (sizeof(data_type) <= 1))>> : select_config<256, 24> {}; +// Based on data_type = double +template +struct default_select_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) + && (sizeof(data_type) > 4))>> : select_config<512, 7> +{}; + +// Based on data_type = float +template +struct default_select_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) + && (sizeof(data_type) > 2))>> : select_config<512, 15> +{}; + +// Based on data_type = rocprim::half +template +struct default_select_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2))>> : select_config<512, 28> +{}; + +// Based on data_type = int64_t +template +struct default_select_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> + : select_config<512, 7> +{}; + +// Based on data_type = int +template +struct default_select_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> + : select_config<512, 15> +{}; + +// Based on data_type = short +template +struct default_select_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> + : select_config<512, 30> +{}; + +// Based on data_type = int8_t +template +struct default_select_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 1))>> : select_config<512, 24> +{}; + } // end namespace detail END_ROCPRIM_NAMESPACE diff --git a/rocprim/include/rocprim/device/detail/config/device_select_predicate.hpp b/rocprim/include/rocprim/device/detail/config/device_select_predicate.hpp index ae05a7773..84c2e635c 100644 --- a/rocprim/include/rocprim/device/detail/config/device_select_predicate.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_select_predicate.hpp @@ -379,7 +379,7 @@ struct default_select_predicate_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<128, 7> + && (sizeof(data_type) > 4))>> : select_config<128, 6> {}; // Based on data_type = float @@ -388,7 +388,7 @@ struct default_select_predicate_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<256, 15> + && (sizeof(data_type) > 2))>> : select_config<512, 10> {}; // Based on data_type = rocprim::half @@ -397,7 +397,7 @@ struct default_select_predicate_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 26> + && (sizeof(data_type) <= 2))>> : select_config<256, 20> {}; // Based on data_type = int64_t @@ -407,7 +407,7 @@ struct default_select_predicate_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<128, 7> + : select_config<512, 5> {}; // Based on data_type = int @@ -417,7 +417,7 @@ struct default_select_predicate_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<256, 14> + : select_config<512, 10> {}; // Based on data_type = short @@ -427,7 +427,7 @@ struct default_select_predicate_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 26> + : select_config<256, 20> {}; // Based on data_type = int8_t @@ -439,6 +439,72 @@ struct default_select_predicate_config< && (sizeof(data_type) <= 1))>> : select_config<256, 24> {}; +// Based on data_type = double +template +struct default_select_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) + && (sizeof(data_type) > 4))>> : select_config<512, 7> +{}; + +// Based on data_type = float +template +struct default_select_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) + && (sizeof(data_type) > 2))>> : select_config<512, 15> +{}; + +// Based on data_type = rocprim::half +template +struct default_select_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2))>> : select_config<512, 30> +{}; + +// Based on data_type = int64_t +template +struct default_select_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> + : select_config<512, 7> +{}; + +// Based on data_type = int +template +struct default_select_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> + : select_config<512, 15> +{}; + +// Based on data_type = short +template +struct default_select_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> + : select_config<512, 28> +{}; + +// Based on data_type = int8_t +template +struct default_select_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 1))>> : select_config<512, 24> +{}; + } // end namespace detail END_ROCPRIM_NAMESPACE diff --git a/rocprim/include/rocprim/device/detail/config/device_select_predicated_flag.hpp b/rocprim/include/rocprim/device/detail/config/device_select_predicated_flag.hpp index 9b093ae3b..79acf6b67 100644 --- a/rocprim/include/rocprim/device/detail/config/device_select_predicated_flag.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_select_predicated_flag.hpp @@ -1631,7 +1631,7 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 8) - && (sizeof(flag_type) > 4))>> : select_config<128, 6> + && (sizeof(flag_type) > 4))>> : select_config<192, 4> {}; // Based on data_type = double, flag_type = int @@ -1642,7 +1642,7 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 4) - && (sizeof(flag_type) > 2))>> : select_config<128, 6> + && (sizeof(flag_type) > 2))>> : select_config<512, 5> {}; // Based on data_type = double, flag_type = short @@ -1653,7 +1653,7 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 2) - && (sizeof(flag_type) > 1))>> : select_config<128, 6> + && (sizeof(flag_type) > 1))>> : select_config<192, 4> {}; // Based on data_type = double, flag_type = int8_t @@ -1664,7 +1664,7 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 1))>> - : select_config<128, 7> + : select_config<192, 4> {}; // Based on data_type = float, flag_type = int64_t @@ -1675,7 +1675,7 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 8) - && (sizeof(flag_type) > 4))>> : select_config<256, 6> + && (sizeof(flag_type) > 4))>> : select_config<192, 5> {}; // Based on data_type = float, flag_type = int @@ -1686,7 +1686,7 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 4) - && (sizeof(flag_type) > 2))>> : select_config<128, 11> + && (sizeof(flag_type) > 2))>> : select_config<256, 6> {}; // Based on data_type = float, flag_type = short @@ -1697,7 +1697,7 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 2) - && (sizeof(flag_type) > 1))>> : select_config<256, 10> + && (sizeof(flag_type) > 1))>> : select_config<192, 10> {}; // Based on data_type = float, flag_type = int8_t @@ -1708,7 +1708,7 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 1))>> - : select_config<256, 11> + : select_config<256, 7> {}; // Based on data_type = rocprim::half, flag_type = int64_t @@ -1730,7 +1730,7 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> - : select_config<256, 12> + : select_config<256, 10> {}; // Based on data_type = rocprim::half, flag_type = short @@ -1741,7 +1741,7 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> - : select_config<512, 18> + : select_config<256, 18> {}; // Based on data_type = rocprim::half, flag_type = int8_t @@ -1751,7 +1751,7 @@ struct default_select_predicated_flag_config< data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) - && (sizeof(flag_type) <= 1))>> : select_config<256, 18> + && (sizeof(flag_type) <= 1))>> : select_config<256, 20> {}; // Based on data_type = int64_t, flag_type = int64_t @@ -1763,7 +1763,7 @@ struct default_select_predicated_flag_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 8) && (sizeof(flag_type) > 4))>> - : select_config<128, 6> + : select_config<192, 4> {}; // Based on data_type = int64_t, flag_type = int @@ -1775,7 +1775,7 @@ struct default_select_predicated_flag_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> - : select_config<128, 6> + : select_config<192, 4> {}; // Based on data_type = int64_t, flag_type = short @@ -1787,7 +1787,7 @@ struct default_select_predicated_flag_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> - : select_config<128, 6> + : select_config<192, 4> {}; // Based on data_type = int64_t, flag_type = int8_t @@ -1798,7 +1798,7 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) - && (sizeof(flag_type) <= 1))>> : select_config<128, 7> + && (sizeof(flag_type) <= 1))>> : select_config<192, 5> {}; // Based on data_type = int, flag_type = int64_t @@ -1810,7 +1810,7 @@ struct default_select_predicated_flag_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 8) && (sizeof(flag_type) > 4))>> - : select_config<256, 6> + : select_config<192, 5> {}; // Based on data_type = int, flag_type = int @@ -1822,7 +1822,7 @@ struct default_select_predicated_flag_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> - : select_config<128, 11> + : select_config<256, 6> {}; // Based on data_type = int, flag_type = short @@ -1834,7 +1834,7 @@ struct default_select_predicated_flag_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> - : select_config<256, 10> + : select_config<192, 10> {}; // Based on data_type = int, flag_type = int8_t @@ -1845,7 +1845,7 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) - && (sizeof(flag_type) <= 1))>> : select_config<256, 11> + && (sizeof(flag_type) <= 1))>> : select_config<192, 10> {}; // Based on data_type = short, flag_type = int64_t @@ -1869,7 +1869,7 @@ struct default_select_predicated_flag_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1) && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> - : select_config<256, 12> + : select_config<256, 10> {}; // Based on data_type = short, flag_type = short @@ -1881,7 +1881,7 @@ struct default_select_predicated_flag_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1) && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> - : select_config<512, 18> + : select_config<256, 20> {}; // Based on data_type = short, flag_type = int8_t @@ -1892,7 +1892,7 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1) - && (sizeof(flag_type) <= 1))>> : select_config<256, 18> + && (sizeof(flag_type) <= 1))>> : select_config<256, 20> {}; // Based on data_type = int8_t, flag_type = int64_t @@ -1903,7 +1903,7 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 1) && (sizeof(flag_type) <= 8) - && (sizeof(flag_type) > 4))>> : select_config<256, 6> + && (sizeof(flag_type) > 4))>> : select_config<192, 5> {}; // Based on data_type = int8_t, flag_type = int @@ -1914,7 +1914,7 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 1) && (sizeof(flag_type) <= 4) - && (sizeof(flag_type) > 2))>> : select_config<256, 11> + && (sizeof(flag_type) > 2))>> : select_config<256, 12> {}; // Based on data_type = int8_t, flag_type = short @@ -1925,7 +1925,7 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 1) && (sizeof(flag_type) <= 2) - && (sizeof(flag_type) > 1))>> : select_config<256, 24> + && (sizeof(flag_type) > 1))>> : select_config<192, 20> {}; // Based on data_type = int8_t, flag_type = int8_t @@ -1939,6 +1939,322 @@ struct default_select_predicated_flag_config< : select_config<256, 24> {}; +// Based on data_type = double, flag_type = int64_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) + && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 8) + && (sizeof(flag_type) > 4))>> : select_config<512, 5> +{}; + +// Based on data_type = double, flag_type = int +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) + && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 4) + && (sizeof(flag_type) > 2))>> : select_config<512, 7> +{}; + +// Based on data_type = double, flag_type = short +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) + && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 2) + && (sizeof(flag_type) > 1))>> : select_config<512, 7> +{}; + +// Based on data_type = double, flag_type = int8_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) + && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 1))>> + : select_config<512, 7> +{}; + +// Based on data_type = float, flag_type = int64_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) + && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 8) + && (sizeof(flag_type) > 4))>> : select_config<512, 7> +{}; + +// Based on data_type = float, flag_type = int +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) + && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 4) + && (sizeof(flag_type) > 2))>> : select_config<512, 15> +{}; + +// Based on data_type = float, flag_type = short +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) + && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 2) + && (sizeof(flag_type) > 1))>> : select_config<512, 14> +{}; + +// Based on data_type = float, flag_type = int8_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) + && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 1))>> + : select_config<512, 15> +{}; + +// Based on data_type = rocprim::half, flag_type = int64_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) + && (sizeof(flag_type) <= 8) && (sizeof(flag_type) > 4))>> + : select_config<512, 7> +{}; + +// Based on data_type = rocprim::half, flag_type = int +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) + && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> + : select_config<512, 14> +{}; + +// Based on data_type = rocprim::half, flag_type = short +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) + && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> + : select_config<512, 30> +{}; + +// Based on data_type = rocprim::half, flag_type = int8_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) + && (sizeof(flag_type) <= 1))>> : select_config<512, 20> +{}; + +// Based on data_type = int64_t, flag_type = int64_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) + && (sizeof(flag_type) <= 8) && (sizeof(flag_type) > 4))>> + : select_config<512, 5> +{}; + +// Based on data_type = int64_t, flag_type = int +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) + && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> + : select_config<512, 7> +{}; + +// Based on data_type = int64_t, flag_type = short +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) + && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> + : select_config<512, 7> +{}; + +// Based on data_type = int64_t, flag_type = int8_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) + && (sizeof(flag_type) <= 1))>> : select_config<512, 5> +{}; + +// Based on data_type = int, flag_type = int64_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) + && (sizeof(flag_type) <= 8) && (sizeof(flag_type) > 4))>> + : select_config<512, 7> +{}; + +// Based on data_type = int, flag_type = int +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) + && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> + : select_config<512, 15> +{}; + +// Based on data_type = int, flag_type = short +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) + && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> + : select_config<512, 14> +{}; + +// Based on data_type = int, flag_type = int8_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) + && (sizeof(flag_type) <= 1))>> : select_config<512, 15> +{}; + +// Based on data_type = short, flag_type = int64_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1) + && (sizeof(flag_type) <= 8) && (sizeof(flag_type) > 4))>> + : select_config<512, 7> +{}; + +// Based on data_type = short, flag_type = int +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1) + && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> + : select_config<512, 14> +{}; + +// Based on data_type = short, flag_type = short +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1) + && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> + : select_config<512, 30> +{}; + +// Based on data_type = short, flag_type = int8_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1) + && (sizeof(flag_type) <= 1))>> : select_config<512, 20> +{}; + +// Based on data_type = int8_t, flag_type = int64_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 1) && (sizeof(flag_type) <= 8) + && (sizeof(flag_type) > 4))>> : select_config<512, 7> +{}; + +// Based on data_type = int8_t, flag_type = int +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 1) && (sizeof(flag_type) <= 4) + && (sizeof(flag_type) > 2))>> : select_config<512, 15> +{}; + +// Based on data_type = int8_t, flag_type = short +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 1) && (sizeof(flag_type) <= 2) + && (sizeof(flag_type) > 1))>> : select_config<512, 20> +{}; + +// Based on data_type = int8_t, flag_type = int8_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 1) && (sizeof(flag_type) <= 1))>> + : select_config<512, 24> +{}; + } // end namespace detail END_ROCPRIM_NAMESPACE @@ -1946,4 +2262,4 @@ END_ROCPRIM_NAMESPACE /// @} // end of group primitivesmodule_deviceconfigs -#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_SELECT_PREDICATED_FLAG_HPP_ \ No newline at end of file +#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_SELECT_PREDICATED_FLAG_HPP_ diff --git a/rocprim/include/rocprim/device/detail/config/device_select_unique.hpp b/rocprim/include/rocprim/device/detail/config/device_select_unique.hpp index 71d0fb28a..2ed70b13c 100644 --- a/rocprim/include/rocprim/device/detail/config/device_select_unique.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_select_unique.hpp @@ -379,7 +379,7 @@ struct default_select_unique_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<512, 7> + && (sizeof(data_type) > 4))>> : select_config<256, 5> {}; // Based on data_type = float @@ -388,7 +388,7 @@ struct default_select_unique_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<512, 11> + && (sizeof(data_type) > 2))>> : select_config<256, 10> {}; // Based on data_type = rocprim::half @@ -397,7 +397,7 @@ struct default_select_unique_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 30> + && (sizeof(data_type) <= 2))>> : select_config<256, 22> {}; // Based on data_type = int64_t @@ -407,7 +407,7 @@ struct default_select_unique_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<512, 7> + : select_config<256, 5> {}; // Based on data_type = int @@ -417,7 +417,7 @@ struct default_select_unique_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<512, 11> + : select_config<256, 10> {}; // Based on data_type = short @@ -427,7 +427,7 @@ struct default_select_unique_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 30> + : select_config<256, 22> {}; // Based on data_type = int8_t @@ -436,7 +436,73 @@ struct default_select_unique_config< static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 1))>> : select_config<256, 28> + && (sizeof(data_type) <= 1))>> : select_config<128, 32> +{}; + +// Based on data_type = double +template +struct default_select_unique_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) + && (sizeof(data_type) > 4))>> : select_config<512, 5> +{}; + +// Based on data_type = float +template +struct default_select_unique_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) + && (sizeof(data_type) > 2))>> : select_config<512, 15> +{}; + +// Based on data_type = rocprim::half +template +struct default_select_unique_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2))>> : select_config<512, 30> +{}; + +// Based on data_type = int64_t +template +struct default_select_unique_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> + : select_config<512, 7> +{}; + +// Based on data_type = int +template +struct default_select_unique_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> + : select_config<512, 15> +{}; + +// Based on data_type = short +template +struct default_select_unique_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> + : select_config<512, 30> +{}; + +// Based on data_type = int8_t +template +struct default_select_unique_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 1))>> : select_config<512, 32> {}; } // end namespace detail diff --git a/rocprim/include/rocprim/device/detail/config/device_select_unique_by_key.hpp b/rocprim/include/rocprim/device/detail/config/device_select_unique_by_key.hpp index 0cfaf26ab..423e88f5d 100644 --- a/rocprim/include/rocprim/device/detail/config/device_select_unique_by_key.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_select_unique_by_key.hpp @@ -1581,7 +1581,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : select_config<128, 5> + && (sizeof(value_type) > 4))>> : select_config<128, 4> {}; // Based on key_type = double, value_type = int @@ -1592,7 +1592,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : select_config<128, 6> + && (sizeof(value_type) > 2))>> : select_config<192, 4> {}; // Based on key_type = double, value_type = short @@ -1603,7 +1603,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : select_config<128, 6> + && (sizeof(value_type) > 1))>> : select_config<256, 4> {}; // Based on key_type = double, value_type = int8_t @@ -1614,7 +1614,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1))>> - : select_config<192, 8> + : select_config<256, 5> {}; // Based on key_type = float, value_type = int64_t @@ -1625,7 +1625,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : select_config<128, 6> + && (sizeof(value_type) > 4))>> : select_config<192, 5> {}; // Based on key_type = float, value_type = int @@ -1636,7 +1636,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : select_config<256, 13> + && (sizeof(value_type) > 2))>> : select_config<192, 7> {}; // Based on key_type = float, value_type = short @@ -1647,7 +1647,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : select_config<256, 14> + && (sizeof(value_type) > 1))>> : select_config<192, 10> {}; // Based on key_type = float, value_type = int8_t @@ -1658,7 +1658,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1))>> - : select_config<256, 16> + : select_config<192, 9> {}; // Based on key_type = rocprim::half, value_type = int64_t @@ -1669,7 +1669,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : select_config<256, 6> + : select_config<192, 5> {}; // Based on key_type = rocprim::half, value_type = int @@ -1680,7 +1680,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : select_config<256, 12> + : select_config<192, 10> {}; // Based on key_type = rocprim::half, value_type = short @@ -1691,7 +1691,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : select_config<256, 14> + : select_config<256, 24> {}; // Based on key_type = rocprim::half, value_type = int8_t @@ -1701,7 +1701,7 @@ struct default_select_unique_by_key_config< key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(value_type) <= 1))>> : select_config<256, 16> + && (sizeof(value_type) <= 1))>> : select_config<256, 24> {}; // Based on key_type = int64_t, value_type = int64_t @@ -1712,7 +1712,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : select_config<128, 5> + && (sizeof(value_type) > 4))>> : select_config<128, 4> {}; // Based on key_type = int64_t, value_type = int @@ -1723,7 +1723,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : select_config<128, 6> + && (sizeof(value_type) > 2))>> : select_config<192, 4> {}; // Based on key_type = int64_t, value_type = short @@ -1734,7 +1734,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : select_config<192, 7> + && (sizeof(value_type) > 1))>> : select_config<256, 4> {}; // Based on key_type = int64_t, value_type = int8_t @@ -1745,7 +1745,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1))>> - : select_config<192, 8> + : select_config<256, 5> {}; // Based on key_type = int, value_type = int64_t @@ -1756,7 +1756,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : select_config<128, 6> + && (sizeof(value_type) > 4))>> : select_config<192, 5> {}; // Based on key_type = int, value_type = int @@ -1767,7 +1767,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : select_config<256, 13> + && (sizeof(value_type) > 2))>> : select_config<192, 7> {}; // Based on key_type = int, value_type = short @@ -1778,7 +1778,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : select_config<256, 14> + && (sizeof(value_type) > 1))>> : select_config<192, 10> {}; // Based on key_type = int, value_type = int8_t @@ -1789,7 +1789,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1))>> - : select_config<192, 16> + : select_config<192, 9> {}; // Based on key_type = short, value_type = int64_t @@ -1800,7 +1800,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : select_config<256, 6> + && (sizeof(value_type) > 4))>> : select_config<192, 5> {}; // Based on key_type = short, value_type = int @@ -1811,7 +1811,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : select_config<256, 12> + && (sizeof(value_type) > 2))>> : select_config<192, 10> {}; // Based on key_type = short, value_type = short @@ -1822,7 +1822,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : select_config<256, 14> + && (sizeof(value_type) > 1))>> : select_config<256, 24> {}; // Based on key_type = short, value_type = int8_t @@ -1833,7 +1833,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1))>> - : select_config<256, 14> + : select_config<256, 24> {}; // Based on key_type = int8_t, value_type = int64_t @@ -1844,7 +1844,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : select_config<192, 7> + : select_config<256, 6> {}; // Based on key_type = int8_t, value_type = int @@ -1855,7 +1855,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : select_config<256, 13> + : select_config<192, 10> {}; // Based on key_type = int8_t, value_type = short @@ -1866,7 +1866,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : select_config<256, 14> + : select_config<256, 28> {}; // Based on key_type = int8_t, value_type = int8_t @@ -1876,7 +1876,313 @@ struct default_select_unique_by_key_config< key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) - && (sizeof(value_type) <= 1))>> : select_config<192, 17> + && (sizeof(value_type) <= 1))>> : select_config<192, 28> +{}; + +// Based on key_type = double, value_type = int64_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : select_config<512, 5> +{}; + +// Based on key_type = double, value_type = int +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : select_config<512, 7> +{}; + +// Based on key_type = double, value_type = short +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : select_config<512, 7> +{}; + +// Based on key_type = double, value_type = int8_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1))>> + : select_config<512, 5> +{}; + +// Based on key_type = float, value_type = int64_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : select_config<512, 7> +{}; + +// Based on key_type = float, value_type = int +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : select_config<512, 15> +{}; + +// Based on key_type = float, value_type = short +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : select_config<512, 14> +{}; + +// Based on key_type = float, value_type = int8_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1))>> + : select_config<512, 15> +{}; + +// Based on key_type = rocprim::half, value_type = int64_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : select_config<512, 7> +{}; + +// Based on key_type = rocprim::half, value_type = int +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : select_config<512, 14> +{}; + +// Based on key_type = rocprim::half, value_type = short +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : select_config<512, 22> +{}; + +// Based on key_type = rocprim::half, value_type = int8_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 1))>> : select_config<512, 24> +{}; + +// Based on key_type = int64_t, value_type = int64_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : select_config<512, 5> +{}; + +// Based on key_type = int64_t, value_type = int +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : select_config<512, 7> +{}; + +// Based on key_type = int64_t, value_type = short +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : select_config<512, 7> +{}; + +// Based on key_type = int64_t, value_type = int8_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1))>> + : select_config<512, 5> +{}; + +// Based on key_type = int, value_type = int64_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : select_config<512, 7> +{}; + +// Based on key_type = int, value_type = int +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : select_config<512, 15> +{}; + +// Based on key_type = int, value_type = short +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : select_config<512, 14> +{}; + +// Based on key_type = int, value_type = int8_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1))>> + : select_config<512, 15> +{}; + +// Based on key_type = short, value_type = int64_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : select_config<512, 7> +{}; + +// Based on key_type = short, value_type = int +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : select_config<512, 14> +{}; + +// Based on key_type = short, value_type = short +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : select_config<512, 22> +{}; + +// Based on key_type = short, value_type = int8_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1))>> + : select_config<512, 24> +{}; + +// Based on key_type = int8_t, value_type = int64_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : select_config<512, 7> +{}; + +// Based on key_type = int8_t, value_type = int +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : select_config<512, 15> +{}; + +// Based on key_type = int8_t, value_type = short +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : select_config<512, 24> +{}; + +// Based on key_type = int8_t, value_type = int8_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 1))>> : select_config<512, 24> {}; } // end namespace detail diff --git a/rocprim/include/rocprim/device/detail/config/device_transform.hpp b/rocprim/include/rocprim/device/detail/config/device_transform.hpp index 908fa398c..3eeaf3efb 100644 --- a/rocprim/include/rocprim/device/detail/config/device_transform.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_transform.hpp @@ -389,7 +389,7 @@ struct default_transform_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : transform_config<128, 1> + : transform_config<256, 2> {}; // Based on value_type = float @@ -399,7 +399,7 @@ struct default_transform_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : transform_config<128, 2> + : transform_config<1024, 2> {}; // Based on value_type = rocprim::half @@ -408,7 +408,7 @@ struct default_transform_config< static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2))>> : transform_config<128, 4> + && (sizeof(value_type) <= 2))>> : transform_config<64, 8> {}; // Based on value_type = int64_t @@ -418,7 +418,7 @@ struct default_transform_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : transform_config<128, 1> + : transform_config<256, 2> {}; // Based on value_type = int @@ -428,7 +428,7 @@ struct default_transform_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : transform_config<128, 2> + : transform_config<1024, 2> {}; // Based on value_type = short @@ -438,7 +438,7 @@ struct default_transform_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : transform_config<128, 4> + : transform_config<64, 8> {}; // Based on value_type = int8_t @@ -447,7 +447,65 @@ struct default_transform_config< static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1))>> : transform_config<128, 8> + && (sizeof(value_type) <= 1))>> : transform_config<64, 16> +{}; + +// Based on value_type = float +template +struct default_transform_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : transform_config<256, 4> +{}; + +// Based on value_type = rocprim::half +template +struct default_transform_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2))>> : transform_config<128, 16> +{}; + +// Based on value_type = int64_t +template +struct default_transform_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : transform_config<1024, 2> +{}; + +// Based on value_type = int +template +struct default_transform_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : transform_config<1024, 4> +{}; + +// Based on value_type = short +template +struct default_transform_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : transform_config<512, 8> +{}; + +// Based on value_type = int8_t +template +struct default_transform_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1))>> : transform_config<512, 16> {}; } // end namespace detail @@ -457,4 +515,4 @@ END_ROCPRIM_NAMESPACE /// @} // end of group primitivesmodule_deviceconfigs -#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_TRANSFORM_HPP_ \ No newline at end of file +#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_TRANSFORM_HPP_ diff --git a/rocprim/include/rocprim/device/detail/config/device_upper_bound.hpp b/rocprim/include/rocprim/device/detail/config/device_upper_bound.hpp index 0f552b869..b1b0096ab 100644 --- a/rocprim/include/rocprim/device/detail/config/device_upper_bound.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_upper_bound.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2023-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -1989,7 +1989,7 @@ struct default_upper_bound_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : upper_bound_config<256, 8> + : upper_bound_config<64, 4> {}; // Based on value_type = double, output_type = int @@ -2001,7 +2001,7 @@ struct default_upper_bound_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : upper_bound_config<64, 8> + : upper_bound_config<64, 4> {}; // Based on value_type = double, output_type = short @@ -2013,7 +2013,7 @@ struct default_upper_bound_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : upper_bound_config<256, 16> + : upper_bound_config<64, 4> {}; // Based on value_type = double, output_type = int8_t @@ -2024,7 +2024,7 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> + && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 4> {}; // Based on value_type = float, output_type = int64_t @@ -2082,7 +2082,7 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : upper_bound_config<64, 4> + && (sizeof(output_type) > 4))>> : upper_bound_config<128, 4> {}; // Based on value_type = rocprim::half, output_type = int @@ -2093,7 +2093,7 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : upper_bound_config<64, 4> + && (sizeof(output_type) > 2))>> : upper_bound_config<64, 2> {}; // Based on value_type = rocprim::half, output_type = short @@ -2104,7 +2104,7 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : upper_bound_config<64, 4> + && (sizeof(output_type) > 1))>> : upper_bound_config<64, 2> {}; // Based on value_type = rocprim::half, output_type = int8_t @@ -2127,7 +2127,7 @@ struct default_upper_bound_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : upper_bound_config<128, 1> + : upper_bound_config<64, 4> {}; // Based on value_type = int64_t, output_type = int @@ -2139,7 +2139,7 @@ struct default_upper_bound_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : upper_bound_config<64, 1> + : upper_bound_config<64, 4> {}; // Based on value_type = int64_t, output_type = short @@ -2151,7 +2151,7 @@ struct default_upper_bound_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : upper_bound_config<256, 16> + : upper_bound_config<64, 4> {}; // Based on value_type = int64_t, output_type = int8_t @@ -2162,7 +2162,7 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> + && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 4> {}; // Based on value_type = int, output_type = int64_t @@ -2209,7 +2209,7 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) - && (sizeof(output_type) <= 1))>> : upper_bound_config<128, 1> + && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> {}; // Based on value_type = short, output_type = int64_t @@ -2221,7 +2221,7 @@ struct default_upper_bound_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : upper_bound_config<128, 1> + : upper_bound_config<256, 1> {}; // Based on value_type = short, output_type = int @@ -2233,7 +2233,7 @@ struct default_upper_bound_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : upper_bound_config<128, 1> + : upper_bound_config<256, 1> {}; // Based on value_type = short, output_type = short @@ -2245,7 +2245,7 @@ struct default_upper_bound_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : upper_bound_config<64, 4> + : upper_bound_config<256, 1> {}; // Based on value_type = short, output_type = int8_t @@ -2256,7 +2256,7 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 1))>> : upper_bound_config<64, 4> + && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> {}; // Based on value_type = int8_t, output_type = int64_t @@ -2267,7 +2267,7 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : upper_bound_config<128, 1> + && (sizeof(output_type) > 4))>> : upper_bound_config<256, 2> {}; // Based on value_type = int8_t, output_type = int @@ -2278,7 +2278,7 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : upper_bound_config<128, 1> + && (sizeof(output_type) > 2))>> : upper_bound_config<256, 2> {}; // Based on value_type = int8_t, output_type = short @@ -2289,7 +2289,7 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : upper_bound_config<128, 1> + && (sizeof(output_type) > 1))>> : upper_bound_config<256, 2> {}; // Based on value_type = int8_t, output_type = int8_t @@ -2300,9 +2300,332 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> + : upper_bound_config<256, 2> +{}; + +// Based on value_type = double, output_type = int64_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : upper_bound_config<256, 1> +{}; + +// Based on value_type = double, output_type = int +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : upper_bound_config<256, 1> +{}; + +// Based on value_type = double, output_type = short +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : upper_bound_config<256, 1> +{}; + +// Based on value_type = double, output_type = int8_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> +{}; + +// Based on value_type = float, output_type = int64_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : upper_bound_config<256, 1> +{}; + +// Based on value_type = float, output_type = int +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : upper_bound_config<256, 1> +{}; + +// Based on value_type = float, output_type = short +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : upper_bound_config<256, 1> +{}; + +// Based on value_type = float, output_type = int8_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> +{}; + +// Based on value_type = rocprim::half, output_type = int64_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) + && (sizeof(output_type) > 4))>> : upper_bound_config<256, 8> +{}; + +// Based on value_type = rocprim::half, output_type = int +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) + && (sizeof(output_type) > 2))>> : upper_bound_config<256, 8> +{}; + +// Based on value_type = rocprim::half, output_type = short +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) + && (sizeof(output_type) > 1))>> : upper_bound_config<256, 4> +{}; + +// Based on value_type = rocprim::half, output_type = int8_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 4> {}; +// Based on value_type = int64_t, output_type = int64_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : upper_bound_config<256, 1> +{}; + +// Based on value_type = int64_t, output_type = int +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : upper_bound_config<256, 1> +{}; + +// Based on value_type = int64_t, output_type = short +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : upper_bound_config<256, 1> +{}; + +// Based on value_type = int64_t, output_type = int8_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> +{}; + +// Based on value_type = int, output_type = int64_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : upper_bound_config<256, 1> +{}; + +// Based on value_type = int, output_type = int +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : upper_bound_config<256, 1> +{}; + +// Based on value_type = int, output_type = short +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : upper_bound_config<256, 1> +{}; + +// Based on value_type = int, output_type = int8_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> +{}; + +// Based on value_type = short, output_type = int64_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : upper_bound_config<256, 1> +{}; + +// Based on value_type = short, output_type = int +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : upper_bound_config<256, 1> +{}; + +// Based on value_type = short, output_type = short +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : upper_bound_config<256, 1> +{}; + +// Based on value_type = short, output_type = int8_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> +{}; + +// Based on value_type = int8_t, output_type = int64_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) + && (sizeof(output_type) > 4))>> : upper_bound_config<256, 16> +{}; + +// Based on value_type = int8_t, output_type = int +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) + && (sizeof(output_type) > 2))>> : upper_bound_config<256, 4> +{}; + +// Based on value_type = int8_t, output_type = short +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) + && (sizeof(output_type) > 1))>> : upper_bound_config<64, 16> +{}; + +// Based on value_type = int8_t, output_type = int8_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> + : upper_bound_config<64, 16> +{}; + } // end namespace detail END_ROCPRIM_NAMESPACE diff --git a/rocprim/include/rocprim/device/detail/device_config_helper.hpp b/rocprim/include/rocprim/device/detail/device_config_helper.hpp index 50db4283b..553d290b9 100644 --- a/rocprim/include/rocprim/device/detail/device_config_helper.hpp +++ b/rocprim/include/rocprim/device/detail/device_config_helper.hpp @@ -53,7 +53,9 @@ struct merge_sort_block_sort_config_params // Necessary to construct a parameterized type of `merge_sort_block_sort_config_params`. // Used in passing to host-side sub-algorithms and GPU kernels so non-default parameters can be available during compile-time. -template +template struct merge_sort_block_sort_config : rocprim::detail::merge_sort_block_sort_config_params { using sort_config = kernel_config; @@ -964,9 +966,9 @@ struct reduce_by_key_config_params /** * \brief Configuration of device-level reduce-by-key operation. - * + * * \tparam BlockSize number of threads in a block. - * \tparam ItemsPerThread number of items processed by each thread per tile. + * \tparam ItemsPerThread number of items processed by each thread per tile. * \tparam LoadKeysMethod method of loading keys * \tparam LoadValuesMethod method of loading values * \tparam ScanAlgorithm block level scan algorithm to use @@ -1307,9 +1309,9 @@ struct merge_config_params /** * \brief Configuration of device-level merge operation. - * + * * \tparam BlockSize number of threads in a block. - * \tparam ItemsPerThread number of items processed by each thread per tile. + * \tparam ItemsPerThread number of items processed by each thread per tile. */ template struct merge_config : public detail::merge_config_params diff --git a/scripts/autotune/templates/mergesort_block_sort_config_template b/scripts/autotune/templates/mergesort_block_sort_config_template index 230f4e0a1..f3931f2c7 100644 --- a/scripts/autotune/templates/mergesort_block_sort_config_template +++ b/scripts/autotune/templates/mergesort_block_sort_config_template @@ -5,7 +5,7 @@ ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_MERGE_SORT_BLOCK_SORT_HPP_ {%- endmacro %} {% macro kernel_configuration(measurement) -%} -merge_sort_block_sort_config<{{ measurement['cfg']['bs'] }}, {{ measurement['cfg']['ipt'] }}, block_sort_algorithm::{{ measurement['cfg']['method'] }}> { }; +merge_sort_block_sort_config<{{ measurement['cfg']['bs'] }}, {{ measurement['cfg']['ipt'] }}> { }; {%- endmacro %} {% macro general_case() -%} @@ -18,6 +18,6 @@ merge_sort_block_sort_config_base::type {}; template struct default_merge_sort_block_sort_config({{ benchmark_of_architecture.name }}), key_type, value_type, {{ fallback_selection_criteria }}> : {%- endmacro %} - + From b60bbabf6e16316cc42c1dec70e15b5c9e7b771d Mon Sep 17 00:00:00 2001 From: Robin Voetter Date: Thu, 16 Jan 2025 10:34:44 +0000 Subject: [PATCH 09/84] Optimizations for MI300 unique performance --- benchmark/ConfigAutotuneSettings.cmake | 12 +- ...hmark_device_reduce_by_key.parallel.cpp.in | 4 +- ...enchmark_device_reduce_by_key.parallel.hpp | 10 +- ...k_device_run_length_encode.parallel.cpp.in | 5 +- ...mark_device_run_length_encode.parallel.hpp | 11 +- .../block/detail/block_radix_rank_match.hpp | 18 +- rocprim/include/rocprim/config.hpp | 54 +- .../config/device_adjacent_difference.hpp | 217 +- .../device_adjacent_difference_inplace.hpp | 221 +- .../detail/config/device_adjacent_find.hpp | 188 +- .../detail/config/device_binary_search.hpp | 1573 +++++-- .../detail/config/device_find_first_of.hpp | 109 +- .../device/detail/config/device_histogram.hpp | 1353 +++--- .../detail/config/device_lower_bound.hpp | 1597 ++++--- .../device/detail/config/device_merge.hpp | 1273 +++++- .../config/device_merge_sort_block_merge.hpp | 1738 ++++++-- .../config/device_merge_sort_block_sort.hpp | 2332 ++++++++--- .../detail/config/device_partition_flag.hpp | 140 +- .../config/device_partition_predicate.hpp | 144 +- .../config/device_partition_three_way.hpp | 146 +- .../config/device_partition_two_way_flag.hpp | 142 +- .../device_partition_two_way_predicate.hpp | 138 +- .../config/device_radix_sort_block_sort.hpp | 1415 ++++++- .../config/device_radix_sort_onesweep.hpp | 2124 ++++++++-- .../device/detail/config/device_reduce.hpp | 231 +- .../detail/config/device_reduce_by_key.hpp | 774 ++-- .../config/device_run_length_encode.hpp | 342 +- .../device_run_length_encode_non_trivial.hpp | 215 +- .../device/detail/config/device_scan.hpp | 321 +- .../detail/config/device_scan_by_key.hpp | 2192 ++++++++-- .../config/device_segmented_radix_sort.hpp | 3718 +++++++++++------ .../detail/config/device_segmented_reduce.hpp | 533 +++ .../detail/config/device_select_flag.hpp | 138 +- .../detail/config/device_select_predicate.hpp | 136 +- .../config/device_select_predicated_flag.hpp | 1245 +++++- .../detail/config/device_select_unique.hpp | 140 +- .../config/device_select_unique_by_key.hpp | 1238 +++++- .../device/detail/config/device_transform.hpp | 177 +- .../detail/config/device_upper_bound.hpp | 1581 +++++-- .../device/detail/device_config_helper.hpp | 6 +- .../device/detail/device_radix_sort.hpp | 156 +- .../device/detail/device_reduce_by_key.hpp | 90 +- .../device/detail/lookback_scan_state.hpp | 13 +- .../rocprim/device/device_reduce_by_key.hpp | 115 +- .../device/device_run_length_encode.hpp | 34 +- rocprim/include/rocprim/intrinsics/atomic.hpp | 160 +- scripts/autotune/templates/config_template | 2 +- .../templates/reduce_by_key_config_template | 13 +- .../run_length_encode_config_template | 15 +- test/rocprim/CMakeLists.txt | 1 + test/rocprim/test_device_reduce_by_key.cpp | 5 +- test/rocprim/test_intrinsics_atomic.cpp | 178 + 52 files changed, 21297 insertions(+), 7436 deletions(-) create mode 100644 rocprim/include/rocprim/device/detail/config/device_segmented_reduce.hpp create mode 100644 test/rocprim/test_intrinsics_atomic.cpp diff --git a/benchmark/ConfigAutotuneSettings.cmake b/benchmark/ConfigAutotuneSettings.cmake index b4c25e53c..37f3a97bc 100644 --- a/benchmark/ConfigAutotuneSettings.cmake +++ b/benchmark/ConfigAutotuneSettings.cmake @@ -112,17 +112,17 @@ DataType;BlockSize;" PARENT_SCOPE) set(list_across "${TUNING_TYPES};rocprim::empty_type ${LIMITED_TUNING_TYPES};128 192 256 384 512" PARENT_SCOPE) set(output_pattern_suffix "@KeyType@_@ValueType@_@BlockSize@" PARENT_SCOPE) elseif(file STREQUAL "benchmark_device_reduce_by_key") - set(list_across_names "KeyType;ValueType;BlockSize;TilesPerBlock" PARENT_SCOPE) - set(list_across "${LIMITED_TUNING_TYPES};${TUNING_TYPES};128 192 256 384 512;1 2" PARENT_SCOPE) - set(output_pattern_suffix "@KeyType@_@ValueType@_@BlockSize@_@TilesPerBlock@" PARENT_SCOPE) + set(list_across_names "KeyType;ValueType;BlockSize" PARENT_SCOPE) + set(list_across "${LIMITED_TUNING_TYPES};${TUNING_TYPES};128 192 256 384 512" PARENT_SCOPE) + set(output_pattern_suffix "@KeyType@_@ValueType@_@BlockSize@" PARENT_SCOPE) elseif(file STREQUAL "benchmark_device_find_first_of") set(list_across_names "DataType;BlockSize" PARENT_SCOPE) set(list_across "${LIMITED_TUNING_TYPES};32 64 128 256 512 1024" PARENT_SCOPE) set(output_pattern_suffix "@DataType@_@BlockSize@" PARENT_SCOPE) elseif(file STREQUAL "benchmark_device_run_length_encode") - set(list_across_names "KeyType;BlockSize;TilesPerBlock" PARENT_SCOPE) - set(list_across "${TUNING_TYPES};128 192 256 384 512;1 2" PARENT_SCOPE) - set(output_pattern_suffix "@KeyType@_@BlockSize@_@TilesPerBlock@" PARENT_SCOPE) + set(list_across_names "KeyType;BlockSize" PARENT_SCOPE) + set(list_across "${TUNING_TYPES};128 192 256 384 512" PARENT_SCOPE) + set(output_pattern_suffix "@KeyType@_@BlockSize@" PARENT_SCOPE) elseif(file STREQUAL "benchmark_device_run_length_encode_non_trivial_runs") set(list_across_names "KeyType;BlockSize;BlockLoadMethod" PARENT_SCOPE) set(list_across "${TUNING_TYPES};64 128 256 512 1024;block_load_vectorize block_load_warp_transpose" PARENT_SCOPE) diff --git a/benchmark/benchmark_device_reduce_by_key.parallel.cpp.in b/benchmark/benchmark_device_reduce_by_key.parallel.cpp.in index 4942bb2bd..50c5fb2e2 100644 --- a/benchmark/benchmark_device_reduce_by_key.parallel.cpp.in +++ b/benchmark/benchmark_device_reduce_by_key.parallel.cpp.in @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -27,5 +27,5 @@ namespace { auto benchmarks = config_autotune_register::create_bulk( - device_reduce_by_key_benchmark_generator<@KeyType@, @ValueType@, @BlockSize@, @TilesPerBlock@>::create); + device_reduce_by_key_benchmark_generator<@KeyType@, @ValueType@, @BlockSize@>::create); } diff --git a/benchmark/benchmark_device_reduce_by_key.parallel.hpp b/benchmark/benchmark_device_reduce_by_key.parallel.hpp index 100cf8c1c..edfd5e99f 100644 --- a/benchmark/benchmark_device_reduce_by_key.parallel.hpp +++ b/benchmark/benchmark_device_reduce_by_key.parallel.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -44,8 +44,7 @@ std::string config_name() { const rocprim::detail::reduce_by_key_config_params params = Config(); return "{bs:" + std::to_string(params.kernel_config.block_size) - + ",ipt:" + std::to_string(params.kernel_config.items_per_thread) - + ",tpb:" + std::to_string(params.tiles_per_block) + "}"; + + ",ipt:" + std::to_string(params.kernel_config.items_per_thread) + "}"; } template<> @@ -224,7 +223,7 @@ struct device_reduce_by_key_benchmark : public config_autotune_interface #ifdef BENCHMARK_CONFIG_TUNING -template +template struct device_reduce_by_key_benchmark_generator { template @@ -237,8 +236,7 @@ struct device_reduce_by_key_benchmark_generator ItemsPerThread, rocprim::block_load_method::block_load_transpose, rocprim::block_load_method::block_load_transpose, - rocprim::block_scan_algorithm::using_warp_scan, - TilesPerBlock>; + rocprim::block_scan_algorithm::using_warp_scan>; // max segment length argument is irrelevant, tuning overrides segment length storage.emplace_back( std::make_unique< diff --git a/benchmark/benchmark_device_run_length_encode.parallel.cpp.in b/benchmark/benchmark_device_run_length_encode.parallel.cpp.in index b1588898b..7515e7586 100644 --- a/benchmark/benchmark_device_run_length_encode.parallel.cpp.in +++ b/benchmark/benchmark_device_run_length_encode.parallel.cpp.in @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -29,6 +29,5 @@ namespace { auto benchmarks = config_autotune_register::create_bulk( device_run_length_encode_benchmark_generator< @KeyType@, - @BlockSize@, - @TilesPerBlock@>::create); + @BlockSize@>::create); } // namespace diff --git a/benchmark/benchmark_device_run_length_encode.parallel.hpp b/benchmark/benchmark_device_run_length_encode.parallel.hpp index d02003400..ccc61f55d 100644 --- a/benchmark/benchmark_device_run_length_encode.parallel.hpp +++ b/benchmark/benchmark_device_run_length_encode.parallel.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -48,8 +48,7 @@ std::string run_length_encode_config_name() { const rocprim::detail::reduce_by_key_config_params config = Config(); return "{bs:" + std::to_string(config.kernel_config.block_size) - + ",ipt:" + std::to_string(config.kernel_config.items_per_thread) - + ",tpb:" + std::to_string(config.tiles_per_block) + "}"; + + ",ipt:" + std::to_string(config.kernel_config.items_per_thread) + "}"; } template<> @@ -195,7 +194,7 @@ struct device_run_length_encode_benchmark : public config_autotune_interface #ifdef BENCHMARK_CONFIG_TUNING -template +template struct device_run_length_encode_benchmark_generator { template @@ -208,8 +207,8 @@ struct device_run_length_encode_benchmark_generator ItemsPerThread, rocprim::block_load_method::block_load_transpose, rocprim::block_load_method::block_load_transpose, - rocprim::block_scan_algorithm::using_warp_scan, - TilesPerBlock>; + rocprim::block_scan_algorithm::using_warp_scan>; + storage.emplace_back( std::make_unique>()); storage.emplace_back( diff --git a/rocprim/include/rocprim/block/detail/block_radix_rank_match.hpp b/rocprim/include/rocprim/block/detail/block_radix_rank_match.hpp index d8d84f3e4..f92b3bad1 100644 --- a/rocprim/include/rocprim/block/detail/block_radix_rank_match.hpp +++ b/rocprim/include/rocprim/block/detail/block_radix_rank_match.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -104,10 +104,10 @@ class block_radix_rank_match digit_counter_type counters[counters]; }; - ROCPRIM_DEVICE ROCPRIM_INLINE digit_counter_type& - get_digit_counter(const unsigned int digit, const unsigned int warp, storage_type_& storage) + ROCPRIM_DEVICE ROCPRIM_INLINE + unsigned int get_digit_counter(const unsigned int digit, const unsigned int warp) { - return storage.counters[digit * warps + warp]; + return digit * warps + warp; } template @@ -136,7 +136,7 @@ class block_radix_rank_match const unsigned int digit = digit_extractor(keys[i]); // Get the digit counter for this key on the current warp. - digit_counters[i] = &get_digit_counter(digit, warp_id, storage); + digit_counters[i] = &storage.counters[get_digit_counter(digit, warp_id)]; // Read the prefix sum of that digit. We already know it's 0 on the first iteration. So // we can skip a read-after-write dependency. The conditional gets optimized out due to @@ -233,12 +233,12 @@ class block_radix_rank_match if(radix_digits % block_size == 0 || digit < radix_digits) { // The counter for warp 0 holds the prefix of all the digits at this point. - prefix[i] = get_digit_counter(digit, 0, storage); + prefix[i] = storage.counters[get_digit_counter(digit, 0)]; // To find the count, subtract the prefix of the next digit with that of the // current digit. - const unsigned int next_prefix = digit + 1 == radix_digits - ? block_size * ItemsPerThread - : get_digit_counter(digit + 1, 0, storage); + const unsigned int next_prefix + = digit + 1 == radix_digits ? block_size * ItemsPerThread + : storage.counters[get_digit_counter(digit + 1, 0)]; counts[i] = next_prefix - prefix[i]; } } diff --git a/rocprim/include/rocprim/config.hpp b/rocprim/include/rocprim/config.hpp index e968f2793..7c96b54cc 100644 --- a/rocprim/include/rocprim/config.hpp +++ b/rocprim/include/rocprim/config.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -68,6 +68,39 @@ #define ROCPRIM_FORCE_INLINE __attribute__((always_inline)) #endif +#undef ROCPRIM_TARGET_GCN3 +#undef ROCPRIM_TARGET_GCN5 +#undef ROCPRIM_TARGET_RDNA1 +#undef ROCPRIM_TARGET_RDNA2 +#undef ROCPRIM_TARGET_RDNA3 +#undef ROCPRIM_TARGET_RDNA4 +#undef ROCPRIM_TARGET_CDNA1 +#undef ROCPRIM_TARGET_CDNA2 +#undef ROCPRIM_TARGET_CDNA3 + +// See https://llvm.org/docs/AMDGPUUsage.html#instructions +#if defined(__gfx940__) || defined(__gfx941__) || defined(__gfx942__) + #define ROCPRIM_TARGET_CDNA3 1 +#elif defined(__gfx90a__) + #define ROCPRIM_TARGET_CDNA2 1 +#elif defined(__gfx908__) + #define ROCPRIM_TARGET_CDNA1 1 +#elif defined(__gfx900__) || defined(__gfx902__) || defined(__gfx904__) || defined(__gfx906__) \ + || defined(__gfx90c__) + #define ROCPRIM_TARGET_GCN5 1 +#elif defined(__GFX12__) + #define ROCPRIM_TARGET_RDNA4 1 +#elif defined(__GFX11__) + #define ROCPRIM_TARGET_RDNA3 1 +#elif defined(__gfx1030__) || defined(__gfx1031__) || defined(__gfx1032__) || defined(__gfx1033__) \ + || defined(__gfx1034__) || defined(__gfx1035__) || defined(__gfx1036__) + #define ROCPRIM_TARGET_RDNA2 1 +#elif defined(__gfx1010__) || defined(__gfx1011__) || defined(__gfx1012__) || defined(__gfx1013__) + #define ROCPRIM_TARGET_RDNA1 1 +#elif defined(__GFX8__) + #define ROCPRIM_TARGET_GCN3 1 +#endif + // DPP is supported only after Volcanic Islands (GFX8+) // Only defined when support is present, in contrast to ROCPRIM_DETAIL_USE_DPP, which should be // always defined @@ -110,7 +143,8 @@ #endif #ifndef ROCPRIM_NAVI - #if defined(__HIP_DEVICE_COMPILE__) && (defined(__GFX10__) || defined(__GFX11__) || defined(__GFX12__)) + #if defined(__HIP_DEVICE_COMPILE__) \ + && (defined(__GFX10__) || defined(__GFX11__) || defined(__GFX12__)) #define ROCPRIM_NAVI 1 #else #define ROCPRIM_NAVI 0 @@ -123,22 +157,22 @@ #define ROCPRIM_WARP_SIZE_64 64u #define ROCPRIM_MAX_WARP_SIZE ROCPRIM_WARP_SIZE_64 -#if (defined(_MSC_VER) && !defined(__clang__)) || (defined(__GNUC__) && !defined(__clang__)) -#define ROCPRIM_UNROLL -#define ROCPRIM_NO_UNROLL +#if(defined(_MSC_VER) && !defined(__clang__)) || (defined(__GNUC__) && !defined(__clang__)) + #define ROCPRIM_UNROLL + #define ROCPRIM_NO_UNROLL #else -#define ROCPRIM_UNROLL _Pragma("unroll") -#define ROCPRIM_NO_UNROLL _Pragma("nounroll") + #define ROCPRIM_UNROLL _Pragma("unroll") + #define ROCPRIM_NO_UNROLL _Pragma("nounroll") #endif #ifndef ROCPRIM_GRID_SIZE_LIMIT -#define ROCPRIM_GRID_SIZE_LIMIT std::numeric_limits::max() + #define ROCPRIM_GRID_SIZE_LIMIT std::numeric_limits::max() #endif #if __cpp_if_constexpr >= 201606 -#define ROCPRIM_IF_CONSTEXPR constexpr + #define ROCPRIM_IF_CONSTEXPR constexpr #else -#define ROCPRIM_IF_CONSTEXPR + #define ROCPRIM_IF_CONSTEXPR #endif // Copyright 2001 John Maddock. diff --git a/rocprim/include/rocprim/device/detail/config/device_adjacent_difference.hpp b/rocprim/include/rocprim/device/detail/config/device_adjacent_difference.hpp index 0f8cbc51b..039af66b8 100644 --- a/rocprim/include/rocprim/device/detail/config/device_adjacent_difference.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_adjacent_difference.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,8 +21,11 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_ADJACENT_DIFFERENCE_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_ADJACENT_DIFFERENCE_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" + #include /* DO NOT EDIT THIS FILE @@ -46,7 +49,7 @@ struct default_adjacent_difference_config // Based on value_type = double template struct default_adjacent_difference_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1030), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> @@ -56,26 +59,26 @@ struct default_adjacent_difference_config< // Based on value_type = float template struct default_adjacent_difference_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1030), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<1024, 2> + : adjacent_difference_config<1024, 1> {}; // Based on value_type = rocprim::half template struct default_adjacent_difference_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1030), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2))>> : adjacent_difference_config<512, 11> + && (sizeof(value_type) <= 2))>> : adjacent_difference_config<1024, 2> {}; // Based on value_type = int64_t template struct default_adjacent_difference_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1030), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> @@ -85,56 +88,66 @@ struct default_adjacent_difference_config< // Based on value_type = int template struct default_adjacent_difference_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1030), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<1024, 2> + : adjacent_difference_config<1024, 1> {}; // Based on value_type = short template struct default_adjacent_difference_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1030), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : adjacent_difference_config<256, 11> + : adjacent_difference_config<1024, 5> {}; // Based on value_type = int8_t template struct default_adjacent_difference_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1030), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1))>> : adjacent_difference_config<64, 19> + && (sizeof(value_type) <= 1))>> : adjacent_difference_config<64, 7> +{}; + +// Based on value_type = rocprim::int128_t +template +struct default_adjacent_difference_config< + static_cast(target_arch::gfx1030), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : adjacent_difference_config<512, 1> {}; // Based on value_type = double template struct default_adjacent_difference_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : adjacent_difference_config<128, 1> + : adjacent_difference_config<1024, 2> {}; // Based on value_type = float template struct default_adjacent_difference_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<128, 3> + : adjacent_difference_config<1024, 5> {}; // Based on value_type = rocprim::half template struct default_adjacent_difference_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2))>> : adjacent_difference_config<64, 7> @@ -143,27 +156,27 @@ struct default_adjacent_difference_config< // Based on value_type = int64_t template struct default_adjacent_difference_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : adjacent_difference_config<128, 1> + : adjacent_difference_config<512, 1> {}; // Based on value_type = int template struct default_adjacent_difference_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<128, 2> + : adjacent_difference_config<1024, 5> {}; // Based on value_type = short template struct default_adjacent_difference_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> @@ -173,10 +186,20 @@ struct default_adjacent_difference_config< // Based on value_type = int8_t template struct default_adjacent_difference_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1))>> : adjacent_difference_config<64, 19> + && (sizeof(value_type) <= 1))>> : adjacent_difference_config<64, 17> +{}; + +// Based on value_type = rocprim::int128_t +template +struct default_adjacent_difference_config< + static_cast(target_arch::gfx1100), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : adjacent_difference_config<256, 1> {}; // Based on value_type = double @@ -247,210 +270,250 @@ struct default_adjacent_difference_config< && (sizeof(value_type) <= 1))>> : adjacent_difference_config<64, 19> {}; +// Based on value_type = rocprim::int128_t +template +struct default_adjacent_difference_config< + static_cast(target_arch::gfx906), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : adjacent_difference_config<512, 1> +{}; + // Based on value_type = double template struct default_adjacent_difference_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : adjacent_difference_config<1024, 1> + : adjacent_difference_config<128, 1> {}; // Based on value_type = float template struct default_adjacent_difference_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<1024, 1> + : adjacent_difference_config<128, 3> {}; // Based on value_type = rocprim::half template struct default_adjacent_difference_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2))>> : adjacent_difference_config<1024, 5> + && (sizeof(value_type) <= 2))>> : adjacent_difference_config<128, 7> {}; // Based on value_type = int64_t template struct default_adjacent_difference_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : adjacent_difference_config<1024, 1> + : adjacent_difference_config<128, 1> {}; // Based on value_type = int template struct default_adjacent_difference_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<1024, 1> + : adjacent_difference_config<128, 2> {}; // Based on value_type = short template struct default_adjacent_difference_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : adjacent_difference_config<1024, 5> + : adjacent_difference_config<64, 7> {}; // Based on value_type = int8_t template struct default_adjacent_difference_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1))>> : adjacent_difference_config<64, 7> + && (sizeof(value_type) <= 1))>> : adjacent_difference_config<64, 19> +{}; + +// Based on value_type = rocprim::int128_t +template +struct default_adjacent_difference_config< + static_cast(target_arch::gfx908), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : adjacent_difference_config<32, 17> {}; // Based on value_type = double template struct default_adjacent_difference_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : adjacent_difference_config<128, 1> + : adjacent_difference_config<256, 2> {}; // Based on value_type = float template struct default_adjacent_difference_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<128, 3> + : adjacent_difference_config<512, 5> {}; // Based on value_type = rocprim::half template struct default_adjacent_difference_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2))>> : adjacent_difference_config<64, 7> + && (sizeof(value_type) <= 2))>> : adjacent_difference_config<256, 11> {}; // Based on value_type = int64_t template struct default_adjacent_difference_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : adjacent_difference_config<128, 1> + : adjacent_difference_config<256, 2> {}; // Based on value_type = int template struct default_adjacent_difference_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<128, 2> + : adjacent_difference_config<512, 5> {}; // Based on value_type = short template struct default_adjacent_difference_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : adjacent_difference_config<64, 7> + : adjacent_difference_config<64, 17> {}; // Based on value_type = int8_t template struct default_adjacent_difference_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1))>> : adjacent_difference_config<64, 19> {}; -// Based on value_type = double +// Based on value_type = rocprim::int128_t template struct default_adjacent_difference_config< static_cast(target_arch::gfx90a), value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : adjacent_difference_config<256, 1> +{}; + +// Based on value_type = double +template +struct default_adjacent_difference_config< + static_cast(target_arch::unknown), + value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : adjacent_difference_config<256, 2> + : adjacent_difference_config<128, 1> {}; // Based on value_type = float template struct default_adjacent_difference_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<512, 5> + : adjacent_difference_config<128, 3> {}; // Based on value_type = rocprim::half template struct default_adjacent_difference_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2))>> : adjacent_difference_config<256, 11> + && (sizeof(value_type) <= 2))>> : adjacent_difference_config<128, 7> {}; // Based on value_type = int64_t template struct default_adjacent_difference_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : adjacent_difference_config<256, 2> + : adjacent_difference_config<128, 1> {}; // Based on value_type = int template struct default_adjacent_difference_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<512, 5> + : adjacent_difference_config<128, 2> {}; // Based on value_type = short template struct default_adjacent_difference_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : adjacent_difference_config<64, 17> + : adjacent_difference_config<64, 7> {}; // Based on value_type = int8_t template struct default_adjacent_difference_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1))>> : adjacent_difference_config<64, 19> {}; +// Based on value_type = rocprim::int128_t +template +struct default_adjacent_difference_config< + static_cast(target_arch::unknown), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : adjacent_difference_config<32, 17> +{}; + // Based on value_type = double template struct default_adjacent_difference_config< @@ -458,7 +521,7 @@ struct default_adjacent_difference_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : adjacent_difference_config<256, 7> + : adjacent_difference_config<64, 13> {}; // Based on value_type = float @@ -468,7 +531,7 @@ struct default_adjacent_difference_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<256, 11> + : adjacent_difference_config<64, 31> {}; // Based on value_type = rocprim::half @@ -480,6 +543,16 @@ struct default_adjacent_difference_config< && (sizeof(value_type) <= 2))>> : adjacent_difference_config<64, 19> {}; +// Based on value_type = rocprim::int128_t +template +struct default_adjacent_difference_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : adjacent_difference_config<512, 2> +{}; + // Based on value_type = int64_t template struct default_adjacent_difference_config< @@ -487,7 +560,7 @@ struct default_adjacent_difference_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : adjacent_difference_config<1024, 2> + : adjacent_difference_config<64, 13> {}; // Based on value_type = int @@ -497,7 +570,7 @@ struct default_adjacent_difference_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<1024, 5> + : adjacent_difference_config<64, 29> {}; // Based on value_type = short @@ -507,7 +580,7 @@ struct default_adjacent_difference_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : adjacent_difference_config<128, 17> + : adjacent_difference_config<256, 17> {}; // Based on value_type = int8_t @@ -516,7 +589,7 @@ struct default_adjacent_difference_config< static_cast(target_arch::gfx942), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1))>> : adjacent_difference_config<64, 23> + && (sizeof(value_type) <= 1))>> : adjacent_difference_config<128, 19> {}; } // end namespace detail diff --git a/rocprim/include/rocprim/device/detail/config/device_adjacent_difference_inplace.hpp b/rocprim/include/rocprim/device/detail/config/device_adjacent_difference_inplace.hpp index f6c9d3cf8..6eb95f43d 100644 --- a/rocprim/include/rocprim/device/detail/config/device_adjacent_difference_inplace.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_adjacent_difference_inplace.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,8 +21,11 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_ADJACENT_DIFFERENCE_INPLACE_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_ADJACENT_DIFFERENCE_INPLACE_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" + #include /* DO NOT EDIT THIS FILE @@ -46,7 +49,7 @@ struct default_adjacent_difference_inplace_config // Based on value_type = double template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1030), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> @@ -56,26 +59,26 @@ struct default_adjacent_difference_inplace_config< // Based on value_type = float template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1030), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<64, 17> + : adjacent_difference_config<32, 17> {}; // Based on value_type = rocprim::half template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1030), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2))>> : adjacent_difference_config<128, 17> + && (sizeof(value_type) <= 2))>> : adjacent_difference_config<1024, 13> {}; // Based on value_type = int64_t template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1030), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> @@ -85,98 +88,118 @@ struct default_adjacent_difference_inplace_config< // Based on value_type = int template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1030), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<64, 17> + : adjacent_difference_config<32, 19> {}; // Based on value_type = short template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1030), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : adjacent_difference_config<128, 17> + : adjacent_difference_config<512, 11> {}; // Based on value_type = int8_t template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1030), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1))>> : adjacent_difference_config<256, 17> + && (sizeof(value_type) <= 1))>> : adjacent_difference_config<32, 23> +{}; + +// Based on value_type = rocprim::int128_t +template +struct default_adjacent_difference_inplace_config< + static_cast(target_arch::gfx1030), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : adjacent_difference_config<64, 29> {}; // Based on value_type = double template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : adjacent_difference_config<512, 2> + : adjacent_difference_config<128, 29> {}; // Based on value_type = float template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<1024, 5> + : adjacent_difference_config<128, 29> {}; // Based on value_type = rocprim::half template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2))>> : adjacent_difference_config<32, 23> + && (sizeof(value_type) <= 2))>> : adjacent_difference_config<128, 23> {}; // Based on value_type = int64_t template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : adjacent_difference_config<512, 2> + : adjacent_difference_config<128, 29> {}; // Based on value_type = int template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<1024, 5> + : adjacent_difference_config<128, 29> {}; // Based on value_type = short template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : adjacent_difference_config<32, 23> + : adjacent_difference_config<128, 23> {}; // Based on value_type = int8_t template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1))>> : adjacent_difference_config<64, 19> + && (sizeof(value_type) <= 1))>> : adjacent_difference_config<512, 31> +{}; + +// Based on value_type = rocprim::int128_t +template +struct default_adjacent_difference_inplace_config< + static_cast(target_arch::gfx1100), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : adjacent_difference_config<32, 29> {}; // Based on value_type = double @@ -205,7 +228,7 @@ struct default_adjacent_difference_inplace_config< static_cast(target_arch::gfx906), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2))>> : adjacent_difference_config<64, 17> + && (sizeof(value_type) <= 2))>> : adjacent_difference_config<64, 31> {}; // Based on value_type = int64_t @@ -247,78 +270,98 @@ struct default_adjacent_difference_inplace_config< && (sizeof(value_type) <= 1))>> : adjacent_difference_config<64, 19> {}; +// Based on value_type = rocprim::int128_t +template +struct default_adjacent_difference_inplace_config< + static_cast(target_arch::gfx906), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : adjacent_difference_config<64, 29> +{}; + // Based on value_type = double template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : adjacent_difference_config<32, 17> + : adjacent_difference_config<512, 2> {}; // Based on value_type = float template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<32, 17> + : adjacent_difference_config<1024, 5> {}; // Based on value_type = rocprim::half template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2))>> : adjacent_difference_config<512, 11> + && (sizeof(value_type) <= 2))>> : adjacent_difference_config<512, 7> {}; // Based on value_type = int64_t template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : adjacent_difference_config<32, 17> + : adjacent_difference_config<512, 2> {}; // Based on value_type = int template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<32, 19> + : adjacent_difference_config<1024, 5> {}; // Based on value_type = short template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : adjacent_difference_config<512, 11> + : adjacent_difference_config<32, 23> {}; // Based on value_type = int8_t template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1))>> : adjacent_difference_config<32, 23> + && (sizeof(value_type) <= 1))>> : adjacent_difference_config<64, 19> +{}; + +// Based on value_type = rocprim::int128_t +template +struct default_adjacent_difference_inplace_config< + static_cast(target_arch::gfx908), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : adjacent_difference_config<32, 19> {}; // Based on value_type = double template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> @@ -328,26 +371,26 @@ struct default_adjacent_difference_inplace_config< // Based on value_type = float template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<1024, 5> + : adjacent_difference_config<512, 5> {}; // Based on value_type = rocprim::half template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2))>> : adjacent_difference_config<32, 23> + && (sizeof(value_type) <= 2))>> : adjacent_difference_config<256, 11> {}; // Based on value_type = int64_t template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> @@ -357,37 +400,47 @@ struct default_adjacent_difference_inplace_config< // Based on value_type = int template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<1024, 5> + : adjacent_difference_config<512, 5> {}; // Based on value_type = short template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : adjacent_difference_config<32, 23> + : adjacent_difference_config<64, 17> {}; // Based on value_type = int8_t template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1))>> : adjacent_difference_config<64, 19> + && (sizeof(value_type) <= 1))>> : adjacent_difference_config<128, 19> {}; -// Based on value_type = double +// Based on value_type = rocprim::int128_t template struct default_adjacent_difference_inplace_config< static_cast(target_arch::gfx90a), value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : adjacent_difference_config<512, 2> +{}; + +// Based on value_type = double +template +struct default_adjacent_difference_inplace_config< + static_cast(target_arch::unknown), + value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : adjacent_difference_config<512, 2> @@ -396,26 +449,26 @@ struct default_adjacent_difference_inplace_config< // Based on value_type = float template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<512, 5> + : adjacent_difference_config<1024, 5> {}; // Based on value_type = rocprim::half template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2))>> : adjacent_difference_config<256, 11> + && (sizeof(value_type) <= 2))>> : adjacent_difference_config<512, 7> {}; // Based on value_type = int64_t template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> @@ -425,30 +478,40 @@ struct default_adjacent_difference_inplace_config< // Based on value_type = int template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<512, 5> + : adjacent_difference_config<1024, 5> {}; // Based on value_type = short template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : adjacent_difference_config<64, 17> + : adjacent_difference_config<32, 23> {}; // Based on value_type = int8_t template struct default_adjacent_difference_inplace_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1))>> : adjacent_difference_config<128, 19> + && (sizeof(value_type) <= 1))>> : adjacent_difference_config<64, 19> +{}; + +// Based on value_type = rocprim::int128_t +template +struct default_adjacent_difference_inplace_config< + static_cast(target_arch::unknown), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : adjacent_difference_config<32, 19> {}; // Based on value_type = double @@ -458,7 +521,7 @@ struct default_adjacent_difference_inplace_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : adjacent_difference_config<128, 11> + : adjacent_difference_config<128, 2> {}; // Based on value_type = float @@ -468,7 +531,7 @@ struct default_adjacent_difference_inplace_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<512, 7> + : adjacent_difference_config<256, 3> {}; // Based on value_type = rocprim::half @@ -477,7 +540,17 @@ struct default_adjacent_difference_inplace_config< static_cast(target_arch::gfx942), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2))>> : adjacent_difference_config<128, 17> + && (sizeof(value_type) <= 2))>> : adjacent_difference_config<512, 13> +{}; + +// Based on value_type = rocprim::int128_t +template +struct default_adjacent_difference_inplace_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : adjacent_difference_config<32, 17> {}; // Based on value_type = int64_t @@ -487,7 +560,7 @@ struct default_adjacent_difference_inplace_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : adjacent_difference_config<128, 11> + : adjacent_difference_config<256, 11> {}; // Based on value_type = int @@ -497,7 +570,7 @@ struct default_adjacent_difference_inplace_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : adjacent_difference_config<512, 7> + : adjacent_difference_config<512, 5> {}; // Based on value_type = short @@ -507,7 +580,7 @@ struct default_adjacent_difference_inplace_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : adjacent_difference_config<128, 17> + : adjacent_difference_config<64, 31> {}; // Based on value_type = int8_t @@ -516,7 +589,7 @@ struct default_adjacent_difference_inplace_config< static_cast(target_arch::gfx942), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1))>> : adjacent_difference_config<128, 19> + && (sizeof(value_type) <= 1))>> : adjacent_difference_config<128, 23> {}; } // end namespace detail diff --git a/rocprim/include/rocprim/device/detail/config/device_adjacent_find.hpp b/rocprim/include/rocprim/device/detail/config/device_adjacent_find.hpp index e16c29597..2b3609c60 100644 --- a/rocprim/include/rocprim/device/detail/config/device_adjacent_find.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_adjacent_find.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,7 +21,9 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_ADJACENT_FIND_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_ADJACENT_FIND_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" #include @@ -111,6 +113,16 @@ struct default_adjacent_find_config< && (sizeof(input_type) <= 1))>> : adjacent_find_config<128, 32> {}; +// Based on input_type = rocprim::int128_t +template +struct default_adjacent_find_config< + static_cast(target_arch::gfx1030), + input_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(input_type) <= 16) && (sizeof(input_type) > 8))>> + : adjacent_find_config<256, 2> +{}; + // Based on input_type = double template struct default_adjacent_find_config< @@ -179,6 +191,16 @@ struct default_adjacent_find_config< && (sizeof(input_type) <= 1))>> : adjacent_find_config<128, 32> {}; +// Based on input_type = rocprim::int128_t +template +struct default_adjacent_find_config< + static_cast(target_arch::gfx1100), + input_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(input_type) <= 16) && (sizeof(input_type) > 8))>> + : adjacent_find_config<1024, 4> +{}; + // Based on input_type = double template struct default_adjacent_find_config< @@ -205,7 +227,7 @@ struct default_adjacent_find_config< static_cast(target_arch::gfx906), input_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(input_type) <= 2))>> : adjacent_find_config<64, 4> + && (sizeof(input_type) <= 2))>> : adjacent_find_config<64, 8> {}; // Based on input_type = int64_t @@ -247,6 +269,16 @@ struct default_adjacent_find_config< && (sizeof(input_type) <= 1))>> : adjacent_find_config<128, 2> {}; +// Based on input_type = rocprim::int128_t +template +struct default_adjacent_find_config< + static_cast(target_arch::gfx906), + input_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(input_type) <= 16) && (sizeof(input_type) > 8))>> + : adjacent_find_config<1024, 4> +{}; + // Based on input_type = double template struct default_adjacent_find_config< @@ -315,69 +347,89 @@ struct default_adjacent_find_config< && (sizeof(input_type) <= 1))>> : adjacent_find_config<64, 16> {}; +// Based on input_type = rocprim::int128_t +template +struct default_adjacent_find_config< + static_cast(target_arch::gfx908), + input_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(input_type) <= 16) && (sizeof(input_type) > 8))>> + : adjacent_find_config<1024, 4> +{}; + // Based on input_type = double template struct default_adjacent_find_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), input_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(input_type) <= 8) && (sizeof(input_type) > 4))>> - : adjacent_find_config<128, 8> + : adjacent_find_config<64, 8> {}; // Based on input_type = float template struct default_adjacent_find_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), input_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(input_type) <= 4) && (sizeof(input_type) > 2))>> - : adjacent_find_config<512, 4> + : adjacent_find_config<64, 16> {}; // Based on input_type = rocprim::half template struct default_adjacent_find_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), input_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(input_type) <= 2))>> : adjacent_find_config<256, 16> + && (sizeof(input_type) <= 2))>> : adjacent_find_config<128, 16> +{}; + +// Based on input_type = rocprim::int128_t +template +struct default_adjacent_find_config< + static_cast(target_arch::gfx90a), + input_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(input_type) <= 16) && (sizeof(input_type) > 8))>> + : adjacent_find_config<128, 2> {}; // Based on input_type = int64_t template struct default_adjacent_find_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), input_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(input_type) <= 8) && (sizeof(input_type) > 4))>> - : adjacent_find_config<512, 2> + : adjacent_find_config<64, 8> {}; // Based on input_type = int template struct default_adjacent_find_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), input_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(input_type) <= 4) && (sizeof(input_type) > 2))>> - : adjacent_find_config<64, 64> + : adjacent_find_config<64, 16> {}; // Based on input_type = short template struct default_adjacent_find_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), input_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(input_type) <= 2) && (sizeof(input_type) > 1))>> - : adjacent_find_config<256, 16> + : adjacent_find_config<128, 16> {}; // Based on input_type = int8_t template struct default_adjacent_find_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), input_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(input_type) <= 1))>> : adjacent_find_config<64, 16> @@ -386,7 +438,85 @@ struct default_adjacent_find_config< // Based on input_type = double template struct default_adjacent_find_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::gfx942), + input_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(input_type) <= 8) && (sizeof(input_type) > 4))>> + : adjacent_find_config<1024, 16> +{}; + +// Based on input_type = float +template +struct default_adjacent_find_config< + static_cast(target_arch::gfx942), + input_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(input_type) <= 4) && (sizeof(input_type) > 2))>> + : adjacent_find_config<512, 32> +{}; + +// Based on input_type = rocprim::half +template +struct default_adjacent_find_config< + static_cast(target_arch::gfx942), + input_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(input_type) <= 2))>> : adjacent_find_config<512, 32> +{}; + +// Based on input_type = rocprim::int128_t +template +struct default_adjacent_find_config< + static_cast(target_arch::gfx942), + input_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(input_type) <= 16) && (sizeof(input_type) > 8))>> + : adjacent_find_config<1024, 16> +{}; + +// Based on input_type = int64_t +template +struct default_adjacent_find_config< + static_cast(target_arch::gfx942), + input_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(input_type) <= 8) && (sizeof(input_type) > 4))>> + : adjacent_find_config<1024, 16> +{}; + +// Based on input_type = int +template +struct default_adjacent_find_config< + static_cast(target_arch::gfx942), + input_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(input_type) <= 4) && (sizeof(input_type) > 2))>> + : adjacent_find_config<512, 32> +{}; + +// Based on input_type = short +template +struct default_adjacent_find_config< + static_cast(target_arch::gfx942), + input_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(input_type) <= 2) && (sizeof(input_type) > 1))>> + : adjacent_find_config<512, 32> +{}; + +// Based on input_type = int8_t +template +struct default_adjacent_find_config< + static_cast(target_arch::gfx942), + input_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(input_type) <= 1))>> : adjacent_find_config<512, 32> +{}; + +// Based on input_type = double +template +struct default_adjacent_find_config< + static_cast(target_arch::unknown), input_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(input_type) <= 8) && (sizeof(input_type) > 4))>> @@ -396,7 +526,7 @@ struct default_adjacent_find_config< // Based on input_type = float template struct default_adjacent_find_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), input_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(input_type) <= 4) && (sizeof(input_type) > 2))>> @@ -406,7 +536,7 @@ struct default_adjacent_find_config< // Based on input_type = rocprim::half template struct default_adjacent_find_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), input_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(input_type) <= 2))>> : adjacent_find_config<256, 16> @@ -415,7 +545,7 @@ struct default_adjacent_find_config< // Based on input_type = int64_t template struct default_adjacent_find_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), input_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(input_type) <= 8) && (sizeof(input_type) > 4))>> @@ -425,7 +555,7 @@ struct default_adjacent_find_config< // Based on input_type = int template struct default_adjacent_find_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), input_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(input_type) <= 4) && (sizeof(input_type) > 2))>> @@ -435,7 +565,7 @@ struct default_adjacent_find_config< // Based on input_type = short template struct default_adjacent_find_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), input_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(input_type) <= 2) && (sizeof(input_type) > 1))>> @@ -445,12 +575,22 @@ struct default_adjacent_find_config< // Based on input_type = int8_t template struct default_adjacent_find_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), input_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(input_type) <= 1))>> : adjacent_find_config<64, 16> {}; +// Based on input_type = rocprim::int128_t +template +struct default_adjacent_find_config< + static_cast(target_arch::unknown), + input_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(input_type) <= 16) && (sizeof(input_type) > 8))>> + : adjacent_find_config<1024, 4> +{}; + } // end namespace detail END_ROCPRIM_NAMESPACE @@ -458,4 +598,4 @@ END_ROCPRIM_NAMESPACE /// @} // end of group primitivesmodule_deviceconfigs -#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_ADJACENT_FIND_HPP_ \ No newline at end of file +#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_ADJACENT_FIND_HPP_ diff --git a/rocprim/include/rocprim/device/detail/config/device_binary_search.hpp b/rocprim/include/rocprim/device/detail/config/device_binary_search.hpp index 86ff91c82..ba9ae94d3 100644 --- a/rocprim/include/rocprim/device/detail/config/device_binary_search.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_binary_search.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2023-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,8 +21,11 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_BINARY_SEARCH_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_BINARY_SEARCH_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" + #include /* DO NOT EDIT THIS FILE @@ -89,6 +92,18 @@ struct default_binary_search_config< && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> {}; +// Based on value_type = double, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<128, 1> +{}; + // Based on value_type = float, output_type = int64_t template struct default_binary_search_config< @@ -136,6 +151,18 @@ struct default_binary_search_config< && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> {}; +// Based on value_type = float, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<128, 1> +{}; + // Based on value_type = rocprim::half, output_type = int64_t template struct default_binary_search_config< @@ -155,7 +182,7 @@ struct default_binary_search_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : binary_search_config<64, 4> + && (sizeof(output_type) > 2))>> : binary_search_config<64, 8> {}; // Based on value_type = rocprim::half, output_type = short @@ -166,7 +193,7 @@ struct default_binary_search_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : binary_search_config<64, 4> + && (sizeof(output_type) > 1))>> : binary_search_config<64, 8> {}; // Based on value_type = rocprim::half, output_type = int8_t @@ -177,7 +204,18 @@ struct default_binary_search_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> - : binary_search_config<64, 4> + : binary_search_config<64, 8> +{}; + +// Based on value_type = rocprim::half, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : binary_search_config<64, 8> {}; // Based on value_type = int64_t, output_type = int64_t @@ -227,6 +265,18 @@ struct default_binary_search_config< && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> {}; +// Based on value_type = int64_t, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<128, 1> +{}; + // Based on value_type = int, output_type = int64_t template struct default_binary_search_config< @@ -274,6 +324,18 @@ struct default_binary_search_config< && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> {}; +// Based on value_type = int, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<128, 1> +{}; + // Based on value_type = short, output_type = int64_t template struct default_binary_search_config< @@ -321,6 +383,18 @@ struct default_binary_search_config< && (sizeof(output_type) <= 1))>> : binary_search_config<128, 4> {}; +// Based on value_type = short, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 1> +{}; + // Based on value_type = int8_t, output_type = int64_t template struct default_binary_search_config< @@ -365,10 +439,80 @@ struct default_binary_search_config< : binary_search_config<128, 2> {}; +// Based on value_type = int8_t, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : binary_search_config<256, 4> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int64_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : binary_search_config<256, 1> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int +template +struct default_binary_search_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : binary_search_config<128, 1> +{}; + +// Based on value_type = rocprim::int128_t, output_type = short +template +struct default_binary_search_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : binary_search_config<128, 1> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int8_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 1))>> : binary_search_config<128, 1> +{}; + +// Based on value_type = rocprim::int128_t, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<128, 1> +{}; + // Based on value_type = double, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -380,42 +524,54 @@ struct default_binary_search_config< // Based on value_type = double, output_type = int template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : binary_search_config<256, 1> + : binary_search_config<256, 16> {}; // Based on value_type = double, output_type = short template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<256, 1> + : binary_search_config<256, 16> {}; // Based on value_type = double, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> + && (sizeof(output_type) <= 1))>> : binary_search_config<64, 16> +{}; + +// Based on value_type = double, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<64, 16> {}; // Based on value_type = float, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -427,7 +583,7 @@ struct default_binary_search_config< // Based on value_type = float, output_type = int template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -439,7 +595,7 @@ struct default_binary_search_config< // Based on value_type = float, output_type = short template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -451,7 +607,7 @@ struct default_binary_search_config< // Based on value_type = float, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -459,10 +615,22 @@ struct default_binary_search_config< && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> {}; +// Based on value_type = float, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<64, 16> +{}; + // Based on value_type = rocprim::half, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -473,40 +641,51 @@ struct default_binary_search_config< // Based on value_type = rocprim::half, output_type = int template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : binary_search_config<128, 8> + && (sizeof(output_type) > 2))>> : binary_search_config<64, 16> {}; // Based on value_type = rocprim::half, output_type = short template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : binary_search_config<64, 4> + && (sizeof(output_type) > 1))>> : binary_search_config<64, 8> {}; // Based on value_type = rocprim::half, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> - : binary_search_config<64, 4> + : binary_search_config<64, 8> +{}; + +// Based on value_type = rocprim::half, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : binary_search_config<64, 8> {}; // Based on value_type = int64_t, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -518,42 +697,54 @@ struct default_binary_search_config< // Based on value_type = int64_t, output_type = int template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : binary_search_config<256, 1> + : binary_search_config<256, 16> {}; // Based on value_type = int64_t, output_type = short template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<256, 1> + : binary_search_config<256, 16> {}; // Based on value_type = int64_t, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> + && (sizeof(output_type) <= 1))>> : binary_search_config<64, 16> +{}; + +// Based on value_type = int64_t, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 16> {}; // Based on value_type = int, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -565,7 +756,7 @@ struct default_binary_search_config< // Based on value_type = int, output_type = int template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -577,7 +768,7 @@ struct default_binary_search_config< // Based on value_type = int, output_type = short template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -589,7 +780,7 @@ struct default_binary_search_config< // Based on value_type = int, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -597,10 +788,22 @@ struct default_binary_search_config< && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> {}; +// Based on value_type = int, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<64, 16> +{}; + // Based on value_type = short, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -612,19 +815,19 @@ struct default_binary_search_config< // Based on value_type = short, output_type = int template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : binary_search_config<128, 8> + : binary_search_config<64, 8> {}; // Based on value_type = short, output_type = short template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -636,51 +839,63 @@ struct default_binary_search_config< // Based on value_type = short, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 1))>> : binary_search_config<64, 4> + && (sizeof(output_type) <= 1))>> : binary_search_config<64, 8> +{}; + +// Based on value_type = short, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 4> {}; // Based on value_type = int8_t, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : binary_search_config<128, 1> + && (sizeof(output_type) > 4))>> : binary_search_config<64, 8> {}; // Based on value_type = int8_t, output_type = int template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : binary_search_config<64, 4> + && (sizeof(output_type) > 2))>> : binary_search_config<128, 4> {}; // Based on value_type = int8_t, output_type = short template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : binary_search_config<128, 8> + && (sizeof(output_type) > 1))>> : binary_search_config<64, 8> {}; // Based on value_type = int8_t, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -688,650 +903,538 @@ struct default_binary_search_config< : binary_search_config<64, 8> {}; -// Based on value_type = double, output_type = int64_t +// Based on value_type = int8_t, output_type = rocprim::int128_t template struct default_binary_search_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : binary_search_config<256, 8> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int64_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> : binary_search_config<256, 1> {}; -// Based on value_type = double, output_type = int +// Based on value_type = rocprim::int128_t, output_type = int template struct default_binary_search_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : binary_search_config<256, 1> + : binary_search_config<64, 16> {}; -// Based on value_type = double, output_type = short +// Based on value_type = rocprim::int128_t, output_type = short template struct default_binary_search_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<256, 1> + : binary_search_config<64, 16> {}; -// Based on value_type = double, output_type = int8_t +// Based on value_type = rocprim::int128_t, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 1))>> : binary_search_config<64, 16> {}; -// Based on value_type = float, output_type = int64_t +// Based on value_type = rocprim::int128_t, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<64, 16> +{}; + +// Based on value_type = double, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> : binary_search_config<256, 1> {}; -// Based on value_type = float, output_type = int +// Based on value_type = double, output_type = int template struct default_binary_search_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> : binary_search_config<256, 1> {}; -// Based on value_type = float, output_type = short +// Based on value_type = double, output_type = short template struct default_binary_search_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> : binary_search_config<256, 1> {}; -// Based on value_type = float, output_type = int8_t +// Based on value_type = double, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> {}; -// Based on value_type = rocprim::half, output_type = int64_t +// Based on value_type = double, output_type = rocprim::int128_t template struct default_binary_search_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : binary_search_config<256, 1> + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 16> {}; -// Based on value_type = rocprim::half, output_type = int +// Based on value_type = float, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : binary_search_config<256, 1> -{}; - -// Based on value_type = rocprim::half, output_type = short -template -struct default_binary_search_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : binary_search_config<256, 1> -{}; - -// Based on value_type = rocprim::half, output_type = int8_t -template -struct default_binary_search_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> - : binary_search_config<256, 1> -{}; - -// Based on value_type = int64_t, output_type = int64_t -template -struct default_binary_search_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : binary_search_config<256, 1> -{}; - -// Based on value_type = int64_t, output_type = int -template -struct default_binary_search_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : binary_search_config<256, 1> -{}; - -// Based on value_type = int64_t, output_type = short -template -struct default_binary_search_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<256, 1> -{}; - -// Based on value_type = int64_t, output_type = int8_t -template -struct default_binary_search_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> -{}; - -// Based on value_type = int, output_type = int64_t -template -struct default_binary_search_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> : binary_search_config<256, 1> {}; -// Based on value_type = int, output_type = int +// Based on value_type = float, output_type = int template struct default_binary_search_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> : binary_search_config<256, 1> {}; -// Based on value_type = int, output_type = short +// Based on value_type = float, output_type = short template struct default_binary_search_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> : binary_search_config<256, 1> {}; -// Based on value_type = int, output_type = int8_t +// Based on value_type = float, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> {}; -// Based on value_type = short, output_type = int64_t +// Based on value_type = float, output_type = rocprim::int128_t template struct default_binary_search_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : binary_search_config<256, 1> -{}; - -// Based on value_type = short, output_type = int -template -struct default_binary_search_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> : binary_search_config<256, 1> {}; -// Based on value_type = short, output_type = short +// Based on value_type = rocprim::half, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<256, 1> + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) + && (sizeof(output_type) > 4))>> : binary_search_config<256, 1> {}; -// Based on value_type = short, output_type = int8_t +// Based on value_type = rocprim::half, output_type = int template struct default_binary_search_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) + && (sizeof(output_type) > 2))>> : binary_search_config<64, 4> {}; -// Based on value_type = int8_t, output_type = int64_t +// Based on value_type = rocprim::half, output_type = short template struct default_binary_search_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : binary_search_config<256, 1> + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) + && (sizeof(output_type) > 1))>> : binary_search_config<256, 4> {}; -// Based on value_type = int8_t, output_type = int +// Based on value_type = rocprim::half, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : binary_search_config<256, 1> + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> + : binary_search_config<256, 4> {}; -// Based on value_type = int8_t, output_type = short +// Based on value_type = rocprim::half, output_type = rocprim::int128_t template struct default_binary_search_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : binary_search_config<256, 1> + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : binary_search_config<256, 4> {}; -// Based on value_type = int8_t, output_type = int8_t +// Based on value_type = int64_t, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> : binary_search_config<256, 1> {}; -// Based on value_type = double, output_type = int64_t +// Based on value_type = int64_t, output_type = int template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> : binary_search_config<256, 1> {}; -// Based on value_type = double, output_type = int +// Based on value_type = int64_t, output_type = short template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> : binary_search_config<256, 1> {}; -// Based on value_type = double, output_type = short +// Based on value_type = int64_t, output_type = int8_t template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<256, 1> + && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> {}; -// Based on value_type = double, output_type = int8_t +// Based on value_type = int64_t, output_type = rocprim::int128_t template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 16> {}; -// Based on value_type = float, output_type = int64_t +// Based on value_type = int, output_type = int64_t template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> : binary_search_config<256, 1> {}; -// Based on value_type = float, output_type = int +// Based on value_type = int, output_type = int template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> : binary_search_config<256, 1> {}; -// Based on value_type = float, output_type = short +// Based on value_type = int, output_type = short template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> : binary_search_config<256, 1> {}; -// Based on value_type = float, output_type = int8_t +// Based on value_type = int, output_type = int8_t template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> {}; -// Based on value_type = rocprim::half, output_type = int64_t +// Based on value_type = int, output_type = rocprim::int128_t template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : binary_search_config<256, 4> -{}; - -// Based on value_type = rocprim::half, output_type = int -template -struct default_binary_search_config< - static_cast(target_arch::gfx906), - value_type, - output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : binary_search_config<256, 4> -{}; - -// Based on value_type = rocprim::half, output_type = short -template -struct default_binary_search_config< - static_cast(target_arch::gfx906), - value_type, - output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : binary_search_config<256, 4> -{}; - -// Based on value_type = rocprim::half, output_type = int8_t -template -struct default_binary_search_config< - static_cast(target_arch::gfx906), - value_type, - output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> - : binary_search_config<256, 4> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 1> {}; -// Based on value_type = int64_t, output_type = int64_t +// Based on value_type = short, output_type = int64_t template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : binary_search_config<256, 1> + : binary_search_config<256, 4> {}; -// Based on value_type = int64_t, output_type = int +// Based on value_type = short, output_type = int template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : binary_search_config<256, 1> + : binary_search_config<256, 4> {}; -// Based on value_type = int64_t, output_type = short +// Based on value_type = short, output_type = short template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<256, 1> -{}; - -// Based on value_type = int64_t, output_type = int8_t -template -struct default_binary_search_config< - static_cast(target_arch::gfx906), - value_type, - output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> + : binary_search_config<256, 4> {}; -// Based on value_type = int, output_type = int64_t +// Based on value_type = short, output_type = int8_t template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) - && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : binary_search_config<256, 1> + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 1))>> : binary_search_config<256, 4> {}; -// Based on value_type = int, output_type = int +// Based on value_type = short, output_type = rocprim::int128_t template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) - && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : binary_search_config<256, 1> + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 16> {}; -// Based on value_type = int, output_type = short +// Based on value_type = int8_t, output_type = int64_t template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) - && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<256, 1> + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) + && (sizeof(output_type) > 4))>> : binary_search_config<256, 1> {}; -// Based on value_type = int, output_type = int8_t +// Based on value_type = int8_t, output_type = int template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) - && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) + && (sizeof(output_type) > 2))>> : binary_search_config<256, 4> {}; -// Based on value_type = short, output_type = int64_t +// Based on value_type = int8_t, output_type = short template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : binary_search_config<256, 4> + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) + && (sizeof(output_type) > 1))>> : binary_search_config<256, 4> {}; -// Based on value_type = short, output_type = int +// Based on value_type = int8_t, output_type = int8_t template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> : binary_search_config<256, 4> {}; -// Based on value_type = short, output_type = short +// Based on value_type = int8_t, output_type = rocprim::int128_t template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<256, 4> + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : binary_search_config<64, 1> {}; -// Based on value_type = short, output_type = int8_t +// Based on value_type = rocprim::int128_t, output_type = int64_t template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 1))>> : binary_search_config<256, 4> + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : binary_search_config<256, 1> {}; -// Based on value_type = int8_t, output_type = int64_t +// Based on value_type = rocprim::int128_t, output_type = int template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : binary_search_config<256, 4> + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : binary_search_config<256, 8> {}; -// Based on value_type = int8_t, output_type = int +// Based on value_type = rocprim::int128_t, output_type = short template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : binary_search_config<256, 4> + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : binary_search_config<256, 16> {}; -// Based on value_type = int8_t, output_type = short +// Based on value_type = rocprim::int128_t, output_type = int8_t template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : binary_search_config<256, 4> + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 1))>> : binary_search_config<256, 16> {}; -// Based on value_type = int8_t, output_type = int8_t +// Based on value_type = rocprim::int128_t, output_type = rocprim::int128_t template struct default_binary_search_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> - : binary_search_config<256, 4> + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 16> {}; // Based on value_type = double, output_type = int64_t @@ -1381,6 +1484,18 @@ struct default_binary_search_config< && (sizeof(output_type) <= 1))>> : binary_search_config<128, 1> {}; +// Based on value_type = double, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 16> +{}; + // Based on value_type = float, output_type = int64_t template struct default_binary_search_config< @@ -1428,6 +1543,18 @@ struct default_binary_search_config< && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> {}; +// Based on value_type = float, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 16> +{}; + // Based on value_type = rocprim::half, output_type = int64_t template struct default_binary_search_config< @@ -1436,7 +1563,7 @@ struct default_binary_search_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : binary_search_config<256, 4> + && (sizeof(output_type) > 4))>> : binary_search_config<64, 4> {}; // Based on value_type = rocprim::half, output_type = int @@ -1447,7 +1574,7 @@ struct default_binary_search_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : binary_search_config<64, 4> + && (sizeof(output_type) > 2))>> : binary_search_config<256, 1> {}; // Based on value_type = rocprim::half, output_type = short @@ -1458,7 +1585,7 @@ struct default_binary_search_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : binary_search_config<256, 4> + && (sizeof(output_type) > 1))>> : binary_search_config<64, 4> {}; // Based on value_type = rocprim::half, output_type = int8_t @@ -1472,6 +1599,17 @@ struct default_binary_search_config< : binary_search_config<256, 4> {}; +// Based on value_type = rocprim::half, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : binary_search_config<64, 4> +{}; + // Based on value_type = int64_t, output_type = int64_t template struct default_binary_search_config< @@ -1519,6 +1657,18 @@ struct default_binary_search_config< && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> {}; +// Based on value_type = int64_t, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 16> +{}; + // Based on value_type = int, output_type = int64_t template struct default_binary_search_config< @@ -1566,6 +1716,18 @@ struct default_binary_search_config< && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> {}; +// Based on value_type = int, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 16> +{}; + // Based on value_type = short, output_type = int64_t template struct default_binary_search_config< @@ -1613,6 +1775,18 @@ struct default_binary_search_config< && (sizeof(output_type) <= 1))>> : binary_search_config<128, 1> {}; +// Based on value_type = short, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 16> +{}; + // Based on value_type = int8_t, output_type = int64_t template struct default_binary_search_config< @@ -1657,57 +1831,139 @@ struct default_binary_search_config< : binary_search_config<256, 4> {}; +// Based on value_type = int8_t, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : binary_search_config<256, 4> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int64_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : binary_search_config<256, 1> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int +template +struct default_binary_search_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : binary_search_config<256, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = short +template +struct default_binary_search_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : binary_search_config<256, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int8_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 1))>> : binary_search_config<256, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 16> +{}; + // Based on value_type = double, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : binary_search_config<256, 1> + : binary_search_config<64, 2> {}; // Based on value_type = double, output_type = int template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : binary_search_config<256, 1> + : binary_search_config<64, 2> {}; // Based on value_type = double, output_type = short template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<256, 16> + : binary_search_config<64, 2> {}; // Based on value_type = double, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : binary_search_config<128, 1> + && (sizeof(output_type) <= 1))>> : binary_search_config<64, 2> +{}; + +// Based on value_type = double, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<64, 2> {}; // Based on value_type = float, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -1719,7 +1975,7 @@ struct default_binary_search_config< // Based on value_type = float, output_type = int template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -1731,19 +1987,19 @@ struct default_binary_search_config< // Based on value_type = float, output_type = short template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<128, 1> + : binary_search_config<256, 1> {}; // Based on value_type = float, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -1751,101 +2007,136 @@ struct default_binary_search_config< && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> {}; +// Based on value_type = float, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 1> +{}; + // Based on value_type = rocprim::half, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : binary_search_config<256, 4> + && (sizeof(output_type) > 4))>> : binary_search_config<256, 1> {}; // Based on value_type = rocprim::half, output_type = int template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : binary_search_config<64, 4> + && (sizeof(output_type) > 2))>> : binary_search_config<64, 2> {}; // Based on value_type = rocprim::half, output_type = short template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : binary_search_config<256, 4> + && (sizeof(output_type) > 1))>> : binary_search_config<64, 2> {}; // Based on value_type = rocprim::half, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> - : binary_search_config<256, 4> + : binary_search_config<64, 2> +{}; + +// Based on value_type = rocprim::half, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : binary_search_config<128, 2> {}; // Based on value_type = int64_t, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : binary_search_config<256, 1> + : binary_search_config<64, 2> {}; // Based on value_type = int64_t, output_type = int template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : binary_search_config<256, 1> + : binary_search_config<256, 2> {}; // Based on value_type = int64_t, output_type = short template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<256, 16> + : binary_search_config<256, 2> {}; // Based on value_type = int64_t, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> + && (sizeof(output_type) <= 1))>> : binary_search_config<64, 2> +{}; + +// Based on value_type = int64_t, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<64, 2> {}; // Based on value_type = int, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -1857,7 +2148,7 @@ struct default_binary_search_config< // Based on value_type = int, output_type = int template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -1869,19 +2160,19 @@ struct default_binary_search_config< // Based on value_type = int, output_type = short template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<256, 8> + : binary_search_config<256, 1> {}; // Based on value_type = int, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -1889,148 +2180,254 @@ struct default_binary_search_config< && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> {}; +// Based on value_type = int, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 1> +{}; + // Based on value_type = short, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : binary_search_config<128, 1> + : binary_search_config<256, 1> {}; // Based on value_type = short, output_type = int template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : binary_search_config<128, 1> + : binary_search_config<256, 1> {}; // Based on value_type = short, output_type = short template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<64, 4> + : binary_search_config<256, 1> {}; // Based on value_type = short, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 1))>> : binary_search_config<128, 1> + && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> +{}; + +// Based on value_type = short, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 1> {}; // Based on value_type = int8_t, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : binary_search_config<128, 2> + && (sizeof(output_type) > 4))>> : binary_search_config<256, 1> {}; // Based on value_type = int8_t, output_type = int template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : binary_search_config<128, 1> + && (sizeof(output_type) > 2))>> : binary_search_config<256, 1> {}; // Based on value_type = int8_t, output_type = short template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : binary_search_config<256, 4> + && (sizeof(output_type) > 1))>> : binary_search_config<256, 1> {}; // Based on value_type = int8_t, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> - : binary_search_config<256, 4> + : binary_search_config<256, 1> {}; -// Based on value_type = double, output_type = int64_t +// Based on value_type = int8_t, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : binary_search_config<256, 1> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int64_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : binary_search_config<256, 1> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int +template +struct default_binary_search_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : binary_search_config<64, 4> +{}; + +// Based on value_type = rocprim::int128_t, output_type = short +template +struct default_binary_search_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : binary_search_config<128, 4> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int8_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 1))>> : binary_search_config<64, 4> +{}; + +// Based on value_type = rocprim::int128_t, output_type = rocprim::int128_t template struct default_binary_search_config< static_cast(target_arch::gfx90a), value_type, output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<64, 2> +{}; + +// Based on value_type = double, output_type = int64_t +template +struct default_binary_search_config< + static_cast(target_arch::unknown), + value_type, + output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : binary_search_config<64, 2> + : binary_search_config<256, 1> {}; // Based on value_type = double, output_type = int template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : binary_search_config<64, 2> + : binary_search_config<256, 1> {}; // Based on value_type = double, output_type = short template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<64, 2> + : binary_search_config<256, 16> {}; // Based on value_type = double, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : binary_search_config<64, 2> + && (sizeof(output_type) <= 1))>> : binary_search_config<128, 1> +{}; + +// Based on value_type = double, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 16> {}; // Based on value_type = float, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -2042,7 +2439,7 @@ struct default_binary_search_config< // Based on value_type = float, output_type = int template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -2054,19 +2451,19 @@ struct default_binary_search_config< // Based on value_type = float, output_type = short template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<256, 1> + : binary_search_config<128, 1> {}; // Based on value_type = float, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -2074,101 +2471,136 @@ struct default_binary_search_config< && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> {}; +// Based on value_type = float, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 16> +{}; + // Based on value_type = rocprim::half, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : binary_search_config<128, 4> + && (sizeof(output_type) > 4))>> : binary_search_config<64, 4> {}; // Based on value_type = rocprim::half, output_type = int template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : binary_search_config<64, 2> + && (sizeof(output_type) > 2))>> : binary_search_config<256, 1> {}; // Based on value_type = rocprim::half, output_type = short template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : binary_search_config<64, 2> + && (sizeof(output_type) > 1))>> : binary_search_config<64, 4> {}; // Based on value_type = rocprim::half, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> - : binary_search_config<64, 4> + : binary_search_config<256, 4> +{}; + +// Based on value_type = rocprim::half, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : binary_search_config<64, 4> {}; // Based on value_type = int64_t, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : binary_search_config<64, 2> + : binary_search_config<256, 1> {}; // Based on value_type = int64_t, output_type = int template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : binary_search_config<256, 2> + : binary_search_config<256, 1> {}; // Based on value_type = int64_t, output_type = short template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : binary_search_config<256, 16> +{}; + +// Based on value_type = int64_t, output_type = int8_t +template +struct default_binary_search_config< + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<256, 2> + && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> {}; -// Based on value_type = int64_t, output_type = int8_t +// Based on value_type = int64_t, output_type = rocprim::int128_t template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : binary_search_config<64, 2> + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 16> {}; // Based on value_type = int, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -2180,7 +2612,7 @@ struct default_binary_search_config< // Based on value_type = int, output_type = int template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -2192,19 +2624,19 @@ struct default_binary_search_config< // Based on value_type = int, output_type = short template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<256, 1> + : binary_search_config<256, 8> {}; // Based on value_type = int, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -2212,94 +2644,200 @@ struct default_binary_search_config< && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> {}; +// Based on value_type = int, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 16> +{}; + // Based on value_type = short, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : binary_search_config<256, 1> + : binary_search_config<128, 1> {}; // Based on value_type = short, output_type = int template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : binary_search_config<256, 1> + : binary_search_config<128, 1> {}; // Based on value_type = short, output_type = short template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<256, 1> + : binary_search_config<64, 4> {}; // Based on value_type = short, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> + && (sizeof(output_type) <= 1))>> : binary_search_config<128, 1> +{}; + +// Based on value_type = short, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 16> {}; // Based on value_type = int8_t, output_type = int64_t template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : binary_search_config<256, 1> + && (sizeof(output_type) > 4))>> : binary_search_config<128, 2> {}; // Based on value_type = int8_t, output_type = int template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : binary_search_config<256, 1> + && (sizeof(output_type) > 2))>> : binary_search_config<128, 1> {}; // Based on value_type = int8_t, output_type = short template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : binary_search_config<256, 1> + && (sizeof(output_type) > 1))>> : binary_search_config<256, 4> {}; // Based on value_type = int8_t, output_type = int8_t template struct default_binary_search_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> + : binary_search_config<256, 4> +{}; + +// Based on value_type = int8_t, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : binary_search_config<256, 4> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int64_t +template +struct default_binary_search_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : binary_search_config<256, 1> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int +template +struct default_binary_search_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : binary_search_config<256, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = short +template +struct default_binary_search_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : binary_search_config<256, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int8_t +template +struct default_binary_search_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 1))>> : binary_search_config<256, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 16> +{}; + +// Based on value_type = double, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> : binary_search_config<256, 1> {}; @@ -2324,7 +2862,7 @@ struct default_binary_search_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : binary_search_config<64, 16> + : binary_search_config<128, 1> {}; // Based on value_type = double, output_type = short @@ -2336,7 +2874,7 @@ struct default_binary_search_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<64, 16> + : binary_search_config<256, 1> {}; // Based on value_type = double, output_type = int8_t @@ -2347,7 +2885,19 @@ struct default_binary_search_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : binary_search_config<64, 16> + && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> +{}; + +// Based on value_type = float, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 1> {}; // Based on value_type = float, output_type = int64_t @@ -2383,7 +2933,7 @@ struct default_binary_search_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : binary_search_config<256, 1> + : binary_search_config<128, 1> {}; // Based on value_type = float, output_type = int8_t @@ -2394,7 +2944,18 @@ struct default_binary_search_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) - && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> + && (sizeof(output_type) <= 1))>> : binary_search_config<128, 1> +{}; + +// Based on value_type = rocprim::half, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : binary_search_config<256, 8> {}; // Based on value_type = rocprim::half, output_type = int64_t @@ -2416,7 +2977,7 @@ struct default_binary_search_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : binary_search_config<256, 4> + && (sizeof(output_type) > 2))>> : binary_search_config<64, 8> {}; // Based on value_type = rocprim::half, output_type = short @@ -2427,7 +2988,7 @@ struct default_binary_search_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : binary_search_config<256, 1> + && (sizeof(output_type) > 1))>> : binary_search_config<64, 8> {}; // Based on value_type = rocprim::half, output_type = int8_t @@ -2441,6 +3002,77 @@ struct default_binary_search_config< : binary_search_config<256, 4> {}; +// Based on value_type = rocprim::int128_t, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<64, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int64_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : binary_search_config<128, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : binary_search_config<64, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = short +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : binary_search_config<64, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int8_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 1))>> : binary_search_config<128, 16> +{}; + +// Based on value_type = int64_t, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 1> +{}; + // Based on value_type = int64_t, output_type = int64_t template struct default_binary_search_config< @@ -2488,6 +3120,18 @@ struct default_binary_search_config< && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> {}; +// Based on value_type = int, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 1> +{}; + // Based on value_type = int, output_type = int64_t template struct default_binary_search_config< @@ -2535,6 +3179,18 @@ struct default_binary_search_config< && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> {}; +// Based on value_type = short, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : binary_search_config<256, 1> +{}; + // Based on value_type = short, output_type = int64_t template struct default_binary_search_config< @@ -2582,6 +3238,17 @@ struct default_binary_search_config< && (sizeof(output_type) <= 1))>> : binary_search_config<256, 1> {}; +// Based on value_type = int8_t, output_type = rocprim::int128_t +template +struct default_binary_search_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : binary_search_config<256, 8> +{}; + // Based on value_type = int8_t, output_type = int64_t template struct default_binary_search_config< @@ -2590,7 +3257,7 @@ struct default_binary_search_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : binary_search_config<256, 16> + && (sizeof(output_type) > 4))>> : binary_search_config<128, 4> {}; // Based on value_type = int8_t, output_type = int @@ -2601,7 +3268,7 @@ struct default_binary_search_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : binary_search_config<128, 4> + && (sizeof(output_type) > 2))>> : binary_search_config<128, 1> {}; // Based on value_type = int8_t, output_type = short @@ -2612,7 +3279,7 @@ struct default_binary_search_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : binary_search_config<64, 16> + && (sizeof(output_type) > 1))>> : binary_search_config<64, 4> {}; // Based on value_type = int8_t, output_type = int8_t @@ -2623,7 +3290,7 @@ struct default_binary_search_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> - : binary_search_config<64, 16> + : binary_search_config<256, 4> {}; } // end namespace detail diff --git a/rocprim/include/rocprim/device/detail/config/device_find_first_of.hpp b/rocprim/include/rocprim/device/detail/config/device_find_first_of.hpp index 4886d50af..b908f06fc 100644 --- a/rocprim/include/rocprim/device/detail/config/device_find_first_of.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_find_first_of.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,7 +21,9 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_FIND_FIRST_OF_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_FIND_FIRST_OF_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" #include @@ -78,6 +80,15 @@ struct default_find_first_of_config(target_arch::gfx10 : find_first_of_config<64, 15> {}; +// Based on value_type = rocprim::int128_t +template +struct default_find_first_of_config< + static_cast(target_arch::gfx1030), + value_type, + std::enable_if_t<((sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : find_first_of_config<256, 4> +{}; + // Based on value_type = int64_t template struct default_find_first_of_config< @@ -113,6 +124,15 @@ struct default_find_first_of_config(target_arch::gfx11 : find_first_of_config<64, 13> {}; +// Based on value_type = rocprim::int128_t +template +struct default_find_first_of_config< + static_cast(target_arch::gfx1100), + value_type, + std::enable_if_t<((sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : find_first_of_config<256, 4> +{}; + // Based on value_type = int64_t template struct default_find_first_of_config< @@ -148,6 +168,15 @@ struct default_find_first_of_config(target_arch::gfx90 : find_first_of_config<256, 11> {}; +// Based on value_type = rocprim::int128_t +template +struct default_find_first_of_config< + static_cast(target_arch::gfx906), + value_type, + std::enable_if_t<((sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : find_first_of_config<256, 4> +{}; + // Based on value_type = int64_t template struct default_find_first_of_config< @@ -183,74 +212,110 @@ struct default_find_first_of_config(target_arch::gfx90 : find_first_of_config<256, 10> {}; +// Based on value_type = rocprim::int128_t +template +struct default_find_first_of_config< + static_cast(target_arch::gfx908), + value_type, + std::enable_if_t<((sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : find_first_of_config<256, 4> +{}; + // Based on value_type = int64_t template struct default_find_first_of_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<((sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : find_first_of_config<256, 8> + : find_first_of_config<256, 6> {}; // Based on value_type = int template struct default_find_first_of_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<((sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : find_first_of_config<256, 10> + : find_first_of_config<128, 9> {}; // Based on value_type = short template struct default_find_first_of_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<((sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : find_first_of_config<256, 11> + : find_first_of_config<256, 15> {}; // Based on value_type = int8_t template -struct default_find_first_of_config(target_arch::unknown), +struct default_find_first_of_config(target_arch::gfx90a), value_type, std::enable_if_t<((sizeof(value_type) <= 1))>> - : find_first_of_config<256, 10> + : find_first_of_config<512, 1> {}; -// Based on value_type = int64_t +// Based on value_type = rocprim::int128_t template struct default_find_first_of_config< static_cast(target_arch::gfx90a), value_type, + std::enable_if_t<((sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : find_first_of_config<256, 4> +{}; + +// Based on value_type = int64_t +template +struct default_find_first_of_config< + static_cast(target_arch::unknown), + value_type, std::enable_if_t<((sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : find_first_of_config<256, 6> + : find_first_of_config<256, 8> {}; // Based on value_type = int template struct default_find_first_of_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<((sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : find_first_of_config<128, 9> + : find_first_of_config<256, 10> {}; // Based on value_type = short template struct default_find_first_of_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<((sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : find_first_of_config<256, 15> + : find_first_of_config<256, 11> {}; // Based on value_type = int8_t template -struct default_find_first_of_config(target_arch::gfx90a), +struct default_find_first_of_config(target_arch::unknown), value_type, std::enable_if_t<((sizeof(value_type) <= 1))>> - : find_first_of_config<512, 1> + : find_first_of_config<256, 10> +{}; + +// Based on value_type = rocprim::int128_t +template +struct default_find_first_of_config< + static_cast(target_arch::unknown), + value_type, + std::enable_if_t<((sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : find_first_of_config<256, 4> +{}; + +// Based on value_type = rocprim::int128_t +template +struct default_find_first_of_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<((sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : find_first_of_config<1024, 6> {}; // Based on value_type = int64_t @@ -259,7 +324,7 @@ struct default_find_first_of_config< static_cast(target_arch::gfx942), value_type, std::enable_if_t<((sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : find_first_of_config<1024, 7> + : find_first_of_config<1024, 6> {}; // Based on value_type = int @@ -268,7 +333,7 @@ struct default_find_first_of_config< static_cast(target_arch::gfx942), value_type, std::enable_if_t<((sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : find_first_of_config<1024, 7> + : find_first_of_config<1024, 6> {}; // Based on value_type = short @@ -277,7 +342,7 @@ struct default_find_first_of_config< static_cast(target_arch::gfx942), value_type, std::enable_if_t<((sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : find_first_of_config<1024, 9> + : find_first_of_config<512, 15> {}; // Based on value_type = int8_t @@ -285,7 +350,7 @@ template struct default_find_first_of_config(target_arch::gfx942), value_type, std::enable_if_t<((sizeof(value_type) <= 1))>> - : find_first_of_config<1024, 12> + : find_first_of_config<1024, 11> {}; } // end namespace detail diff --git a/rocprim/include/rocprim/device/detail/config/device_histogram.hpp b/rocprim/include/rocprim/device/detail/config/device_histogram.hpp index 33c8269ae..70cb3da03 100644 --- a/rocprim/include/rocprim/device/detail/config/device_histogram.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_histogram.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,8 +21,11 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_HISTOGRAM_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_HISTOGRAM_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" + #include /* DO NOT EDIT THIS FILE @@ -198,7 +201,7 @@ struct default_histogram_config< active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 3, active_channels = 3 @@ -210,7 +213,7 @@ struct default_histogram_config< active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 4, active_channels = 3 @@ -222,7 +225,7 @@ struct default_histogram_config< active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 4, active_channels = 4 @@ -234,7 +237,72 @@ struct default_histogram_config< active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = rocprim::int128_t, channels = 1, active_channels = 1 +template +struct default_histogram_config< + static_cast(target_arch::gfx1030), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 1) + && (active_channels == 1))>> + : histogram_config, 2048, 2048, 2> +{}; + +// Based on value_type = rocprim::int128_t, channels = 2, active_channels = 2 +template +struct default_histogram_config< + static_cast(target_arch::gfx1030), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 2) + && (active_channels == 2))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = rocprim::int128_t, channels = 3, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx1030), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 3) + && (active_channels == 3))>> + : histogram_config, 2048, 2048, 3> +{}; + +// Based on value_type = rocprim::int128_t, channels = 4, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx1030), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 4) + && (active_channels == 3))>> + : histogram_config, 2048, 2048, 3> +{}; + +// Based on value_type = rocprim::int128_t, channels = 4, active_channels = 4 +template +struct default_histogram_config< + static_cast(target_arch::gfx1030), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 4) + && (active_channels == 4))>> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int64_t, channels = 1, active_channels = 1 @@ -495,161 +563,161 @@ struct default_histogram_config< // Based on value_type = double, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 3> + : histogram_config, 1024, 2048, 3> {}; // Based on value_type = double, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 2> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = double, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = double, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 3> + : histogram_config, 2048, 2048, 2> {}; // Based on value_type = double, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 3> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = float, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 1024, 2048, 3> {}; // Based on value_type = float, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = float, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 3> + : histogram_config, 1024, 2048, 3> {}; // Based on value_type = float, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = float, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 2> + : histogram_config, 2048, 2048, 2> {}; // Based on value_type = rocprim::half, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 1024, 2048, 3> {}; // Based on value_type = rocprim::half, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, @@ -661,970 +729,655 @@ struct default_histogram_config< // Based on value_type = rocprim::half, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 3> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = rocprim::int128_t, channels = 1, active_channels = 1 +template +struct default_histogram_config< + static_cast(target_arch::gfx1100), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 1) + && (active_channels == 1))>> + : histogram_config, 1024, 2048, 3> +{}; + +// Based on value_type = rocprim::int128_t, channels = 2, active_channels = 2 +template +struct default_histogram_config< + static_cast(target_arch::gfx1100), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 2) + && (active_channels == 2))>> + : histogram_config, 2048, 2048, 3> +{}; + +// Based on value_type = rocprim::int128_t, channels = 3, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx1100), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 3) + && (active_channels == 3))>> + : histogram_config, 2048, 2048, 2> +{}; + +// Based on value_type = rocprim::int128_t, channels = 4, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx1100), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 4) + && (active_channels == 3))>> + : histogram_config, 2048, 2048, 2> +{}; + +// Based on value_type = rocprim::int128_t, channels = 4, active_channels = 4 +template +struct default_histogram_config< + static_cast(target_arch::gfx1100), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 4) + && (active_channels == 4))>> + : histogram_config, 2048, 2048, 2> {}; // Based on value_type = int64_t, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 2> + : histogram_config, 1024, 2048, 3> {}; // Based on value_type = int64_t, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = int64_t, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 2> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = int64_t, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 3> + : histogram_config, 2048, 2048, 2> {}; // Based on value_type = int64_t, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 2> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 1024, 2048, 3> {}; // Based on value_type = int, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 2> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = int, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 3> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = short, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 1024, 2048, 3> {}; // Based on value_type = short, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = short, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = short, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 1024, 2048, 3> {}; // Based on value_type = short, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int8_t, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int8_t, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int8_t, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int8_t, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 1024, 2048, 3> {}; // Based on value_type = int8_t, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = double, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 2> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = double, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 2> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = double, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = double, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 2> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = double, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 2> {}; // Based on value_type = float, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = float, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = float, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = float, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = float, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 3> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 3> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; -// Based on value_type = int64_t, channels = 1, active_channels = 1 +// Based on value_type = rocprim::int128_t, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 3> + : histogram_config, 1024, 2048, 3> {}; -// Based on value_type = int64_t, channels = 2, active_channels = 2 +// Based on value_type = rocprim::int128_t, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 3> {}; -// Based on value_type = int64_t, channels = 3, active_channels = 3 +// Based on value_type = rocprim::int128_t, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 3) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; -// Based on value_type = int64_t, channels = 4, active_channels = 3 +// Based on value_type = rocprim::int128_t, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 1024, 2048, 3> {}; -// Based on value_type = int64_t, channels = 4, active_channels = 4 +// Based on value_type = rocprim::int128_t, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 3> {}; -// Based on value_type = int, channels = 1, active_channels = 1 +// Based on value_type = int64_t, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 1) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; -// Based on value_type = int, channels = 2, active_channels = 2 +// Based on value_type = int64_t, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 2) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; -// Based on value_type = int, channels = 3, active_channels = 3 +// Based on value_type = int64_t, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 3) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 2> + : histogram_config, 2048, 2048, 4> {}; -// Based on value_type = int, channels = 4, active_channels = 3 +// Based on value_type = int64_t, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 4) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 2> + : histogram_config, 2048, 2048, 2> {}; -// Based on value_type = int, channels = 4, active_channels = 4 -template -struct default_histogram_config< - static_cast(target_arch::gfx900), - value_type, - channels, - active_channels, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 4) - && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = short, channels = 1, active_channels = 1 -template -struct default_histogram_config< - static_cast(target_arch::gfx900), - value_type, - channels, - active_channels, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 1) - && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = short, channels = 2, active_channels = 2 -template -struct default_histogram_config< - static_cast(target_arch::gfx900), - value_type, - channels, - active_channels, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 2) - && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = short, channels = 3, active_channels = 3 -template -struct default_histogram_config< - static_cast(target_arch::gfx900), - value_type, - channels, - active_channels, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 3) - && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = short, channels = 4, active_channels = 3 -template -struct default_histogram_config< - static_cast(target_arch::gfx900), - value_type, - channels, - active_channels, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 4) - && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = short, channels = 4, active_channels = 4 -template -struct default_histogram_config< - static_cast(target_arch::gfx900), - value_type, - channels, - active_channels, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 4) - && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = int8_t, channels = 1, active_channels = 1 -template -struct default_histogram_config< - static_cast(target_arch::gfx900), - value_type, - channels, - active_channels, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = int8_t, channels = 2, active_channels = 2 -template -struct default_histogram_config< - static_cast(target_arch::gfx900), - value_type, - channels, - active_channels, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = int8_t, channels = 3, active_channels = 3 -template -struct default_histogram_config< - static_cast(target_arch::gfx900), - value_type, - channels, - active_channels, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = int8_t, channels = 4, active_channels = 3 -template -struct default_histogram_config< - static_cast(target_arch::gfx900), - value_type, - channels, - active_channels, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = int8_t, channels = 4, active_channels = 4 -template -struct default_histogram_config< - static_cast(target_arch::gfx900), - value_type, - channels, - active_channels, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = double, channels = 1, active_channels = 1 -template -struct default_histogram_config< - static_cast(target_arch::gfx906), - value_type, - channels, - active_channels, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 1) - && (active_channels == 1))>> - : histogram_config, 2048, 2048, 3> -{}; - -// Based on value_type = double, channels = 2, active_channels = 2 -template -struct default_histogram_config< - static_cast(target_arch::gfx906), - value_type, - channels, - active_channels, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 2) - && (active_channels == 2))>> - : histogram_config, 2048, 2048, 3> -{}; - -// Based on value_type = double, channels = 3, active_channels = 3 -template -struct default_histogram_config< - static_cast(target_arch::gfx906), - value_type, - channels, - active_channels, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 3) - && (active_channels == 3))>> - : histogram_config, 2048, 2048, 3> -{}; - -// Based on value_type = double, channels = 4, active_channels = 3 -template -struct default_histogram_config< - static_cast(target_arch::gfx906), - value_type, - channels, - active_channels, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) - && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = double, channels = 4, active_channels = 4 -template -struct default_histogram_config< - static_cast(target_arch::gfx906), - value_type, - channels, - active_channels, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) - && (active_channels == 4))>> - : histogram_config, 2048, 2048, 2> -{}; - -// Based on value_type = float, channels = 1, active_channels = 1 -template -struct default_histogram_config< - static_cast(target_arch::gfx906), - value_type, - channels, - active_channels, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 1) - && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = float, channels = 2, active_channels = 2 -template -struct default_histogram_config< - static_cast(target_arch::gfx906), - value_type, - channels, - active_channels, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 2) - && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = float, channels = 3, active_channels = 3 -template -struct default_histogram_config< - static_cast(target_arch::gfx906), - value_type, - channels, - active_channels, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 3) - && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = float, channels = 4, active_channels = 3 -template -struct default_histogram_config< - static_cast(target_arch::gfx906), - value_type, - channels, - active_channels, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 4) - && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = float, channels = 4, active_channels = 4 -template -struct default_histogram_config< - static_cast(target_arch::gfx906), - value_type, - channels, - active_channels, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 4) - && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = rocprim::half, channels = 1, active_channels = 1 -template -struct default_histogram_config< - static_cast(target_arch::gfx906), - value_type, - channels, - active_channels, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = rocprim::half, channels = 2, active_channels = 2 -template -struct default_histogram_config< - static_cast(target_arch::gfx906), - value_type, - channels, - active_channels, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = rocprim::half, channels = 3, active_channels = 3 -template -struct default_histogram_config< - static_cast(target_arch::gfx906), - value_type, - channels, - active_channels, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = rocprim::half, channels = 4, active_channels = 3 -template -struct default_histogram_config< - static_cast(target_arch::gfx906), - value_type, - channels, - active_channels, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = rocprim::half, channels = 4, active_channels = 4 -template -struct default_histogram_config< - static_cast(target_arch::gfx906), - value_type, - channels, - active_channels, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = int64_t, channels = 1, active_channels = 1 -template -struct default_histogram_config< - static_cast(target_arch::gfx906), - value_type, - channels, - active_channels, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 1) - && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = int64_t, channels = 2, active_channels = 2 -template -struct default_histogram_config< - static_cast(target_arch::gfx906), - value_type, - channels, - active_channels, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 2) - && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = int64_t, channels = 3, active_channels = 3 -template -struct default_histogram_config< - static_cast(target_arch::gfx906), - value_type, - channels, - active_channels, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 3) - && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> -{}; - -// Based on value_type = int64_t, channels = 4, active_channels = 3 -template -struct default_histogram_config< - static_cast(target_arch::gfx906), - value_type, - channels, - active_channels, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) - && (active_channels == 3))>> - : histogram_config, 2048, 2048, 2> -{}; - -// Based on value_type = int64_t, channels = 4, active_channels = 4 +// Based on value_type = int64_t, channels = 4, active_channels = 4 template struct default_histogram_config< static_cast(target_arch::gfx906), @@ -1990,7 +1743,7 @@ struct default_histogram_config< active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 4, active_channels = 3 @@ -2002,7 +1755,7 @@ struct default_histogram_config< active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 4, active_channels = 4 @@ -2014,7 +1767,72 @@ struct default_histogram_config< active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = rocprim::int128_t, channels = 1, active_channels = 1 +template +struct default_histogram_config< + static_cast(target_arch::gfx908), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 1) + && (active_channels == 1))>> + : histogram_config, 1024, 2048, 3> +{}; + +// Based on value_type = rocprim::int128_t, channels = 2, active_channels = 2 +template +struct default_histogram_config< + static_cast(target_arch::gfx908), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 2) + && (active_channels == 2))>> + : histogram_config, 2048, 2048, 2> +{}; + +// Based on value_type = rocprim::int128_t, channels = 3, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx908), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 3) + && (active_channels == 3))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = rocprim::int128_t, channels = 4, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx908), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 4) + && (active_channels == 3))>> + : histogram_config, 1024, 2048, 3> +{}; + +// Based on value_type = rocprim::int128_t, channels = 4, active_channels = 4 +template +struct default_histogram_config< + static_cast(target_arch::gfx908), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 4) + && (active_channels == 4))>> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = int64_t, channels = 1, active_channels = 1 @@ -2275,111 +2093,111 @@ struct default_histogram_config< // Based on value_type = double, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = double, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 3> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = double, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = double, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = double, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = float, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = float, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = float, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = float, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, @@ -2392,32 +2210,32 @@ struct default_histogram_config< // Based on value_type = float, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, @@ -2429,147 +2247,212 @@ struct default_histogram_config< // Based on value_type = rocprim::half, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = rocprim::int128_t, channels = 1, active_channels = 1 +template +struct default_histogram_config< + static_cast(target_arch::gfx90a), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 1) + && (active_channels == 1))>> + : histogram_config, 1024, 2048, 3> +{}; + +// Based on value_type = rocprim::int128_t, channels = 2, active_channels = 2 +template +struct default_histogram_config< + static_cast(target_arch::gfx90a), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 2) + && (active_channels == 2))>> + : histogram_config, 2048, 2048, 2> +{}; + +// Based on value_type = rocprim::int128_t, channels = 3, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx90a), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 3) + && (active_channels == 3))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = rocprim::int128_t, channels = 4, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx90a), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 4) + && (active_channels == 3))>> + : histogram_config, 1024, 2048, 3> +{}; + +// Based on value_type = rocprim::int128_t, channels = 4, active_channels = 4 +template +struct default_histogram_config< + static_cast(target_arch::gfx90a), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 4) + && (active_channels == 4))>> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int64_t, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = int64_t, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 2> + : histogram_config, 2048, 2048, 2> {}; // Based on value_type = int64_t, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 2> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int64_t, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 2> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = int64_t, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 2> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, @@ -2582,7 +2465,7 @@ struct default_histogram_config< // Based on value_type = int, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, @@ -2595,84 +2478,84 @@ struct default_histogram_config< // Based on value_type = short, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = short, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = short, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = short, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = short, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int8_t, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int8_t, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, @@ -2684,7 +2567,7 @@ struct default_histogram_config< // Based on value_type = int8_t, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, @@ -2696,7 +2579,7 @@ struct default_histogram_config< // Based on value_type = int8_t, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, @@ -2708,7 +2591,7 @@ struct default_histogram_config< // Based on value_type = int8_t, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, channels, active_channels, @@ -2720,111 +2603,111 @@ struct default_histogram_config< // Based on value_type = double, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 3> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = double, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = double, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 3> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = double, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 3> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = double, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = float, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = float, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = float, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = float, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, @@ -2837,32 +2720,32 @@ struct default_histogram_config< // Based on value_type = float, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, @@ -2874,19 +2757,19 @@ struct default_histogram_config< // Based on value_type = rocprim::half, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, @@ -2898,123 +2781,188 @@ struct default_histogram_config< // Based on value_type = rocprim::half, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = rocprim::int128_t, channels = 1, active_channels = 1 +template +struct default_histogram_config< + static_cast(target_arch::unknown), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 1) + && (active_channels == 1))>> + : histogram_config, 1024, 2048, 3> +{}; + +// Based on value_type = rocprim::int128_t, channels = 2, active_channels = 2 +template +struct default_histogram_config< + static_cast(target_arch::unknown), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 2) + && (active_channels == 2))>> + : histogram_config, 2048, 2048, 2> +{}; + +// Based on value_type = rocprim::int128_t, channels = 3, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::unknown), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 3) + && (active_channels == 3))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = rocprim::int128_t, channels = 4, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::unknown), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 4) + && (active_channels == 3))>> + : histogram_config, 1024, 2048, 3> +{}; + +// Based on value_type = rocprim::int128_t, channels = 4, active_channels = 4 +template +struct default_histogram_config< + static_cast(target_arch::unknown), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 4) + && (active_channels == 4))>> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = int64_t, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 3> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int64_t, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 2> + : histogram_config, 2048, 2048, 2> {}; // Based on value_type = int64_t, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 2> {}; // Based on value_type = int64_t, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 3> + : histogram_config, 2048, 2048, 2> {}; // Based on value_type = int64_t, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 2> {}; // Based on value_type = int, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, @@ -3027,7 +2975,7 @@ struct default_histogram_config< // Based on value_type = int, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, @@ -3040,84 +2988,84 @@ struct default_histogram_config< // Based on value_type = short, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = short, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = short, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = short, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = short, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int8_t, channels = 1, active_channels = 1 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int8_t, channels = 2, active_channels = 2 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, @@ -3129,7 +3077,7 @@ struct default_histogram_config< // Based on value_type = int8_t, channels = 3, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, @@ -3141,7 +3089,7 @@ struct default_histogram_config< // Based on value_type = int8_t, channels = 4, active_channels = 3 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, @@ -3153,7 +3101,7 @@ struct default_histogram_config< // Based on value_type = int8_t, channels = 4, active_channels = 4 template struct default_histogram_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, channels, active_channels, @@ -3172,7 +3120,7 @@ struct default_histogram_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = double, channels = 2, active_channels = 2 @@ -3185,7 +3133,7 @@ struct default_histogram_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = double, channels = 3, active_channels = 3 @@ -3198,7 +3146,7 @@ struct default_histogram_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 3> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = double, channels = 4, active_channels = 3 @@ -3224,7 +3172,7 @@ struct default_histogram_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 3> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = float, channels = 1, active_channels = 1 @@ -3237,7 +3185,7 @@ struct default_histogram_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = float, channels = 2, active_channels = 2 @@ -3250,7 +3198,7 @@ struct default_histogram_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = float, channels = 3, active_channels = 3 @@ -3276,7 +3224,7 @@ struct default_histogram_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = float, channels = 4, active_channels = 4 @@ -3301,7 +3249,7 @@ struct default_histogram_config< active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 2, active_channels = 2 @@ -3313,7 +3261,7 @@ struct default_histogram_config< active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = rocprim::half, channels = 3, active_channels = 3 @@ -3349,9 +3297,74 @@ struct default_histogram_config< active_channels, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (channels == 4) && (active_channels == 4))>> + : histogram_config, 2048, 2048, 4> +{}; + +// Based on value_type = rocprim::int128_t, channels = 1, active_channels = 1 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 1) + && (active_channels == 1))>> + : histogram_config, 2048, 2048, 2> +{}; + +// Based on value_type = rocprim::int128_t, channels = 2, active_channels = 2 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 2) + && (active_channels == 2))>> + : histogram_config, 2048, 2048, 2> +{}; + +// Based on value_type = rocprim::int128_t, channels = 3, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 3) + && (active_channels == 3))>> : histogram_config, 2048, 2048, 4> {}; +// Based on value_type = rocprim::int128_t, channels = 4, active_channels = 3 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 4) + && (active_channels == 3))>> + : histogram_config, 2048, 2048, 3> +{}; + +// Based on value_type = rocprim::int128_t, channels = 4, active_channels = 4 +template +struct default_histogram_config< + static_cast(target_arch::gfx942), + value_type, + channels, + active_channels, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (channels == 4) + && (active_channels == 4))>> + : histogram_config, 2048, 2048, 3> +{}; + // Based on value_type = int64_t, channels = 1, active_channels = 1 template struct default_histogram_config< @@ -3362,7 +3375,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 3> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = int64_t, channels = 2, active_channels = 2 @@ -3375,7 +3388,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 3> + : histogram_config, 2048, 2048, 3> {}; // Based on value_type = int64_t, channels = 3, active_channels = 3 @@ -3388,7 +3401,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int64_t, channels = 4, active_channels = 3 @@ -3401,7 +3414,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int64_t, channels = 4, active_channels = 4 @@ -3414,7 +3427,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int, channels = 1, active_channels = 1 @@ -3427,7 +3440,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int, channels = 2, active_channels = 2 @@ -3440,7 +3453,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int, channels = 3, active_channels = 3 @@ -3466,7 +3479,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int, channels = 4, active_channels = 4 @@ -3505,7 +3518,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 2) && (active_channels == 2))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = short, channels = 3, active_channels = 3 @@ -3518,7 +3531,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 3) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = short, channels = 4, active_channels = 3 @@ -3531,7 +3544,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 4) && (active_channels == 3))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = short, channels = 4, active_channels = 4 @@ -3544,7 +3557,7 @@ struct default_histogram_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (channels == 4) && (active_channels == 4))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int8_t, channels = 1, active_channels = 1 @@ -3556,7 +3569,7 @@ struct default_histogram_config< active_channels, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (channels == 1) && (active_channels == 1))>> - : histogram_config, 2048, 2048, 4> + : histogram_config, 2048, 2048, 4> {}; // Based on value_type = int8_t, channels = 2, active_channels = 2 diff --git a/rocprim/include/rocprim/device/detail/config/device_lower_bound.hpp b/rocprim/include/rocprim/device/detail/config/device_lower_bound.hpp index 91599aeac..2f6f83a9a 100644 --- a/rocprim/include/rocprim/device/detail/config/device_lower_bound.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_lower_bound.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2023-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,8 +21,11 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_LOWER_BOUND_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_LOWER_BOUND_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" + #include /* DO NOT EDIT THIS FILE @@ -89,6 +92,18 @@ struct default_lower_bound_config< && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> {}; +// Based on value_type = double, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<128, 1> +{}; + // Based on value_type = float, output_type = int64_t template struct default_lower_bound_config< @@ -136,6 +151,18 @@ struct default_lower_bound_config< && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> {}; +// Based on value_type = float, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<128, 1> +{}; + // Based on value_type = rocprim::half, output_type = int64_t template struct default_lower_bound_config< @@ -144,7 +171,7 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : lower_bound_config<64, 4> + && (sizeof(output_type) > 4))>> : lower_bound_config<64, 8> {}; // Based on value_type = rocprim::half, output_type = int @@ -155,7 +182,7 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : lower_bound_config<64, 4> + && (sizeof(output_type) > 2))>> : lower_bound_config<64, 8> {}; // Based on value_type = rocprim::half, output_type = short @@ -166,7 +193,7 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : lower_bound_config<64, 4> + && (sizeof(output_type) > 1))>> : lower_bound_config<64, 8> {}; // Based on value_type = rocprim::half, output_type = int8_t @@ -180,6 +207,17 @@ struct default_lower_bound_config< : lower_bound_config<64, 4> {}; +// Based on value_type = rocprim::half, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : lower_bound_config<64, 8> +{}; + // Based on value_type = int64_t, output_type = int64_t template struct default_lower_bound_config< @@ -227,6 +265,18 @@ struct default_lower_bound_config< && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> {}; +// Based on value_type = int64_t, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<128, 1> +{}; + // Based on value_type = int, output_type = int64_t template struct default_lower_bound_config< @@ -274,6 +324,18 @@ struct default_lower_bound_config< && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> {}; +// Based on value_type = int, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<128, 1> +{}; + // Based on value_type = short, output_type = int64_t template struct default_lower_bound_config< @@ -321,6 +383,18 @@ struct default_lower_bound_config< && (sizeof(output_type) <= 1))>> : lower_bound_config<128, 4> {}; +// Based on value_type = short, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<256, 1> +{}; + // Based on value_type = int8_t, output_type = int64_t template struct default_lower_bound_config< @@ -365,10 +439,80 @@ struct default_lower_bound_config< : lower_bound_config<128, 4> {}; +// Based on value_type = int8_t, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : lower_bound_config<256, 4> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int64_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : lower_bound_config<256, 1> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int +template +struct default_lower_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : lower_bound_config<128, 1> +{}; + +// Based on value_type = rocprim::int128_t, output_type = short +template +struct default_lower_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : lower_bound_config<128, 1> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int8_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 1))>> : lower_bound_config<128, 1> +{}; + +// Based on value_type = rocprim::int128_t, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<128, 1> +{}; + // Based on value_type = double, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -380,42 +524,54 @@ struct default_lower_bound_config< // Based on value_type = double, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : lower_bound_config<256, 1> + : lower_bound_config<256, 16> {}; // Based on value_type = double, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : lower_bound_config<256, 1> + : lower_bound_config<64, 16> {}; // Based on value_type = double, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> + && (sizeof(output_type) <= 1))>> : lower_bound_config<64, 16> +{}; + +// Based on value_type = double, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<64, 16> {}; // Based on value_type = float, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -427,42 +583,54 @@ struct default_lower_bound_config< // Based on value_type = float, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : lower_bound_config<256, 1> + : lower_bound_config<128, 1> {}; // Based on value_type = float, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : lower_bound_config<256, 1> + : lower_bound_config<128, 1> {}; // Based on value_type = float, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) - && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> + && (sizeof(output_type) <= 1))>> : lower_bound_config<128, 1> +{}; + +// Based on value_type = float, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<128, 16> {}; // Based on value_type = rocprim::half, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -473,40 +641,51 @@ struct default_lower_bound_config< // Based on value_type = rocprim::half, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : lower_bound_config<64, 4> + && (sizeof(output_type) > 2))>> : lower_bound_config<128, 8> {}; // Based on value_type = rocprim::half, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : lower_bound_config<64, 4> + && (sizeof(output_type) > 1))>> : lower_bound_config<128, 8> {}; // Based on value_type = rocprim::half, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> - : lower_bound_config<64, 4> + : lower_bound_config<128, 8> +{}; + +// Based on value_type = rocprim::half, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : lower_bound_config<64, 8> {}; // Based on value_type = int64_t, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -518,42 +697,54 @@ struct default_lower_bound_config< // Based on value_type = int64_t, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : lower_bound_config<256, 1> + : lower_bound_config<64, 16> {}; // Based on value_type = int64_t, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : lower_bound_config<256, 1> + : lower_bound_config<256, 16> {}; // Based on value_type = int64_t, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> + && (sizeof(output_type) <= 1))>> : lower_bound_config<64, 16> +{}; + +// Based on value_type = int64_t, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<128, 16> {}; // Based on value_type = int, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -565,773 +756,685 @@ struct default_lower_bound_config< // Based on value_type = int, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : lower_bound_config<256, 1> + : lower_bound_config<128, 1> {}; // Based on value_type = int, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : lower_bound_config<256, 1> + : lower_bound_config<128, 1> {}; // Based on value_type = int, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) - && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> + && (sizeof(output_type) <= 1))>> : lower_bound_config<128, 1> +{}; + +// Based on value_type = int, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<64, 16> {}; // Based on value_type = short, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : lower_bound_config<64, 8> + : lower_bound_config<64, 16> {}; // Based on value_type = short, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : lower_bound_config<64, 4> + : lower_bound_config<64, 8> {}; // Based on value_type = short, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : lower_bound_config<64, 4> + : lower_bound_config<64, 8> {}; // Based on value_type = short, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 1))>> : lower_bound_config<128, 4> + && (sizeof(output_type) <= 1))>> : lower_bound_config<64, 8> +{}; + +// Based on value_type = short, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<256, 8> {}; // Based on value_type = int8_t, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : lower_bound_config<128, 4> + && (sizeof(output_type) > 4))>> : lower_bound_config<256, 8> {}; // Based on value_type = int8_t, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : lower_bound_config<64, 4> + && (sizeof(output_type) > 2))>> : lower_bound_config<128, 4> {}; // Based on value_type = int8_t, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : lower_bound_config<256, 4> + && (sizeof(output_type) > 1))>> : lower_bound_config<256, 8> {}; // Based on value_type = int8_t, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> - : lower_bound_config<128, 8> + : lower_bound_config<64, 8> {}; -// Based on value_type = double, output_type = int64_t +// Based on value_type = int8_t, output_type = rocprim::int128_t template struct default_lower_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : lower_bound_config<256, 8> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int64_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> : lower_bound_config<256, 1> {}; -// Based on value_type = double, output_type = int +// Based on value_type = rocprim::int128_t, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : lower_bound_config<256, 1> + : lower_bound_config<64, 16> {}; -// Based on value_type = double, output_type = short +// Based on value_type = rocprim::int128_t, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : lower_bound_config<256, 1> + : lower_bound_config<64, 16> {}; -// Based on value_type = double, output_type = int8_t +// Based on value_type = rocprim::int128_t, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 1))>> : lower_bound_config<64, 16> {}; -// Based on value_type = float, output_type = int64_t +// Based on value_type = rocprim::int128_t, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<64, 16> +{}; + +// Based on value_type = double, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> : lower_bound_config<256, 1> {}; -// Based on value_type = float, output_type = int +// Based on value_type = double, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> : lower_bound_config<256, 1> {}; -// Based on value_type = float, output_type = short +// Based on value_type = double, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> : lower_bound_config<256, 1> {}; -// Based on value_type = float, output_type = int8_t +// Based on value_type = double, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> {}; -// Based on value_type = rocprim::half, output_type = int64_t -template -struct default_lower_bound_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : lower_bound_config<256, 1> -{}; - -// Based on value_type = rocprim::half, output_type = int -template -struct default_lower_bound_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : lower_bound_config<256, 1> -{}; - -// Based on value_type = rocprim::half, output_type = short +// Based on value_type = double, output_type = rocprim::int128_t template struct default_lower_bound_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : lower_bound_config<256, 1> -{}; - -// Based on value_type = rocprim::half, output_type = int8_t -template -struct default_lower_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> - : lower_bound_config<64, 1> -{}; - -// Based on value_type = int64_t, output_type = int64_t -template -struct default_lower_bound_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : lower_bound_config<256, 1> -{}; - -// Based on value_type = int64_t, output_type = int -template -struct default_lower_bound_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : lower_bound_config<256, 1> -{}; - -// Based on value_type = int64_t, output_type = short -template -struct default_lower_bound_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : lower_bound_config<256, 1> -{}; - -// Based on value_type = int64_t, output_type = int8_t -template -struct default_lower_bound_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<256, 16> {}; -// Based on value_type = int, output_type = int64_t +// Based on value_type = float, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> : lower_bound_config<256, 1> {}; -// Based on value_type = int, output_type = int +// Based on value_type = float, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> : lower_bound_config<256, 1> {}; -// Based on value_type = int, output_type = short +// Based on value_type = float, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> : lower_bound_config<256, 1> {}; -// Based on value_type = int, output_type = int8_t +// Based on value_type = float, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> {}; -// Based on value_type = short, output_type = int64_t +// Based on value_type = float, output_type = rocprim::int128_t template struct default_lower_bound_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : lower_bound_config<256, 1> -{}; - -// Based on value_type = short, output_type = int -template -struct default_lower_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> : lower_bound_config<256, 1> {}; -// Based on value_type = short, output_type = short +// Based on value_type = rocprim::half, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : lower_bound_config<256, 1> + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) + && (sizeof(output_type) > 4))>> : lower_bound_config<256, 1> {}; -// Based on value_type = short, output_type = int8_t +// Based on value_type = rocprim::half, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) + && (sizeof(output_type) > 2))>> : lower_bound_config<64, 4> {}; -// Based on value_type = int8_t, output_type = int64_t +// Based on value_type = rocprim::half, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : lower_bound_config<256, 1> + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) + && (sizeof(output_type) > 1))>> : lower_bound_config<256, 4> {}; -// Based on value_type = int8_t, output_type = int +// Based on value_type = rocprim::half, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : lower_bound_config<256, 1> + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> + : lower_bound_config<256, 4> {}; -// Based on value_type = int8_t, output_type = short +// Based on value_type = rocprim::half, output_type = rocprim::int128_t template struct default_lower_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : lower_bound_config<256, 1> + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : lower_bound_config<256, 1> {}; -// Based on value_type = int8_t, output_type = int8_t +// Based on value_type = int64_t, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> - : lower_bound_config<64, 1> + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : lower_bound_config<256, 1> {}; -// Based on value_type = double, output_type = int64_t +// Based on value_type = int64_t, output_type = int template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> : lower_bound_config<256, 1> {}; -// Based on value_type = double, output_type = int +// Based on value_type = int64_t, output_type = short template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> : lower_bound_config<256, 1> {}; -// Based on value_type = double, output_type = short +// Based on value_type = int64_t, output_type = int8_t template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : lower_bound_config<256, 1> + && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> {}; -// Based on value_type = double, output_type = int8_t +// Based on value_type = int64_t, output_type = rocprim::int128_t template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<256, 16> {}; -// Based on value_type = float, output_type = int64_t +// Based on value_type = int, output_type = int64_t template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> : lower_bound_config<256, 1> {}; -// Based on value_type = float, output_type = int +// Based on value_type = int, output_type = int template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> : lower_bound_config<256, 1> {}; -// Based on value_type = float, output_type = short +// Based on value_type = int, output_type = short template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> : lower_bound_config<256, 1> {}; -// Based on value_type = float, output_type = int8_t +// Based on value_type = int, output_type = int8_t template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> {}; -// Based on value_type = rocprim::half, output_type = int64_t -template -struct default_lower_bound_config< - static_cast(target_arch::gfx906), - value_type, - output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : lower_bound_config<256, 4> -{}; - -// Based on value_type = rocprim::half, output_type = int +// Based on value_type = int, output_type = rocprim::int128_t template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : lower_bound_config<256, 4> -{}; - -// Based on value_type = rocprim::half, output_type = short -template -struct default_lower_bound_config< - static_cast(target_arch::gfx906), - value_type, - output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : lower_bound_config<256, 4> -{}; - -// Based on value_type = rocprim::half, output_type = int8_t -template -struct default_lower_bound_config< - static_cast(target_arch::gfx906), - value_type, - output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> - : lower_bound_config<256, 4> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<256, 1> {}; -// Based on value_type = int64_t, output_type = int64_t +// Based on value_type = short, output_type = int64_t template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : lower_bound_config<256, 1> + : lower_bound_config<256, 4> {}; -// Based on value_type = int64_t, output_type = int +// Based on value_type = short, output_type = int template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : lower_bound_config<256, 1> + : lower_bound_config<256, 4> {}; -// Based on value_type = int64_t, output_type = short +// Based on value_type = short, output_type = short template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : lower_bound_config<256, 1> -{}; - -// Based on value_type = int64_t, output_type = int8_t -template -struct default_lower_bound_config< - static_cast(target_arch::gfx906), - value_type, - output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> + : lower_bound_config<256, 4> {}; -// Based on value_type = int, output_type = int64_t +// Based on value_type = short, output_type = int8_t template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) - && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : lower_bound_config<256, 1> + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 4> {}; -// Based on value_type = int, output_type = int +// Based on value_type = short, output_type = rocprim::int128_t template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) - && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : lower_bound_config<256, 1> + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<128, 16> {}; -// Based on value_type = int, output_type = short +// Based on value_type = int8_t, output_type = int64_t template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) - && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : lower_bound_config<256, 1> + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) + && (sizeof(output_type) > 4))>> : lower_bound_config<256, 4> {}; -// Based on value_type = int, output_type = int8_t +// Based on value_type = int8_t, output_type = int template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) - && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) + && (sizeof(output_type) > 2))>> : lower_bound_config<256, 4> {}; -// Based on value_type = short, output_type = int64_t +// Based on value_type = int8_t, output_type = short template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : lower_bound_config<256, 4> + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) + && (sizeof(output_type) > 1))>> : lower_bound_config<256, 4> {}; -// Based on value_type = short, output_type = int +// Based on value_type = int8_t, output_type = int8_t template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 4> {}; -// Based on value_type = short, output_type = short +// Based on value_type = int8_t, output_type = rocprim::int128_t template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : lower_bound_config<256, 4> + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : lower_bound_config<64, 4> {}; -// Based on value_type = short, output_type = int8_t +// Based on value_type = rocprim::int128_t, output_type = int64_t template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 4> + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : lower_bound_config<256, 1> {}; -// Based on value_type = int8_t, output_type = int64_t +// Based on value_type = rocprim::int128_t, output_type = int template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : lower_bound_config<256, 4> + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : lower_bound_config<256, 8> {}; -// Based on value_type = int8_t, output_type = int +// Based on value_type = rocprim::int128_t, output_type = short template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : lower_bound_config<256, 4> + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : lower_bound_config<256, 16> {}; -// Based on value_type = int8_t, output_type = short +// Based on value_type = rocprim::int128_t, output_type = int8_t template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : lower_bound_config<256, 4> + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 16> {}; -// Based on value_type = int8_t, output_type = int8_t +// Based on value_type = rocprim::int128_t, output_type = rocprim::int128_t template struct default_lower_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> - : lower_bound_config<256, 4> + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<256, 16> {}; // Based on value_type = double, output_type = int64_t @@ -1381,6 +1484,18 @@ struct default_lower_bound_config< && (sizeof(output_type) <= 1))>> : lower_bound_config<128, 1> {}; +// Based on value_type = double, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<256, 16> +{}; + // Based on value_type = float, output_type = int64_t template struct default_lower_bound_config< @@ -1428,6 +1543,18 @@ struct default_lower_bound_config< && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> {}; +// Based on value_type = float, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<256, 16> +{}; + // Based on value_type = rocprim::half, output_type = int64_t template struct default_lower_bound_config< @@ -1436,7 +1563,7 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : lower_bound_config<256, 4> + && (sizeof(output_type) > 4))>> : lower_bound_config<256, 2> {}; // Based on value_type = rocprim::half, output_type = int @@ -1447,7 +1574,7 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : lower_bound_config<256, 4> + && (sizeof(output_type) > 2))>> : lower_bound_config<64, 4> {}; // Based on value_type = rocprim::half, output_type = short @@ -1458,7 +1585,7 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : lower_bound_config<256, 4> + && (sizeof(output_type) > 1))>> : lower_bound_config<64, 4> {}; // Based on value_type = rocprim::half, output_type = int8_t @@ -1469,7 +1596,18 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> - : lower_bound_config<128, 1> + : lower_bound_config<256, 4> +{}; + +// Based on value_type = rocprim::half, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : lower_bound_config<256, 1> {}; // Based on value_type = int64_t, output_type = int64_t @@ -1519,6 +1657,18 @@ struct default_lower_bound_config< && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> {}; +// Based on value_type = int64_t, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<256, 16> +{}; + // Based on value_type = int, output_type = int64_t template struct default_lower_bound_config< @@ -1566,6 +1716,18 @@ struct default_lower_bound_config< && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 2> {}; +// Based on value_type = int, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<256, 16> +{}; + // Based on value_type = short, output_type = int64_t template struct default_lower_bound_config< @@ -1613,6 +1775,18 @@ struct default_lower_bound_config< && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 4> {}; +// Based on value_type = short, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<256, 16> +{}; + // Based on value_type = int8_t, output_type = int64_t template struct default_lower_bound_config< @@ -1657,57 +1831,139 @@ struct default_lower_bound_config< : lower_bound_config<256, 4> {}; +// Based on value_type = int8_t, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : lower_bound_config<64, 4> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int64_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : lower_bound_config<256, 1> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int +template +struct default_lower_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : lower_bound_config<256, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = short +template +struct default_lower_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : lower_bound_config<256, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int8_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<256, 16> +{}; + // Based on value_type = double, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : lower_bound_config<64, 8> + : lower_bound_config<64, 4> {}; // Based on value_type = double, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : lower_bound_config<64, 8> + : lower_bound_config<64, 4> {}; // Based on value_type = double, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : lower_bound_config<256, 16> + : lower_bound_config<256, 4> {}; // Based on value_type = double, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> + && (sizeof(output_type) <= 1))>> : lower_bound_config<64, 4> +{}; + +// Based on value_type = double, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<64, 4> {}; // Based on value_type = float, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -1719,7 +1975,7 @@ struct default_lower_bound_config< // Based on value_type = float, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -1731,7 +1987,7 @@ struct default_lower_bound_config< // Based on value_type = float, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -1743,133 +1999,168 @@ struct default_lower_bound_config< // Based on value_type = float, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> +{}; + +// Based on value_type = float, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) - && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<256, 1> {}; // Based on value_type = rocprim::half, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : lower_bound_config<64, 4> + && (sizeof(output_type) > 4))>> : lower_bound_config<128, 4> {}; // Based on value_type = rocprim::half, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : lower_bound_config<64, 4> + && (sizeof(output_type) > 2))>> : lower_bound_config<64, 2> {}; // Based on value_type = rocprim::half, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : lower_bound_config<64, 4> + && (sizeof(output_type) > 1))>> : lower_bound_config<64, 2> {}; // Based on value_type = rocprim::half, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> - : lower_bound_config<64, 4> + : lower_bound_config<64, 2> +{}; + +// Based on value_type = rocprim::half, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : lower_bound_config<128, 4> {}; // Based on value_type = int64_t, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : lower_bound_config<128, 1> + : lower_bound_config<128, 4> {}; // Based on value_type = int64_t, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : lower_bound_config<64, 8> + : lower_bound_config<64, 4> {}; // Based on value_type = int64_t, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : lower_bound_config<256, 16> + : lower_bound_config<128, 4> {}; // Based on value_type = int64_t, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 2> + && (sizeof(output_type) <= 1))>> : lower_bound_config<64, 4> +{}; + +// Based on value_type = int64_t, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<64, 4> {}; // Based on value_type = int, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : lower_bound_config<128, 1> + : lower_bound_config<256, 1> {}; // Based on value_type = int, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : lower_bound_config<128, 1> + : lower_bound_config<256, 1> {}; // Based on value_type = int, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -1881,7 +2172,7 @@ struct default_lower_bound_config< // Based on value_type = int, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -1889,148 +2180,254 @@ struct default_lower_bound_config< && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> {}; +// Based on value_type = int, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<256, 1> +{}; + // Based on value_type = short, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : lower_bound_config<128, 1> + : lower_bound_config<256, 1> {}; // Based on value_type = short, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : lower_bound_config<128, 1> + : lower_bound_config<256, 1> {}; // Based on value_type = short, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : lower_bound_config<256, 4> + : lower_bound_config<256, 1> {}; // Based on value_type = short, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 1))>> : lower_bound_config<64, 4> + && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> +{}; + +// Based on value_type = short, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<256, 1> {}; // Based on value_type = int8_t, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : lower_bound_config<128, 1> + && (sizeof(output_type) > 4))>> : lower_bound_config<128, 2> {}; // Based on value_type = int8_t, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : lower_bound_config<128, 1> + && (sizeof(output_type) > 2))>> : lower_bound_config<256, 2> {}; // Based on value_type = int8_t, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : lower_bound_config<128, 1> + && (sizeof(output_type) > 1))>> : lower_bound_config<256, 2> {}; // Based on value_type = int8_t, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> + : lower_bound_config<128, 2> +{}; + +// Based on value_type = int8_t, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : lower_bound_config<256, 2> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int64_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : lower_bound_config<256, 1> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int +template +struct default_lower_bound_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> : lower_bound_config<64, 4> {}; -// Based on value_type = double, output_type = int64_t +// Based on value_type = rocprim::int128_t, output_type = short +template +struct default_lower_bound_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : lower_bound_config<64, 4> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int8_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 1))>> : lower_bound_config<64, 4> +{}; + +// Based on value_type = rocprim::int128_t, output_type = rocprim::int128_t template struct default_lower_bound_config< static_cast(target_arch::gfx90a), value_type, output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<64, 4> +{}; + +// Based on value_type = double, output_type = int64_t +template +struct default_lower_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : lower_bound_config<64, 4> + : lower_bound_config<64, 8> {}; // Based on value_type = double, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : lower_bound_config<64, 4> + : lower_bound_config<64, 8> {}; // Based on value_type = double, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : lower_bound_config<256, 4> + : lower_bound_config<256, 16> {}; // Based on value_type = double, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : lower_bound_config<64, 4> + && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> +{}; + +// Based on value_type = double, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<256, 16> {}; // Based on value_type = float, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -2042,7 +2439,7 @@ struct default_lower_bound_config< // Based on value_type = float, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -2054,7 +2451,7 @@ struct default_lower_bound_config< // Based on value_type = float, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -2066,7 +2463,7 @@ struct default_lower_bound_config< // Based on value_type = float, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -2074,125 +2471,160 @@ struct default_lower_bound_config< && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> {}; +// Based on value_type = float, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<256, 16> +{}; + // Based on value_type = rocprim::half, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : lower_bound_config<128, 4> + && (sizeof(output_type) > 4))>> : lower_bound_config<256, 2> {}; // Based on value_type = rocprim::half, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : lower_bound_config<64, 2> + && (sizeof(output_type) > 2))>> : lower_bound_config<64, 4> {}; // Based on value_type = rocprim::half, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : lower_bound_config<64, 2> + && (sizeof(output_type) > 1))>> : lower_bound_config<64, 4> {}; // Based on value_type = rocprim::half, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> - : lower_bound_config<64, 2> + : lower_bound_config<256, 4> +{}; + +// Based on value_type = rocprim::half, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : lower_bound_config<256, 1> {}; // Based on value_type = int64_t, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : lower_bound_config<128, 4> + : lower_bound_config<128, 1> {}; // Based on value_type = int64_t, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : lower_bound_config<64, 4> + : lower_bound_config<64, 8> +{}; + +// Based on value_type = int64_t, output_type = short +template +struct default_lower_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : lower_bound_config<256, 16> {}; -// Based on value_type = int64_t, output_type = short +// Based on value_type = int64_t, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : lower_bound_config<128, 4> + && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 2> {}; -// Based on value_type = int64_t, output_type = int8_t +// Based on value_type = int64_t, output_type = rocprim::int128_t template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : lower_bound_config<64, 4> + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<256, 16> {}; // Based on value_type = int, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : lower_bound_config<256, 1> + : lower_bound_config<128, 1> {}; // Based on value_type = int, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : lower_bound_config<256, 1> + : lower_bound_config<128, 1> {}; // Based on value_type = int, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -2204,7 +2636,7 @@ struct default_lower_bound_config< // Based on value_type = int, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -2212,95 +2644,201 @@ struct default_lower_bound_config< && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> {}; +// Based on value_type = int, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<256, 16> +{}; + // Based on value_type = short, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : lower_bound_config<256, 1> + : lower_bound_config<128, 1> {}; // Based on value_type = short, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : lower_bound_config<256, 1> + : lower_bound_config<128, 1> {}; // Based on value_type = short, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : lower_bound_config<256, 1> + : lower_bound_config<256, 4> {}; // Based on value_type = short, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> + && (sizeof(output_type) <= 1))>> : lower_bound_config<64, 4> +{}; + +// Based on value_type = short, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<256, 16> {}; // Based on value_type = int8_t, output_type = int64_t template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : lower_bound_config<128, 2> + && (sizeof(output_type) > 4))>> : lower_bound_config<128, 1> {}; // Based on value_type = int8_t, output_type = int template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : lower_bound_config<256, 2> + && (sizeof(output_type) > 2))>> : lower_bound_config<128, 1> {}; // Based on value_type = int8_t, output_type = short template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : lower_bound_config<256, 2> + && (sizeof(output_type) > 1))>> : lower_bound_config<128, 1> {}; // Based on value_type = int8_t, output_type = int8_t template struct default_lower_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> - : lower_bound_config<128, 2> + : lower_bound_config<64, 4> +{}; + +// Based on value_type = int8_t, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : lower_bound_config<64, 4> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int64_t +template +struct default_lower_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : lower_bound_config<256, 1> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int +template +struct default_lower_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : lower_bound_config<256, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = short +template +struct default_lower_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : lower_bound_config<256, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int8_t +template +struct default_lower_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<256, 16> +{}; + +// Based on value_type = double, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<128, 16> {}; // Based on value_type = double, output_type = int64_t @@ -2324,7 +2862,7 @@ struct default_lower_bound_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : lower_bound_config<256, 1> + : lower_bound_config<64, 16> {}; // Based on value_type = double, output_type = short @@ -2336,7 +2874,7 @@ struct default_lower_bound_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : lower_bound_config<256, 1> + : lower_bound_config<64, 16> {}; // Based on value_type = double, output_type = int8_t @@ -2347,7 +2885,19 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : lower_bound_config<128, 1> + && (sizeof(output_type) <= 1))>> : lower_bound_config<64, 16> +{}; + +// Based on value_type = float, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<256, 1> {}; // Based on value_type = float, output_type = int64_t @@ -2397,6 +2947,17 @@ struct default_lower_bound_config< && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> {}; +// Based on value_type = rocprim::half, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : lower_bound_config<128, 8> +{}; + // Based on value_type = rocprim::half, output_type = int64_t template struct default_lower_bound_config< @@ -2427,7 +2988,7 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : lower_bound_config<256, 4> + && (sizeof(output_type) > 1))>> : lower_bound_config<64, 8> {}; // Based on value_type = rocprim::half, output_type = int8_t @@ -2441,6 +3002,77 @@ struct default_lower_bound_config< : lower_bound_config<256, 4> {}; +// Based on value_type = rocprim::int128_t, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<64, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int64_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : lower_bound_config<128, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : lower_bound_config<128, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = short +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : lower_bound_config<128, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int8_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 1))>> : lower_bound_config<128, 16> +{}; + +// Based on value_type = int64_t, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<64, 16> +{}; + // Based on value_type = int64_t, output_type = int64_t template struct default_lower_bound_config< @@ -2462,7 +3094,7 @@ struct default_lower_bound_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : lower_bound_config<128, 1> + : lower_bound_config<64, 16> {}; // Based on value_type = int64_t, output_type = short @@ -2485,7 +3117,19 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> + && (sizeof(output_type) <= 1))>> : lower_bound_config<64, 16> +{}; + +// Based on value_type = int, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<128, 1> {}; // Based on value_type = int, output_type = int64_t @@ -2532,7 +3176,19 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) - && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> + && (sizeof(output_type) <= 1))>> : lower_bound_config<128, 1> +{}; + +// Based on value_type = short, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : lower_bound_config<256, 1> {}; // Based on value_type = short, output_type = int64_t @@ -2582,6 +3238,17 @@ struct default_lower_bound_config< && (sizeof(output_type) <= 1))>> : lower_bound_config<256, 1> {}; +// Based on value_type = int8_t, output_type = rocprim::int128_t +template +struct default_lower_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : lower_bound_config<256, 4> +{}; + // Based on value_type = int8_t, output_type = int64_t template struct default_lower_bound_config< @@ -2601,7 +3268,7 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : lower_bound_config<128, 4> + && (sizeof(output_type) > 2))>> : lower_bound_config<128, 1> {}; // Based on value_type = int8_t, output_type = short @@ -2612,7 +3279,7 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : lower_bound_config<64, 16> + && (sizeof(output_type) > 1))>> : lower_bound_config<64, 8> {}; // Based on value_type = int8_t, output_type = int8_t @@ -2623,7 +3290,7 @@ struct default_lower_bound_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> - : lower_bound_config<64, 16> + : lower_bound_config<128, 4> {}; } // end namespace detail diff --git a/rocprim/include/rocprim/device/detail/config/device_merge.hpp b/rocprim/include/rocprim/device/detail/config/device_merge.hpp index ec7e922b8..8e15cbd7f 100644 --- a/rocprim/include/rocprim/device/detail/config/device_merge.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_merge.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,7 +21,9 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_MERGE_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_MERGE_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" #include @@ -88,6 +90,17 @@ struct default_merge_config< : merge_config<1024, 4> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<512, 4> +{}; + // Based on key_type = double, value_type = empty_type template struct default_merge_config< @@ -145,6 +158,17 @@ struct default_merge_config< : merge_config<1024, 4> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 2> +{}; + // Based on key_type = float, value_type = empty_type template struct default_merge_config< @@ -202,6 +226,17 @@ struct default_merge_config< : merge_config<256, 10> {}; +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : merge_config<1024, 1> +{}; + // Based on key_type = rocprim::half, value_type = empty_type template struct default_merge_config< @@ -258,6 +293,17 @@ struct default_merge_config< : merge_config<1024, 4> {}; +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<512, 4> +{}; + // Based on key_type = int64_t, value_type = empty_type template struct default_merge_config< @@ -315,6 +361,17 @@ struct default_merge_config< : merge_config<1024, 4> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 2> +{}; + // Based on key_type = int, value_type = empty_type template struct default_merge_config< @@ -372,6 +429,17 @@ struct default_merge_config< : merge_config<1024, 8> {}; +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 1> +{}; + // Based on key_type = short, value_type = empty_type template struct default_merge_config< @@ -429,6 +497,17 @@ struct default_merge_config< : merge_config<256, 10> {}; +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : merge_config<1024, 1> +{}; + // Based on key_type = int8_t, value_type = empty_type template struct default_merge_config< @@ -440,6 +519,74 @@ struct default_merge_config< : merge_config<256, 11> {}; +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_merge_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_config<256, 8> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_merge_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_config<1024, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_merge_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_config<1024, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_merge_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_config<1024, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_merge_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : merge_config<256, 8> +{}; + // Based on key_type = double, value_type = int64_t template struct default_merge_config< @@ -485,6 +632,17 @@ struct default_merge_config< : merge_config<1024, 4> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<512, 4> +{}; + // Based on key_type = double, value_type = empty_type template struct default_merge_config< @@ -542,6 +700,17 @@ struct default_merge_config< : merge_config<1024, 8> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 2> +{}; + // Based on key_type = float, value_type = empty_type template struct default_merge_config< @@ -562,7 +731,7 @@ struct default_merge_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_config<512, 8> + : merge_config<1024, 4> {}; // Based on key_type = rocprim::half, value_type = int @@ -584,7 +753,7 @@ struct default_merge_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_config<1024, 8> + : merge_config<256, 10> {}; // Based on key_type = rocprim::half, value_type = int8_t @@ -596,7 +765,18 @@ struct default_merge_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_config<1024, 8> + : merge_config<256, 10> +{}; + +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : merge_config<1024, 2> {}; // Based on key_type = rocprim::half, value_type = empty_type @@ -655,6 +835,17 @@ struct default_merge_config< : merge_config<1024, 4> {}; +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<512, 4> +{}; + // Based on key_type = int64_t, value_type = empty_type template struct default_merge_config< @@ -712,6 +903,17 @@ struct default_merge_config< : merge_config<1024, 8> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 2> +{}; + // Based on key_type = int, value_type = empty_type template struct default_merge_config< @@ -769,6 +971,17 @@ struct default_merge_config< : merge_config<512, 16> {}; +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<512, 4> +{}; + // Based on key_type = short, value_type = empty_type template struct default_merge_config< @@ -826,6 +1039,17 @@ struct default_merge_config< : merge_config<512, 8> {}; +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : merge_config<1024, 2> +{}; + // Based on key_type = int8_t, value_type = empty_type template struct default_merge_config< @@ -837,6 +1061,74 @@ struct default_merge_config< : merge_config<512, 16> {}; +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_merge_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_config<256, 8> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_merge_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_config<256, 8> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_merge_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_config<1024, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_merge_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_config<1024, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_merge_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : merge_config<1024, 2> +{}; + // Based on key_type = double, value_type = int64_t template struct default_merge_config< @@ -882,6 +1174,17 @@ struct default_merge_config< : merge_config<256, 5> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 1> +{}; + // Based on key_type = double, value_type = empty_type template struct default_merge_config< @@ -939,6 +1242,17 @@ struct default_merge_config< : merge_config<512, 4> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 2> +{}; + // Based on key_type = float, value_type = empty_type template struct default_merge_config< @@ -993,7 +1307,18 @@ struct default_merge_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_config<256, 10> + : merge_config<512, 4> +{}; + +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : merge_config<1024, 2> {}; // Based on key_type = rocprim::half, value_type = empty_type @@ -1052,6 +1377,17 @@ struct default_merge_config< : merge_config<256, 5> {}; +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 1> +{}; + // Based on key_type = int64_t, value_type = empty_type template struct default_merge_config< @@ -1109,19 +1445,30 @@ struct default_merge_config< : merge_config<512, 4> {}; -// Based on key_type = int, value_type = empty_type +// Based on key_type = int, value_type = rocprim::int128_t template struct default_merge_config< static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) - && (std::is_same::value))>> - : merge_config<256, 10> + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 2> {}; -// Based on key_type = short, value_type = int64_t +// Based on key_type = int, value_type = empty_type +template +struct default_merge_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) + && (std::is_same::value))>> + : merge_config<256, 10> +{}; + +// Based on key_type = short, value_type = int64_t template struct default_merge_config< static_cast(target_arch::gfx906), @@ -1166,6 +1513,17 @@ struct default_merge_config< : merge_config<256, 10> {}; +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 2> +{}; + // Based on key_type = short, value_type = empty_type template struct default_merge_config< @@ -1223,6 +1581,17 @@ struct default_merge_config< : merge_config<512, 4> {}; +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : merge_config<1024, 2> +{}; + // Based on key_type = int8_t, value_type = empty_type template struct default_merge_config< @@ -1234,6 +1603,74 @@ struct default_merge_config< : merge_config<256, 11> {}; +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_merge_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_config<512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_merge_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_config<512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_merge_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_config<512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_merge_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_config<512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_merge_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : merge_config<512, 2> +{}; + // Based on key_type = double, value_type = int64_t template struct default_merge_config< @@ -1279,6 +1716,17 @@ struct default_merge_config< : merge_config<256, 5> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 1> +{}; + // Based on key_type = double, value_type = empty_type template struct default_merge_config< @@ -1336,6 +1784,17 @@ struct default_merge_config< : merge_config<512, 4> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 2> +{}; + // Based on key_type = float, value_type = empty_type template struct default_merge_config< @@ -1393,6 +1852,17 @@ struct default_merge_config< : merge_config<256, 4> {}; +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : merge_config<1024, 2> +{}; + // Based on key_type = rocprim::half, value_type = empty_type template struct default_merge_config< @@ -1449,6 +1919,17 @@ struct default_merge_config< : merge_config<256, 5> {}; +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 1> +{}; + // Based on key_type = int64_t, value_type = empty_type template struct default_merge_config< @@ -1506,6 +1987,17 @@ struct default_merge_config< : merge_config<512, 4> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 2> +{}; + // Based on key_type = int, value_type = empty_type template struct default_merge_config< @@ -1563,6 +2055,17 @@ struct default_merge_config< : merge_config<512, 4> {}; +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 2> +{}; + // Based on key_type = short, value_type = empty_type template struct default_merge_config< @@ -1620,6 +2123,17 @@ struct default_merge_config< : merge_config<512, 4> {}; +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : merge_config<1024, 2> +{}; + // Based on key_type = int8_t, value_type = empty_type template struct default_merge_config< @@ -1631,32 +2145,100 @@ struct default_merge_config< : merge_config<512, 8> {}; +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_merge_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_config<512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_merge_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_config<512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_merge_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_config<512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_merge_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_config<512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_merge_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : merge_config<512, 2> +{}; + // Based on key_type = double, value_type = int64_t template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : merge_config<1024, 2> + && (sizeof(value_type) > 4))>> : merge_config<512, 2> {}; // Based on key_type = double, value_type = int template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : merge_config<512, 2> + && (sizeof(value_type) > 2))>> : merge_config<256, 5> {}; // Based on key_type = double, value_type = short template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1667,7 +2249,7 @@ struct default_merge_config< // Based on key_type = double, value_type = int8_t template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1676,22 +2258,33 @@ struct default_merge_config< : merge_config<256, 5> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 1> +{}; + // Based on key_type = double, value_type = empty_type template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_config<256, 7> + : merge_config<256, 4> {}; // Based on key_type = float, value_type = int64_t template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1702,18 +2295,18 @@ struct default_merge_config< // Based on key_type = float, value_type = int template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : merge_config<512, 2> + && (sizeof(value_type) > 2))>> : merge_config<512, 4> {}; // Based on key_type = float, value_type = short template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1724,7 +2317,7 @@ struct default_merge_config< // Based on key_type = float, value_type = int8_t template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1733,22 +2326,33 @@ struct default_merge_config< : merge_config<512, 4> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 1> +{}; + // Based on key_type = float, value_type = empty_type template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : merge_config<512, 4> + : merge_config<256, 10> {}; // Based on key_type = rocprim::half, value_type = int64_t template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1759,74 +2363,85 @@ struct default_merge_config< // Based on key_type = rocprim::half, value_type = int template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_config<1024, 2> + : merge_config<512, 2> {}; // Based on key_type = rocprim::half, value_type = short template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_config<256, 4> + : merge_config<512, 4> {}; // Based on key_type = rocprim::half, value_type = int8_t template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_config<256, 4> + : merge_config<256, 10> +{}; + +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : merge_config<1024, 1> {}; // Based on key_type = rocprim::half, value_type = empty_type template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (std::is_same::value))>> - : merge_config<256, 8> + : merge_config<256, 11> {}; // Based on key_type = int64_t, value_type = int64_t template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : merge_config<1024, 2> + && (sizeof(value_type) > 4))>> : merge_config<512, 2> {}; // Based on key_type = int64_t, value_type = int template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : merge_config<512, 2> + && (sizeof(value_type) > 2))>> : merge_config<256, 5> {}; // Based on key_type = int64_t, value_type = short template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1837,7 +2452,7 @@ struct default_merge_config< // Based on key_type = int64_t, value_type = int8_t template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1846,22 +2461,33 @@ struct default_merge_config< : merge_config<256, 5> {}; +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 1> +{}; + // Based on key_type = int64_t, value_type = empty_type template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_config<256, 7> + : merge_config<256, 4> {}; // Based on key_type = int, value_type = int64_t template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1872,18 +2498,18 @@ struct default_merge_config< // Based on key_type = int, value_type = int template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : merge_config<512, 2> + && (sizeof(value_type) > 2))>> : merge_config<1024, 2> {}; // Based on key_type = int, value_type = short template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1894,7 +2520,7 @@ struct default_merge_config< // Based on key_type = int, value_type = int8_t template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1903,10 +2529,21 @@ struct default_merge_config< : merge_config<512, 4> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 1> +{}; + // Based on key_type = int, value_type = empty_type template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1918,29 +2555,29 @@ struct default_merge_config< // Based on key_type = short, value_type = int64_t template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : merge_config<1024, 2> + && (sizeof(value_type) > 4))>> : merge_config<512, 2> {}; // Based on key_type = short, value_type = int template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : merge_config<512, 2> + && (sizeof(value_type) > 2))>> : merge_config<512, 4> {}; // Based on key_type = short, value_type = short template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1951,31 +2588,42 @@ struct default_merge_config< // Based on key_type = short, value_type = int8_t template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_config<512, 4> + : merge_config<256, 10> +{}; + +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 2> {}; // Based on key_type = short, value_type = empty_type template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (std::is_same::value))>> - : merge_config<512, 8> + : merge_config<256, 16> {}; // Based on key_type = int8_t, value_type = int64_t template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -1986,7 +2634,7 @@ struct default_merge_config< // Based on key_type = int8_t, value_type = int template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -1997,7 +2645,7 @@ struct default_merge_config< // Based on key_type = int8_t, value_type = short template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -2008,52 +2656,131 @@ struct default_merge_config< // Based on key_type = int8_t, value_type = int8_t template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_config<512, 4> + : merge_config<256, 10> +{}; + +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : merge_config<1024, 1> {}; // Based on key_type = int8_t, value_type = empty_type template struct default_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (std::is_same::value))>> - : merge_config<512, 8> + : merge_config<256, 16> {}; -// Based on key_type = double, value_type = int64_t +// Based on key_type = rocprim::int128_t, value_type = int64_t template struct default_merge_config< static_cast(target_arch::gfx90a), key_type, value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_config<512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_config<512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_config<512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_config<512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<512, 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : merge_config<512, 2> +{}; + +// Based on key_type = double, value_type = int64_t +template +struct default_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : merge_config<512, 2> + && (sizeof(value_type) > 4))>> : merge_config<1024, 2> {}; // Based on key_type = double, value_type = int template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : merge_config<256, 5> + && (sizeof(value_type) > 2))>> : merge_config<512, 2> {}; // Based on key_type = double, value_type = short template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -2064,7 +2791,7 @@ struct default_merge_config< // Based on key_type = double, value_type = int8_t template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -2073,22 +2800,33 @@ struct default_merge_config< : merge_config<256, 5> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 1> +{}; + // Based on key_type = double, value_type = empty_type template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_config<256, 4> + : merge_config<256, 7> {}; // Based on key_type = float, value_type = int64_t template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -2099,18 +2837,18 @@ struct default_merge_config< // Based on key_type = float, value_type = int template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : merge_config<512, 4> + && (sizeof(value_type) > 2))>> : merge_config<512, 2> {}; // Based on key_type = float, value_type = short template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -2121,7 +2859,7 @@ struct default_merge_config< // Based on key_type = float, value_type = int8_t template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -2130,22 +2868,33 @@ struct default_merge_config< : merge_config<512, 4> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 2> +{}; + // Based on key_type = float, value_type = empty_type template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : merge_config<256, 10> + : merge_config<512, 4> {}; // Based on key_type = rocprim::half, value_type = int64_t template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -2156,74 +2905,85 @@ struct default_merge_config< // Based on key_type = rocprim::half, value_type = int template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_config<512, 2> + : merge_config<1024, 2> {}; // Based on key_type = rocprim::half, value_type = short template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_config<512, 4> + : merge_config<256, 4> {}; // Based on key_type = rocprim::half, value_type = int8_t template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_config<256, 10> + : merge_config<256, 4> +{}; + +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : merge_config<1024, 2> {}; // Based on key_type = rocprim::half, value_type = empty_type template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (std::is_same::value))>> - : merge_config<256, 11> + : merge_config<256, 8> {}; // Based on key_type = int64_t, value_type = int64_t template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : merge_config<512, 2> + && (sizeof(value_type) > 4))>> : merge_config<1024, 2> {}; // Based on key_type = int64_t, value_type = int template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : merge_config<256, 5> + && (sizeof(value_type) > 2))>> : merge_config<512, 2> {}; // Based on key_type = int64_t, value_type = short template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -2234,7 +2994,7 @@ struct default_merge_config< // Based on key_type = int64_t, value_type = int8_t template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -2243,22 +3003,33 @@ struct default_merge_config< : merge_config<256, 5> {}; +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 1> +{}; + // Based on key_type = int64_t, value_type = empty_type template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_config<256, 4> + : merge_config<256, 7> {}; // Based on key_type = int, value_type = int64_t template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -2269,18 +3040,18 @@ struct default_merge_config< // Based on key_type = int, value_type = int template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : merge_config<1024, 2> + && (sizeof(value_type) > 2))>> : merge_config<512, 2> {}; // Based on key_type = int, value_type = short template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -2291,7 +3062,7 @@ struct default_merge_config< // Based on key_type = int, value_type = int8_t template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -2300,10 +3071,21 @@ struct default_merge_config< : merge_config<512, 4> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 2> +{}; + // Based on key_type = int, value_type = empty_type template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -2315,29 +3097,29 @@ struct default_merge_config< // Based on key_type = short, value_type = int64_t template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : merge_config<512, 2> + && (sizeof(value_type) > 4))>> : merge_config<1024, 2> {}; // Based on key_type = short, value_type = int template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : merge_config<512, 4> + && (sizeof(value_type) > 2))>> : merge_config<512, 2> {}; // Based on key_type = short, value_type = short template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -2348,31 +3130,42 @@ struct default_merge_config< // Based on key_type = short, value_type = int8_t template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_config<256, 10> + : merge_config<512, 4> +{}; + +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<1024, 2> {}; // Based on key_type = short, value_type = empty_type template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (std::is_same::value))>> - : merge_config<256, 16> + : merge_config<512, 8> {}; // Based on key_type = int8_t, value_type = int64_t template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -2383,7 +3176,7 @@ struct default_merge_config< // Based on key_type = int8_t, value_type = int template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -2394,7 +3187,7 @@ struct default_merge_config< // Based on key_type = int8_t, value_type = short template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -2405,24 +3198,114 @@ struct default_merge_config< // Based on key_type = int8_t, value_type = int8_t template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_config<256, 10> + : merge_config<512, 4> +{}; + +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : merge_config<1024, 2> {}; // Based on key_type = int8_t, value_type = empty_type template struct default_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (std::is_same::value))>> - : merge_config<256, 16> + : merge_config<512, 8> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_config<512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_config<512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_config<512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_config<512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : merge_config<512, 2> +{}; + +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<512, 2> {}; // Based on key_type = double, value_type = int64_t @@ -2444,7 +3327,7 @@ struct default_merge_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : merge_config<256, 4> + && (sizeof(value_type) > 2))>> : merge_config<256, 5> {}; // Based on key_type = double, value_type = short @@ -2479,7 +3362,18 @@ struct default_merge_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_config<256, 7> + : merge_config<256, 4> +{}; + +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<512, 2> {}; // Based on key_type = float, value_type = int64_t @@ -2490,7 +3384,7 @@ struct default_merge_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : merge_config<256, 4> + && (sizeof(value_type) > 4))>> : merge_config<512, 4> {}; // Based on key_type = float, value_type = int @@ -2524,7 +3418,7 @@ struct default_merge_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_config<256, 8> + : merge_config<256, 10> {}; // Based on key_type = float, value_type = empty_type @@ -2539,6 +3433,17 @@ struct default_merge_config< : merge_config<256, 10> {}; +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : merge_config<512, 2> +{}; + // Based on key_type = rocprim::half, value_type = int64_t template struct default_merge_config< @@ -2595,6 +3500,85 @@ struct default_merge_config< : merge_config<256, 16> {}; +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<256, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_config<256, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_config<256, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_config<256, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_config<256, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : merge_config<256, 2> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<512, 2> +{}; + // Based on key_type = int64_t, value_type = int64_t template struct default_merge_config< @@ -2614,7 +3598,7 @@ struct default_merge_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : merge_config<256, 4> + && (sizeof(value_type) > 2))>> : merge_config<256, 5> {}; // Based on key_type = int64_t, value_type = short @@ -2649,7 +3633,18 @@ struct default_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_config<256, 7> + : merge_config<256, 4> +{}; + +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<512, 2> {}; // Based on key_type = int, value_type = int64_t @@ -2660,7 +3655,7 @@ struct default_merge_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : merge_config<256, 4> + && (sizeof(value_type) > 4))>> : merge_config<512, 4> {}; // Based on key_type = int, value_type = int @@ -2694,7 +3689,7 @@ struct default_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_config<256, 8> + : merge_config<256, 10> {}; // Based on key_type = int, value_type = empty_type @@ -2709,6 +3704,17 @@ struct default_merge_config< : merge_config<256, 10> {}; +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_config<512, 2> +{}; + // Based on key_type = short, value_type = int64_t template struct default_merge_config< @@ -2717,7 +3723,7 @@ struct default_merge_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : merge_config<256, 4> + && (sizeof(value_type) > 4))>> : merge_config<512, 4> {}; // Based on key_type = short, value_type = int @@ -2766,6 +3772,17 @@ struct default_merge_config< : merge_config<256, 16> {}; +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : merge_config<1024, 1> +{}; + // Based on key_type = int8_t, value_type = int64_t template struct default_merge_config< diff --git a/rocprim/include/rocprim/device/detail/config/device_merge_sort_block_merge.hpp b/rocprim/include/rocprim/device/detail/config/device_merge_sort_block_merge.hpp index 39c44610d..07948451b 100644 --- a/rocprim/include/rocprim/device/detail/config/device_merge_sort_block_merge.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_merge_sort_block_merge.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,8 +21,11 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_MERGE_SORT_BLOCK_MERGE_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_MERGE_SORT_BLOCK_MERGE_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" + #include /* DO NOT EDIT THIS FILE @@ -46,7 +49,7 @@ struct default_merge_sort_block_merge_config : merge_sort_block_merge_config_base::type {}; -// Based on key_type = double, value_type = custom_type +// Based on key_type = double, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< static_cast(target_arch::gfx1030), @@ -118,7 +121,7 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> {}; -// Based on key_type = float, value_type = custom_type +// Based on key_type = float, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< static_cast(target_arch::gfx1030), @@ -190,7 +193,7 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; -// Based on key_type = rocprim::half, value_type = custom_type +// Based on key_type = rocprim::half, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< static_cast(target_arch::gfx1030), @@ -198,7 +201,7 @@ struct default_merge_sort_block_merge_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 1> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> {}; // Based on key_type = rocprim::half, value_type = int64_t @@ -209,7 +212,7 @@ struct default_merge_sort_block_merge_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> {}; // Based on key_type = rocprim::half, value_type = int @@ -220,7 +223,7 @@ struct default_merge_sort_block_merge_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; // Based on key_type = rocprim::half, value_type = short @@ -243,7 +246,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = rocprim::half, value_type = empty_type @@ -257,7 +260,79 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = int64_t, value_type = custom_type +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< static_cast(target_arch::gfx1030), @@ -329,7 +404,7 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> {}; -// Based on key_type = int, value_type = custom_type +// Based on key_type = int, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< static_cast(target_arch::gfx1030), @@ -401,7 +476,7 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; -// Based on key_type = short, value_type = custom_type +// Based on key_type = short, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< static_cast(target_arch::gfx1030), @@ -410,7 +485,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> {}; // Based on key_type = short, value_type = int64_t @@ -473,7 +548,7 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = int8_t, value_type = custom_type +// Based on key_type = int8_t, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< static_cast(target_arch::gfx1030), @@ -481,7 +556,7 @@ struct default_merge_sort_block_merge_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> {}; // Based on key_type = int8_t, value_type = int64_t @@ -540,34 +615,34 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = double, value_type = custom_type +// Based on key_type = double, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> {}; // Based on key_type = double, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> {}; // Based on key_type = double, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -579,55 +654,55 @@ struct default_merge_sort_block_merge_config< // Based on key_type = double, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; // Based on key_type = double, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; // Based on key_type = double, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; -// Based on key_type = float, value_type = custom_type +// Based on key_type = float, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> {}; // Based on key_type = float, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -639,66 +714,66 @@ struct default_merge_sort_block_merge_config< // Based on key_type = float, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; // Based on key_type = float, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = float, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = float, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; -// Based on key_type = rocprim::half, value_type = custom_type +// Based on key_type = rocprim::half, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 1> {}; // Based on key_type = rocprim::half, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -709,29 +784,29 @@ struct default_merge_sort_block_merge_config< // Based on key_type = rocprim::half, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = rocprim::half, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; // Based on key_type = rocprim::half, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -743,7 +818,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = rocprim::half, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -751,10 +826,82 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = int64_t, value_type = custom_type +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -766,19 +913,19 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int64_t, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> {}; // Based on key_type = int64_t, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -790,55 +937,55 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int64_t, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; // Based on key_type = int64_t, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; // Based on key_type = int64_t, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; -// Based on key_type = int, value_type = custom_type +// Based on key_type = int, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> {}; // Based on key_type = int, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -850,31 +997,31 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; // Based on key_type = int, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = int, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -886,67 +1033,67 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; -// Based on key_type = short, value_type = custom_type +// Based on key_type = short, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 1> {}; // Based on key_type = short, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; // Based on key_type = short, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = short, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; // Based on key_type = short, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -958,7 +1105,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = short, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -967,43 +1114,43 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = int8_t, value_type = custom_type +// Based on key_type = int8_t, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = int8_t, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; // Based on key_type = int8_t, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = int8_t, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -1014,7 +1161,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int8_t, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -1026,7 +1173,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int8_t, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -1034,22 +1181,22 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = double, value_type = custom_type +// Based on key_type = double, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> {}; // Based on key_type = double, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1061,7 +1208,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = double, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1073,7 +1220,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = double, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1085,7 +1232,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = double, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1097,31 +1244,31 @@ struct default_merge_sort_block_merge_config< // Based on key_type = double, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = float, value_type = custom_type +// Based on key_type = float, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; // Based on key_type = float, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1133,7 +1280,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = float, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1145,19 +1292,19 @@ struct default_merge_sort_block_merge_config< // Based on key_type = float, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; // Based on key_type = float, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1169,7 +1316,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = float, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1178,10 +1325,10 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = rocprim::half, value_type = custom_type +// Based on key_type = rocprim::half, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1192,40 +1339,40 @@ struct default_merge_sort_block_merge_config< // Based on key_type = rocprim::half, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = rocprim::half, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; // Based on key_type = rocprim::half, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; // Based on key_type = rocprim::half, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1237,7 +1384,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = rocprim::half, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1245,70 +1392,142 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = int64_t, value_type = custom_type +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> {}; -// Based on key_type = int64_t, value_type = int64_t +// Based on key_type = rocprim::int128_t, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; -// Based on key_type = int64_t, value_type = int +// Based on key_type = rocprim::int128_t, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; -// Based on key_type = int64_t, value_type = short +// Based on key_type = rocprim::int128_t, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; -// Based on key_type = int64_t, value_type = int8_t +// Based on key_type = rocprim::int128_t, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; -// Based on key_type = int64_t, value_type = empty_type +// Based on key_type = rocprim::int128_t, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> +{}; + +// Based on key_type = int64_t, value_type = int64_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = int64_t, value_type = int +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = int64_t, value_type = short +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = int64_t, value_type = int8_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = int64_t, value_type = empty_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1317,10 +1536,10 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = int, value_type = custom_type +// Based on key_type = int, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1332,43 +1551,43 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<128, 1, (1 << 17) + 70000, 128, 128, 4> {}; // Based on key_type = int, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; // Based on key_type = int, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; // Based on key_type = int, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1380,7 +1599,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1389,10 +1608,10 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = short, value_type = custom_type +// Based on key_type = short, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1404,31 +1623,31 @@ struct default_merge_sort_block_merge_config< // Based on key_type = short, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; // Based on key_type = short, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; // Based on key_type = short, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1440,7 +1659,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = short, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1452,7 +1671,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = short, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1461,10 +1680,10 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = int8_t, value_type = custom_type +// Based on key_type = int8_t, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -1475,29 +1694,29 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int8_t, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; // Based on key_type = int8_t, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; // Based on key_type = int8_t, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -1508,7 +1727,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int8_t, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -1520,7 +1739,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int8_t, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -1528,10 +1747,10 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = double, value_type = custom_type +// Based on key_type = double, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1543,7 +1762,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = double, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1555,7 +1774,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = double, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1567,7 +1786,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = double, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1579,7 +1798,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = double, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1591,7 +1810,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = double, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1600,22 +1819,22 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; -// Based on key_type = float, value_type = custom_type +// Based on key_type = float, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; // Based on key_type = float, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1627,7 +1846,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = float, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1639,7 +1858,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = float, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1651,7 +1870,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = float, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1663,7 +1882,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = float, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1672,10 +1891,10 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = rocprim::half, value_type = custom_type +// Based on key_type = rocprim::half, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1686,40 +1905,40 @@ struct default_merge_sort_block_merge_config< // Based on key_type = rocprim::half, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = rocprim::half, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; // Based on key_type = rocprim::half, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; // Based on key_type = rocprim::half, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1731,7 +1950,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = rocprim::half, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1739,22 +1958,94 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = int64_t, value_type = custom_type +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> {}; // Based on key_type = int64_t, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1766,7 +2057,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int64_t, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1778,7 +2069,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int64_t, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1790,7 +2081,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int64_t, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1802,7 +2093,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int64_t, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1811,10 +2102,10 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = int, value_type = custom_type +// Based on key_type = int, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1826,7 +2117,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1838,7 +2129,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1850,7 +2141,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1862,7 +2153,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1874,7 +2165,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1883,10 +2174,10 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = short, value_type = custom_type +// Based on key_type = short, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1898,7 +2189,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = short, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1910,7 +2201,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = short, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1922,7 +2213,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = short, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1934,7 +2225,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = short, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1946,7 +2237,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = short, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1955,10 +2246,10 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = int8_t, value_type = custom_type +// Based on key_type = int8_t, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -1969,7 +2260,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int8_t, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -1980,7 +2271,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int8_t, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -1991,7 +2282,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int8_t, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -2002,7 +2293,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int8_t, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -2014,7 +2305,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int8_t, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -2022,7 +2313,7 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = double, value_type = custom_type +// Based on key_type = double, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< static_cast(target_arch::gfx90a), @@ -2031,7 +2322,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; // Based on key_type = double, value_type = int64_t @@ -2094,7 +2385,7 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; -// Based on key_type = float, value_type = custom_type +// Based on key_type = float, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< static_cast(target_arch::gfx90a), @@ -2166,7 +2457,7 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = rocprim::half, value_type = custom_type +// Based on key_type = rocprim::half, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< static_cast(target_arch::gfx90a), @@ -2196,7 +2487,7 @@ struct default_merge_sort_block_merge_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = rocprim::half, value_type = short @@ -2233,12 +2524,650 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = int64_t, value_type = custom_type +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = int64_t, value_type = int64_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = int64_t, value_type = int +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> +{}; + +// Based on key_type = int64_t, value_type = short +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> +{}; + +// Based on key_type = int64_t, value_type = int8_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> +{}; + +// Based on key_type = int64_t, value_type = empty_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) + && (std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> +{}; + +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = int, value_type = int64_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> +{}; + +// Based on key_type = int, value_type = int +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> +{}; + +// Based on key_type = int, value_type = short +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = int, value_type = int8_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = int, value_type = empty_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) + && (std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = short, value_type = int64_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> +{}; + +// Based on key_type = short, value_type = int +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = short, value_type = short template struct default_merge_sort_block_merge_config< static_cast(target_arch::gfx90a), key_type, value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = short, value_type = int8_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = short, value_type = empty_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) + && (std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = int8_t, value_type = int64_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> +{}; + +// Based on key_type = int8_t, value_type = int +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = int8_t, value_type = short +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = int8_t, value_type = int8_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = int8_t, value_type = empty_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = double, value_type = int64_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = double, value_type = int +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = double, value_type = short +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = double, value_type = int8_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = double, value_type = empty_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) + && (std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> +{}; + +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = float, value_type = int64_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = float, value_type = int +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = float, value_type = short +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> +{}; + +// Based on key_type = float, value_type = int8_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = float, value_type = empty_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) + && (std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = rocprim::half, value_type = int64_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> +{}; + +// Based on key_type = rocprim::half, value_type = int +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = rocprim::half, value_type = short +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = rocprim::half, value_type = int8_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = rocprim::half, value_type = empty_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 1024, 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::unknown), + key_type, + value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> @@ -2248,67 +3177,67 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int64_t, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; // Based on key_type = int64_t, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; // Based on key_type = int64_t, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; // Based on key_type = int64_t, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; // Based on key_type = int64_t, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = int, value_type = custom_type +// Based on key_type = int, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -2320,19 +3249,19 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; // Based on key_type = int, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -2344,19 +3273,19 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = int, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -2368,7 +3297,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -2377,10 +3306,10 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = short, value_type = custom_type +// Based on key_type = short, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -2392,31 +3321,31 @@ struct default_merge_sort_block_merge_config< // Based on key_type = short, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; // Based on key_type = short, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = short, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -2428,7 +3357,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = short, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -2440,7 +3369,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = short, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -2449,10 +3378,10 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = int8_t, value_type = custom_type +// Based on key_type = int8_t, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -2463,29 +3392,29 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int8_t, value_type = int64_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; // Based on key_type = int8_t, value_type = int template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; // Based on key_type = int8_t, value_type = short template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -2496,7 +3425,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int8_t, value_type = int8_t template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -2508,168 +3437,7 @@ struct default_merge_sort_block_merge_config< // Based on key_type = int8_t, value_type = empty_type template struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx90a), - key_type, - value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) - && (std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> -{}; - -template<> -struct default_merge_sort_block_merge_config(target_arch::gfx906), double> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> -{}; - -template<> -struct default_merge_sort_block_merge_config(target_arch::gfx906), float> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> -{}; - -template<> -struct default_merge_sort_block_merge_config(target_arch::gfx906), - int, - float> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> -{}; - -template<> -struct default_merge_sort_block_merge_config(target_arch::gfx906), int> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> -{}; - -template<> -struct default_merge_sort_block_merge_config(target_arch::gfx906), - int64_t, - double> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> -{}; - -template<> -struct default_merge_sort_block_merge_config(target_arch::gfx906), - int64_t> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> -{}; - -template<> -struct default_merge_sort_block_merge_config(target_arch::gfx906), - int8_t, - int8_t> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> -{}; - -template<> -struct default_merge_sort_block_merge_config(target_arch::gfx906), int8_t> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> -{}; - -template<> -struct default_merge_sort_block_merge_config(target_arch::gfx906), - rocprim::half> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> -{}; - -template<> -struct default_merge_sort_block_merge_config(target_arch::gfx906), - rocprim::half, - rocprim::half> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> -{}; - -template<> -struct default_merge_sort_block_merge_config(target_arch::gfx906), short> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> -{}; - -template<> -struct default_merge_sort_block_merge_config(target_arch::gfx906), - short, - short> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> -{}; - -template<> -struct default_merge_sort_block_merge_config(target_arch::gfx906), - uint8_t, - uint8_t> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> -{}; - -// Based on key_type = double, value_type = rocprim::empty_type -template -struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx906), - key_type, - value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) - && (std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> -{}; - -// Based on key_type = float, value_type = rocprim::empty_type -template -struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx906), - key_type, - value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) - && (std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> -{}; - -// Based on key_type = rocprim::half, value_type = rocprim::empty_type -template -struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx906), - key_type, - value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> -{}; - -// Based on key_type = int64_t, value_type = rocprim::empty_type -template -struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx906), - key_type, - value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) - && (std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> -{}; - -// Based on key_type = int, value_type = rocprim::empty_type -template -struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx906), - key_type, - value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) - && (std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> -{}; - -// Based on key_type = short, value_type = rocprim::empty_type -template -struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx906), - key_type, - value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(key_type) > 1) - && (std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> -{}; - -// Based on key_type = int8_t, value_type = rocprim::empty_type -template -struct default_merge_sort_block_merge_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -2677,7 +3445,7 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = double, value_type = custom_type +// Based on key_type = double, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< static_cast(target_arch::gfx942), @@ -2686,7 +3454,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; // Based on key_type = double, value_type = int64_t @@ -2710,7 +3478,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = double, value_type = short @@ -2749,7 +3517,7 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; -// Based on key_type = float, value_type = custom_type +// Based on key_type = float, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< static_cast(target_arch::gfx942), @@ -2821,7 +3589,7 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = rocprim::half, value_type = custom_type +// Based on key_type = rocprim::half, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< static_cast(target_arch::gfx942), @@ -2840,7 +3608,7 @@ struct default_merge_sort_block_merge_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = rocprim::half, value_type = int @@ -2888,7 +3656,79 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = int64_t, value_type = custom_type +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_merge_sort_block_merge_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 4> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< static_cast(target_arch::gfx942), @@ -2897,7 +3737,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 512, 2> {}; // Based on key_type = int64_t, value_type = int64_t @@ -2909,7 +3749,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = int64_t, value_type = int @@ -2921,7 +3761,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = int64_t, value_type = short @@ -2945,7 +3785,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = int64_t, value_type = empty_type @@ -2957,10 +3797,10 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = int, value_type = custom_type +// Based on key_type = int, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< static_cast(target_arch::gfx942), @@ -2981,7 +3821,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = int, value_type = int @@ -3032,7 +3872,7 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = short, value_type = custom_type +// Based on key_type = short, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< static_cast(target_arch::gfx942), @@ -3053,7 +3893,7 @@ struct default_merge_sort_block_merge_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> + : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 256, 4> {}; // Based on key_type = short, value_type = int @@ -3104,7 +3944,7 @@ struct default_merge_sort_block_merge_config< : merge_sort_block_merge_config<256, 1, (1 << 17) + 70000, 128, 128, 8> {}; -// Based on key_type = int8_t, value_type = custom_type +// Based on key_type = int8_t, value_type = rocprim::int128_t template struct default_merge_sort_block_merge_config< static_cast(target_arch::gfx942), diff --git a/rocprim/include/rocprim/device/detail/config/device_merge_sort_block_sort.hpp b/rocprim/include/rocprim/device/detail/config/device_merge_sort_block_sort.hpp index 3343376cc..59c9287fe 100644 --- a/rocprim/include/rocprim/device/detail/config/device_merge_sort_block_sort.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_merge_sort_block_sort.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,8 +21,11 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_MERGE_SORT_BLOCK_SORT_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_MERGE_SORT_BLOCK_SORT_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" + #include /* DO NOT EDIT THIS FILE @@ -46,7 +49,7 @@ struct default_merge_sort_block_sort_config : merge_sort_block_sort_config_base::type {}; -// Based on key_type = double, value_type = custom_type +// Based on key_type = double, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< static_cast(target_arch::gfx1030), @@ -54,8 +57,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = double, value_type = int64_t @@ -66,8 +68,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = double, value_type = int @@ -78,8 +79,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = double, value_type = short @@ -90,8 +90,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<512, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<512, 8> {}; // Based on key_type = double, value_type = int8_t @@ -103,7 +102,7 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = double, value_type = empty_type @@ -115,10 +114,10 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 4> {}; -// Based on key_type = float, value_type = custom_type +// Based on key_type = float, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< static_cast(target_arch::gfx1030), @@ -126,8 +125,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = float, value_type = int64_t @@ -138,8 +136,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = float, value_type = int @@ -150,8 +147,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = float, value_type = short @@ -162,8 +158,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<512, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<512, 8> {}; // Based on key_type = float, value_type = int8_t @@ -175,7 +170,7 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<1024, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<1024, 4> {}; // Based on key_type = float, value_type = empty_type @@ -187,10 +182,10 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; -// Based on key_type = rocprim::half, value_type = custom_type +// Based on key_type = rocprim::half, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< static_cast(target_arch::gfx1030), @@ -198,7 +193,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = rocprim::half, value_type = int64_t @@ -209,7 +204,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<1024, 4> {}; // Based on key_type = rocprim::half, value_type = int @@ -220,7 +215,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = rocprim::half, value_type = short @@ -231,7 +226,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<512, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = rocprim::half, value_type = int8_t @@ -243,7 +238,7 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = rocprim::half, value_type = empty_type @@ -254,10 +249,78 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (std::is_same::value))>> - : merge_sort_block_sort_config<512, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 8> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<512, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<512, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<512, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_sort_config<512, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : merge_sort_block_sort_config<256, 4> {}; -// Based on key_type = int64_t, value_type = custom_type +// Based on key_type = int64_t, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< static_cast(target_arch::gfx1030), @@ -265,8 +328,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = int64_t, value_type = int64_t @@ -277,8 +339,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int64_t, value_type = int @@ -289,8 +350,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int64_t, value_type = short @@ -301,8 +361,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<512, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<512, 8> {}; // Based on key_type = int64_t, value_type = int8_t @@ -314,7 +373,7 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int64_t, value_type = empty_type @@ -326,10 +385,10 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 4> {}; -// Based on key_type = int, value_type = custom_type +// Based on key_type = int, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< static_cast(target_arch::gfx1030), @@ -337,8 +396,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = int, value_type = int64_t @@ -349,8 +407,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = int, value_type = int @@ -361,8 +418,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int, value_type = short @@ -373,8 +429,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = int, value_type = int8_t @@ -386,7 +441,7 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<1024, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<1024, 4> {}; // Based on key_type = int, value_type = empty_type @@ -398,10 +453,10 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; -// Based on key_type = short, value_type = custom_type +// Based on key_type = short, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< static_cast(target_arch::gfx1030), @@ -409,8 +464,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = short, value_type = int64_t @@ -421,8 +475,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = short, value_type = int @@ -433,8 +486,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<512, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<512, 8> {}; // Based on key_type = short, value_type = short @@ -445,8 +497,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 16, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 16> {}; // Based on key_type = short, value_type = int8_t @@ -458,7 +509,7 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = short, value_type = empty_type @@ -470,10 +521,10 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (std::is_same::value))>> - : merge_sort_block_sort_config<512, 16, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 16> {}; -// Based on key_type = int8_t, value_type = custom_type +// Based on key_type = int8_t, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< static_cast(target_arch::gfx1030), @@ -481,7 +532,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = int8_t, value_type = int64_t @@ -492,7 +543,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = int8_t, value_type = int @@ -503,7 +554,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<512, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 8> {}; // Based on key_type = int8_t, value_type = short @@ -514,7 +565,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<512, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 8> {}; // Based on key_type = int8_t, value_type = int8_t @@ -526,7 +577,7 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 16, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 16> {}; // Based on key_type = int8_t, value_type = empty_type @@ -537,1492 +588,1636 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 32, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 32> {}; -// Based on key_type = double, value_type = custom_type +// Based on key_type = double, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = double, value_type = int64_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = double, value_type = int template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<512, 8> {}; // Based on key_type = double, value_type = short template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<512, 8> {}; // Based on key_type = double, value_type = int8_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<1024, 4> {}; // Based on key_type = double, value_type = empty_type template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 8> {}; -// Based on key_type = float, value_type = custom_type +// Based on key_type = float, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<1024, 2, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = float, value_type = int64_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<1024, 4> {}; // Based on key_type = float, value_type = int template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 16> {}; // Based on key_type = float, value_type = short template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<512, 8> {}; // Based on key_type = float, value_type = int8_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<1024, 4> {}; // Based on key_type = float, value_type = empty_type template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 16> {}; -// Based on key_type = rocprim::half, value_type = custom_type +// Based on key_type = rocprim::half, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = rocprim::half, value_type = int64_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = rocprim::half, value_type = int template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<1024, 8> {}; // Based on key_type = rocprim::half, value_type = short template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = rocprim::half, value_type = int8_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 8> {}; // Based on key_type = rocprim::half, value_type = empty_type template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 16, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<512, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<512, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<512, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_sort_config<512, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : merge_sort_block_sort_config<256, 4> {}; -// Based on key_type = int64_t, value_type = custom_type +// Based on key_type = int64_t, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int64_t, value_type = int64_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int64_t, value_type = int template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<512, 8> {}; // Based on key_type = int64_t, value_type = short template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<512, 8> {}; // Based on key_type = int64_t, value_type = int8_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<1024, 4> {}; // Based on key_type = int64_t, value_type = empty_type template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 4> {}; -// Based on key_type = int, value_type = custom_type +// Based on key_type = int, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<1024, 2, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = int, value_type = int64_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int, value_type = int template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = int, value_type = short template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<512, 8> {}; // Based on key_type = int, value_type = int8_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<1024, 4> {}; // Based on key_type = int, value_type = empty_type template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; -// Based on key_type = short, value_type = custom_type +// Based on key_type = short, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = short, value_type = int64_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = short, value_type = int template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = short, value_type = short template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = short, value_type = int8_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<1024, 8> {}; // Based on key_type = short, value_type = empty_type template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 16, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; -// Based on key_type = int8_t, value_type = custom_type +// Based on key_type = int8_t, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = int8_t, value_type = int64_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 8> {}; // Based on key_type = int8_t, value_type = int template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<1024, 8> {}; // Based on key_type = int8_t, value_type = short template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<1024, 8> {}; // Based on key_type = int8_t, value_type = int8_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = int8_t, value_type = empty_type template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 32, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 32> {}; -// Based on key_type = double, value_type = custom_type +// Based on key_type = double, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = double, value_type = int64_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = double, value_type = int template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = double, value_type = short template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = double, value_type = int8_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = double, value_type = empty_type template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 4> {}; -// Based on key_type = float, value_type = custom_type +// Based on key_type = float, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = float, value_type = int64_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = float, value_type = int template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 16> {}; // Based on key_type = float, value_type = short template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<512, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = float, value_type = int8_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = float, value_type = empty_type template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : merge_sort_block_sort_config<512, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; -// Based on key_type = rocprim::half, value_type = custom_type +// Based on key_type = rocprim::half, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = rocprim::half, value_type = int64_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = rocprim::half, value_type = int template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = rocprim::half, value_type = short template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = rocprim::half, value_type = int8_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<1024, 8> {}; // Based on key_type = rocprim::half, value_type = empty_type template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 16, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 16> {}; -// Based on key_type = int64_t, value_type = custom_type +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int64_t, value_type = int64_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int64_t, value_type = int template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int64_t, value_type = short template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int64_t, value_type = int8_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int64_t, value_type = empty_type template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 4> {}; -// Based on key_type = int, value_type = custom_type +// Based on key_type = int, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = int, value_type = int64_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int, value_type = int template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<512, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int, value_type = short template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<512, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int, value_type = int8_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = int, value_type = empty_type template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; -// Based on key_type = short, value_type = custom_type +// Based on key_type = short, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = short, value_type = int64_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = short, value_type = int template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = short, value_type = short template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 16, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = short, value_type = int8_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<1024, 8> {}; // Based on key_type = short, value_type = empty_type template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 16, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 16> {}; -// Based on key_type = int8_t, value_type = custom_type +// Based on key_type = int8_t, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = int8_t, value_type = int64_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int8_t, value_type = int template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<512, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int8_t, value_type = short template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<1024, 8> {}; // Based on key_type = int8_t, value_type = int8_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = int8_t, value_type = empty_type template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 32, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 32> {}; -// Based on key_type = double, value_type = custom_type +// Based on key_type = double, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = double, value_type = int64_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = double, value_type = int template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = double, value_type = short template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = double, value_type = int8_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = double, value_type = empty_type template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 4> {}; -// Based on key_type = float, value_type = custom_type +// Based on key_type = float, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = float, value_type = int64_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = float, value_type = int template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = float, value_type = short template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<512, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<512, 8> {}; // Based on key_type = float, value_type = int8_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = float, value_type = empty_type template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : merge_sort_block_sort_config<512, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 8> {}; -// Based on key_type = rocprim::half, value_type = custom_type +// Based on key_type = rocprim::half, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = rocprim::half, value_type = int64_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = rocprim::half, value_type = int template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<1024, 8> {}; // Based on key_type = rocprim::half, value_type = short template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = rocprim::half, value_type = int8_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<1024, 8> {}; // Based on key_type = rocprim::half, value_type = empty_type template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 16, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 16> {}; -// Based on key_type = int64_t, value_type = custom_type +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<256, 4> {}; -// Based on key_type = int64_t, value_type = int64_t +// Based on key_type = rocprim::int128_t, value_type = int64_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = int64_t, value_type = int64_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int64_t, value_type = int template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int64_t, value_type = short template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = int64_t, value_type = int8_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = int64_t, value_type = empty_type template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 4> {}; -// Based on key_type = int, value_type = custom_type +// Based on key_type = int, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = int, value_type = int64_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = int, value_type = int template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<512, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<512, 8> {}; // Based on key_type = int, value_type = short template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<512, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<512, 8> {}; // Based on key_type = int, value_type = int8_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = int, value_type = empty_type template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; -// Based on key_type = short, value_type = custom_type +// Based on key_type = short, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = short, value_type = int64_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = short, value_type = int template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = short, value_type = short template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 16, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 16> {}; // Based on key_type = short, value_type = int8_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = short, value_type = empty_type template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 16, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 16> {}; -// Based on key_type = int8_t, value_type = custom_type +// Based on key_type = int8_t, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = int8_t, value_type = int64_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = int8_t, value_type = int template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<512, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 8> {}; // Based on key_type = int8_t, value_type = short template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = int8_t, value_type = int8_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = int8_t, value_type = empty_type template struct default_merge_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 32, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 32> {}; -// Based on key_type = double, value_type = custom_type +// Based on key_type = double, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< static_cast(target_arch::gfx90a), @@ -2030,8 +2225,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = double, value_type = int64_t @@ -2042,8 +2236,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<512, 4> {}; // Based on key_type = double, value_type = int @@ -2054,8 +2247,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = double, value_type = short @@ -2066,8 +2258,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = double, value_type = int8_t @@ -2079,7 +2270,7 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = double, value_type = empty_type @@ -2091,10 +2282,10 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 4> {}; -// Based on key_type = float, value_type = custom_type +// Based on key_type = float, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< static_cast(target_arch::gfx90a), @@ -2102,8 +2293,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = float, value_type = int64_t @@ -2114,8 +2304,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = float, value_type = int @@ -2126,8 +2315,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = float, value_type = short @@ -2138,8 +2326,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<512, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<512, 8> {}; // Based on key_type = float, value_type = int8_t @@ -2151,7 +2338,7 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = float, value_type = empty_type @@ -2163,10 +2350,10 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : merge_sort_block_sort_config<512, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 8> {}; -// Based on key_type = rocprim::half, value_type = custom_type +// Based on key_type = rocprim::half, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< static_cast(target_arch::gfx90a), @@ -2174,7 +2361,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = rocprim::half, value_type = int64_t @@ -2185,7 +2372,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = rocprim::half, value_type = int @@ -2196,7 +2383,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = rocprim::half, value_type = short @@ -2207,7 +2394,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = rocprim::half, value_type = int8_t @@ -2219,7 +2406,7 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 16> {}; // Based on key_type = rocprim::half, value_type = empty_type @@ -2230,10 +2417,78 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 16, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 16> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : merge_sort_block_sort_config<256, 4> {}; -// Based on key_type = int64_t, value_type = custom_type +// Based on key_type = int64_t, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< static_cast(target_arch::gfx90a), @@ -2241,8 +2496,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int64_t, value_type = int64_t @@ -2253,8 +2507,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int64_t, value_type = int @@ -2265,8 +2518,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int64_t, value_type = short @@ -2277,8 +2529,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = int64_t, value_type = int8_t @@ -2290,7 +2541,7 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = int64_t, value_type = empty_type @@ -2302,10 +2553,10 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 4> {}; -// Based on key_type = int, value_type = custom_type +// Based on key_type = int, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< static_cast(target_arch::gfx90a), @@ -2313,8 +2564,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int, value_type = int64_t @@ -2325,8 +2575,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = int, value_type = int @@ -2337,8 +2586,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<512, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<512, 8> {}; // Based on key_type = int, value_type = short @@ -2349,8 +2597,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<512, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<512, 8> {}; // Based on key_type = int, value_type = int8_t @@ -2362,7 +2609,7 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = int, value_type = empty_type @@ -2374,10 +2621,10 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; -// Based on key_type = short, value_type = custom_type +// Based on key_type = short, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< static_cast(target_arch::gfx90a), @@ -2385,8 +2632,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) - && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = short, value_type = int64_t @@ -2397,8 +2643,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = short, value_type = int @@ -2409,8 +2654,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = short, value_type = short @@ -2421,8 +2665,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 16, block_sort_algorithm::stable_merge_sort> + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 16> {}; // Based on key_type = short, value_type = int8_t @@ -2434,7 +2677,7 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = short, value_type = empty_type @@ -2446,10 +2689,10 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 16, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 16> {}; -// Based on key_type = int8_t, value_type = custom_type +// Based on key_type = int8_t, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< static_cast(target_arch::gfx90a), @@ -2457,7 +2700,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> - : merge_sort_block_sort_config<512, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 4> {}; // Based on key_type = int8_t, value_type = int64_t @@ -2468,7 +2711,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = int8_t, value_type = int @@ -2479,7 +2722,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : merge_sort_block_sort_config<512, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 8> {}; // Based on key_type = int8_t, value_type = short @@ -2490,7 +2733,7 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = int8_t, value_type = int8_t @@ -2502,7 +2745,7 @@ struct default_merge_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 8> {}; // Based on key_type = int8_t, value_type = empty_type @@ -2513,180 +2756,1091 @@ struct default_merge_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 32, block_sort_algorithm::stable_merge_sort> -{}; - -template<> -struct default_merge_sort_block_sort_config(target_arch::gfx906), - double, - empty_type> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> -{}; - -template<> -struct default_merge_sort_block_sort_config(target_arch::gfx906), - float, - empty_type> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> -{}; - -template<> -struct default_merge_sort_block_sort_config(target_arch::gfx906), - int, - empty_type> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> -{}; - -template<> -struct default_merge_sort_block_sort_config(target_arch::gfx906), - int, - float> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> -{}; - -template<> -struct default_merge_sort_block_sort_config(target_arch::gfx906), - int64_t, - double> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 32> {}; -template<> -struct default_merge_sort_block_sort_config(target_arch::gfx906), - int64_t, - empty_type> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<256, 4> {}; -template<> -struct default_merge_sort_block_sort_config(target_arch::gfx906), - int8_t, - empty_type> - : merge_sort_block_sort_config<256, 32, block_sort_algorithm::stable_merge_sort> +// Based on key_type = double, value_type = int64_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<512, 4> {}; -template<> -struct default_merge_sort_block_sort_config(target_arch::gfx906), - int8_t, - int8_t> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> +// Based on key_type = double, value_type = int +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 4> {}; -template<> -struct default_merge_sort_block_sort_config(target_arch::gfx906), - rocprim::half, - empty_type> - : merge_sort_block_sort_config<256, 16, block_sort_algorithm::stable_merge_sort> +// Based on key_type = double, value_type = short +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 8> {}; -template<> -struct default_merge_sort_block_sort_config(target_arch::gfx906), - rocprim::half, - rocprim::half> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> +// Based on key_type = double, value_type = int8_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_sort_config<256, 8> {}; -template<> -struct default_merge_sort_block_sort_config(target_arch::gfx906), - short, - empty_type> - : merge_sort_block_sort_config<256, 16, block_sort_algorithm::stable_merge_sort> +// Based on key_type = double, value_type = empty_type +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) + && (std::is_same::value))>> + : merge_sort_block_sort_config<512, 4> {}; -template<> -struct default_merge_sort_block_sort_config(target_arch::gfx906), - short, - short> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<512, 4> {}; -template<> -struct default_merge_sort_block_sort_config(target_arch::gfx906), - uint8_t, - uint8_t> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> +// Based on key_type = float, value_type = int64_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> {}; -// Based on key_type = double, value_type = rocprim::empty_type +// Based on key_type = float, value_type = int template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) - && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 8> {}; -// Based on key_type = float, value_type = rocprim::empty_type +// Based on key_type = float, value_type = short template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) - && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<512, 8> {}; -// Based on key_type = rocprim::half, value_type = rocprim::empty_type +// Based on key_type = float, value_type = int8_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 16, block_sort_algorithm::stable_merge_sort> + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_sort_config<256, 8> {}; -// Based on key_type = int64_t, value_type = rocprim::empty_type +// Based on key_type = float, value_type = empty_type template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 4, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<512, 8> {}; -// Based on key_type = int, value_type = rocprim::empty_type +// Based on key_type = rocprim::half, value_type = rocprim::int128_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) - && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 8, block_sort_algorithm::stable_merge_sort> + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : merge_sort_block_sort_config<512, 4> {}; -// Based on key_type = short, value_type = rocprim::empty_type +// Based on key_type = rocprim::half, value_type = int64_t template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(key_type) > 1) - && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 16, block_sort_algorithm::stable_merge_sort> + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : merge_sort_block_sort_config<256, 4> {}; -// Based on key_type = int8_t, value_type = rocprim::empty_type +// Based on key_type = rocprim::half, value_type = int template struct default_merge_sort_block_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : merge_sort_block_sort_config<1024, 8> +{}; + +// Based on key_type = rocprim::half, value_type = short +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : merge_sort_block_sort_config<256, 8> +{}; + +// Based on key_type = rocprim::half, value_type = int8_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_sort_config<1024, 8> +{}; + +// Based on key_type = rocprim::half, value_type = empty_type +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (std::is_same::value))>> + : merge_sort_block_sort_config<256, 16> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = int64_t, value_type = int64_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = int64_t, value_type = int +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = int64_t, value_type = short +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 8> +{}; + +// Based on key_type = int64_t, value_type = int8_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_sort_config<256, 8> +{}; + +// Based on key_type = int64_t, value_type = empty_type +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) + && (std::is_same::value))>> + : merge_sort_block_sort_config<512, 4> +{}; + +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<512, 4> +{}; + +// Based on key_type = int, value_type = int64_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 8> +{}; + +// Based on key_type = int, value_type = int +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<512, 8> +{}; + +// Based on key_type = int, value_type = short +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<512, 8> +{}; + +// Based on key_type = int, value_type = int8_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_sort_config<256, 8> +{}; + +// Based on key_type = int, value_type = empty_type +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) + && (std::is_same::value))>> + : merge_sort_block_sort_config<256, 8> +{}; + +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<512, 4> +{}; + +// Based on key_type = short, value_type = int64_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 8> +{}; + +// Based on key_type = short, value_type = int +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 8> +{}; + +// Based on key_type = short, value_type = short +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 16> +{}; + +// Based on key_type = short, value_type = int8_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_sort_config<256, 8> +{}; + +// Based on key_type = short, value_type = empty_type +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) + && (std::is_same::value))>> + : merge_sort_block_sort_config<256, 16> +{}; + +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : merge_sort_block_sort_config<512, 4> +{}; + +// Based on key_type = int8_t, value_type = int64_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : merge_sort_block_sort_config<256, 8> +{}; + +// Based on key_type = int8_t, value_type = int +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : merge_sort_block_sort_config<512, 8> +{}; + +// Based on key_type = int8_t, value_type = short +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : merge_sort_block_sort_config<256, 8> +{}; + +// Based on key_type = int8_t, value_type = int8_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_sort_config<256, 8> +{}; + +// Based on key_type = int8_t, value_type = empty_type +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (std::is_same::value))>> + : merge_sort_block_sort_config<256, 32> +{}; + +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = double, value_type = int64_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = double, value_type = int +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = double, value_type = short +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 8> +{}; + +// Based on key_type = double, value_type = int8_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_sort_config<512, 4> +{}; + +// Based on key_type = double, value_type = empty_type +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) + && (std::is_same::value))>> + : merge_sort_block_sort_config<256, 8> +{}; + +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<512, 4> +{}; + +// Based on key_type = float, value_type = int64_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = float, value_type = int +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<1024, 4> +{}; + +// Based on key_type = float, value_type = short +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<1024, 4> +{}; + +// Based on key_type = float, value_type = int8_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_sort_config<1024, 4> +{}; + +// Based on key_type = float, value_type = empty_type +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) + && (std::is_same::value))>> + : merge_sort_block_sort_config<256, 8> +{}; + +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : merge_sort_block_sort_config<512, 4> +{}; + +// Based on key_type = rocprim::half, value_type = int64_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::half, value_type = int +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : merge_sort_block_sort_config<256, 8> +{}; + +// Based on key_type = rocprim::half, value_type = short +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : merge_sort_block_sort_config<1024, 8> +{}; + +// Based on key_type = rocprim::half, value_type = int8_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_sort_config<256, 16> +{}; + +// Based on key_type = rocprim::half, value_type = empty_type +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (std::is_same::value))>> + : merge_sort_block_sort_config<256, 16> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = int64_t, value_type = int64_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = int64_t, value_type = int +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = int64_t, value_type = short +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 8> +{}; + +// Based on key_type = int64_t, value_type = int8_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_sort_config<512, 4> +{}; + +// Based on key_type = int64_t, value_type = empty_type +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) + && (std::is_same::value))>> + : merge_sort_block_sort_config<256, 8> +{}; + +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<512, 4> +{}; + +// Based on key_type = int, value_type = int64_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<512, 4> +{}; + +// Based on key_type = int, value_type = int +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<256, 8> +{}; + +// Based on key_type = int, value_type = short +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 8> +{}; + +// Based on key_type = int, value_type = int8_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_sort_config<1024, 4> +{}; + +// Based on key_type = int, value_type = empty_type +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) + && (std::is_same::value))>> + : merge_sort_block_sort_config<256, 8> +{}; + +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : merge_sort_block_sort_config<512, 4> +{}; + +// Based on key_type = short, value_type = int64_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = short, value_type = int +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : merge_sort_block_sort_config<512, 4> +{}; + +// Based on key_type = short, value_type = short +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : merge_sort_block_sort_config<256, 16> +{}; + +// Based on key_type = short, value_type = int8_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_sort_config<256, 16> +{}; + +// Based on key_type = short, value_type = empty_type +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) + && (std::is_same::value))>> + : merge_sort_block_sort_config<256, 32> +{}; + +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : merge_sort_block_sort_config<512, 4> +{}; + +// Based on key_type = int8_t, value_type = int64_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : merge_sort_block_sort_config<256, 4> +{}; + +// Based on key_type = int8_t, value_type = int +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : merge_sort_block_sort_config<256, 8> +{}; + +// Based on key_type = int8_t, value_type = short +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : merge_sort_block_sort_config<256, 16> +{}; + +// Based on key_type = int8_t, value_type = int8_t +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : merge_sort_block_sort_config<1024, 8> +{}; + +// Based on key_type = int8_t, value_type = empty_type +template +struct default_merge_sort_block_sort_config< + static_cast(target_arch::gfx942), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (std::is_same::value))>> - : merge_sort_block_sort_config<256, 32, block_sort_algorithm::stable_merge_sort> + : merge_sort_block_sort_config<256, 32> {}; } // end namespace detail diff --git a/rocprim/include/rocprim/device/detail/config/device_partition_flag.hpp b/rocprim/include/rocprim/device/detail/config/device_partition_flag.hpp index 56f548238..3cf0f0d18 100644 --- a/rocprim/include/rocprim/device/detail/config/device_partition_flag.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_partition_flag.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,7 +21,9 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_PARTITION_FLAG_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_PARTITION_FLAG_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" #include @@ -67,7 +69,17 @@ struct default_partition_flag_config< static_cast(target_arch::gfx1030), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<512, 12> + && (sizeof(data_type) <= 2))>> : select_config<512, 8> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_flag_config< + static_cast(target_arch::gfx1030), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 4> {}; // Based on data_type = int64_t @@ -133,7 +145,17 @@ struct default_partition_flag_config< static_cast(target_arch::gfx1100), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<128, 19> + && (sizeof(data_type) <= 2))>> : select_config<256, 20> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_flag_config< + static_cast(target_arch::gfx1100), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<512, 4> {}; // Based on data_type = int64_t @@ -202,6 +224,16 @@ struct default_partition_flag_config< && (sizeof(data_type) <= 2))>> : select_config<256, 28> {}; +// Based on data_type = rocprim::int128_t +template +struct default_partition_flag_config< + static_cast(target_arch::gfx906), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<192, 4> +{}; + // Based on data_type = int64_t template struct default_partition_flag_config< @@ -268,6 +300,16 @@ struct default_partition_flag_config< && (sizeof(data_type) <= 2))>> : select_config<256, 28> {}; +// Based on data_type = rocprim::int128_t +template +struct default_partition_flag_config< + static_cast(target_arch::gfx908), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<128, 4> +{}; + // Based on data_type = int64_t template struct default_partition_flag_config< @@ -310,133 +352,153 @@ struct default_partition_flag_config< // Based on data_type = double template struct default_partition_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<128, 7> + && (sizeof(data_type) > 4))>> : select_config<192, 4> {}; // Based on data_type = float template struct default_partition_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<128, 12> + && (sizeof(data_type) > 2))>> : select_config<256, 10> {}; // Based on data_type = rocprim::half template struct default_partition_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 28> + && (sizeof(data_type) <= 2))>> : select_config<256, 20> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_flag_config< + static_cast(target_arch::gfx90a), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 3> {}; // Based on data_type = int64_t template struct default_partition_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<128, 7> + : select_config<192, 4> {}; // Based on data_type = int template struct default_partition_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<128, 12> + : select_config<256, 10> {}; // Based on data_type = short template struct default_partition_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 28> + : select_config<256, 20> {}; // Based on data_type = int8_t template struct default_partition_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 1))>> : select_config<256, 28> + && (sizeof(data_type) <= 1))>> : select_config<256, 24> {}; // Based on data_type = double template struct default_partition_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<192, 4> + && (sizeof(data_type) > 4))>> : select_config<128, 7> {}; // Based on data_type = float template struct default_partition_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<256, 10> + && (sizeof(data_type) > 2))>> : select_config<128, 12> {}; // Based on data_type = rocprim::half template struct default_partition_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 20> + && (sizeof(data_type) <= 2))>> : select_config<256, 28> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_flag_config< + static_cast(target_arch::unknown), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<128, 4> {}; // Based on data_type = int64_t template struct default_partition_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<192, 4> + : select_config<128, 7> {}; // Based on data_type = int template struct default_partition_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<256, 10> + : select_config<128, 12> {}; // Based on data_type = short template struct default_partition_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 20> + : select_config<256, 28> {}; // Based on data_type = int8_t template struct default_partition_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 1))>> : select_config<256, 24> + && (sizeof(data_type) <= 1))>> : select_config<256, 28> {}; // Based on data_type = double @@ -445,7 +507,7 @@ struct default_partition_flag_config< static_cast(target_arch::gfx942), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<512, 5> + && (sizeof(data_type) > 4))>> : select_config<512, 7> {}; // Based on data_type = float @@ -466,6 +528,16 @@ struct default_partition_flag_config< && (sizeof(data_type) <= 2))>> : select_config<512, 28> {}; +// Based on data_type = rocprim::int128_t +template +struct default_partition_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<384, 4> +{}; + // Based on data_type = int64_t template struct default_partition_flag_config< @@ -473,7 +545,7 @@ struct default_partition_flag_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<512, 5> + : select_config<512, 7> {}; // Based on data_type = int diff --git a/rocprim/include/rocprim/device/detail/config/device_partition_predicate.hpp b/rocprim/include/rocprim/device/detail/config/device_partition_predicate.hpp index b77fb0d67..fdd747029 100644 --- a/rocprim/include/rocprim/device/detail/config/device_partition_predicate.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_partition_predicate.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,7 +21,9 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_PARTITION_PREDICATE_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_PARTITION_PREDICATE_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" #include @@ -67,7 +69,17 @@ struct default_partition_predicate_config< static_cast(target_arch::gfx1030), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 30> + && (sizeof(data_type) <= 2))>> : select_config<512, 8> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_predicate_config< + static_cast(target_arch::gfx1030), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 4> {}; // Based on data_type = int64_t @@ -133,7 +145,17 @@ struct default_partition_predicate_config< static_cast(target_arch::gfx1100), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<128, 30> + && (sizeof(data_type) <= 2))>> : select_config<384, 18> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_predicate_config< + static_cast(target_arch::gfx1100), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<512, 4> {}; // Based on data_type = int64_t @@ -199,7 +221,17 @@ struct default_partition_predicate_config< static_cast(target_arch::gfx906), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 24> + && (sizeof(data_type) <= 2))>> : select_config<256, 26> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_predicate_config< + static_cast(target_arch::gfx906), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 4> {}; // Based on data_type = int64_t @@ -265,7 +297,17 @@ struct default_partition_predicate_config< static_cast(target_arch::gfx908), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 24> + && (sizeof(data_type) <= 2))>> : select_config<256, 26> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_predicate_config< + static_cast(target_arch::gfx908), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<128, 4> {}; // Based on data_type = int64_t @@ -310,133 +352,153 @@ struct default_partition_predicate_config< // Based on data_type = double template struct default_partition_predicate_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<128, 7> + && (sizeof(data_type) > 4))>> : select_config<192, 4> {}; // Based on data_type = float template struct default_partition_predicate_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<256, 15> + && (sizeof(data_type) > 2))>> : select_config<256, 10> {}; // Based on data_type = rocprim::half template struct default_partition_predicate_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 24> + && (sizeof(data_type) <= 2))>> : select_config<256, 20> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_predicate_config< + static_cast(target_arch::gfx90a), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 3> {}; // Based on data_type = int64_t template struct default_partition_predicate_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<192, 7> + : select_config<192, 4> {}; // Based on data_type = int template struct default_partition_predicate_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<256, 15> + : select_config<256, 10> {}; // Based on data_type = short template struct default_partition_predicate_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 24> + : select_config<256, 20> {}; // Based on data_type = int8_t template struct default_partition_predicate_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 1))>> : select_config<256, 24> + && (sizeof(data_type) <= 1))>> : select_config<512, 24> {}; // Based on data_type = double template struct default_partition_predicate_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<192, 4> + && (sizeof(data_type) > 4))>> : select_config<128, 7> {}; // Based on data_type = float template struct default_partition_predicate_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<256, 10> + && (sizeof(data_type) > 2))>> : select_config<256, 15> {}; // Based on data_type = rocprim::half template struct default_partition_predicate_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 20> + && (sizeof(data_type) <= 2))>> : select_config<256, 26> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_predicate_config< + static_cast(target_arch::unknown), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<128, 4> {}; // Based on data_type = int64_t template struct default_partition_predicate_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<192, 4> + : select_config<192, 7> {}; // Based on data_type = int template struct default_partition_predicate_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<256, 10> + : select_config<256, 15> {}; // Based on data_type = short template struct default_partition_predicate_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 20> + : select_config<256, 24> {}; // Based on data_type = int8_t template struct default_partition_predicate_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 1))>> : select_config<512, 24> + && (sizeof(data_type) <= 1))>> : select_config<256, 24> {}; // Based on data_type = double @@ -445,7 +507,7 @@ struct default_partition_predicate_config< static_cast(target_arch::gfx942), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<512, 5> + && (sizeof(data_type) > 4))>> : select_config<512, 7> {}; // Based on data_type = float @@ -466,6 +528,16 @@ struct default_partition_predicate_config< && (sizeof(data_type) <= 2))>> : select_config<512, 30> {}; +// Based on data_type = rocprim::int128_t +template +struct default_partition_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<384, 4> +{}; + // Based on data_type = int64_t template struct default_partition_predicate_config< @@ -473,7 +545,7 @@ struct default_partition_predicate_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<512, 5> + : select_config<512, 7> {}; // Based on data_type = int diff --git a/rocprim/include/rocprim/device/detail/config/device_partition_three_way.hpp b/rocprim/include/rocprim/device/detail/config/device_partition_three_way.hpp index 7538d8c87..6bbe6bd4a 100644 --- a/rocprim/include/rocprim/device/detail/config/device_partition_three_way.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_partition_three_way.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,7 +21,9 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_PARTITION_THREE_WAY_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_PARTITION_THREE_WAY_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" #include @@ -67,7 +69,17 @@ struct default_partition_three_way_config< static_cast(target_arch::gfx1030), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 30> + && (sizeof(data_type) <= 2))>> : select_config<512, 12> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_three_way_config< + static_cast(target_arch::gfx1030), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 4> {}; // Based on data_type = int64_t @@ -133,7 +145,17 @@ struct default_partition_three_way_config< static_cast(target_arch::gfx1100), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 18> + && (sizeof(data_type) <= 2))>> : select_config<256, 17> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_three_way_config< + static_cast(target_arch::gfx1100), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<512, 4> {}; // Based on data_type = int64_t @@ -202,6 +224,16 @@ struct default_partition_three_way_config< && (sizeof(data_type) <= 2))>> : select_config<256, 14> {}; +// Based on data_type = rocprim::int128_t +template +struct default_partition_three_way_config< + static_cast(target_arch::gfx906), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 4> +{}; + // Based on data_type = int64_t template struct default_partition_three_way_config< @@ -268,6 +300,16 @@ struct default_partition_three_way_config< && (sizeof(data_type) <= 2))>> : select_config<256, 14> {}; +// Based on data_type = rocprim::int128_t +template +struct default_partition_three_way_config< + static_cast(target_arch::gfx908), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 3> +{}; + // Based on data_type = int64_t template struct default_partition_three_way_config< @@ -310,133 +352,153 @@ struct default_partition_three_way_config< // Based on data_type = double template struct default_partition_three_way_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<192, 6> + && (sizeof(data_type) > 4))>> : select_config<256, 6> {}; // Based on data_type = float template struct default_partition_three_way_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<256, 15> + && (sizeof(data_type) > 2))>> : select_config<256, 11> {}; // Based on data_type = rocprim::half template struct default_partition_three_way_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 14> + && (sizeof(data_type) <= 2))>> : select_config<256, 16> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_three_way_config< + static_cast(target_arch::gfx90a), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 3> {}; // Based on data_type = int64_t template struct default_partition_three_way_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<256, 7> + : select_config<256, 6> {}; // Based on data_type = int template struct default_partition_three_way_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<256, 15> + : select_config<256, 11> {}; // Based on data_type = short template struct default_partition_three_way_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 14> + : select_config<256, 16> {}; // Based on data_type = int8_t template struct default_partition_three_way_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 1))>> : select_config<256, 15> + && (sizeof(data_type) <= 1))>> : select_config<256, 17> {}; // Based on data_type = double template struct default_partition_three_way_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<256, 6> + && (sizeof(data_type) > 4))>> : select_config<192, 6> {}; // Based on data_type = float template struct default_partition_three_way_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<256, 11> + && (sizeof(data_type) > 2))>> : select_config<256, 15> {}; // Based on data_type = rocprim::half template struct default_partition_three_way_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 16> + && (sizeof(data_type) <= 2))>> : select_config<256, 14> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_three_way_config< + static_cast(target_arch::unknown), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 3> {}; // Based on data_type = int64_t template struct default_partition_three_way_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<256, 6> + : select_config<256, 7> {}; // Based on data_type = int template struct default_partition_three_way_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<256, 11> + : select_config<256, 15> {}; // Based on data_type = short template struct default_partition_three_way_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 16> + : select_config<256, 14> {}; // Based on data_type = int8_t template struct default_partition_three_way_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 1))>> : select_config<256, 17> + && (sizeof(data_type) <= 1))>> : select_config<256, 15> {}; // Based on data_type = double @@ -445,7 +507,7 @@ struct default_partition_three_way_config< static_cast(target_arch::gfx942), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<512, 5> + && (sizeof(data_type) > 4))>> : select_config<512, 7> {}; // Based on data_type = float @@ -463,7 +525,17 @@ struct default_partition_three_way_config< static_cast(target_arch::gfx942), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 26> + && (sizeof(data_type) <= 2))>> : select_config<256, 18> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_three_way_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<384, 4> {}; // Based on data_type = int64_t @@ -473,7 +545,7 @@ struct default_partition_three_way_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<512, 5> + : select_config<512, 7> {}; // Based on data_type = int @@ -493,7 +565,7 @@ struct default_partition_three_way_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<512, 31> + : select_config<256, 18> {}; // Based on data_type = int8_t @@ -502,7 +574,7 @@ struct default_partition_three_way_config< static_cast(target_arch::gfx942), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 1))>> : select_config<512, 26> + && (sizeof(data_type) <= 1))>> : select_config<256, 18> {}; } // end namespace detail diff --git a/rocprim/include/rocprim/device/detail/config/device_partition_two_way_flag.hpp b/rocprim/include/rocprim/device/detail/config/device_partition_two_way_flag.hpp index 309213176..87362f2fd 100644 --- a/rocprim/include/rocprim/device/detail/config/device_partition_two_way_flag.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_partition_two_way_flag.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,7 +21,9 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_PARTITION_TWO_WAY_FLAG_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_PARTITION_TWO_WAY_FLAG_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" #include @@ -67,7 +69,17 @@ struct default_partition_two_way_flag_config< static_cast(target_arch::gfx1030), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<512, 12> + && (sizeof(data_type) <= 2))>> : select_config<512, 6> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_two_way_flag_config< + static_cast(target_arch::gfx1030), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 4> {}; // Based on data_type = int64_t @@ -133,7 +145,17 @@ struct default_partition_two_way_flag_config< static_cast(target_arch::gfx1100), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<128, 15> + && (sizeof(data_type) <= 2))>> : select_config<384, 18> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_two_way_flag_config< + static_cast(target_arch::gfx1100), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<512, 4> {}; // Based on data_type = int64_t @@ -202,6 +224,16 @@ struct default_partition_two_way_flag_config< && (sizeof(data_type) <= 2))>> : select_config<256, 28> {}; +// Based on data_type = rocprim::int128_t +template +struct default_partition_two_way_flag_config< + static_cast(target_arch::gfx906), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<192, 4> +{}; + // Based on data_type = int64_t template struct default_partition_two_way_flag_config< @@ -268,6 +300,16 @@ struct default_partition_two_way_flag_config< && (sizeof(data_type) <= 2))>> : select_config<256, 28> {}; +// Based on data_type = rocprim::int128_t +template +struct default_partition_two_way_flag_config< + static_cast(target_arch::gfx908), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<128, 4> +{}; + // Based on data_type = int64_t template struct default_partition_two_way_flag_config< @@ -310,133 +352,153 @@ struct default_partition_two_way_flag_config< // Based on data_type = double template struct default_partition_two_way_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<128, 7> + && (sizeof(data_type) > 4))>> : select_config<192, 4> {}; // Based on data_type = float template struct default_partition_two_way_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<256, 13> + && (sizeof(data_type) > 2))>> : select_config<256, 10> {}; // Based on data_type = rocprim::half template struct default_partition_two_way_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 28> + && (sizeof(data_type) <= 2))>> : select_config<256, 20> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_two_way_flag_config< + static_cast(target_arch::gfx90a), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 3> {}; // Based on data_type = int64_t template struct default_partition_two_way_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<128, 7> + : select_config<192, 4> {}; // Based on data_type = int template struct default_partition_two_way_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<256, 12> + : select_config<256, 7> {}; // Based on data_type = short template struct default_partition_two_way_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 28> + : select_config<256, 20> {}; // Based on data_type = int8_t template struct default_partition_two_way_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 1))>> : select_config<256, 28> + && (sizeof(data_type) <= 1))>> : select_config<256, 24> {}; // Based on data_type = double template struct default_partition_two_way_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<192, 4> + && (sizeof(data_type) > 4))>> : select_config<128, 7> {}; // Based on data_type = float template struct default_partition_two_way_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<256, 10> + && (sizeof(data_type) > 2))>> : select_config<256, 13> {}; // Based on data_type = rocprim::half template struct default_partition_two_way_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 20> + && (sizeof(data_type) <= 2))>> : select_config<256, 28> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_two_way_flag_config< + static_cast(target_arch::unknown), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<128, 4> {}; // Based on data_type = int64_t template struct default_partition_two_way_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<192, 4> + : select_config<128, 7> {}; // Based on data_type = int template struct default_partition_two_way_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<256, 7> + : select_config<256, 12> {}; // Based on data_type = short template struct default_partition_two_way_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 20> + : select_config<256, 28> {}; // Based on data_type = int8_t template struct default_partition_two_way_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 1))>> : select_config<256, 24> + && (sizeof(data_type) <= 1))>> : select_config<256, 28> {}; // Based on data_type = double @@ -445,7 +507,7 @@ struct default_partition_two_way_flag_config< static_cast(target_arch::gfx942), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<512, 5> + && (sizeof(data_type) > 4))>> : select_config<512, 7> {}; // Based on data_type = float @@ -463,7 +525,17 @@ struct default_partition_two_way_flag_config< static_cast(target_arch::gfx942), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<512, 28> + && (sizeof(data_type) <= 2))>> : select_config<512, 30> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_two_way_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<384, 4> {}; // Based on data_type = int64_t @@ -473,7 +545,7 @@ struct default_partition_two_way_flag_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<512, 5> + : select_config<512, 7> {}; // Based on data_type = int diff --git a/rocprim/include/rocprim/device/detail/config/device_partition_two_way_predicate.hpp b/rocprim/include/rocprim/device/detail/config/device_partition_two_way_predicate.hpp index 246fe23c7..e24efd16d 100644 --- a/rocprim/include/rocprim/device/detail/config/device_partition_two_way_predicate.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_partition_two_way_predicate.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,7 +21,9 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_PARTITION_TWO_WAY_PREDICATE_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_PARTITION_TWO_WAY_PREDICATE_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" #include @@ -68,7 +70,17 @@ struct default_partition_two_way_predicate_config< static_cast(target_arch::gfx1030), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<384, 22> + && (sizeof(data_type) <= 2))>> : select_config<512, 8> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_two_way_predicate_config< + static_cast(target_arch::gfx1030), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 4> {}; // Based on data_type = int64_t @@ -134,7 +146,17 @@ struct default_partition_two_way_predicate_config< static_cast(target_arch::gfx1100), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<128, 18> + && (sizeof(data_type) <= 2))>> : select_config<192, 12> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_two_way_predicate_config< + static_cast(target_arch::gfx1100), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<512, 4> {}; // Based on data_type = int64_t @@ -200,7 +222,17 @@ struct default_partition_two_way_predicate_config< static_cast(target_arch::gfx906), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 24> + && (sizeof(data_type) <= 2))>> : select_config<256, 26> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_two_way_predicate_config< + static_cast(target_arch::gfx906), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 4> {}; // Based on data_type = int64_t @@ -266,7 +298,17 @@ struct default_partition_two_way_predicate_config< static_cast(target_arch::gfx908), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 24> + && (sizeof(data_type) <= 2))>> : select_config<256, 26> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_two_way_predicate_config< + static_cast(target_arch::gfx908), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<128, 4> {}; // Based on data_type = int64_t @@ -311,64 +353,74 @@ struct default_partition_two_way_predicate_config< // Based on data_type = double template struct default_partition_two_way_predicate_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<128, 7> + && (sizeof(data_type) > 4))>> : select_config<128, 6> {}; // Based on data_type = float template struct default_partition_two_way_predicate_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<256, 15> + && (sizeof(data_type) > 2))>> : select_config<256, 10> {}; // Based on data_type = rocprim::half template struct default_partition_two_way_predicate_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 24> + && (sizeof(data_type) <= 2))>> : select_config<256, 20> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_two_way_predicate_config< + static_cast(target_arch::gfx90a), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<128, 4> {}; // Based on data_type = int64_t template struct default_partition_two_way_predicate_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<128, 7> + : select_config<192, 4> {}; // Based on data_type = int template struct default_partition_two_way_predicate_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<256, 13> + : select_config<256, 10> {}; // Based on data_type = short template struct default_partition_two_way_predicate_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 24> + : select_config<256, 28> {}; // Based on data_type = int8_t template struct default_partition_two_way_predicate_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 1))>> : select_config<256, 24> @@ -377,64 +429,74 @@ struct default_partition_two_way_predicate_config< // Based on data_type = double template struct default_partition_two_way_predicate_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<128, 6> + && (sizeof(data_type) > 4))>> : select_config<128, 7> {}; // Based on data_type = float template struct default_partition_two_way_predicate_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<256, 10> + && (sizeof(data_type) > 2))>> : select_config<256, 15> {}; // Based on data_type = rocprim::half template struct default_partition_two_way_predicate_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 20> + && (sizeof(data_type) <= 2))>> : select_config<256, 26> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_partition_two_way_predicate_config< + static_cast(target_arch::unknown), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<128, 4> {}; // Based on data_type = int64_t template struct default_partition_two_way_predicate_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<192, 4> + : select_config<128, 7> {}; // Based on data_type = int template struct default_partition_two_way_predicate_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<256, 10> + : select_config<256, 13> {}; // Based on data_type = short template struct default_partition_two_way_predicate_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 28> + : select_config<256, 24> {}; // Based on data_type = int8_t template struct default_partition_two_way_predicate_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 1))>> : select_config<256, 24> @@ -467,6 +529,16 @@ struct default_partition_two_way_predicate_config< && (sizeof(data_type) <= 2))>> : select_config<512, 30> {}; +// Based on data_type = rocprim::int128_t +template +struct default_partition_two_way_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<384, 4> +{}; + // Based on data_type = int64_t template struct default_partition_two_way_predicate_config< @@ -474,7 +546,7 @@ struct default_partition_two_way_predicate_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<512, 5> + : select_config<512, 7> {}; // Based on data_type = int diff --git a/rocprim/include/rocprim/device/detail/config/device_radix_sort_block_sort.hpp b/rocprim/include/rocprim/device/detail/config/device_radix_sort_block_sort.hpp index 02897f37f..377aee0d3 100644 --- a/rocprim/include/rocprim/device/detail/config/device_radix_sort_block_sort.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_radix_sort_block_sort.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,8 +21,11 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RADIX_SORT_BLOCK_SORT_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RADIX_SORT_BLOCK_SORT_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" + #include /* DO NOT EDIT THIS FILE @@ -46,6 +49,17 @@ struct default_radix_sort_block_sort_config : radix_sort_block_sort_config_base::type {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<128, 16> +{}; + // Based on key_type = double, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -103,6 +117,17 @@ struct default_radix_sort_block_sort_config< : kernel_config<64, 32> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 15> +{}; + // Based on key_type = float, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -160,6 +185,17 @@ struct default_radix_sort_block_sort_config< : kernel_config<64, 32> {}; +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : kernel_config<256, 15> +{}; + // Based on key_type = rocprim::half, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -168,7 +204,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : kernel_config<256, 25> + : kernel_config<256, 26> {}; // Based on key_type = rocprim::half, value_type = int @@ -216,6 +252,85 @@ struct default_radix_sort_block_sort_config< : kernel_config<128, 32> {}; +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 13> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : kernel_config<256, 15> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : kernel_config<128, 16> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : kernel_config<128, 16> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : kernel_config<128, 16> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : kernel_config<128, 16> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<128, 16> +{}; + // Based on key_type = int64_t, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -273,6 +388,17 @@ struct default_radix_sort_block_sort_config< : kernel_config<64, 28> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 4> +{}; + // Based on key_type = int, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -330,6 +456,17 @@ struct default_radix_sort_block_sort_config< : kernel_config<64, 32> {}; +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 15> +{}; + // Based on key_type = short, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -387,6 +524,17 @@ struct default_radix_sort_block_sort_config< : kernel_config<64, 32> {}; +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : kernel_config<256, 15> +{}; + // Based on key_type = int8_t, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -443,6 +591,17 @@ struct default_radix_sort_block_sort_config< : kernel_config<64, 32> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<128, 16> +{}; + // Based on key_type = double, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -500,6 +659,17 @@ struct default_radix_sort_block_sort_config< : kernel_config<128, 32> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 15> +{}; + // Based on key_type = float, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -557,6 +727,17 @@ struct default_radix_sort_block_sort_config< : kernel_config<128, 32> {}; +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : kernel_config<256, 15> +{}; + // Based on key_type = rocprim::half, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -613,6 +794,85 @@ struct default_radix_sort_block_sort_config< : kernel_config<64, 32> {}; +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 10> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : kernel_config<256, 15> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : kernel_config<256, 15> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : kernel_config<256, 15> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : kernel_config<256, 15> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : kernel_config<256, 15> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<128, 16> +{}; + // Based on key_type = int64_t, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -670,6 +930,17 @@ struct default_radix_sort_block_sort_config< : kernel_config<128, 32> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 4> +{}; + // Based on key_type = int, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -727,6 +998,17 @@ struct default_radix_sort_block_sort_config< : kernel_config<64, 30> {}; +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 15> +{}; + // Based on key_type = short, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -784,6 +1066,17 @@ struct default_radix_sort_block_sort_config< : kernel_config<64, 32> {}; +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : kernel_config<256, 15> +{}; + // Based on key_type = int8_t, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -840,6 +1133,17 @@ struct default_radix_sort_block_sort_config< : kernel_config<64, 32> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 7> +{}; + // Based on key_type = double, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -897,6 +1201,17 @@ struct default_radix_sort_block_sort_config< : kernel_config<256, 16> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 8> +{}; + // Based on key_type = float, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -954,6 +1269,17 @@ struct default_radix_sort_block_sort_config< : kernel_config<512, 23> {}; +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : kernel_config<512, 4> +{}; + // Based on key_type = rocprim::half, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -996,7 +1322,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<512, 17> + : kernel_config<256, 15> {}; // Based on key_type = rocprim::half, value_type = empty_type @@ -1007,52 +1333,131 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (std::is_same::value))>> - : kernel_config<512, 23> + : kernel_config<256, 22> {}; -// Based on key_type = int64_t, value_type = int64_t +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t template struct default_radix_sort_block_sort_config< static_cast(target_arch::gfx906), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 12> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 4> {}; -// Based on key_type = int64_t, value_type = int +// Based on key_type = rocprim::int128_t, value_type = int64_t template struct default_radix_sort_block_sort_config< static_cast(target_arch::gfx906), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<256, 12> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : kernel_config<256, 7> {}; -// Based on key_type = int64_t, value_type = short +// Based on key_type = rocprim::int128_t, value_type = int template struct default_radix_sort_block_sort_config< static_cast(target_arch::gfx906), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<256, 13> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : kernel_config<256, 8> {}; -// Based on key_type = int64_t, value_type = int8_t +// Based on key_type = rocprim::int128_t, value_type = short template struct default_radix_sort_block_sort_config< static_cast(target_arch::gfx906), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) - && (!std::is_same::value))>> - : kernel_config<256, 12> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : kernel_config<256, 8> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : kernel_config<256, 8> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : kernel_config<256, 4> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 4> +{}; + +// Based on key_type = int64_t, value_type = int64_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : kernel_config<256, 12> +{}; + +// Based on key_type = int64_t, value_type = int +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : kernel_config<256, 12> +{}; + +// Based on key_type = int64_t, value_type = short +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : kernel_config<256, 13> +{}; + +// Based on key_type = int64_t, value_type = int8_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : kernel_config<256, 12> {}; // Based on key_type = int64_t, value_type = empty_type @@ -1067,6 +1472,17 @@ struct default_radix_sort_block_sort_config< : kernel_config<256, 16> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 4> +{}; + // Based on key_type = int, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -1124,6 +1540,17 @@ struct default_radix_sort_block_sort_config< : kernel_config<512, 23> {}; +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 8> +{}; + // Based on key_type = short, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -1181,6 +1608,17 @@ struct default_radix_sort_block_sort_config< : kernel_config<256, 23> {}; +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : kernel_config<256, 15> +{}; + // Based on key_type = int8_t, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -1237,6 +1675,17 @@ struct default_radix_sort_block_sort_config< : kernel_config<256, 22> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 7> +{}; + // Based on key_type = double, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -1294,6 +1743,17 @@ struct default_radix_sort_block_sort_config< : kernel_config<256, 16> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 8> +{}; + // Based on key_type = float, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -1351,6 +1811,17 @@ struct default_radix_sort_block_sort_config< : kernel_config<512, 31> {}; +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : kernel_config<512, 4> +{}; + // Based on key_type = rocprim::half, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -1393,7 +1864,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<512, 31> + : kernel_config<512, 27> {}; // Based on key_type = rocprim::half, value_type = empty_type @@ -1404,7 +1875,86 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (std::is_same::value))>> - : kernel_config<1024, 23> + : kernel_config<1024, 22> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : kernel_config<256, 7> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : kernel_config<256, 8> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : kernel_config<256, 8> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : kernel_config<256, 8> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : kernel_config<256, 4> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 4> {}; // Based on key_type = int64_t, value_type = int64_t @@ -1464,6 +2014,17 @@ struct default_radix_sort_block_sort_config< : kernel_config<256, 16> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 4> +{}; + // Based on key_type = int, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -1521,6 +2082,17 @@ struct default_radix_sort_block_sort_config< : kernel_config<512, 31> {}; +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<512, 4> +{}; + // Based on key_type = short, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -1578,6 +2150,17 @@ struct default_radix_sort_block_sort_config< : kernel_config<1024, 23> {}; +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : kernel_config<256, 15> +{}; + // Based on key_type = int8_t, value_type = int64_t template struct default_radix_sort_block_sort_config< @@ -1634,798 +2217,1099 @@ struct default_radix_sort_block_sort_config< : kernel_config<1024, 23> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 8> +{}; + // Based on key_type = double, value_type = int64_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 12> + && (sizeof(value_type) > 4))>> : kernel_config<256, 15> {}; // Based on key_type = double, value_type = int template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<256, 12> + && (sizeof(value_type) > 2))>> : kernel_config<256, 15> {}; // Based on key_type = double, value_type = short template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<256, 13> + && (sizeof(value_type) > 1))>> : kernel_config<256, 15> {}; // Based on key_type = double, value_type = int8_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<256, 12> + : kernel_config<256, 15> {}; // Based on key_type = double, value_type = empty_type template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : kernel_config<256, 16> + : kernel_config<256, 15> +{}; + +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 4> {}; // Based on key_type = float, value_type = int64_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 12> + && (sizeof(value_type) > 4))>> : kernel_config<256, 15> {}; // Based on key_type = float, value_type = int template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<512, 31> + && (sizeof(value_type) > 2))>> : kernel_config<256, 21> {}; // Based on key_type = float, value_type = short template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<512, 31> + && (sizeof(value_type) > 1))>> : kernel_config<512, 15> {}; // Based on key_type = float, value_type = int8_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<512, 31> + : kernel_config<256, 21> {}; // Based on key_type = float, value_type = empty_type template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : kernel_config<512, 31> + : kernel_config<256, 21> {}; -// Based on key_type = rocprim::half, value_type = int64_t +// Based on key_type = rocprim::half, value_type = rocprim::int128_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : kernel_config<256, 8> +{}; + +// Based on key_type = rocprim::half, value_type = int64_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : kernel_config<256, 12> + : kernel_config<256, 10> {}; // Based on key_type = rocprim::half, value_type = int template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : kernel_config<512, 31> + : kernel_config<256, 31> {}; // Based on key_type = rocprim::half, value_type = short template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : kernel_config<512, 32> + : kernel_config<512, 19> {}; // Based on key_type = rocprim::half, value_type = int8_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<512, 31> + : kernel_config<256, 32> {}; // Based on key_type = rocprim::half, value_type = empty_type template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (std::is_same::value))>> - : kernel_config<1024, 23> + : kernel_config<256, 32> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : kernel_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : kernel_config<256, 8> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : kernel_config<256, 8> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : kernel_config<256, 8> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : kernel_config<256, 4> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 4> {}; // Based on key_type = int64_t, value_type = int64_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 12> + && (sizeof(value_type) > 4))>> : kernel_config<256, 15> {}; // Based on key_type = int64_t, value_type = int template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<256, 12> + && (sizeof(value_type) > 2))>> : kernel_config<256, 15> {}; // Based on key_type = int64_t, value_type = short template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<256, 8> + && (sizeof(value_type) > 1))>> : kernel_config<256, 15> {}; // Based on key_type = int64_t, value_type = int8_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<256, 12> + : kernel_config<512, 15> {}; // Based on key_type = int64_t, value_type = empty_type template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : kernel_config<256, 16> + : kernel_config<256, 15> +{}; + +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 4> {}; // Based on key_type = int, value_type = int64_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 12> + && (sizeof(value_type) > 4))>> : kernel_config<256, 15> {}; // Based on key_type = int, value_type = int template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<512, 31> + && (sizeof(value_type) > 2))>> : kernel_config<256, 21> {}; // Based on key_type = int, value_type = short template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<512, 31> + && (sizeof(value_type) > 1))>> : kernel_config<512, 15> {}; // Based on key_type = int, value_type = int8_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<512, 31> + : kernel_config<256, 21> {}; // Based on key_type = int, value_type = empty_type template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : kernel_config<512, 31> + : kernel_config<256, 21> +{}; + +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 8> {}; // Based on key_type = short, value_type = int64_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 12> + && (sizeof(value_type) > 4))>> : kernel_config<256, 10> {}; // Based on key_type = short, value_type = int template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<512, 31> + && (sizeof(value_type) > 2))>> : kernel_config<512, 15> {}; // Based on key_type = short, value_type = short template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<512, 32> + && (sizeof(value_type) > 1))>> : kernel_config<256, 32> {}; // Based on key_type = short, value_type = int8_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<512, 32> + : kernel_config<256, 32> {}; // Based on key_type = short, value_type = empty_type template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (std::is_same::value))>> - : kernel_config<1024, 23> + : kernel_config<256, 32> +{}; + +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : kernel_config<512, 4> {}; // Based on key_type = int8_t, value_type = int64_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : kernel_config<256, 13> + : kernel_config<512, 5> {}; // Based on key_type = int8_t, value_type = int template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : kernel_config<512, 31> + : kernel_config<512, 13> {}; // Based on key_type = int8_t, value_type = short template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : kernel_config<512, 32> + : kernel_config<512, 16> {}; // Based on key_type = int8_t, value_type = int8_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<512, 32> + : kernel_config<256, 32> {}; // Based on key_type = int8_t, value_type = empty_type template struct default_radix_sort_block_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (std::is_same::value))>> - : kernel_config<1024, 23> + : kernel_config<512, 32> +{}; + +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 7> {}; // Based on key_type = double, value_type = int64_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 15> + && (sizeof(value_type) > 4))>> : kernel_config<256, 12> {}; // Based on key_type = double, value_type = int template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<256, 15> + && (sizeof(value_type) > 2))>> : kernel_config<256, 12> {}; // Based on key_type = double, value_type = short template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<256, 15> + && (sizeof(value_type) > 1))>> : kernel_config<256, 13> {}; // Based on key_type = double, value_type = int8_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<256, 15> + : kernel_config<256, 12> {}; // Based on key_type = double, value_type = empty_type template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : kernel_config<256, 15> + : kernel_config<256, 16> +{}; + +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 8> {}; // Based on key_type = float, value_type = int64_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 15> + && (sizeof(value_type) > 4))>> : kernel_config<256, 12> {}; // Based on key_type = float, value_type = int template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<256, 21> + && (sizeof(value_type) > 2))>> : kernel_config<512, 31> {}; // Based on key_type = float, value_type = short template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<512, 15> + && (sizeof(value_type) > 1))>> : kernel_config<512, 31> {}; // Based on key_type = float, value_type = int8_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<256, 21> + : kernel_config<512, 31> {}; // Based on key_type = float, value_type = empty_type template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : kernel_config<256, 21> + : kernel_config<512, 31> +{}; + +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : kernel_config<512, 4> {}; // Based on key_type = rocprim::half, value_type = int64_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : kernel_config<256, 10> + : kernel_config<256, 12> {}; // Based on key_type = rocprim::half, value_type = int template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : kernel_config<512, 15> + : kernel_config<512, 31> {}; // Based on key_type = rocprim::half, value_type = short template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : kernel_config<512, 16> + : kernel_config<512, 32> {}; // Based on key_type = rocprim::half, value_type = int8_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<256, 32> + : kernel_config<512, 27> {}; // Based on key_type = rocprim::half, value_type = empty_type template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (std::is_same::value))>> - : kernel_config<256, 32> + : kernel_config<1024, 22> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : kernel_config<256, 7> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : kernel_config<256, 8> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : kernel_config<256, 8> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : kernel_config<256, 8> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : kernel_config<256, 4> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 4> {}; // Based on key_type = int64_t, value_type = int64_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 15> + && (sizeof(value_type) > 4))>> : kernel_config<256, 12> {}; // Based on key_type = int64_t, value_type = int template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<256, 15> + && (sizeof(value_type) > 2))>> : kernel_config<256, 12> {}; // Based on key_type = int64_t, value_type = short template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<256, 15> + && (sizeof(value_type) > 1))>> : kernel_config<256, 8> {}; // Based on key_type = int64_t, value_type = int8_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<512, 15> + : kernel_config<256, 12> {}; // Based on key_type = int64_t, value_type = empty_type template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : kernel_config<256, 15> + : kernel_config<256, 16> +{}; + +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 4> {}; // Based on key_type = int, value_type = int64_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 15> + && (sizeof(value_type) > 4))>> : kernel_config<256, 12> {}; // Based on key_type = int, value_type = int template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<256, 21> + && (sizeof(value_type) > 2))>> : kernel_config<512, 31> {}; // Based on key_type = int, value_type = short template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<512, 15> + && (sizeof(value_type) > 1))>> : kernel_config<512, 31> {}; // Based on key_type = int, value_type = int8_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<256, 21> + : kernel_config<512, 31> {}; // Based on key_type = int, value_type = empty_type template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : kernel_config<256, 21> + : kernel_config<512, 31> +{}; + +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<512, 4> {}; // Based on key_type = short, value_type = int64_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 10> + && (sizeof(value_type) > 4))>> : kernel_config<256, 12> {}; // Based on key_type = short, value_type = int template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<512, 15> + && (sizeof(value_type) > 2))>> : kernel_config<512, 31> {}; // Based on key_type = short, value_type = short template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<256, 32> + && (sizeof(value_type) > 1))>> : kernel_config<512, 32> {}; // Based on key_type = short, value_type = int8_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<256, 32> + : kernel_config<512, 32> {}; // Based on key_type = short, value_type = empty_type template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (std::is_same::value))>> - : kernel_config<256, 32> + : kernel_config<1024, 23> +{}; + +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : kernel_config<256, 15> {}; // Based on key_type = int8_t, value_type = int64_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : kernel_config<512, 5> + : kernel_config<256, 13> {}; // Based on key_type = int8_t, value_type = int template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : kernel_config<512, 13> + : kernel_config<512, 31> {}; // Based on key_type = int8_t, value_type = short template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : kernel_config<512, 16> + : kernel_config<512, 32> {}; // Based on key_type = int8_t, value_type = int8_t template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<256, 32> + : kernel_config<512, 32> {}; // Based on key_type = int8_t, value_type = empty_type template struct default_radix_sort_block_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (std::is_same::value))>> - : kernel_config<512, 32> + : kernel_config<1024, 23> +{}; + +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 8> {}; // Based on key_type = double, value_type = int64_t @@ -2436,7 +3320,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 15> + && (sizeof(value_type) > 4))>> : kernel_config<256, 16> {}; // Based on key_type = double, value_type = int @@ -2447,7 +3331,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<512, 15> + && (sizeof(value_type) > 2))>> : kernel_config<256, 16> {}; // Based on key_type = double, value_type = short @@ -2458,7 +3342,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<512, 14> + && (sizeof(value_type) > 1))>> : kernel_config<256, 16> {}; // Based on key_type = double, value_type = int8_t @@ -2470,7 +3354,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<512, 15> + : kernel_config<256, 16> {}; // Based on key_type = double, value_type = empty_type @@ -2482,7 +3366,18 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : kernel_config<256, 15> + : kernel_config<256, 16> +{}; + +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 8> {}; // Based on key_type = float, value_type = int64_t @@ -2493,7 +3388,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 15> + && (sizeof(value_type) > 4))>> : kernel_config<256, 8> {}; // Based on key_type = float, value_type = int @@ -2504,7 +3399,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<256, 19> + && (sizeof(value_type) > 2))>> : kernel_config<512, 16> {}; // Based on key_type = float, value_type = short @@ -2515,7 +3410,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<256, 16> + && (sizeof(value_type) > 1))>> : kernel_config<256, 32> {}; // Based on key_type = float, value_type = int8_t @@ -2527,7 +3422,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<512, 31> + : kernel_config<256, 32> {}; // Based on key_type = float, value_type = empty_type @@ -2539,7 +3434,18 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : kernel_config<512, 31> + : kernel_config<256, 32> +{}; + +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : kernel_config<256, 8> {}; // Based on key_type = rocprim::half, value_type = int64_t @@ -2550,7 +3456,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : kernel_config<256, 15> + : kernel_config<512, 8> {}; // Based on key_type = rocprim::half, value_type = int @@ -2561,7 +3467,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : kernel_config<512, 15> + : kernel_config<256, 32> {}; // Based on key_type = rocprim::half, value_type = short @@ -2572,7 +3478,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : kernel_config<512, 16> + : kernel_config<512, 18> {}; // Based on key_type = rocprim::half, value_type = int8_t @@ -2595,7 +3501,86 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (std::is_same::value))>> - : kernel_config<256, 32> + : kernel_config<512, 22> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 8> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : kernel_config<256, 8> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : kernel_config<256, 8> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : kernel_config<256, 8> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : kernel_config<256, 8> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : kernel_config<256, 8> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 8> {}; // Based on key_type = int64_t, value_type = int64_t @@ -2606,7 +3591,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<128, 21> + && (sizeof(value_type) > 4))>> : kernel_config<256, 16> {}; // Based on key_type = int64_t, value_type = int @@ -2617,7 +3602,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<512, 15> + && (sizeof(value_type) > 2))>> : kernel_config<256, 16> {}; // Based on key_type = int64_t, value_type = short @@ -2628,7 +3613,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<512, 14> + && (sizeof(value_type) > 1))>> : kernel_config<256, 16> {}; // Based on key_type = int64_t, value_type = int8_t @@ -2640,7 +3625,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<512, 15> + : kernel_config<256, 16> {}; // Based on key_type = int64_t, value_type = empty_type @@ -2652,7 +3637,18 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : kernel_config<256, 15> + : kernel_config<256, 16> +{}; + +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 8> {}; // Based on key_type = int, value_type = int64_t @@ -2663,7 +3659,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 15> + && (sizeof(value_type) > 4))>> : kernel_config<256, 10> {}; // Based on key_type = int, value_type = int @@ -2674,7 +3670,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<256, 21> + && (sizeof(value_type) > 2))>> : kernel_config<256, 32> {}; // Based on key_type = int, value_type = short @@ -2685,7 +3681,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<512, 31> + && (sizeof(value_type) > 1))>> : kernel_config<256, 32> {}; // Based on key_type = int, value_type = int8_t @@ -2697,7 +3693,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<512, 31> + : kernel_config<256, 32> {}; // Based on key_type = int, value_type = empty_type @@ -2709,7 +3705,18 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : kernel_config<256, 31> + : kernel_config<256, 32> +{}; + +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : kernel_config<256, 8> {}; // Based on key_type = short, value_type = int64_t @@ -2720,7 +3727,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : kernel_config<256, 15> + && (sizeof(value_type) > 4))>> : kernel_config<256, 16> {}; // Based on key_type = short, value_type = int @@ -2731,7 +3738,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : kernel_config<256, 30> + && (sizeof(value_type) > 2))>> : kernel_config<256, 32> {}; // Based on key_type = short, value_type = short @@ -2742,7 +3749,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : kernel_config<512, 16> + && (sizeof(value_type) > 1))>> : kernel_config<512, 19> {}; // Based on key_type = short, value_type = int8_t @@ -2766,7 +3773,18 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (std::is_same::value))>> - : kernel_config<256, 32> + : kernel_config<512, 23> +{}; + +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_radix_sort_block_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : kernel_config<512, 7> {}; // Based on key_type = int8_t, value_type = int64_t @@ -2777,7 +3795,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : kernel_config<512, 5> + : kernel_config<1024, 4> {}; // Based on key_type = int8_t, value_type = int @@ -2788,7 +3806,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : kernel_config<512, 13> + : kernel_config<256, 21> {}; // Based on key_type = int8_t, value_type = short @@ -2799,7 +3817,7 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : kernel_config<512, 16> + : kernel_config<256, 21> {}; // Based on key_type = int8_t, value_type = int8_t @@ -2811,7 +3829,7 @@ struct default_radix_sort_block_sort_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : kernel_config<1024, 20> + : kernel_config<256, 23> {}; // Based on key_type = int8_t, value_type = empty_type @@ -2822,8 +3840,9 @@ struct default_radix_sort_block_sort_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (std::is_same::value))>> - : kernel_config<512, 32> + : kernel_config<512, 22> {}; + } // end namespace detail END_ROCPRIM_NAMESPACE diff --git a/rocprim/include/rocprim/device/detail/config/device_radix_sort_onesweep.hpp b/rocprim/include/rocprim/device/detail/config/device_radix_sort_onesweep.hpp index fa9d53ac0..843f94487 100644 --- a/rocprim/include/rocprim/device/detail/config/device_radix_sort_onesweep.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_radix_sort_onesweep.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,8 +21,11 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RADIX_SORT_ONESWEEP_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RADIX_SORT_ONESWEEP_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" + #include /* DO NOT EDIT THIS FILE @@ -46,6 +49,21 @@ struct default_radix_sort_onesweep_config : radix_sort_onesweep_config_base::type {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<1024, 1>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = double, value_type = int64_t template struct default_radix_sort_onesweep_config< @@ -121,6 +139,21 @@ struct default_radix_sort_onesweep_config< block_radix_rank_algorithm::match> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<1024, 1>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = float, value_type = int64_t template struct default_radix_sort_onesweep_config< @@ -196,6 +229,20 @@ struct default_radix_sort_onesweep_config< block_radix_rank_algorithm::match> {}; +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<128, 22>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = rocprim::half, value_type = int64_t template struct default_radix_sort_onesweep_config< @@ -204,8 +251,8 @@ struct default_radix_sort_onesweep_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, - kernel_config<512, 12>, + : radix_sort_onesweep_config, + kernel_config<1024, 4>, 8, block_radix_rank_algorithm::match> {}; @@ -218,8 +265,8 @@ struct default_radix_sort_onesweep_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, - kernel_config<512, 16>, + : radix_sort_onesweep_config, + kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -232,8 +279,8 @@ struct default_radix_sort_onesweep_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<512, 16>, + : radix_sort_onesweep_config, + kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -247,8 +294,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 18>, + : radix_sort_onesweep_config, + kernel_config<512, 22>, 8, block_radix_rank_algorithm::match> {}; @@ -261,8 +308,113 @@ struct default_radix_sort_onesweep_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<512, 12>, + : radix_sort_onesweep_config, + kernel_config<256, 22>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<1024, 1>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : radix_sort_onesweep_config, + kernel_config<1024, 1>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : radix_sort_onesweep_config, + kernel_config<256, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : radix_sort_onesweep_config, + kernel_config<256, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : radix_sort_onesweep_config, + kernel_config<256, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : radix_sort_onesweep_config, + kernel_config<256, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<1024, 1>, 8, block_radix_rank_algorithm::match> {}; @@ -342,6 +494,21 @@ struct default_radix_sort_onesweep_config< block_radix_rank_algorithm::match> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<1024, 1>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = int, value_type = int64_t template struct default_radix_sort_onesweep_config< @@ -417,6 +584,21 @@ struct default_radix_sort_onesweep_config< block_radix_rank_algorithm::match> {}; +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<128, 22>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = short, value_type = int64_t template struct default_radix_sort_onesweep_config< @@ -492,6 +674,20 @@ struct default_radix_sort_onesweep_config< block_radix_rank_algorithm::match> {}; +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<128, 22>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = int8_t, value_type = int64_t template struct default_radix_sort_onesweep_config< @@ -563,17 +759,32 @@ struct default_radix_sort_onesweep_config< block_radix_rank_algorithm::match> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<128, 16>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = double, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, - kernel_config<512, 8>, + : radix_sort_onesweep_config, + kernel_config<1024, 4>, 8, block_radix_rank_algorithm::match> {}; @@ -581,14 +792,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = double, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, - kernel_config<512, 4>, + : radix_sort_onesweep_config, + kernel_config<1024, 4>, 8, block_radix_rank_algorithm::match> {}; @@ -596,14 +807,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = double, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<512, 12>, + : radix_sort_onesweep_config, + kernel_config<1024, 4>, 8, block_radix_rank_algorithm::match> {}; @@ -611,14 +822,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = double, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<512, 6>, + : radix_sort_onesweep_config, + kernel_config<512, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -626,14 +837,29 @@ struct default_radix_sort_onesweep_config< // Based on key_type = double, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<512, 12>, + : radix_sort_onesweep_config, + kernel_config<1024, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<128, 22>, 8, block_radix_rank_algorithm::match> {}; @@ -641,14 +867,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = float, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, - kernel_config<512, 8>, + : radix_sort_onesweep_config, + kernel_config<1024, 4>, 8, block_radix_rank_algorithm::match> {}; @@ -656,14 +882,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = float, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, - kernel_config<512, 16>, + : radix_sort_onesweep_config, + kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -671,14 +897,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = float, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<512, 6>, + : radix_sort_onesweep_config, + kernel_config<1024, 6>, 8, block_radix_rank_algorithm::match> {}; @@ -686,14 +912,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = float, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<512, 8>, + : radix_sort_onesweep_config, + kernel_config<1024, 8>, 8, block_radix_rank_algorithm::match> {}; @@ -701,14 +927,28 @@ struct default_radix_sort_onesweep_config< // Based on key_type = float, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<512, 8>, + : radix_sort_onesweep_config, + kernel_config<512, 22>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<128, 22>, 8, block_radix_rank_algorithm::match> {}; @@ -716,13 +956,13 @@ struct default_radix_sort_onesweep_config< // Based on key_type = rocprim::half, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, - kernel_config<512, 6>, + : radix_sort_onesweep_config, + kernel_config<1024, 4>, 8, block_radix_rank_algorithm::match> {}; @@ -730,13 +970,13 @@ struct default_radix_sort_onesweep_config< // Based on key_type = rocprim::half, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<1024, 6>, 8, block_radix_rank_algorithm::match> {}; @@ -744,13 +984,13 @@ struct default_radix_sort_onesweep_config< // Based on key_type = rocprim::half, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -758,14 +998,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = rocprim::half, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<512, 18>, 8, block_radix_rank_algorithm::match> {}; @@ -773,28 +1013,133 @@ struct default_radix_sort_onesweep_config< // Based on key_type = rocprim::half, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<512, 18>, 8, block_radix_rank_algorithm::match> {}; -// Based on key_type = int64_t, value_type = int64_t +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<1024, 1>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, - kernel_config<256, 22>, + : radix_sort_onesweep_config, + kernel_config<128, 16>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : radix_sort_onesweep_config, + kernel_config<512, 4>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : radix_sort_onesweep_config, + kernel_config<512, 4>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<128, 22>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = int64_t, value_type = int64_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : radix_sort_onesweep_config, + kernel_config<1024, 4>, 8, block_radix_rank_algorithm::match> {}; @@ -802,14 +1147,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int64_t, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, - kernel_config<512, 8>, + : radix_sort_onesweep_config, + kernel_config<1024, 4>, 8, block_radix_rank_algorithm::match> {}; @@ -817,14 +1162,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int64_t, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<512, 12>, + : radix_sort_onesweep_config, + kernel_config<1024, 4>, 8, block_radix_rank_algorithm::match> {}; @@ -832,14 +1177,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int64_t, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 18>, + : radix_sort_onesweep_config, + kernel_config<256, 16>, 8, block_radix_rank_algorithm::match> {}; @@ -847,7 +1192,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int64_t, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -859,17 +1204,32 @@ struct default_radix_sort_onesweep_config< block_radix_rank_algorithm::match> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<128, 22>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = int, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<1024, 4>, 8, block_radix_rank_algorithm::match> {}; @@ -877,29 +1237,29 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, - kernel_config<1024, 12>, - 4, - block_radix_rank_algorithm::basic> + : radix_sort_onesweep_config, + kernel_config<1024, 4>, + 8, + block_radix_rank_algorithm::match> {}; // Based on key_type = int, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<512, 16>, + : radix_sort_onesweep_config, + kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -907,7 +1267,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -922,14 +1282,29 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 18>, + : radix_sort_onesweep_config, + kernel_config<512, 22>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<128, 22>, 8, block_radix_rank_algorithm::match> {}; @@ -937,14 +1312,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = short, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, - kernel_config<256, 12>, + : radix_sort_onesweep_config, + kernel_config<1024, 4>, 8, block_radix_rank_algorithm::match> {}; @@ -952,14 +1327,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = short, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, - kernel_config<256, 8>, + : radix_sort_onesweep_config, + kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -967,14 +1342,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = short, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<256, 18>, + : radix_sort_onesweep_config, + kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -982,14 +1357,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = short, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<512, 8>, + : radix_sort_onesweep_config, + kernel_config<512, 16>, 8, block_radix_rank_algorithm::match> {}; @@ -997,22 +1372,36 @@ struct default_radix_sort_onesweep_config< // Based on key_type = short, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<512, 12>, - 4, + : radix_sort_onesweep_config, + kernel_config<512, 18>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<128, 22>, + 8, block_radix_rank_algorithm::match> {}; // Based on key_type = int8_t, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -1026,13 +1415,13 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int8_t, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, - kernel_config<512, 22>, + : radix_sort_onesweep_config, + kernel_config<1024, 8>, 8, block_radix_rank_algorithm::match> {}; @@ -1040,13 +1429,13 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int8_t, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<256, 18>, + : radix_sort_onesweep_config, + kernel_config<512, 16>, 8, block_radix_rank_algorithm::match> {}; @@ -1054,14 +1443,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int8_t, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 22>, + : radix_sort_onesweep_config, + kernel_config<512, 18>, 8, block_radix_rank_algorithm::match> {}; @@ -1069,13 +1458,28 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int8_t, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<512, 18>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<256, 6>, 8, block_radix_rank_algorithm::match> {}; @@ -1083,14 +1487,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = double, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, - kernel_config<1024, 6>, + : radix_sort_onesweep_config, + kernel_config<256, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -1098,7 +1502,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = double, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1113,14 +1517,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = double, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<256, 12>, + : radix_sort_onesweep_config, + kernel_config<512, 6>, 8, block_radix_rank_algorithm::match> {}; @@ -1128,7 +1532,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = double, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1143,14 +1547,29 @@ struct default_radix_sort_onesweep_config< // Based on key_type = double, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<512, 6>, + : radix_sort_onesweep_config, + kernel_config<256, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<256, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -1158,14 +1577,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = float, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, - kernel_config<1024, 6>, + : radix_sort_onesweep_config, + kernel_config<256, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -1173,7 +1592,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = float, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1188,14 +1607,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = float, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<256, 22>, + : radix_sort_onesweep_config, + kernel_config<256, 16>, 8, block_radix_rank_algorithm::match> {}; @@ -1203,14 +1622,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = float, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 22>, + : radix_sort_onesweep_config, + kernel_config<512, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -1218,56 +1637,70 @@ struct default_radix_sort_onesweep_config< // Based on key_type = float, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 22>, + : radix_sort_onesweep_config, + kernel_config<512, 12>, 8, block_radix_rank_algorithm::match> {}; -// Based on key_type = rocprim::half, value_type = int64_t +// Based on key_type = rocprim::half, value_type = rocprim::int128_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> : radix_sort_onesweep_config, kernel_config<256, 12>, 8, block_radix_rank_algorithm::match> {}; -// Based on key_type = rocprim::half, value_type = int +// Based on key_type = rocprim::half, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : radix_sort_onesweep_config, + kernel_config<256, 12>, 8, block_radix_rank_algorithm::match> {}; -// Based on key_type = rocprim::half, value_type = short +// Based on key_type = rocprim::half, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : radix_sort_onesweep_config, + kernel_config<512, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::half, value_type = short +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<256, 22>, 8, block_radix_rank_algorithm::match> {}; @@ -1275,14 +1708,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = rocprim::half, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<512, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -1290,13 +1723,118 @@ struct default_radix_sort_onesweep_config< // Based on key_type = rocprim::half, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<512, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<256, 6>, 8, block_radix_rank_algorithm::match> {}; @@ -1304,22 +1842,22 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int64_t, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, - kernel_config<1024, 6>, - 6, + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, block_radix_rank_algorithm::match> {}; // Based on key_type = int64_t, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1334,14 +1872,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int64_t, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<256, 12>, + : radix_sort_onesweep_config, + kernel_config<512, 6>, 8, block_radix_rank_algorithm::match> {}; @@ -1349,7 +1887,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int64_t, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1364,7 +1902,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int64_t, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1376,10 +1914,25 @@ struct default_radix_sort_onesweep_config< block_radix_rank_algorithm::match> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<256, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = int, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1394,7 +1947,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1409,14 +1962,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<256, 18>, 8, block_radix_rank_algorithm::match> {}; @@ -1424,14 +1977,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 22>, + : radix_sort_onesweep_config, + kernel_config<256, 18>, 8, block_radix_rank_algorithm::match> {}; @@ -1439,7 +1992,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1447,21 +2000,36 @@ struct default_radix_sort_onesweep_config< && (std::is_same::value))>> : radix_sort_onesweep_config, kernel_config<256, 22>, - 7, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, block_radix_rank_algorithm::match> {}; // Based on key_type = short, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, - kernel_config<256, 18>, + : radix_sort_onesweep_config, + kernel_config<256, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -1469,14 +2037,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = short, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, - kernel_config<256, 18>, + : radix_sort_onesweep_config, + kernel_config<512, 22>, 8, block_radix_rank_algorithm::match> {}; @@ -1484,29 +2052,29 @@ struct default_radix_sort_onesweep_config< // Based on key_type = short, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<1024, 22>, - 6, + : radix_sort_onesweep_config, + kernel_config<512, 12>, + 4, block_radix_rank_algorithm::match> {}; // Based on key_type = short, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 18>, + : radix_sort_onesweep_config, + kernel_config<512, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -1514,14 +2082,28 @@ struct default_radix_sort_onesweep_config< // Based on key_type = short, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 22>, + : radix_sort_onesweep_config, + kernel_config<512, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, 8, block_radix_rank_algorithm::match> {}; @@ -1529,7 +2111,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int8_t, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -1543,13 +2125,13 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int8_t, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, - kernel_config<256, 22>, + : radix_sort_onesweep_config, + kernel_config<256, 16>, 8, block_radix_rank_algorithm::match> {}; @@ -1557,13 +2139,13 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int8_t, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<512, 8>, 8, block_radix_rank_algorithm::match> {}; @@ -1571,14 +2153,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int8_t, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<1024, 22>, + : radix_sort_onesweep_config, + kernel_config<512, 16>, 4, block_radix_rank_algorithm::basic> {}; @@ -1586,13 +2168,28 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int8_t, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<512, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<256, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -1600,7 +2197,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = double, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1615,7 +2212,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = double, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1630,7 +2227,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = double, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1645,7 +2242,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = double, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1660,7 +2257,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = double, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1672,10 +2269,25 @@ struct default_radix_sort_onesweep_config< block_radix_rank_algorithm::match> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = float, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1690,7 +2302,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = float, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1705,7 +2317,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = float, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1720,7 +2332,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = float, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1735,7 +2347,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = float, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1747,10 +2359,24 @@ struct default_radix_sort_onesweep_config< block_radix_rank_algorithm::match> {}; +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<256, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = rocprim::half, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1764,13 +2390,13 @@ struct default_radix_sort_onesweep_config< // Based on key_type = rocprim::half, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<512, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -1778,13 +2404,13 @@ struct default_radix_sort_onesweep_config< // Based on key_type = rocprim::half, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<256, 22>, 8, block_radix_rank_algorithm::match> {}; @@ -1792,28 +2418,133 @@ struct default_radix_sort_onesweep_config< // Based on key_type = rocprim::half, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<512, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::half, value_type = empty_type +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (std::is_same::value))>> + : radix_sort_onesweep_config, + kernel_config<512, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<256, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : radix_sort_onesweep_config, + kernel_config<256, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : radix_sort_onesweep_config, + kernel_config<256, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, 8, block_radix_rank_algorithm::match> {}; -// Based on key_type = rocprim::half, value_type = empty_type +// Based on key_type = int64_t, value_type = rocprim::int128_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, 8, block_radix_rank_algorithm::match> {}; @@ -1821,7 +2552,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int64_t, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1836,7 +2567,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int64_t, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1851,7 +2582,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int64_t, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1866,7 +2597,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int64_t, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1881,7 +2612,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int64_t, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1893,10 +2624,25 @@ struct default_radix_sort_onesweep_config< block_radix_rank_algorithm::match> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = int, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1911,7 +2657,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1926,7 +2672,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1941,7 +2687,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1956,7 +2702,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1968,10 +2714,25 @@ struct default_radix_sort_onesweep_config< block_radix_rank_algorithm::match> {}; +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = short, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1986,7 +2747,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = short, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -2001,7 +2762,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = short, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -2016,7 +2777,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = short, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -2031,7 +2792,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = short, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -2043,10 +2804,24 @@ struct default_radix_sort_onesweep_config< block_radix_rank_algorithm::match> {}; +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<256, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = int8_t, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -2060,7 +2835,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int8_t, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -2074,7 +2849,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int8_t, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -2088,7 +2863,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int8_t, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -2103,7 +2878,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int8_t, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -2114,6 +2889,21 @@ struct default_radix_sort_onesweep_config< block_radix_rank_algorithm::match> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<256, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = double, value_type = int64_t template struct default_radix_sort_onesweep_config< @@ -2189,6 +2979,21 @@ struct default_radix_sort_onesweep_config< block_radix_rank_algorithm::match> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = float, value_type = int64_t template struct default_radix_sort_onesweep_config< @@ -2264,6 +3069,20 @@ struct default_radix_sort_onesweep_config< block_radix_rank_algorithm::match> {}; +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<256, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = rocprim::half, value_type = int64_t template struct default_radix_sort_onesweep_config< @@ -2315,8 +3134,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<512, 16>, + : radix_sort_onesweep_config, + kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -2335,6 +3154,111 @@ struct default_radix_sort_onesweep_config< block_radix_rank_algorithm::match> {}; +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : radix_sort_onesweep_config, + kernel_config<256, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<256, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = int64_t, value_type = int64_t template struct default_radix_sort_onesweep_config< @@ -2410,6 +3334,21 @@ struct default_radix_sort_onesweep_config< block_radix_rank_algorithm::match> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = int, value_type = int64_t template struct default_radix_sort_onesweep_config< @@ -2485,6 +3424,21 @@ struct default_radix_sort_onesweep_config< block_radix_rank_algorithm::match> {}; +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = short, value_type = int64_t template struct default_radix_sort_onesweep_config< @@ -2560,6 +3514,20 @@ struct default_radix_sort_onesweep_config< block_radix_rank_algorithm::match> {}; +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<256, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = int8_t, value_type = int64_t template struct default_radix_sort_onesweep_config< @@ -2631,17 +3599,32 @@ struct default_radix_sort_onesweep_config< block_radix_rank_algorithm::basic> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<256, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = double, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, - kernel_config<256, 12>, + : radix_sort_onesweep_config, + kernel_config<1024, 6>, 8, block_radix_rank_algorithm::match> {}; @@ -2649,7 +3632,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = double, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -2664,14 +3647,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = double, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<512, 6>, + : radix_sort_onesweep_config, + kernel_config<256, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -2679,7 +3662,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = double, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -2694,14 +3677,29 @@ struct default_radix_sort_onesweep_config< // Based on key_type = double, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 12>, + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, 8, block_radix_rank_algorithm::match> {}; @@ -2709,14 +3707,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = float, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, - kernel_config<256, 12>, + : radix_sort_onesweep_config, + kernel_config<1024, 6>, 8, block_radix_rank_algorithm::match> {}; @@ -2724,7 +3722,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = float, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -2739,14 +3737,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = float, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<256, 22>, 8, block_radix_rank_algorithm::match> {}; @@ -2754,14 +3752,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = float, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<512, 12>, + : radix_sort_onesweep_config, + kernel_config<256, 22>, 8, block_radix_rank_algorithm::match> {}; @@ -2769,14 +3767,28 @@ struct default_radix_sort_onesweep_config< // Based on key_type = float, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<512, 12>, + : radix_sort_onesweep_config, + kernel_config<256, 22>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<256, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -2784,13 +3796,13 @@ struct default_radix_sort_onesweep_config< // Based on key_type = rocprim::half, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, - kernel_config<512, 6>, + : radix_sort_onesweep_config, + kernel_config<256, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -2798,13 +3810,13 @@ struct default_radix_sort_onesweep_config< // Based on key_type = rocprim::half, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<512, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -2812,13 +3824,13 @@ struct default_radix_sort_onesweep_config< // Based on key_type = rocprim::half, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<256, 22>, 8, block_radix_rank_algorithm::match> {}; @@ -2826,7 +3838,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = rocprim::half, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -2841,7 +3853,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = rocprim::half, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -2852,25 +3864,130 @@ struct default_radix_sort_onesweep_config< block_radix_rank_algorithm::match> {}; -// Based on key_type = int64_t, value_type = int64_t +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<256, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : radix_sort_onesweep_config, + kernel_config<256, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : radix_sort_onesweep_config, + kernel_config<256, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : radix_sort_onesweep_config, kernel_config<512, 6>, 8, block_radix_rank_algorithm::match> {}; +// Based on key_type = int64_t, value_type = int64_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : radix_sort_onesweep_config, + kernel_config<1024, 6>, + 6, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = int64_t, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -2885,14 +4002,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int64_t, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<512, 6>, + : radix_sort_onesweep_config, + kernel_config<256, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -2900,7 +4017,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int64_t, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -2915,7 +4032,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int64_t, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -2927,10 +4044,25 @@ struct default_radix_sort_onesweep_config< block_radix_rank_algorithm::match> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = int, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -2945,7 +4077,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -2960,14 +4092,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<256, 18>, + : radix_sort_onesweep_config, + kernel_config<256, 16>, 8, block_radix_rank_algorithm::match> {}; @@ -2975,14 +4107,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<256, 18>, + : radix_sort_onesweep_config, + kernel_config<256, 22>, 8, block_radix_rank_algorithm::match> {}; @@ -2990,7 +4122,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -2998,6 +4130,21 @@ struct default_radix_sort_onesweep_config< && (std::is_same::value))>> : radix_sort_onesweep_config, kernel_config<256, 22>, + 7, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, 8, block_radix_rank_algorithm::match> {}; @@ -3005,14 +4152,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = short, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, - kernel_config<256, 12>, + : radix_sort_onesweep_config, + kernel_config<256, 18>, 8, block_radix_rank_algorithm::match> {}; @@ -3020,14 +4167,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = short, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, - kernel_config<512, 22>, + : radix_sort_onesweep_config, + kernel_config<256, 18>, 8, block_radix_rank_algorithm::match> {}; @@ -3035,29 +4182,29 @@ struct default_radix_sort_onesweep_config< // Based on key_type = short, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<512, 12>, - 4, + : radix_sort_onesweep_config, + kernel_config<1024, 22>, + 6, block_radix_rank_algorithm::match> {}; // Based on key_type = short, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<512, 12>, + : radix_sort_onesweep_config, + kernel_config<256, 18>, 8, block_radix_rank_algorithm::match> {}; @@ -3065,14 +4212,28 @@ struct default_radix_sort_onesweep_config< // Based on key_type = short, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<512, 12>, + : radix_sort_onesweep_config, + kernel_config<256, 22>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<256, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -3080,7 +4241,7 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int8_t, value_type = int64_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -3094,13 +4255,13 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int8_t, value_type = int template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, - kernel_config<256, 16>, + : radix_sort_onesweep_config, + kernel_config<256, 22>, 8, block_radix_rank_algorithm::match> {}; @@ -3108,13 +4269,13 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int8_t, value_type = short template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<512, 8>, + : radix_sort_onesweep_config, + kernel_config<256, 16>, 8, block_radix_rank_algorithm::match> {}; @@ -3122,14 +4283,14 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int8_t, value_type = int8_t template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<512, 16>, + : radix_sort_onesweep_config, + kernel_config<1024, 22>, 4, block_radix_rank_algorithm::basic> {}; @@ -3137,13 +4298,28 @@ struct default_radix_sort_onesweep_config< // Based on key_type = int8_t, value_type = empty_type template struct default_radix_sort_onesweep_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<512, 12>, + : radix_sort_onesweep_config, + kernel_config<256, 16>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, 8, block_radix_rank_algorithm::match> {}; @@ -3157,7 +4333,7 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<1024, 6>, 8, block_radix_rank_algorithm::match> @@ -3172,8 +4348,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, - kernel_config<1024, 6>, + : radix_sort_onesweep_config, + kernel_config<512, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -3187,7 +4363,7 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<512, 12>, 8, block_radix_rank_algorithm::match> @@ -3202,7 +4378,7 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<1024, 6>, 8, block_radix_rank_algorithm::match> @@ -3217,12 +4393,27 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<1024, 6>, 8, block_radix_rank_algorithm::match> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = float, value_type = int64_t template struct default_radix_sort_onesweep_config< @@ -3232,7 +4423,7 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<1024, 6>, 8, block_radix_rank_algorithm::match> @@ -3247,7 +4438,7 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> @@ -3262,8 +4453,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<1024, 12>, + : radix_sort_onesweep_config, + kernel_config<512, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -3277,8 +4468,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<1024, 12>, + : radix_sort_onesweep_config, + kernel_config<512, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -3292,12 +4483,26 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> {}; +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = rocprim::half, value_type = int64_t template struct default_radix_sort_onesweep_config< @@ -3306,7 +4511,7 @@ struct default_radix_sort_onesweep_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<1024, 6>, 8, block_radix_rank_algorithm::match> @@ -3320,7 +4525,7 @@ struct default_radix_sort_onesweep_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> @@ -3334,7 +4539,7 @@ struct default_radix_sort_onesweep_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<512, 22>, 8, block_radix_rank_algorithm::match> @@ -3349,7 +4554,7 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<512, 22>, 8, block_radix_rank_algorithm::match> @@ -3363,12 +4568,117 @@ struct default_radix_sort_onesweep_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (std::is_same::value))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> {}; +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : radix_sort_onesweep_config, + kernel_config<256, 12>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = int64_t, value_type = int64_t template struct default_radix_sort_onesweep_config< @@ -3378,7 +4688,7 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<1024, 6>, 8, block_radix_rank_algorithm::match> @@ -3393,7 +4703,7 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<1024, 6>, 8, block_radix_rank_algorithm::match> @@ -3408,7 +4718,7 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<1024, 6>, 8, block_radix_rank_algorithm::match> @@ -3423,7 +4733,7 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<1024, 6>, 8, block_radix_rank_algorithm::match> @@ -3438,12 +4748,27 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (std::is_same::value))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<1024, 6>, 8, block_radix_rank_algorithm::match> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = int, value_type = int64_t template struct default_radix_sort_onesweep_config< @@ -3453,7 +4778,7 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<1024, 6>, 8, block_radix_rank_algorithm::match> @@ -3468,7 +4793,7 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> @@ -3483,7 +4808,7 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> @@ -3498,7 +4823,7 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> @@ -3513,12 +4838,27 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (std::is_same::value))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> {}; +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = short, value_type = int64_t template struct default_radix_sort_onesweep_config< @@ -3528,8 +4868,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, - kernel_config<512, 12>, + : radix_sort_onesweep_config, + kernel_config<1024, 6>, 8, block_radix_rank_algorithm::match> {}; @@ -3543,9 +4883,9 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, - kernel_config<1024, 12>, - 8, + : radix_sort_onesweep_config, + kernel_config<512, 12>, + 6, block_radix_rank_algorithm::match> {}; @@ -3558,9 +4898,9 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, - kernel_config<1024, 22>, - 8, + : radix_sort_onesweep_config, + kernel_config<512, 22>, + 6, block_radix_rank_algorithm::match> {}; @@ -3573,8 +4913,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<1024, 22>, + : radix_sort_onesweep_config, + kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -3588,12 +4928,26 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (std::is_same::value))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> {}; +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_radix_sort_onesweep_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : radix_sort_onesweep_config, + kernel_config<512, 6>, + 8, + block_radix_rank_algorithm::match> +{}; + // Based on key_type = int8_t, value_type = int64_t template struct default_radix_sort_onesweep_config< @@ -3602,7 +4956,7 @@ struct default_radix_sort_onesweep_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<1024, 6>, 8, block_radix_rank_algorithm::match> @@ -3616,7 +4970,7 @@ struct default_radix_sort_onesweep_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> @@ -3630,7 +4984,7 @@ struct default_radix_sort_onesweep_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : radix_sort_onesweep_config, + : radix_sort_onesweep_config, kernel_config<1024, 22>, 8, block_radix_rank_algorithm::match> @@ -3645,8 +4999,8 @@ struct default_radix_sort_onesweep_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<1024, 22>, + : radix_sort_onesweep_config, + kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> {}; @@ -3659,8 +5013,8 @@ struct default_radix_sort_onesweep_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (std::is_same::value))>> - : radix_sort_onesweep_config, - kernel_config<1024, 22>, + : radix_sort_onesweep_config, + kernel_config<1024, 12>, 8, block_radix_rank_algorithm::match> {}; diff --git a/rocprim/include/rocprim/device/detail/config/device_reduce.hpp b/rocprim/include/rocprim/device/detail/config/device_reduce.hpp index b4d506d21..d0da09293 100644 --- a/rocprim/include/rocprim/device/detail/config/device_reduce.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_reduce.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,8 +21,11 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_REDUCE_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_REDUCE_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" + #include /* DO NOT EDIT THIS FILE @@ -71,6 +74,16 @@ struct default_reduce_config(target_arch::gfx1030), : reduce_config<256, 4, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; +// Based on key_type = rocprim::int128_t +template +struct default_reduce_config< + static_cast(target_arch::gfx1030), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : reduce_config<256, 2, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + // Based on key_type = int64_t template struct default_reduce_config< @@ -113,137 +126,79 @@ struct default_reduce_config(target_arch::gfx1030), // Based on key_type = double template struct default_reduce_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4))>> - : reduce_config<256, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<256, 4, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = float template struct default_reduce_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2))>> - : reduce_config<128, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<256, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = rocprim::half template -struct default_reduce_config(target_arch::gfx1102), +struct default_reduce_config(target_arch::gfx1100), key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2))>> : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; -// Based on key_type = int64_t -template -struct default_reduce_config< - static_cast(target_arch::gfx1102), - key_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4))>> - : reduce_config<128, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> -{}; - -// Based on key_type = int -template -struct default_reduce_config< - static_cast(target_arch::gfx1102), - key_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2))>> - : reduce_config<256, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> -{}; - -// Based on key_type = short -template -struct default_reduce_config< - static_cast(target_arch::gfx1102), - key_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(key_type) > 1))>> - : reduce_config<256, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> -{}; - -// Based on key_type = int8_t -template -struct default_reduce_config(target_arch::gfx1102), - key_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(key_type) <= 1))>> - : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> -{}; - -// Based on key_type = double -template -struct default_reduce_config< - static_cast(target_arch::gfx900), - key_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4))>> - : reduce_config<128, 4, ::rocprim::block_reduce_algorithm::using_warp_reduce> -{}; - -// Based on key_type = float +// Based on key_type = rocprim::int128_t template struct default_reduce_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2))>> - : reduce_config<128, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> -{}; - -// Based on key_type = rocprim::half -template -struct default_reduce_config(target_arch::gfx900), - key_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(key_type) <= 2))>> - : reduce_config<128, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : reduce_config<256, 4, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = int64_t template struct default_reduce_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4))>> - : reduce_config<128, 4, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<256, 4, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = int template struct default_reduce_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2))>> - : reduce_config<128, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<256, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = short template struct default_reduce_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1))>> - : reduce_config<128, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = int8_t template -struct default_reduce_config(target_arch::gfx900), +struct default_reduce_config(target_arch::gfx1100), key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1))>> - : reduce_config<64, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = double @@ -272,7 +227,17 @@ struct default_reduce_config(target_arch::gfx906), key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2))>> - : reduce_config<64, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<256, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = rocprim::int128_t +template +struct default_reduce_config< + static_cast(target_arch::gfx906), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : reduce_config<128, 2, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = int64_t @@ -340,7 +305,17 @@ struct default_reduce_config(target_arch::gfx908), key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2))>> - : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<128, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = rocprim::int128_t +template +struct default_reduce_config< + static_cast(target_arch::gfx908), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : reduce_config<128, 2, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = int64_t @@ -385,7 +360,7 @@ struct default_reduce_config(target_arch::gfx908), // Based on key_type = double template struct default_reduce_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4))>> @@ -395,65 +370,75 @@ struct default_reduce_config< // Based on key_type = float template struct default_reduce_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2))>> - : reduce_config<256, 4, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<128, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = rocprim::half template -struct default_reduce_config(target_arch::unknown), +struct default_reduce_config(target_arch::gfx90a), key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2))>> - : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<64, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = rocprim::int128_t +template +struct default_reduce_config< + static_cast(target_arch::gfx90a), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : reduce_config<256, 2, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = int64_t template struct default_reduce_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4))>> - : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<256, 4, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = int template struct default_reduce_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2))>> - : reduce_config<256, 4, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<128, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = short template struct default_reduce_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1))>> - : reduce_config<128, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<64, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = int8_t template -struct default_reduce_config(target_arch::unknown), +struct default_reduce_config(target_arch::gfx90a), key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1))>> - : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<128, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = double template struct default_reduce_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4))>> @@ -463,59 +448,69 @@ struct default_reduce_config< // Based on key_type = float template struct default_reduce_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2))>> - : reduce_config<128, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<256, 4, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = rocprim::half template -struct default_reduce_config(target_arch::gfx90a), +struct default_reduce_config(target_arch::unknown), key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2))>> - : reduce_config<64, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<128, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = rocprim::int128_t +template +struct default_reduce_config< + static_cast(target_arch::unknown), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : reduce_config<128, 2, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = int64_t template struct default_reduce_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4))>> - : reduce_config<256, 4, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = int template struct default_reduce_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2))>> - : reduce_config<128, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<256, 4, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = short template struct default_reduce_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1))>> - : reduce_config<64, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<128, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = int8_t template -struct default_reduce_config(target_arch::gfx90a), +struct default_reduce_config(target_arch::unknown), key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1))>> - : reduce_config<128, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = double @@ -525,7 +520,7 @@ struct default_reduce_config< key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4))>> - : reduce_config<256, 2, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = float @@ -535,7 +530,7 @@ struct default_reduce_config< key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2))>> - : reduce_config<256, 4, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<128, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = rocprim::half @@ -544,7 +539,17 @@ struct default_reduce_config(target_arch::gfx942), key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2))>> - : reduce_config<128, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = rocprim::int128_t +template +struct default_reduce_config< + static_cast(target_arch::gfx942), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = int64_t @@ -554,7 +559,7 @@ struct default_reduce_config< key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4))>> - : reduce_config<256, 2, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = int @@ -564,7 +569,7 @@ struct default_reduce_config< key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2))>> - : reduce_config<256, 4, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<256, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = short @@ -574,7 +579,7 @@ struct default_reduce_config< key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1))>> - : reduce_config<256, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; // Based on key_type = int8_t @@ -583,7 +588,7 @@ struct default_reduce_config(target_arch::gfx942), key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1))>> - : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> + : reduce_config<128, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> {}; } // end namespace detail diff --git a/rocprim/include/rocprim/device/detail/config/device_reduce_by_key.hpp b/rocprim/include/rocprim/device/detail/config/device_reduce_by_key.hpp index 30ab7deb4..c63e26316 100644 --- a/rocprim/include/rocprim/device/detail/config/device_reduce_by_key.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_reduce_by_key.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -56,8 +56,7 @@ struct default_reduce_by_key_config< 8, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = float @@ -73,8 +72,7 @@ struct default_reduce_by_key_config< 8, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = rocprim::half @@ -90,8 +88,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = int64_t @@ -124,8 +121,7 @@ struct default_reduce_by_key_config< 8, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = short @@ -141,8 +137,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = int8_t @@ -158,8 +153,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = double @@ -175,8 +169,7 @@ struct default_reduce_by_key_config< 8, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = float @@ -192,8 +185,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = rocprim::half @@ -209,8 +201,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = int64_t @@ -226,8 +217,7 @@ struct default_reduce_by_key_config< 8, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = int @@ -243,8 +233,7 @@ struct default_reduce_by_key_config< 8, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = short @@ -260,8 +249,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = int8_t @@ -277,8 +265,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = double @@ -294,8 +281,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = float @@ -311,8 +297,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = rocprim::half @@ -328,8 +313,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int64_t @@ -345,8 +329,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int @@ -362,8 +345,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = short @@ -379,8 +361,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int8_t @@ -396,8 +377,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = double @@ -412,8 +392,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = float @@ -428,8 +407,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = rocprim::half @@ -444,8 +422,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int64_t @@ -461,8 +438,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int @@ -478,8 +454,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = short @@ -495,8 +470,7 @@ struct default_reduce_by_key_config< 13, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int8_t @@ -512,8 +486,7 @@ struct default_reduce_by_key_config< 12, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = double @@ -529,8 +502,7 @@ struct default_reduce_by_key_config< 12, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = float @@ -648,8 +620,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = float @@ -682,8 +653,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = int64_t @@ -733,8 +703,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = int8_t @@ -750,8 +719,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = double @@ -767,8 +735,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = float @@ -784,8 +751,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = rocprim::half @@ -801,8 +767,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int64_t @@ -835,8 +800,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = short @@ -852,8 +816,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int8_t @@ -869,8 +832,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = double @@ -901,8 +863,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = rocprim::half @@ -917,8 +878,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int64_t @@ -951,8 +911,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = short @@ -968,8 +927,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int8_t @@ -985,8 +943,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = double @@ -1002,8 +959,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = float @@ -1019,8 +975,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = rocprim::half @@ -1036,8 +991,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = int64_t @@ -1053,8 +1007,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = int @@ -1070,8 +1023,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = short @@ -1087,8 +1039,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = int8_t @@ -1104,8 +1055,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = double @@ -1121,8 +1071,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = float @@ -1138,8 +1087,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = rocprim::half @@ -1155,8 +1103,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = int64_t @@ -1172,8 +1119,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = int @@ -1189,8 +1135,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = short @@ -1206,8 +1151,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = int8_t @@ -1223,8 +1167,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = double @@ -1240,8 +1183,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = float @@ -1257,8 +1199,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = rocprim::half @@ -1274,8 +1215,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int64_t @@ -1291,8 +1231,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int @@ -1308,8 +1247,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = short @@ -1325,8 +1263,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int8_t @@ -1342,8 +1279,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = double @@ -1358,8 +1294,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = float @@ -1374,8 +1309,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = rocprim::half @@ -1390,8 +1324,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int64_t @@ -1407,8 +1340,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int @@ -1424,8 +1356,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = short @@ -1441,8 +1372,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int8_t @@ -1458,8 +1388,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = double @@ -1475,8 +1404,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = float @@ -1492,8 +1420,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = rocprim::half @@ -1509,8 +1436,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = int64_t @@ -1526,8 +1452,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = int @@ -1543,8 +1468,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = short @@ -1560,8 +1484,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = int8_t @@ -1577,8 +1500,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = double @@ -1594,8 +1516,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = float @@ -1611,8 +1532,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = rocprim::half @@ -1628,8 +1548,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = int64_t @@ -1645,8 +1564,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = int @@ -1662,8 +1580,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = short @@ -1679,8 +1596,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = int8_t @@ -1696,8 +1612,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = double @@ -1713,8 +1628,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = float @@ -1730,8 +1644,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = rocprim::half @@ -1747,8 +1660,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int64_t @@ -1764,8 +1676,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int @@ -1781,8 +1692,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = short @@ -1798,8 +1708,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int8_t @@ -1815,8 +1724,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = double @@ -1831,8 +1739,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = float @@ -1847,8 +1754,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = rocprim::half @@ -1863,8 +1769,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int64_t @@ -1880,8 +1785,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int @@ -1897,8 +1801,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = short @@ -1914,8 +1817,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int8_t @@ -1931,8 +1833,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = double @@ -1948,8 +1849,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = float @@ -1965,8 +1865,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = rocprim::half @@ -1982,8 +1881,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = int64_t @@ -1999,8 +1897,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = int @@ -2016,8 +1913,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = short @@ -2033,8 +1929,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = int8_t @@ -2050,8 +1945,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = double @@ -2067,8 +1961,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = float @@ -2084,8 +1977,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = rocprim::half @@ -2101,8 +1993,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = int64_t @@ -2118,8 +2009,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = int @@ -2135,8 +2025,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = short @@ -2152,8 +2041,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = int8_t @@ -2169,8 +2057,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = double @@ -2186,8 +2073,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = float @@ -2203,8 +2089,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = rocprim::half @@ -2220,8 +2105,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int64_t @@ -2237,8 +2121,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int @@ -2254,8 +2137,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = short @@ -2271,8 +2153,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int8_t @@ -2288,8 +2169,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = double @@ -2304,8 +2184,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = float @@ -2320,8 +2199,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = rocprim::half @@ -2336,8 +2214,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int64_t @@ -2353,8 +2230,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int @@ -2370,8 +2246,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = short @@ -2387,8 +2262,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int8_t @@ -2404,8 +2278,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = double @@ -2421,8 +2294,7 @@ struct default_reduce_by_key_config< 7, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = float @@ -2438,8 +2310,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = rocprim::half @@ -2455,8 +2326,7 @@ struct default_reduce_by_key_config< 7, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = int64_t @@ -2472,8 +2342,7 @@ struct default_reduce_by_key_config< 7, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = int @@ -2489,8 +2358,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = short @@ -2506,8 +2374,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = int8_t @@ -2523,8 +2390,7 @@ struct default_reduce_by_key_config< 10, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = double @@ -2540,8 +2406,7 @@ struct default_reduce_by_key_config< 7, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = float @@ -2557,8 +2422,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = rocprim::half @@ -2574,8 +2438,7 @@ struct default_reduce_by_key_config< 12, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = int64_t @@ -2591,8 +2454,7 @@ struct default_reduce_by_key_config< 7, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = int @@ -2608,8 +2470,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = short @@ -2625,8 +2486,7 @@ struct default_reduce_by_key_config< 12, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = int8_t @@ -2642,8 +2502,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = double @@ -2676,8 +2535,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = rocprim::half @@ -2693,8 +2551,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int64_t @@ -2710,8 +2567,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int @@ -2727,8 +2583,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = short @@ -2744,8 +2599,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int8_t @@ -2761,8 +2615,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = double @@ -2777,8 +2630,7 @@ struct default_reduce_by_key_config< 7, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = float @@ -2793,8 +2645,7 @@ struct default_reduce_by_key_config< 12, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = rocprim::half @@ -2809,8 +2660,7 @@ struct default_reduce_by_key_config< 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int64_t @@ -2826,8 +2676,7 @@ struct default_reduce_by_key_config< 7, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int @@ -2843,8 +2692,7 @@ struct default_reduce_by_key_config< 12, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = short @@ -2860,8 +2708,7 @@ struct default_reduce_by_key_config< 13, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int8_t @@ -2877,8 +2724,135 @@ struct default_reduce_by_key_config< 13, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = double +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 16) && (sizeof(key_type) > 8) + && bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : reduce_by_key_config<256, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = float +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 16) && (sizeof(key_type) > 8) + && bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : reduce_by_key_config<512, + 7, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::half +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 16) && (sizeof(key_type) > 8) + && bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2))>> + : reduce_by_key_config<384, + 5, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 16) && (sizeof(key_type) > 8) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : reduce_by_key_config<256, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 16) && (sizeof(key_type) > 8) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : reduce_by_key_config<256, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 16) && (sizeof(key_type) > 8) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : reduce_by_key_config<512, + 7, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 16) && (sizeof(key_type) > 8) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : reduce_by_key_config<384, + 5, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 16) && (sizeof(key_type) > 8) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1))>> + : reduce_by_key_config<256, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = double @@ -2894,8 +2868,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = float @@ -2907,12 +2880,11 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : reduce_by_key_config<512, + : reduce_by_key_config<256, 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = rocprim::half @@ -2924,12 +2896,27 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2))>> - : reduce_by_key_config<512, + : reduce_by_key_config<384, + 10, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 8) && (sizeof(key_type) > 4) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : reduce_by_key_config<256, 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = int64_t @@ -2945,8 +2932,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = int @@ -2958,12 +2944,11 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && !bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : reduce_by_key_config<512, + : reduce_by_key_config<256, 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = short @@ -2975,12 +2960,11 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && !bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : reduce_by_key_config<512, + : reduce_by_key_config<256, 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = int8_t @@ -2992,12 +2976,11 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && !bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1))>> - : reduce_by_key_config<512, + : reduce_by_key_config<256, 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = double @@ -3013,8 +2996,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = float @@ -3030,8 +3012,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = rocprim::half @@ -3047,8 +3028,23 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 4) && (sizeof(key_type) > 2) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : reduce_by_key_config<256, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = int64_t @@ -3064,8 +3060,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = int @@ -3081,8 +3076,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = short @@ -3098,8 +3092,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = int8_t @@ -3115,8 +3108,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = double @@ -3132,8 +3124,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = float @@ -3146,11 +3137,10 @@ struct default_reduce_by_key_config< && bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : reduce_by_key_config<512, - 15, + 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = rocprim::half @@ -3162,12 +3152,27 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2))>> - : reduce_by_key_config<512, + : reduce_by_key_config<192, + 14, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 2) && (sizeof(key_type) > 1) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : reduce_by_key_config<256, 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int64_t @@ -3183,8 +3188,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int @@ -3197,11 +3201,10 @@ struct default_reduce_by_key_config< && !bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : reduce_by_key_config<512, - 15, + 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = short @@ -3213,12 +3216,11 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && !bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : reduce_by_key_config<512, - 15, + : reduce_by_key_config<192, + 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int8_t @@ -3230,12 +3232,11 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && !bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1))>> - : reduce_by_key_config<512, - 15, + : reduce_by_key_config<192, + 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = double @@ -3250,8 +3251,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = float @@ -3266,8 +3266,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = rocprim::half @@ -3278,12 +3277,27 @@ struct default_reduce_by_key_config< value_type, std::enable_if_t<((sizeof(key_type) <= 1) && bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2))>> - : reduce_by_key_config<512, + : reduce_by_key_config<192, 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_reduce_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<((sizeof(key_type) <= 1) + && !bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : reduce_by_key_config<256, + 15, + block_load_method::block_load_transpose, + block_load_method::block_load_transpose, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int64_t @@ -3299,8 +3313,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int @@ -3316,8 +3329,7 @@ struct default_reduce_by_key_config< 15, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = short @@ -3329,12 +3341,11 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 1) && !bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : reduce_by_key_config<512, - 15, + : reduce_by_key_config<192, + 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int8_t @@ -3346,12 +3357,11 @@ struct default_reduce_by_key_config< std::enable_if_t<((sizeof(key_type) <= 1) && !bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1))>> - : reduce_by_key_config<512, - 15, + : reduce_by_key_config<192, + 14, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; } // end namespace detail diff --git a/rocprim/include/rocprim/device/detail/config/device_run_length_encode.hpp b/rocprim/include/rocprim/device/detail/config/device_run_length_encode.hpp index 1b473943f..57c5efc03 100644 --- a/rocprim/include/rocprim/device/detail/config/device_run_length_encode.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_run_length_encode.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -54,9 +54,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<256, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = float @@ -70,9 +69,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<512, 14, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = rocprim::half @@ -86,9 +84,23 @@ struct default_trivial_runs_config< : reduce_by_key_config<384, 14, block_load_method::block_load_transpose, + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : reduce_by_key_config<128, + 15, + block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t @@ -102,9 +114,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<256, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int @@ -118,9 +129,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<512, 14, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short @@ -134,9 +144,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<384, 14, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t @@ -150,9 +159,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<384, 13, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = double @@ -166,9 +174,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<512, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 2> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = float @@ -182,9 +189,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<512, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = rocprim::half @@ -198,9 +204,23 @@ struct default_trivial_runs_config< : reduce_by_key_config<512, 14, block_load_method::block_load_transpose, + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : reduce_by_key_config<384, + 7, + block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t @@ -214,9 +234,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<512, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 2> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int @@ -230,9 +249,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<512, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short @@ -246,9 +264,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<512, 14, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t @@ -262,9 +279,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<384, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = double @@ -278,9 +294,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<256, 10, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = float @@ -294,9 +309,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<256, 14, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = rocprim::half @@ -310,9 +324,23 @@ struct default_trivial_runs_config< : reduce_by_key_config<256, 14, block_load_method::block_load_transpose, + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : reduce_by_key_config<256, + 15, + block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t @@ -326,9 +354,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<256, 10, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int @@ -342,9 +369,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<256, 14, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short @@ -358,9 +384,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<256, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t @@ -374,9 +399,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<192, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = double @@ -390,9 +414,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<256, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = float @@ -406,9 +429,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<256, 14, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = rocprim::half @@ -422,9 +444,23 @@ struct default_trivial_runs_config< : reduce_by_key_config<512, 10, block_load_method::block_load_transpose, + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : reduce_by_key_config<256, + 15, + block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t @@ -438,9 +474,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<256, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int @@ -454,9 +489,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<256, 14, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short @@ -470,9 +504,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<512, 10, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t @@ -486,9 +519,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<192, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = double @@ -502,9 +534,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<256, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = float @@ -518,9 +549,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<256, 14, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = rocprim::half @@ -534,9 +564,23 @@ struct default_trivial_runs_config< : reduce_by_key_config<512, 10, block_load_method::block_load_transpose, + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t +template +struct default_trivial_runs_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : reduce_by_key_config<256, + 15, + block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t @@ -550,9 +594,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<256, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int @@ -566,9 +609,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<256, 14, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short @@ -582,9 +624,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<512, 10, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t @@ -598,9 +639,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<192, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = double @@ -614,9 +654,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<256, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = float @@ -630,9 +669,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<256, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = rocprim::half @@ -646,9 +684,23 @@ struct default_trivial_runs_config< : reduce_by_key_config<192, 14, block_load_method::block_load_transpose, + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : reduce_by_key_config<256, + 7, + block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t @@ -662,9 +714,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<256, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int @@ -678,9 +729,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<256, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short @@ -694,9 +744,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<192, 14, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t @@ -710,9 +759,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<256, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = double @@ -726,9 +774,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<512, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = float @@ -742,9 +789,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<512, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = rocprim::half @@ -756,11 +802,25 @@ struct default_trivial_runs_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2))>> : reduce_by_key_config<512, - 15, + 14, block_load_method::block_load_transpose, + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t +template +struct default_trivial_runs_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : reduce_by_key_config<256, + 15, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t @@ -774,9 +834,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<512, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int @@ -790,9 +849,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<512, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short @@ -804,11 +862,10 @@ struct default_trivial_runs_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1))>> : reduce_by_key_config<512, - 15, - block_load_method::block_load_transpose, + 14, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t @@ -822,9 +879,8 @@ struct default_trivial_runs_config< : reduce_by_key_config<512, 15, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - 1> + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> {}; } // end namespace detail diff --git a/rocprim/include/rocprim/device/detail/config/device_run_length_encode_non_trivial.hpp b/rocprim/include/rocprim/device/detail/config/device_run_length_encode_non_trivial.hpp index a8b6c105f..9b304330a 100644 --- a/rocprim/include/rocprim/device/detail/config/device_run_length_encode_non_trivial.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_run_length_encode_non_trivial.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,7 +21,9 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RUN_LENGTH_ENCODE_NON_TRIVIAL_RUNS_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RUN_LENGTH_ENCODE_NON_TRIVIAL_RUNS_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" #include @@ -39,12 +41,12 @@ BEGIN_ROCPRIM_NAMESPACE namespace detail { -template +template struct default_non_trivial_runs_config : default_non_trivial_runs_config_base::type {}; // Based on key_type = double -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx1030), key_type, @@ -57,7 +59,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = float -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx1030), key_type, @@ -70,7 +72,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = rocprim::half -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx1030), key_type, @@ -82,8 +84,21 @@ struct default_non_trivial_runs_config< ::rocprim::block_scan_algorithm::using_warp_scan> {}; +// Based on key_type = rocprim::int128_t +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx1030), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : non_trivial_runs_config<64, + 16, + ::rocprim::block_load_method::block_load_warp_transpose, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + // Based on key_type = int64_t -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx1030), key_type, @@ -96,7 +111,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = int -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx1030), key_type, @@ -109,7 +124,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = short -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx1030), key_type, @@ -122,7 +137,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = int8_t -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx1030), key_type, @@ -135,7 +150,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = double -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx1100), key_type, @@ -148,7 +163,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = float -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx1100), key_type, @@ -161,7 +176,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = rocprim::half -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx1100), key_type, @@ -173,8 +188,21 @@ struct default_non_trivial_runs_config< ::rocprim::block_scan_algorithm::using_warp_scan> {}; +// Based on key_type = rocprim::int128_t +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx1100), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : non_trivial_runs_config<64, + 32, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + // Based on key_type = int64_t -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx1100), key_type, @@ -187,7 +215,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = int -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx1100), key_type, @@ -200,7 +228,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = short -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx1100), key_type, @@ -213,7 +241,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = int8_t -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx1100), key_type, @@ -226,7 +254,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = double -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx906), key_type, @@ -239,7 +267,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = float -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx906), key_type, @@ -252,7 +280,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = rocprim::half -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx906), key_type, @@ -264,8 +292,21 @@ struct default_non_trivial_runs_config< ::rocprim::block_scan_algorithm::using_warp_scan> {}; +// Based on key_type = rocprim::int128_t +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx906), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : non_trivial_runs_config<64, + 16, + ::rocprim::block_load_method::block_load_warp_transpose, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + // Based on key_type = int64_t -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx906), key_type, @@ -278,7 +319,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = int -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx906), key_type, @@ -291,7 +332,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = short -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx906), key_type, @@ -304,7 +345,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = int8_t -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx906), key_type, @@ -317,7 +358,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = double -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx908), key_type, @@ -330,7 +371,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = float -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx908), key_type, @@ -343,7 +384,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = rocprim::half -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx908), key_type, @@ -355,8 +396,21 @@ struct default_non_trivial_runs_config< ::rocprim::block_scan_algorithm::using_warp_scan> {}; +// Based on key_type = rocprim::int128_t +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx908), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : non_trivial_runs_config<64, + 16, + ::rocprim::block_load_method::block_load_warp_transpose, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + // Based on key_type = int64_t -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx908), key_type, @@ -369,7 +423,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = int -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx908), key_type, @@ -382,7 +436,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = short -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx908), key_type, @@ -395,7 +449,7 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = int8_t -template +template struct default_non_trivial_runs_config< static_cast(target_arch::gfx908), key_type, @@ -408,22 +462,22 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = double -template +template struct default_non_trivial_runs_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4))>> : non_trivial_runs_config<64, 16, - ::rocprim::block_load_method::block_load_warp_transpose, + ::rocprim::block_load_method::block_load_vectorize, ::rocprim::block_scan_algorithm::using_warp_scan> {}; // Based on key_type = float -template +template struct default_non_trivial_runs_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2))>> @@ -434,35 +488,48 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = rocprim::half -template +template struct default_non_trivial_runs_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2))>> - : non_trivial_runs_config<64, - 64, + : non_trivial_runs_config<256, + 16, ::rocprim::block_load_method::block_load_vectorize, ::rocprim::block_scan_algorithm::using_warp_scan> {}; +// Based on key_type = rocprim::int128_t +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx90a), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : non_trivial_runs_config<64, + 16, + ::rocprim::block_load_method::block_load_warp_transpose, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + // Based on key_type = int64_t -template +template struct default_non_trivial_runs_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4))>> : non_trivial_runs_config<64, 16, - ::rocprim::block_load_method::block_load_warp_transpose, + ::rocprim::block_load_method::block_load_vectorize, ::rocprim::block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int -template +template struct default_non_trivial_runs_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2))>> @@ -473,9 +540,9 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = short -template +template struct default_non_trivial_runs_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1))>> @@ -486,9 +553,9 @@ struct default_non_trivial_runs_config< {}; // Based on key_type = int8_t -template +template struct default_non_trivial_runs_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1))>> @@ -501,20 +568,20 @@ struct default_non_trivial_runs_config< // Based on key_type = double template struct default_non_trivial_runs_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4))>> : non_trivial_runs_config<64, 16, - ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_load_method::block_load_warp_transpose, ::rocprim::block_scan_algorithm::using_warp_scan> {}; // Based on key_type = float template struct default_non_trivial_runs_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2))>> @@ -527,33 +594,46 @@ struct default_non_trivial_runs_config< // Based on key_type = rocprim::half template struct default_non_trivial_runs_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2))>> - : non_trivial_runs_config<256, - 16, + : non_trivial_runs_config<64, + 64, ::rocprim::block_load_method::block_load_vectorize, ::rocprim::block_scan_algorithm::using_warp_scan> {}; +// Based on key_type = rocprim::int128_t +template +struct default_non_trivial_runs_config< + static_cast(target_arch::unknown), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : non_trivial_runs_config<64, + 16, + ::rocprim::block_load_method::block_load_warp_transpose, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + // Based on key_type = int64_t template struct default_non_trivial_runs_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4))>> : non_trivial_runs_config<64, 16, - ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_load_method::block_load_warp_transpose, ::rocprim::block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int template struct default_non_trivial_runs_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2))>> @@ -566,7 +646,7 @@ struct default_non_trivial_runs_config< // Based on key_type = short template struct default_non_trivial_runs_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1))>> @@ -579,7 +659,7 @@ struct default_non_trivial_runs_config< // Based on key_type = int8_t template struct default_non_trivial_runs_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1))>> @@ -622,8 +702,21 @@ struct default_non_trivial_runs_config< key_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2))>> + : non_trivial_runs_config<256, + 16, + ::rocprim::block_load_method::block_load_vectorize, + ::rocprim::block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t +template +struct default_non_trivial_runs_config< + static_cast(target_arch::gfx942), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> : non_trivial_runs_config<128, - 32, + 16, ::rocprim::block_load_method::block_load_vectorize, ::rocprim::block_scan_algorithm::using_warp_scan> {}; diff --git a/rocprim/include/rocprim/device/detail/config/device_scan.hpp b/rocprim/include/rocprim/device/detail/config/device_scan.hpp index 44d65fce0..2bf468c29 100644 --- a/rocprim/include/rocprim/device/detail/config/device_scan.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_scan.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,8 +21,11 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_SCAN_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_SCAN_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" + #include /* DO NOT EDIT THIS FILE @@ -45,39 +48,53 @@ struct default_scan_config : default_scan_config_base::type // Based on value_type = double template struct default_scan_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_config<256, - 6, + : scan_config<128, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on value_type = float template struct default_scan_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_config<256, - 14, + : scan_config<64, + 18, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on value_type = rocprim::half template -struct default_scan_config(target_arch::gfx908), +struct default_scan_config(target_arch::gfx1030), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2))>> + : scan_config<64, + 12, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on value_type = rocprim::int128_t +template +struct default_scan_config< + static_cast(target_arch::gfx1030), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> : scan_config<256, - 18, + 2, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -86,12 +103,12 @@ struct default_scan_config(target_arch::gfx908), // Based on value_type = int64_t template struct default_scan_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_config<256, - 6, + 9, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -100,12 +117,12 @@ struct default_scan_config< // Based on value_type = int template struct default_scan_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_config<256, - 14, + : scan_config<64, + 18, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -114,12 +131,12 @@ struct default_scan_config< // Based on value_type = short template struct default_scan_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_config<128, - 14, + : scan_config<256, + 22, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -127,7 +144,7 @@ struct default_scan_config< // Based on value_type = int8_t template -struct default_scan_config(target_arch::gfx908), +struct default_scan_config(target_arch::gfx1030), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1))>> @@ -141,26 +158,26 @@ struct default_scan_config(target_arch::gfx908), // Based on value_type = double template struct default_scan_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_config<256, - 15, + 16, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on value_type = float template struct default_scan_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_config<256, - 10, + : scan_config<64, + 16, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -168,12 +185,26 @@ struct default_scan_config< // Based on value_type = rocprim::half template -struct default_scan_config(target_arch::gfx900), +struct default_scan_config(target_arch::gfx1100), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2))>> + : scan_config<64, + 22, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on value_type = rocprim::int128_t +template +struct default_scan_config< + static_cast(target_arch::gfx1100), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> : scan_config<256, - 24, + 8, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -182,48 +213,48 @@ struct default_scan_config(target_arch::gfx900), // Based on value_type = int64_t template struct default_scan_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_config<256, - 10, + 16, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on value_type = int template struct default_scan_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_config<256, - 10, + : scan_config<64, + 16, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on value_type = short template struct default_scan_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_config<256, - 24, + : scan_config<64, + 22, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on value_type = int8_t template -struct default_scan_config(target_arch::gfx900), +struct default_scan_config(target_arch::gfx1100), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1))>> @@ -272,6 +303,20 @@ struct default_scan_config(target_arch::gfx906), 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on value_type = rocprim::int128_t +template +struct default_scan_config< + static_cast(target_arch::gfx906), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : scan_config<64, + 15, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> {}; @@ -333,39 +378,53 @@ struct default_scan_config(target_arch::gfx906), // Based on value_type = double template struct default_scan_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_config<128, - 14, + : scan_config<256, + 6, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on value_type = float template struct default_scan_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_config<64, - 18, + : scan_config<256, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on value_type = rocprim::half template -struct default_scan_config(target_arch::gfx1030), +struct default_scan_config(target_arch::gfx908), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2))>> : scan_config<256, - 22, + 24, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on value_type = rocprim::int128_t +template +struct default_scan_config< + static_cast(target_arch::gfx908), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : scan_config<128, + 10, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -374,12 +433,12 @@ struct default_scan_config(target_arch::gfx1030), // Based on value_type = int64_t template struct default_scan_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_config<256, - 9, + 6, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -388,12 +447,12 @@ struct default_scan_config< // Based on value_type = int template struct default_scan_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_config<64, - 18, + : scan_config<256, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -402,12 +461,12 @@ struct default_scan_config< // Based on value_type = short template struct default_scan_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_config<256, - 22, + : scan_config<128, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -415,7 +474,7 @@ struct default_scan_config< // Based on value_type = int8_t template -struct default_scan_config(target_arch::gfx1030), +struct default_scan_config(target_arch::gfx908), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1))>> @@ -429,25 +488,25 @@ struct default_scan_config(target_arch::gfx1030), // Based on value_type = double template struct default_scan_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_config<256, - 6, + : scan_config<128, + 10, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on value_type = float template struct default_scan_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_config<256, + : scan_config<128, 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, @@ -456,26 +515,40 @@ struct default_scan_config< // Based on value_type = rocprim::half template -struct default_scan_config(target_arch::unknown), +struct default_scan_config(target_arch::gfx90a), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2))>> - : scan_config<256, - 18, + : scan_config<64, + 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> +{}; + +// Based on value_type = rocprim::int128_t +template +struct default_scan_config< + static_cast(target_arch::gfx90a), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : scan_config<64, + 15, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> {}; // Based on value_type = int64_t template struct default_scan_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_config<256, - 6, + : scan_config<64, + 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -484,34 +557,34 @@ struct default_scan_config< // Based on value_type = int template struct default_scan_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_config<256, - 14, + : scan_config<128, + 10, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on value_type = short template struct default_scan_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_config<128, - 14, + : scan_config<256, + 10, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on value_type = int8_t template -struct default_scan_config(target_arch::unknown), +struct default_scan_config(target_arch::gfx90a), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1))>> @@ -519,31 +592,31 @@ struct default_scan_config(target_arch::unknown), 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on value_type = double template struct default_scan_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_config<128, - 10, + : scan_config<256, + 6, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on value_type = float template struct default_scan_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_config<128, + : scan_config<256, 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, @@ -552,26 +625,40 @@ struct default_scan_config< // Based on value_type = rocprim::half template -struct default_scan_config(target_arch::gfx90a), +struct default_scan_config(target_arch::unknown), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2))>> - : scan_config<64, + : scan_config<256, 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> +{}; + +// Based on value_type = rocprim::int128_t +template +struct default_scan_config< + static_cast(target_arch::unknown), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : scan_config<128, + 10, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> {}; // Based on value_type = int64_t template struct default_scan_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_config<64, - 21, + : scan_config<256, + 6, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -580,34 +667,34 @@ struct default_scan_config< // Based on value_type = int template struct default_scan_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_config<128, - 10, + : scan_config<256, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on value_type = short template struct default_scan_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_config<256, - 10, + : scan_config<128, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on value_type = int8_t template -struct default_scan_config(target_arch::gfx90a), +struct default_scan_config(target_arch::unknown), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1))>> @@ -615,7 +702,7 @@ struct default_scan_config(target_arch::gfx90a), 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on value_type = double @@ -626,7 +713,7 @@ struct default_scan_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_config<256, - 24, + 15, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -640,10 +727,10 @@ struct default_scan_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_config<256, - 21, + 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on value_type = rocprim::half @@ -659,6 +746,20 @@ struct default_scan_config(target_arch::gfx942), block_scan_algorithm::using_warp_scan> {}; +// Based on value_type = rocprim::int128_t +template +struct default_scan_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : scan_config<256, + 14, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + // Based on value_type = int64_t template struct default_scan_config< @@ -667,10 +768,10 @@ struct default_scan_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_config<256, - 24, + 15, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on value_type = int diff --git a/rocprim/include/rocprim/device/detail/config/device_scan_by_key.hpp b/rocprim/include/rocprim/device/detail/config/device_scan_by_key.hpp index 037a224f0..1a380f8b7 100644 --- a/rocprim/include/rocprim/device/detail/config/device_scan_by_key.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_scan_by_key.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,8 +21,11 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_SCAN_BY_KEY_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_SCAN_BY_KEY_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" + #include /* DO NOT EDIT THIS FILE @@ -42,17 +45,33 @@ template::type {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 4, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + // Based on key_type = double, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_by_key_config<256, - 14, + : scan_by_key_config<128, + 23, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -61,14 +80,14 @@ struct default_scan_by_key_config< // Based on key_type = double, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_by_key_config<256, - 14, + : scan_by_key_config<128, + 23, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -77,14 +96,14 @@ struct default_scan_by_key_config< // Based on key_type = double, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<64, - 14, + 22, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -93,29 +112,45 @@ struct default_scan_by_key_config< // Based on key_type = double, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1))>> : scan_by_key_config<64, - 14, + 17, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 2, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = float, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_by_key_config<128, - 10, + : scan_by_key_config<256, + 9, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -124,30 +159,30 @@ struct default_scan_by_key_config< // Based on key_type = float, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<256, - 12, + 15, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = float, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_by_key_config<256, - 14, + : scan_by_key_config<64, + 22, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -156,58 +191,73 @@ struct default_scan_by_key_config< // Based on key_type = float, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1))>> : scan_by_key_config<128, - 14, + 7, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 4, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = rocprim::half, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_by_key_config<64, - 18, + : scan_by_key_config<256, + 4, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = rocprim::half, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<256, - 18, + 8, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = rocprim::half, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_by_key_config<256, - 20, + : scan_by_key_config<64, + 22, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -216,61 +266,156 @@ struct default_scan_by_key_config< // Based on key_type = rocprim::half, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1))>> - : scan_by_key_config<256, + : scan_by_key_config<64, 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 4, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : scan_by_key_config<256, + 8, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> {}; +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : scan_by_key_config<128, + 20, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : scan_by_key_config<256, + 10, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1))>> + : scan_by_key_config<256, + 8, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 4, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + // Based on key_type = int64_t, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_by_key_config<256, - 14, + 13, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_by_key_config<256, - 14, + : scan_by_key_config<128, + 19, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<64, - 14, + 22, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -279,13 +424,29 @@ struct default_scan_by_key_config< // Based on key_type = int64_t, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1))>> + : scan_by_key_config<64, + 18, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : scan_by_key_config<256, - 14, + 5, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -294,14 +455,14 @@ struct default_scan_by_key_config< // Based on key_type = int, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_by_key_config<128, - 10, + : scan_by_key_config<256, + 9, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -310,14 +471,14 @@ struct default_scan_by_key_config< // Based on key_type = int, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_by_key_config<256, - 12, + : scan_by_key_config<128, + 22, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -326,14 +487,14 @@ struct default_scan_by_key_config< // Based on key_type = int, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_by_key_config<256, - 14, + : scan_by_key_config<64, + 22, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -342,106 +503,137 @@ struct default_scan_by_key_config< // Based on key_type = int, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1))>> : scan_by_key_config<128, - 14, + 7, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; -// Based on key_type = short, value_type = int64_t +// Based on key_type = short, value_type = rocprim::int128_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : scan_by_key_config<64, - 18, + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 4, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> {}; -// Based on key_type = short, value_type = int +// Based on key_type = short, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : scan_by_key_config<256, - 18, + 9, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> {}; -// Based on key_type = short, value_type = short +// Based on key_type = short, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : scan_by_key_config<128, + 22, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = short, value_type = short +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_by_key_config<256, - 20, + : scan_by_key_config<128, + 22, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1))>> : scan_by_key_config<256, - 24, + 20, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> {}; +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 4, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + // Based on key_type = int8_t, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_by_key_config<64, - 18, + : scan_by_key_config<128, + 17, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = int8_t, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_by_key_config<128, - 18, + : scan_by_key_config<256, + 12, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -450,44 +642,60 @@ struct default_scan_by_key_config< // Based on key_type = int8_t, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_by_key_config<256, - 20, + : scan_by_key_config<128, + 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1))>> : scan_by_key_config<256, - 24, + 20, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 8, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = double, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_by_key_config<256, - 14, + 16, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -496,14 +704,14 @@ struct default_scan_by_key_config< // Based on key_type = double, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_by_key_config<128, - 19, + : scan_by_key_config<256, + 16, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -512,14 +720,14 @@ struct default_scan_by_key_config< // Based on key_type = double, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_by_key_config<128, - 14, + : scan_by_key_config<256, + 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -528,13 +736,29 @@ struct default_scan_by_key_config< // Based on key_type = double, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1))>> - : scan_by_key_config<128, - 14, + : scan_by_key_config<64, + 24, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 8, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -543,46 +767,46 @@ struct default_scan_by_key_config< // Based on key_type = float, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_by_key_config<256, - 10, + 8, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = float, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<256, - 12, + 9, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = float, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_by_key_config<256, - 14, + : scan_by_key_config<64, + 20, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -591,13 +815,28 @@ struct default_scan_by_key_config< // Based on key_type = float, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1))>> : scan_by_key_config<256, - 14, + 23, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 8, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -606,58 +845,153 @@ struct default_scan_by_key_config< // Based on key_type = rocprim::half, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_by_key_config<64, - 18, + : scan_by_key_config<256, + 16, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = rocprim::half, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<256, - 18, + 16, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = rocprim::half, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<256, - 22, + 17, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = rocprim::half, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1))>> : scan_by_key_config<256, - 20, + 24, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 4, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : scan_by_key_config<256, + 8, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : scan_by_key_config<256, + 11, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : scan_by_key_config<64, + 17, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1))>> + : scan_by_key_config<64, + 15, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 8, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -666,30 +1000,30 @@ struct default_scan_by_key_config< // Based on key_type = int64_t, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_by_key_config<256, - 14, + 16, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = int64_t, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_by_key_config<128, - 19, + : scan_by_key_config<256, + 16, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -698,29 +1032,45 @@ struct default_scan_by_key_config< // Based on key_type = int64_t, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_by_key_config<128, - 14, + : scan_by_key_config<256, + 20, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1))>> - : scan_by_key_config<128, - 14, + : scan_by_key_config<64, + 24, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 8, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -729,14 +1079,14 @@ struct default_scan_by_key_config< // Based on key_type = int, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_by_key_config<256, - 10, + 8, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -745,30 +1095,30 @@ struct default_scan_by_key_config< // Based on key_type = int, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<256, - 12, + 9, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_by_key_config<256, - 14, + : scan_by_key_config<64, + 22, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -777,76 +1127,107 @@ struct default_scan_by_key_config< // Based on key_type = int, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1))>> : scan_by_key_config<256, - 14, + 23, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> {}; -// Based on key_type = short, value_type = int64_t +// Based on key_type = short, value_type = rocprim::int128_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : scan_by_key_config<64, - 18, + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<128, + 16, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> {}; -// Based on key_type = short, value_type = int +// Based on key_type = short, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : scan_by_key_config<256, - 18, + 16, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> {}; +// Based on key_type = short, value_type = int +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : scan_by_key_config<256, + 16, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + // Based on key_type = short, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<256, - 22, + 17, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1))>> : scan_by_key_config<256, - 20, + 24, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 8, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -855,13 +1236,13 @@ struct default_scan_by_key_config< // Based on key_type = int8_t, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_by_key_config<64, - 18, + : scan_by_key_config<256, + 16, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -870,28 +1251,28 @@ struct default_scan_by_key_config< // Based on key_type = int8_t, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<256, - 17, + 16, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = int8_t, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_by_key_config<64, - 24, + : scan_by_key_config<256, + 20, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -900,18 +1281,34 @@ struct default_scan_by_key_config< // Based on key_type = int8_t, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1))>> - : scan_by_key_config<64, + : scan_by_key_config<256, 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 7, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + // Based on key_type = double, value_type = int64_t template struct default_scan_by_key_config< @@ -975,6 +1372,22 @@ struct default_scan_by_key_config< block_scan_algorithm::using_warp_scan> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 7, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + // Based on key_type = float, value_type = int64_t template struct default_scan_by_key_config< @@ -1038,6 +1451,21 @@ struct default_scan_by_key_config< block_scan_algorithm::reduce_then_scan> {}; +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 7, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + // Based on key_type = rocprim::half, value_type = int64_t template struct default_scan_by_key_config< @@ -1047,7 +1475,7 @@ struct default_scan_by_key_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_by_key_config<64, - 18, + 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -1062,10 +1490,10 @@ struct default_scan_by_key_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<256, - 18, + 15, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = rocprim::half, value_type = short @@ -1077,10 +1505,10 @@ struct default_scan_by_key_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<64, - 22, + 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = rocprim::half, value_type = int8_t @@ -1095,6 +1523,101 @@ struct default_scan_by_key_config< 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<64, + 14, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : scan_by_key_config<64, + 19, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : scan_by_key_config<64, + 19, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : scan_by_key_config<64, + 19, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1))>> + : scan_by_key_config<64, + 14, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 7, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> {}; @@ -1161,6 +1684,22 @@ struct default_scan_by_key_config< block_scan_algorithm::using_warp_scan> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 5, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + // Based on key_type = int, value_type = int64_t template struct default_scan_by_key_config< @@ -1224,6 +1763,22 @@ struct default_scan_by_key_config< block_scan_algorithm::using_warp_scan> {}; +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 7, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + // Based on key_type = short, value_type = int64_t template struct default_scan_by_key_config< @@ -1287,6 +1842,21 @@ struct default_scan_by_key_config< block_scan_algorithm::using_warp_scan> {}; +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : scan_by_key_config<64, + 15, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + // Based on key_type = int8_t, value_type = int64_t template struct default_scan_by_key_config< @@ -1347,17 +1917,33 @@ struct default_scan_by_key_config< block_scan_algorithm::reduce_then_scan> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<128, + 6, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + // Based on key_type = double, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_by_key_config<128, - 23, + : scan_by_key_config<256, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -1366,14 +1952,14 @@ struct default_scan_by_key_config< // Based on key_type = double, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_by_key_config<128, - 23, + : scan_by_key_config<256, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -1382,14 +1968,14 @@ struct default_scan_by_key_config< // Based on key_type = double, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<64, - 22, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -1398,61 +1984,77 @@ struct default_scan_by_key_config< // Based on key_type = double, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1))>> : scan_by_key_config<64, - 17, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; -// Based on key_type = float, value_type = int64_t +// Based on key_type = float, value_type = rocprim::int128_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : scan_by_key_config<256, - 9, + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<64, + 10, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; -// Based on key_type = float, value_type = int +// Based on key_type = float, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : scan_by_key_config<128, + 10, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = float, value_type = int +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<256, - 15, + 12, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = float, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_by_key_config<64, - 22, + : scan_by_key_config<256, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -1461,13 +2063,28 @@ struct default_scan_by_key_config< // Based on key_type = float, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1))>> : scan_by_key_config<128, - 7, + 14, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : scan_by_key_config<64, + 12, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -1476,28 +2093,28 @@ struct default_scan_by_key_config< // Based on key_type = rocprim::half, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_by_key_config<64, - 23, + 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = rocprim::half, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<128, - 22, + 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -1506,13 +2123,13 @@ struct default_scan_by_key_config< // Based on key_type = rocprim::half, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_by_key_config<128, - 22, + : scan_by_key_config<256, + 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -1521,61 +2138,156 @@ struct default_scan_by_key_config< // Based on key_type = rocprim::half, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1))>> : scan_by_key_config<256, - 20, + 24, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<64, + 10, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : scan_by_key_config<64, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> {}; +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : scan_by_key_config<64, + 14, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : scan_by_key_config<64, + 19, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1))>> + : scan_by_key_config<64, + 14, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<64, + 9, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + // Based on key_type = int64_t, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_by_key_config<256, - 13, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = int64_t, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_by_key_config<128, - 19, + : scan_by_key_config<256, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = int64_t, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<64, - 22, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -1584,13 +2296,29 @@ struct default_scan_by_key_config< // Based on key_type = int64_t, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1))>> + : scan_by_key_config<256, + 14, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : scan_by_key_config<64, - 18, + 10, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -1599,14 +2327,14 @@ struct default_scan_by_key_config< // Based on key_type = int, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_by_key_config<256, - 9, + : scan_by_key_config<128, + 10, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -1615,14 +2343,14 @@ struct default_scan_by_key_config< // Based on key_type = int, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_by_key_config<128, - 22, + : scan_by_key_config<256, + 12, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -1631,14 +2359,14 @@ struct default_scan_by_key_config< // Based on key_type = int, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_by_key_config<64, - 22, + : scan_by_key_config<256, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -1647,13 +2375,29 @@ struct default_scan_by_key_config< // Based on key_type = int, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1))>> : scan_by_key_config<128, - 7, + 14, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<64, + 12, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -1662,30 +2406,30 @@ struct default_scan_by_key_config< // Based on key_type = short, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_by_key_config<256, - 9, + : scan_by_key_config<64, + 18, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = short, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_by_key_config<128, - 22, + : scan_by_key_config<256, + 18, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -1694,29 +2438,44 @@ struct default_scan_by_key_config< // Based on key_type = short, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_by_key_config<128, - 22, + : scan_by_key_config<256, + 20, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = short, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1))>> : scan_by_key_config<256, - 20, + 24, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : scan_by_key_config<64, + 15, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -1725,28 +2484,28 @@ struct default_scan_by_key_config< // Based on key_type = int8_t, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_by_key_config<128, - 17, + : scan_by_key_config<64, + 18, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_by_key_config<256, - 12, + : scan_by_key_config<128, + 18, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -1755,44 +2514,60 @@ struct default_scan_by_key_config< // Based on key_type = int8_t, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_by_key_config<128, - 24, + : scan_by_key_config<256, + 20, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = int8_t, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1))>> : scan_by_key_config<256, - 20, + 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<128, + 5, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = double, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_by_key_config<256, - 14, + : scan_by_key_config<64, + 19, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -1801,7 +2576,7 @@ struct default_scan_by_key_config< // Based on key_type = double, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1811,51 +2586,67 @@ struct default_scan_by_key_config< 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = double, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<64, - 14, + 19, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = double, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1))>> - : scan_by_key_config<64, + : scan_by_key_config<256, 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<128, + 15, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + // Based on key_type = float, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_by_key_config<128, - 10, + : scan_by_key_config<64, + 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -1864,30 +2655,30 @@ struct default_scan_by_key_config< // Based on key_type = float, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_by_key_config<256, - 12, + : scan_by_key_config<128, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = float, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_by_key_config<256, - 14, + : scan_by_key_config<64, + 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -1896,7 +2687,7 @@ struct default_scan_by_key_config< // Based on key_type = float, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1905,19 +2696,34 @@ struct default_scan_by_key_config< 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 3, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> {}; // Based on key_type = rocprim::half, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_by_key_config<64, - 18, + 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -1926,13 +2732,13 @@ struct default_scan_by_key_config< // Based on key_type = rocprim::half, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<256, - 18, + 9, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -1941,13 +2747,13 @@ struct default_scan_by_key_config< // Based on key_type = rocprim::half, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<256, - 20, + 18, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -1956,38 +2762,133 @@ struct default_scan_by_key_config< // Based on key_type = rocprim::half, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1))>> : scan_by_key_config<256, - 24, + 18, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<64, + 10, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : scan_by_key_config<64, + 14, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : scan_by_key_config<128, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> {}; +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : scan_by_key_config<64, + 19, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1))>> + : scan_by_key_config<64, + 14, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<128, + 5, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + // Based on key_type = int64_t, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_by_key_config<256, - 14, + : scan_by_key_config<64, + 19, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1997,20 +2898,20 @@ struct default_scan_by_key_config< 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int64_t, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<64, - 14, + 19, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -2019,7 +2920,7 @@ struct default_scan_by_key_config< // Based on key_type = int64_t, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -2028,52 +2929,68 @@ struct default_scan_by_key_config< 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 5, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_by_key_config<128, - 10, + : scan_by_key_config<64, + 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = int, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<256, - 12, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_by_key_config<256, - 14, + : scan_by_key_config<64, + 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -2082,7 +2999,7 @@ struct default_scan_by_key_config< // Based on key_type = int, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -2091,52 +3008,68 @@ struct default_scan_by_key_config< 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<128, + 6, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_by_key_config<64, - 18, + 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<256, - 18, + 9, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = short, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<256, - 20, + 22, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -2145,13 +3078,28 @@ struct default_scan_by_key_config< // Based on key_type = short, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1))>> : scan_by_key_config<256, - 24, + 18, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 3, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -2160,28 +3108,28 @@ struct default_scan_by_key_config< // Based on key_type = int8_t, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_by_key_config<64, - 18, + 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = int8_t, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<128, - 18, + 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -2190,13 +3138,13 @@ struct default_scan_by_key_config< // Based on key_type = int8_t, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<256, - 20, + 18, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -2205,7 +3153,7 @@ struct default_scan_by_key_config< // Based on key_type = int8_t, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -2217,17 +3165,33 @@ struct default_scan_by_key_config< block_scan_algorithm::reduce_then_scan> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<128, + 6, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + // Based on key_type = double, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_by_key_config<64, - 19, + : scan_by_key_config<256, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -2236,7 +3200,7 @@ struct default_scan_by_key_config< // Based on key_type = double, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -2246,51 +3210,67 @@ struct default_scan_by_key_config< 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = double, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<64, - 19, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = double, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1))>> - : scan_by_key_config<256, + : scan_by_key_config<64, 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<64, + 10, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + // Based on key_type = float, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_by_key_config<64, - 21, + : scan_by_key_config<128, + 10, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -2299,30 +3279,30 @@ struct default_scan_by_key_config< // Based on key_type = float, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_by_key_config<128, - 14, + : scan_by_key_config<256, + 12, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = float, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_by_key_config<64, - 24, + : scan_by_key_config<256, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -2331,7 +3311,7 @@ struct default_scan_by_key_config< // Based on key_type = float, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -2340,13 +3320,28 @@ struct default_scan_by_key_config< 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : scan_by_key_config<64, + 12, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = rocprim::half, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -2355,63 +3350,111 @@ struct default_scan_by_key_config< 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = rocprim::half, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : scan_by_key_config<256, - 9, + : scan_by_key_config<128, + 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = rocprim::half, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<256, - 22, + 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = rocprim::half, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1))>> : scan_by_key_config<256, - 18, + 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> {}; -// Based on key_type = int64_t, value_type = int64_t +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<64, + 10, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_scan_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : scan_by_key_config<64, + 14, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_scan_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : scan_by_key_config<64, + 14, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_scan_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : scan_by_key_config<64, 19, ::rocprim::block_load_method::block_load_transpose, @@ -2419,10 +3462,57 @@ struct default_scan_by_key_config< block_scan_algorithm::using_warp_scan> {}; +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_scan_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1))>> + : scan_by_key_config<64, + 14, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<64, + 9, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = int64_t, value_type = int64_t +template +struct default_scan_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : scan_by_key_config<256, + 14, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + // Based on key_type = int64_t, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -2432,20 +3522,20 @@ struct default_scan_by_key_config< 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = int64_t, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<64, - 19, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -2454,7 +3544,7 @@ struct default_scan_by_key_config< // Based on key_type = int64_t, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -2463,52 +3553,68 @@ struct default_scan_by_key_config< 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<64, + 10, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = int, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : scan_by_key_config<64, - 21, + : scan_by_key_config<128, + 10, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<256, - 14, + 12, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = int, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : scan_by_key_config<64, - 24, + : scan_by_key_config<256, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -2517,7 +3623,7 @@ struct default_scan_by_key_config< // Based on key_type = int, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -2526,52 +3632,68 @@ struct default_scan_by_key_config< 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<64, + 12, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = short, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_by_key_config<64, - 21, + 18, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = short, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<256, - 9, + 18, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<256, - 22, + 20, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -2580,43 +3702,58 @@ struct default_scan_by_key_config< // Based on key_type = short, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1))>> : scan_by_key_config<256, - 18, + 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : scan_by_key_config<64, + 15, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int64_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_by_key_config<64, - 21, + 18, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<128, - 21, + 18, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -2625,13 +3762,13 @@ struct default_scan_by_key_config< // Based on key_type = int8_t, value_type = short template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<256, - 18, + 20, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -2640,7 +3777,7 @@ struct default_scan_by_key_config< // Based on key_type = int8_t, value_type = int8_t template struct default_scan_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -2652,6 +3789,22 @@ struct default_scan_by_key_config< block_scan_algorithm::reduce_then_scan> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<128, + 21, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + // Based on key_type = double, value_type = int64_t template struct default_scan_by_key_config< @@ -2662,7 +3815,7 @@ struct default_scan_by_key_config< && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_by_key_config<256, - 13, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -2678,7 +3831,7 @@ struct default_scan_by_key_config< && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<256, - 21, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -2715,6 +3868,22 @@ struct default_scan_by_key_config< block_scan_algorithm::using_warp_scan> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<128, + 23, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + // Based on key_type = float, value_type = int64_t template struct default_scan_by_key_config< @@ -2728,7 +3897,7 @@ struct default_scan_by_key_config< 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = float, value_type = int @@ -2741,10 +3910,10 @@ struct default_scan_by_key_config< && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<256, - 19, + 23, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = float, value_type = short @@ -2757,7 +3926,7 @@ struct default_scan_by_key_config< && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<256, - 19, + 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -2772,12 +3941,27 @@ struct default_scan_by_key_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1))>> : scan_by_key_config<256, - 19, + 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> {}; +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 14, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + // Based on key_type = rocprim::half, value_type = int64_t template struct default_scan_by_key_config< @@ -2787,10 +3971,10 @@ struct default_scan_by_key_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_by_key_config<256, - 24, + 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = rocprim::half, value_type = int @@ -2817,10 +4001,10 @@ struct default_scan_by_key_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<256, - 22, + 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = rocprim::half, value_type = int8_t @@ -2835,6 +4019,101 @@ struct default_scan_by_key_config< 23, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<128, + 16, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : scan_by_key_config<128, + 21, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : scan_by_key_config<128, + 14, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : scan_by_key_config<256, + 13, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1))>> + : scan_by_key_config<256, + 14, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<128, + 21, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> {}; @@ -2848,7 +4127,7 @@ struct default_scan_by_key_config< && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_by_key_config<256, - 13, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -2864,10 +4143,10 @@ struct default_scan_by_key_config< && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<256, - 21, + 14, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = int64_t, value_type = short @@ -2901,6 +4180,22 @@ struct default_scan_by_key_config< block_scan_algorithm::using_warp_scan> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<128, + 23, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + // Based on key_type = int, value_type = int64_t template struct default_scan_by_key_config< @@ -2914,7 +4209,7 @@ struct default_scan_by_key_config< 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int, value_type = int @@ -2927,10 +4222,10 @@ struct default_scan_by_key_config< && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : scan_by_key_config<256, - 19, + 23, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = int, value_type = short @@ -2943,7 +4238,7 @@ struct default_scan_by_key_config< && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<256, - 19, + 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> @@ -2961,6 +4256,22 @@ struct default_scan_by_key_config< 19, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::reduce_then_scan> +{}; + +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 14, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::using_warp_scan> {}; @@ -2974,10 +4285,10 @@ struct default_scan_by_key_config< && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_by_key_config<256, - 24, + 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int @@ -3006,10 +4317,10 @@ struct default_scan_by_key_config< && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : scan_by_key_config<256, - 22, + 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = short, value_type = int8_t @@ -3024,7 +4335,22 @@ struct default_scan_by_key_config< 23, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::reduce_then_scan> + block_scan_algorithm::using_warp_scan> +{}; + +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_scan_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : scan_by_key_config<256, + 14, + ::rocprim::block_load_method::block_load_transpose, + ::rocprim::block_store_method::block_store_transpose, + block_scan_algorithm::using_warp_scan> {}; // Based on key_type = int8_t, value_type = int64_t @@ -3036,7 +4362,7 @@ struct default_scan_by_key_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : scan_by_key_config<256, - 24, + 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> @@ -3054,7 +4380,7 @@ struct default_scan_by_key_config< 21, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = int8_t, value_type = short @@ -3069,7 +4395,7 @@ struct default_scan_by_key_config< 24, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, - block_scan_algorithm::using_warp_scan> + block_scan_algorithm::reduce_then_scan> {}; // Based on key_type = int8_t, value_type = int8_t @@ -3081,7 +4407,7 @@ struct default_scan_by_key_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1))>> : scan_by_key_config<256, - 24, + 23, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_store_method::block_store_transpose, block_scan_algorithm::reduce_then_scan> diff --git a/rocprim/include/rocprim/device/detail/config/device_segmented_radix_sort.hpp b/rocprim/include/rocprim/device/detail/config/device_segmented_radix_sort.hpp index 57bbb0194..b1ae4e50f 100644 --- a/rocprim/include/rocprim/device/detail/config/device_segmented_radix_sort.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_segmented_radix_sort.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,8 +21,11 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_SEGMENTED_RADIX_SORT_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_SEGMENTED_RADIX_SORT_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" + #include /* DO NOT EDIT THIS FILE @@ -39,13 +42,32 @@ namespace detail { template -struct default_segmented_radix_sort_config : default_segmented_radix_sort_config_base<7, 6>::type +struct default_segmented_radix_sort_config : default_segmented_radix_sort_config_base<6, 4>::type +{}; + +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> {}; // Based on key_type = double, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -64,7 +86,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = double, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -75,7 +97,7 @@ struct default_segmented_radix_sort_config< 4, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -83,7 +105,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = double, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -94,7 +116,7 @@ struct default_segmented_radix_sort_config< 4, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -102,7 +124,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = double, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -113,7 +135,7 @@ struct default_segmented_radix_sort_config< 4, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -121,7 +143,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = double, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -132,7 +154,26 @@ struct default_segmented_radix_sort_config< 4, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -140,7 +181,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = float, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -148,10 +189,10 @@ struct default_segmented_radix_sort_config< && (sizeof(value_type) > 4))>> : segmented_radix_sort_config< 7, - 6, - kernel_config<256, 17>, + 2, + kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -159,7 +200,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = float, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -167,7 +208,7 @@ struct default_segmented_radix_sort_config< && (sizeof(value_type) > 2))>> : segmented_radix_sort_config< 7, - 4, + 2, kernel_config<256, 16>, typename std::conditional<1, WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, @@ -178,7 +219,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = float, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -186,7 +227,7 @@ struct default_segmented_radix_sort_config< && (sizeof(value_type) > 1))>> : segmented_radix_sort_config< 7, - 4, + 2, kernel_config<256, 16>, typename std::conditional<1, WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, @@ -197,7 +238,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = float, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -205,7 +246,7 @@ struct default_segmented_radix_sort_config< && (!std::is_same::value))>> : segmented_radix_sort_config< 7, - 4, + 2, kernel_config<256, 16>, typename std::conditional<1, WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, @@ -216,7 +257,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = float, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -224,10 +265,28 @@ struct default_segmented_radix_sort_config< && (std::is_same::value))>> : segmented_radix_sort_config< 7, - 6, - kernel_config<256, 17>, + 5, + kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -235,48 +294,50 @@ struct default_segmented_radix_sort_config< // Based on key_type = rocprim::half, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : segmented_radix_sort_config<6, - 4, - kernel_config<256, 8>, - typename std::conditional<1, - WarpSortConfig<4, 4, 256, 5, 8, 8, 256>, - DisabledWarpSortConfig>::type, - 1> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> {}; // Based on key_type = rocprim::half, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : segmented_radix_sort_config<6, - 4, - kernel_config<256, 8>, - typename std::conditional<1, - WarpSortConfig<4, 4, 256, 5, 8, 8, 256>, - DisabledWarpSortConfig>::type, - 1> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 4>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> {}; // Based on key_type = rocprim::half, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : segmented_radix_sort_config< - 7, - 2, + 6, + 4, kernel_config<256, 16>, typename std::conditional<1, WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, @@ -287,18 +348,18 @@ struct default_segmented_radix_sort_config< // Based on key_type = rocprim::half, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> : segmented_radix_sort_config< - 6, - 4, - kernel_config<256, 16>, + 8, + 8, + kernel_config<256, 4>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -306,7 +367,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = rocprim::half, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -321,128 +382,166 @@ struct default_segmented_radix_sort_config< 1> {}; -// Based on key_type = int64_t, value_type = int64_t +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 7, + 6, + kernel_config<128, 17>, + typename std::conditional<1, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 16>, + 8, + 8, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = int64_t, value_type = int +// Based on key_type = rocprim::int128_t, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 16>, + 8, + 8, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = int64_t, value_type = short +// Based on key_type = rocprim::int128_t, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 16>, + 8, + 8, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = int64_t, value_type = int8_t +// Based on key_type = rocprim::int128_t, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 16>, + 8, + 8, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = int64_t, value_type = empty_type +// Based on key_type = rocprim::int128_t, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (std::is_same::value))>> : segmented_radix_sort_config< 7, - 3, - kernel_config<256, 16>, + 6, + kernel_config<128, 17>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = int, value_type = int64_t +// Based on key_type = int64_t, value_type = rocprim::int128_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : segmented_radix_sort_config< 7, 6, - kernel_config<256, 17>, + kernel_config<128, 17>, typename std::conditional<1, WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = int, value_type = int +// Based on key_type = int64_t, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : segmented_radix_sort_config< + 7, + 4, + kernel_config<256, 16>, + typename std::conditional<1, + WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = int64_t, value_type = int +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : segmented_radix_sort_config< 7, @@ -454,33 +553,33 @@ struct default_segmented_radix_sort_config< 1> {}; -// Based on key_type = int, value_type = short +// Based on key_type = int64_t, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : segmented_radix_sort_config< 7, 4, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = int, value_type = int8_t +// Based on key_type = int64_t, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> : segmented_radix_sort_config< 7, @@ -492,29 +591,162 @@ struct default_segmented_radix_sort_config< 1> {}; -// Based on key_type = int, value_type = empty_type +// Based on key_type = int64_t, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (std::is_same::value))>> : segmented_radix_sort_config< 7, - 6, - kernel_config<256, 17>, + 4, + kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = short, value_type = int64_t +// Based on key_type = int, value_type = rocprim::int128_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = int, value_type = int64_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : segmented_radix_sort_config< + 7, + 6, + kernel_config<256, 17>, + typename std::conditional<1, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = int, value_type = int +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : segmented_radix_sort_config< + 7, + 4, + kernel_config<256, 16>, + typename std::conditional<1, + WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = int, value_type = short +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : segmented_radix_sort_config< + 7, + 4, + kernel_config<256, 16>, + typename std::conditional<1, + WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = int, value_type = int8_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : segmented_radix_sort_config< + 7, + 2, + kernel_config<256, 16>, + typename std::conditional<1, + WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = int, value_type = empty_type +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) + && (std::is_same::value))>> + : segmented_radix_sort_config< + 7, + 2, + kernel_config<256, 16>, + typename std::conditional<1, + WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = short, value_type = int64_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -533,7 +765,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = short, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -541,10 +773,10 @@ struct default_segmented_radix_sort_config< && (sizeof(value_type) > 2))>> : segmented_radix_sort_config< 7, - 2, - kernel_config<256, 8>, + 6, + kernel_config<256, 17>, typename std::conditional<1, - WarpSortConfig<16, 4, 256, 5, 32, 8, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -552,7 +784,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = short, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -563,7 +795,7 @@ struct default_segmented_radix_sort_config< 4, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -571,7 +803,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = short, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -579,10 +811,10 @@ struct default_segmented_radix_sort_config< && (!std::is_same::value))>> : segmented_radix_sort_config< 7, - 2, - kernel_config<256, 8>, + 6, + kernel_config<256, 17>, typename std::conditional<1, - WarpSortConfig<16, 4, 256, 5, 32, 8, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -590,7 +822,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = short, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -606,20 +838,38 @@ struct default_segmented_radix_sort_config< 1> {}; +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 4>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + // Based on key_type = int8_t, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : segmented_radix_sort_config< 7, - 4, - kernel_config<256, 4>, + 6, + kernel_config<256, 17>, typename std::conditional<1, - WarpSortConfig<8, 2, 256, 5, 16, 4, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -627,17 +877,17 @@ struct default_segmented_radix_sort_config< // Based on key_type = int8_t, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : segmented_radix_sort_config< + 7, 6, - 4, - kernel_config<256, 4>, + kernel_config<256, 17>, typename std::conditional<1, - WarpSortConfig<8, 2, 256, 5, 16, 4, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -645,24 +895,25 @@ struct default_segmented_radix_sort_config< // Based on key_type = int8_t, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : segmented_radix_sort_config<6, - 4, - kernel_config<256, 8>, - typename std::conditional<1, - WarpSortConfig<4, 4, 256, 5, 8, 8, 256>, - DisabledWarpSortConfig>::type, - 1> + : segmented_radix_sort_config< + 7, + 6, + kernel_config<256, 17>, + typename std::conditional<1, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + DisabledWarpSortConfig>::type, + 1> {}; // Based on key_type = int8_t, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -671,9 +922,9 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 6, 4, - kernel_config<256, 4>, + kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<8, 2, 256, 5, 16, 4, 256>, + WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -681,35 +932,55 @@ struct default_segmented_radix_sort_config< // Based on key_type = int8_t, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1030), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (std::is_same::value))>> - : segmented_radix_sort_config<4, - 4, - kernel_config<256, 8>, - typename std::conditional<1, - WarpSortConfig<4, 4, 256, 5, 8, 8, 256>, - DisabledWarpSortConfig>::type, - 1> + : segmented_radix_sort_config< + 7, + 6, + kernel_config<256, 17>, + typename std::conditional<1, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> {}; // Based on key_type = double, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : segmented_radix_sort_config< - 7, - 2, + 8, + 8, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -717,18 +988,18 @@ struct default_segmented_radix_sort_config< // Based on key_type = double, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : segmented_radix_sort_config< - 7, - 4, + 8, + 8, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -736,18 +1007,18 @@ struct default_segmented_radix_sort_config< // Based on key_type = double, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : segmented_radix_sort_config< - 7, - 4, + 8, + 8, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -755,18 +1026,18 @@ struct default_segmented_radix_sort_config< // Based on key_type = double, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> : segmented_radix_sort_config< - 7, - 2, + 8, + 8, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -774,7 +1045,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = double, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -782,29 +1053,48 @@ struct default_segmented_radix_sort_config< && (std::is_same::value))>> : segmented_radix_sort_config< 7, - 4, - kernel_config<256, 16>, + 6, + kernel_config<128, 17>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = float, value_type = int64_t +// Based on key_type = float, value_type = rocprim::int128_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = float, value_type = int64_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : segmented_radix_sort_config< - 7, - 6, - kernel_config<256, 17>, + 8, + 8, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -812,18 +1102,18 @@ struct default_segmented_radix_sort_config< // Based on key_type = float, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 16>, + 8, + 8, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -831,18 +1121,18 @@ struct default_segmented_radix_sort_config< // Based on key_type = float, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 16>, + 8, + 8, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -850,18 +1140,18 @@ struct default_segmented_radix_sort_config< // Based on key_type = float, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 16>, + 8, + 8, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -869,7 +1159,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = float, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -878,61 +1168,81 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 7, 6, - kernel_config<256, 17>, + kernel_config<128, 17>, typename std::conditional<1, WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + // Based on key_type = rocprim::half, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : segmented_radix_sort_config<7, - 2, - kernel_config<256, 8>, - typename std::conditional<1, - WarpSortConfig<4, 4, 256, 5, 8, 8, 256>, - DisabledWarpSortConfig>::type, - 1> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> {}; // Based on key_type = rocprim::half, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : segmented_radix_sort_config<6, - 4, - kernel_config<256, 8>, - typename std::conditional<1, - WarpSortConfig<4, 4, 256, 5, 8, 8, 256>, - DisabledWarpSortConfig>::type, - 1> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> {}; // Based on key_type = rocprim::half, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : segmented_radix_sort_config< + 7, 6, - 4, - kernel_config<256, 8>, + kernel_config<128, 17>, typename std::conditional<1, - WarpSortConfig<16, 4, 256, 5, 32, 8, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -940,18 +1250,18 @@ struct default_segmented_radix_sort_config< // Based on key_type = rocprim::half, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> : segmented_radix_sort_config< - 7, - 2, - kernel_config<256, 16>, + 8, + 8, + kernel_config<256, 4>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -959,7 +1269,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = rocprim::half, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -967,275 +1277,446 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 7, 6, - kernel_config<256, 17>, + kernel_config<128, 17>, typename std::conditional<1, WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = int64_t, value_type = int64_t +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : segmented_radix_sort_config< 7, - 4, - kernel_config<256, 16>, + 6, + kernel_config<128, 17>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = int64_t, value_type = int +// Based on key_type = rocprim::int128_t, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 4>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 16>, + 8, + 8, + kernel_config<256, 4>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = int64_t, value_type = short +// Based on key_type = rocprim::int128_t, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 16>, + 8, + 8, + kernel_config<256, 4>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = int64_t, value_type = int8_t +// Based on key_type = rocprim::int128_t, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 4>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> : segmented_radix_sort_config< 7, - 4, - kernel_config<256, 16>, + 6, + kernel_config<128, 17>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = int64_t, value_type = empty_type +// Based on key_type = int64_t, value_type = rocprim::int128_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) - && (std::is_same::value))>> + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : segmented_radix_sort_config< 7, - 4, - kernel_config<256, 16>, + 6, + kernel_config<128, 17>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = int, value_type = int64_t +// Based on key_type = int64_t, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : segmented_radix_sort_config< 7, 6, - kernel_config<256, 17>, + kernel_config<128, 17>, typename std::conditional<1, WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = int, value_type = int +// Based on key_type = int64_t, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : segmented_radix_sort_config< - 7, - 4, + 8, + 8, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = int, value_type = short +// Based on key_type = int64_t, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : segmented_radix_sort_config< - 7, - 4, + 8, + 8, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = int, value_type = int8_t +// Based on key_type = int64_t, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> : segmented_radix_sort_config< - 7, - 4, + 8, + 8, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = int, value_type = empty_type +// Based on key_type = int64_t, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (std::is_same::value))>> : segmented_radix_sort_config< - 7, - 6, - kernel_config<256, 17>, + 8, + 0, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = short, value_type = int64_t +// Based on key_type = int, value_type = rocprim::int128_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : segmented_radix_sort_config< - 7, - 6, - kernel_config<256, 17>, + 8, + 8, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = short, value_type = int +// Based on key_type = int, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : segmented_radix_sort_config< + 7, + 6, + kernel_config<128, 17>, + typename std::conditional<1, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = int, value_type = int +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : segmented_radix_sort_config< 7, 6, - kernel_config<256, 17>, + kernel_config<128, 17>, typename std::conditional<1, WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = short, value_type = short +// Based on key_type = int, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = int, value_type = int8_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = int, value_type = empty_type +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) + && (std::is_same::value))>> : segmented_radix_sort_config< 7, 6, - kernel_config<256, 17>, + kernel_config<128, 17>, typename std::conditional<1, WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = short, value_type = int64_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = short, value_type = int +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = short, value_type = short +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + // Based on key_type = short, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> : segmented_radix_sort_config< - 7, - 6, - kernel_config<256, 17>, + 8, + 8, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -1243,18 +1724,36 @@ struct default_segmented_radix_sort_config< // Based on key_type = short, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (std::is_same::value))>> : segmented_radix_sort_config< - 7, - 6, - kernel_config<256, 17>, + 8, + 0, + kernel_config<256, 4>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 4>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -1262,17 +1761,17 @@ struct default_segmented_radix_sort_config< // Based on key_type = int8_t, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 4>, + 8, + 8, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<16, 2, 256, 5, 32, 4, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -1280,34 +1779,35 @@ struct default_segmented_radix_sort_config< // Based on key_type = int8_t, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : segmented_radix_sort_config<6, - 4, - kernel_config<256, 8>, - typename std::conditional<1, - WarpSortConfig<4, 4, 256, 5, 8, 8, 256>, - DisabledWarpSortConfig>::type, - 1> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> {}; // Based on key_type = int8_t, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : segmented_radix_sort_config< - 6, - 4, - kernel_config<256, 8>, + 8, + 8, + kernel_config<256, 4>, typename std::conditional<1, - WarpSortConfig<16, 4, 256, 5, 32, 8, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -1315,18 +1815,18 @@ struct default_segmented_radix_sort_config< // Based on key_type = int8_t, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> : segmented_radix_sort_config< + 7, 6, - 4, - kernel_config<256, 4>, + kernel_config<128, 17>, typename std::conditional<1, - WarpSortConfig<8, 2, 256, 5, 16, 4, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -1334,7 +1834,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int8_t, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx906), + static_cast(target_arch::gfx1100), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -1342,17 +1842,36 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 7, 6, - kernel_config<256, 17>, + kernel_config<128, 17>, typename std::conditional<1, WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + // Based on key_type = double, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1360,10 +1879,10 @@ struct default_segmented_radix_sort_config< && (sizeof(value_type) > 4))>> : segmented_radix_sort_config< 7, - 6, - kernel_config<256, 17>, + 2, + kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -1371,7 +1890,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = double, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1379,10 +1898,10 @@ struct default_segmented_radix_sort_config< && (sizeof(value_type) > 2))>> : segmented_radix_sort_config< 7, - 6, - kernel_config<256, 17>, + 4, + kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -1390,7 +1909,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = double, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1401,7 +1920,7 @@ struct default_segmented_radix_sort_config< 4, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -1409,7 +1928,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = double, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1417,10 +1936,10 @@ struct default_segmented_radix_sort_config< && (!std::is_same::value))>> : segmented_radix_sort_config< 7, - 4, + 2, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -1428,7 +1947,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = double, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1444,34 +1963,34 @@ struct default_segmented_radix_sort_config< 1> {}; -// Based on key_type = float, value_type = int64_t +// Based on key_type = float, value_type = rocprim::int128_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : segmented_radix_sort_config< - 7, - 6, - kernel_config<256, 17>, + 8, + 8, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = float, value_type = int +// Based on key_type = float, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : segmented_radix_sort_config< 7, 6, @@ -1482,10 +2001,29 @@ struct default_segmented_radix_sort_config< 1> {}; +// Based on key_type = float, value_type = int +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : segmented_radix_sort_config< + 7, + 4, + kernel_config<256, 16>, + typename std::conditional<1, + WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + // Based on key_type = float, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1493,10 +2031,10 @@ struct default_segmented_radix_sort_config< && (sizeof(value_type) > 1))>> : segmented_radix_sort_config< 7, - 6, - kernel_config<256, 17>, + 4, + kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -1504,7 +2042,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = float, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1512,10 +2050,10 @@ struct default_segmented_radix_sort_config< && (!std::is_same::value))>> : segmented_radix_sort_config< 7, - 6, - kernel_config<256, 17>, + 4, + kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -1523,7 +2061,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = float, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1539,20 +2077,38 @@ struct default_segmented_radix_sort_config< 1> {}; +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + // Based on key_type = rocprim::half, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : segmented_radix_sort_config< - 7, - 6, - kernel_config<256, 17>, + 8, + 8, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -1560,17 +2116,17 @@ struct default_segmented_radix_sort_config< // Based on key_type = rocprim::half, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : segmented_radix_sort_config< - 7, - 6, - kernel_config<256, 17>, + 8, + 8, + kernel_config<256, 4>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -1578,17 +2134,17 @@ struct default_segmented_radix_sort_config< // Based on key_type = rocprim::half, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : segmented_radix_sort_config< - 7, 6, - kernel_config<256, 17>, + 4, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<16, 4, 256, 5, 32, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -1596,12 +2152,30 @@ struct default_segmented_radix_sort_config< // Based on key_type = rocprim::half, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 4>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::half, value_type = empty_type +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (std::is_same::value))>> : segmented_radix_sort_config< 7, 6, @@ -1612,18 +2186,133 @@ struct default_segmented_radix_sort_config< 1> {}; -// Based on key_type = rocprim::half, value_type = empty_type +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 6, + 4, + kernel_config<128, 17>, + typename std::conditional<1, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (std::is_same::value))>> : segmented_radix_sort_config< 7, 6, - kernel_config<256, 17>, + kernel_config<128, 17>, + typename std::conditional<1, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 7, + 6, + kernel_config<128, 17>, typename std::conditional<1, WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, @@ -1633,7 +2322,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int64_t, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1641,10 +2330,10 @@ struct default_segmented_radix_sort_config< && (sizeof(value_type) > 4))>> : segmented_radix_sort_config< 7, - 6, - kernel_config<256, 17>, + 4, + kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -1652,7 +2341,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int64_t, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1663,7 +2352,7 @@ struct default_segmented_radix_sort_config< 4, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -1671,7 +2360,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int64_t, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1690,7 +2379,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int64_t, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1709,7 +2398,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int64_t, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -1725,10 +2414,29 @@ struct default_segmented_radix_sort_config< 1> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + // Based on key_type = int, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1747,7 +2455,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1758,7 +2466,7 @@ struct default_segmented_radix_sort_config< 4, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -1766,7 +2474,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1774,10 +2482,10 @@ struct default_segmented_radix_sort_config< && (sizeof(value_type) > 1))>> : segmented_radix_sort_config< 7, - 6, - kernel_config<256, 17>, + 4, + kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -1785,7 +2493,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1804,7 +2512,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -1820,17 +2528,36 @@ struct default_segmented_radix_sort_config< 1> {}; -// Based on key_type = short, value_type = int64_t +// Based on key_type = short, value_type = rocprim::int128_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : segmented_radix_sort_config< - 7, + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = short, value_type = int64_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : segmented_radix_sort_config< + 7, 6, kernel_config<256, 17>, typename std::conditional<1, @@ -1842,7 +2569,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = short, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1861,7 +2588,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = short, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1880,7 +2607,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = short, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1899,7 +2626,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = short, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -1915,56 +2642,73 @@ struct default_segmented_radix_sort_config< 1> {}; -// Based on key_type = int8_t, value_type = int64_t +// Based on key_type = int8_t, value_type = rocprim::int128_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> : segmented_radix_sort_config< - 7, - 6, - kernel_config<256, 17>, + 8, + 8, + kernel_config<256, 4>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = int8_t, value_type = int +// Based on key_type = int8_t, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : segmented_radix_sort_config< 7, - 6, - kernel_config<256, 17>, + 4, + kernel_config<256, 4>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<16, 2, 256, 5, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; +// Based on key_type = int8_t, value_type = int +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + : segmented_radix_sort_config<6, + 4, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<4, 4, 256, 5, 8, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + // Based on key_type = int8_t, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : segmented_radix_sort_config< - 7, 6, - kernel_config<256, 17>, + 4, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<16, 4, 256, 5, 32, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -1972,18 +2716,18 @@ struct default_segmented_radix_sort_config< // Based on key_type = int8_t, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> : segmented_radix_sort_config< - 7, 6, - kernel_config<256, 17>, + 4, + kernel_config<256, 4>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 2, 256, 5, 16, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -1991,7 +2735,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int8_t, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx908), + static_cast(target_arch::gfx906), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -2001,490 +2745,38 @@ struct default_segmented_radix_sort_config< 6, kernel_config<256, 17>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = double, value_type = int64_t -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 16>, - typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = double, value_type = int -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 16>, - typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = double, value_type = short -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 16>, - typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = double, value_type = int8_t -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) - && (!std::is_same::value))>> - : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 16>, - typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = double, value_type = empty_type -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) - && (std::is_same::value))>> - : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 16>, - typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = float, value_type = int64_t -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : segmented_radix_sort_config< - 7, - 2, - kernel_config<256, 16>, - typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = float, value_type = int -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : segmented_radix_sort_config< - 7, - 2, - kernel_config<256, 16>, - typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = float, value_type = short -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : segmented_radix_sort_config< - 7, - 2, - kernel_config<256, 16>, - typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = float, value_type = int8_t -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) - && (!std::is_same::value))>> - : segmented_radix_sort_config< - 7, - 2, - kernel_config<256, 16>, - typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = float, value_type = empty_type -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) - && (std::is_same::value))>> - : segmented_radix_sort_config< - 7, - 5, - kernel_config<256, 16>, - typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = rocprim::half, value_type = int64_t -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : segmented_radix_sort_config< - 7, - 6, - kernel_config<256, 17>, - typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = rocprim::half, value_type = int -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : segmented_radix_sort_config< - 7, - 6, - kernel_config<256, 17>, - typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = rocprim::half, value_type = short -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : segmented_radix_sort_config< - 6, - 4, - kernel_config<256, 16>, - typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = rocprim::half, value_type = int8_t -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(value_type) <= 1) - && (!std::is_same::value))>> - : segmented_radix_sort_config< - 7, - 6, - kernel_config<256, 17>, - typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = rocprim::half, value_type = empty_type -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (std::is_same::value))>> - : segmented_radix_sort_config< - 7, - 6, - kernel_config<256, 17>, - typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = int64_t, value_type = int64_t -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 16>, - typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = int64_t, value_type = int -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 16>, - typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = int64_t, value_type = short -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 16>, - typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = int64_t, value_type = int8_t -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) - && (!std::is_same::value))>> - : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 16>, - typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = int64_t, value_type = empty_type -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) - && (std::is_same::value))>> - : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 16>, - typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = int, value_type = int64_t -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> - : segmented_radix_sort_config< - 7, - 6, - kernel_config<256, 17>, - typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = int, value_type = int -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> - : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 16>, - typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = int, value_type = short -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> - : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 16>, - typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = int, value_type = int8_t -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) - && (!std::is_same::value))>> - : segmented_radix_sort_config< - 7, - 2, - kernel_config<256, 16>, - typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = int, value_type = empty_type +// Based on key_type = double, value_type = rocprim::int128_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) - && (std::is_same::value))>> + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : segmented_radix_sort_config< - 7, - 2, - kernel_config<256, 16>, + 8, + 8, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = short, value_type = int64_t +// Based on key_type = double, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : segmented_radix_sort_config< 7, @@ -2496,14 +2788,14 @@ struct default_segmented_radix_sort_config< 1> {}; -// Based on key_type = short, value_type = int +// Based on key_type = double, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : segmented_radix_sort_config< 7, @@ -2515,89 +2807,91 @@ struct default_segmented_radix_sort_config< 1> {}; -// Based on key_type = short, value_type = short +// Based on key_type = double, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : segmented_radix_sort_config< - 6, + 7, 4, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = short, value_type = int8_t +// Based on key_type = double, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1) + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> : segmented_radix_sort_config< 7, - 6, - kernel_config<256, 17>, + 4, + kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = short, value_type = empty_type +// Based on key_type = double, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(key_type) > 1) + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (std::is_same::value))>> : segmented_radix_sort_config< 7, - 6, - kernel_config<256, 17>, + 4, + kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = int8_t, value_type = int64_t +// Based on key_type = float, value_type = rocprim::int128_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : segmented_radix_sort_config< - 7, - 6, - kernel_config<256, 17>, + 8, + 8, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = int8_t, value_type = int +// Based on key_type = float, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : segmented_radix_sort_config< 7, 6, @@ -2608,14 +2902,15 @@ struct default_segmented_radix_sort_config< 1> {}; -// Based on key_type = int8_t, value_type = short +// Based on key_type = float, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : segmented_radix_sort_config< 7, 6, @@ -2626,33 +2921,15 @@ struct default_segmented_radix_sort_config< 1> {}; -// Based on key_type = int8_t, value_type = int8_t -template -struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), - key_type, - value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) - && (sizeof(value_type) <= 1) - && (!std::is_same::value))>> - : segmented_radix_sort_config< - 6, - 4, - kernel_config<256, 16>, - typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, - DisabledWarpSortConfig>::type, - 1> -{}; - -// Based on key_type = int8_t, value_type = empty_type +// Based on key_type = float, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1030), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) - && (std::is_same::value))>> + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : segmented_radix_sort_config< 7, 6, @@ -2663,282 +2940,282 @@ struct default_segmented_radix_sort_config< 1> {}; -// Based on key_type = double, value_type = int64_t +// Based on key_type = float, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> : segmented_radix_sort_config< 7, - 4, - kernel_config<256, 16>, + 6, + kernel_config<256, 17>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = double, value_type = int +// Based on key_type = float, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) + && (std::is_same::value))>> : segmented_radix_sort_config< 7, - 4, - kernel_config<256, 16>, + 6, + kernel_config<256, 17>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = double, value_type = short +// Based on key_type = rocprim::half, value_type = rocprim::int128_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 16>, + 8, + 8, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = double, value_type = int8_t +// Based on key_type = rocprim::half, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1) - && (!std::is_same::value))>> + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : segmented_radix_sort_config< - 7, - 4, + 8, + 8, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = double, value_type = empty_type +// Based on key_type = rocprim::half, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) - && (std::is_same::value))>> + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : segmented_radix_sort_config< - 7, - 4, + 8, + 8, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = float, value_type = int64_t +// Based on key_type = rocprim::half, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : segmented_radix_sort_config< 7, - 4, - kernel_config<256, 16>, + 6, + kernel_config<256, 17>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = float, value_type = int +// Based on key_type = rocprim::half, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> : segmented_radix_sort_config< - 7, - 4, + 8, + 8, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = float, value_type = short +// Based on key_type = rocprim::half, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (std::is_same::value))>> : segmented_radix_sort_config< 7, - 4, - kernel_config<256, 16>, + 6, + kernel_config<256, 17>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = float, value_type = int8_t +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1) - && (!std::is_same::value))>> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : segmented_radix_sort_config< 7, - 4, - kernel_config<256, 16>, + 6, + kernel_config<128, 17>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = float, value_type = empty_type +// Based on key_type = rocprim::int128_t, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) - && (std::is_same::value))>> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : segmented_radix_sort_config< - 7, - 4, - kernel_config<256, 16>, + 8, + 8, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = rocprim::half, value_type = int64_t +// Based on key_type = rocprim::int128_t, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : segmented_radix_sort_config<6, - 4, - kernel_config<128, 8>, - typename std::conditional<1, - WarpSortConfig<4, 4, 128, 5, 8, 8, 128>, - DisabledWarpSortConfig>::type, - 1> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> {}; -// Based on key_type = rocprim::half, value_type = int +// Based on key_type = rocprim::int128_t, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : segmented_radix_sort_config< - 7, - 6, - kernel_config<256, 17>, + 8, + 8, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = rocprim::half, value_type = short +// Based on key_type = rocprim::int128_t, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> : segmented_radix_sort_config< - 6, - 4, - kernel_config<256, 16>, + 8, + 8, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = rocprim::half, value_type = int8_t +// Based on key_type = rocprim::int128_t, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(value_type) <= 1) - && (!std::is_same::value))>> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> : segmented_radix_sort_config< + 7, 6, - 4, - kernel_config<256, 16>, + kernel_config<128, 17>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; -// Based on key_type = rocprim::half, value_type = empty_type +// Based on key_type = int64_t, value_type = rocprim::int128_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (std::is_same::value))>> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : segmented_radix_sort_config< + 7, 6, - 2, - kernel_config<256, 16>, + kernel_config<128, 17>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -2946,7 +3223,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int64_t, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -2954,10 +3231,10 @@ struct default_segmented_radix_sort_config< && (sizeof(value_type) > 4))>> : segmented_radix_sort_config< 7, - 4, - kernel_config<256, 16>, + 6, + kernel_config<256, 17>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -2965,7 +3242,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int64_t, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -2976,7 +3253,7 @@ struct default_segmented_radix_sort_config< 4, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -2984,7 +3261,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int64_t, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -3003,7 +3280,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int64_t, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -3022,7 +3299,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int64_t, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -3038,10 +3315,29 @@ struct default_segmented_radix_sort_config< 1> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + // Based on key_type = int, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -3060,7 +3356,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -3071,7 +3367,7 @@ struct default_segmented_radix_sort_config< 4, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -3079,7 +3375,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -3087,10 +3383,10 @@ struct default_segmented_radix_sort_config< && (sizeof(value_type) > 1))>> : segmented_radix_sort_config< 7, - 2, - kernel_config<256, 16>, + 6, + kernel_config<256, 17>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -3098,7 +3394,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -3117,7 +3413,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -3125,10 +3421,29 @@ struct default_segmented_radix_sort_config< && (std::is_same::value))>> : segmented_radix_sort_config< 7, - 4, - kernel_config<256, 16>, + 6, + kernel_config<256, 17>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -3136,7 +3451,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = short, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -3155,7 +3470,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = short, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -3174,18 +3489,18 @@ struct default_segmented_radix_sort_config< // Based on key_type = short, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> : segmented_radix_sort_config< + 7, 6, - 4, - kernel_config<256, 16>, + kernel_config<256, 17>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -3193,7 +3508,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = short, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -3201,10 +3516,10 @@ struct default_segmented_radix_sort_config< && (!std::is_same::value))>> : segmented_radix_sort_config< 7, - 2, - kernel_config<256, 16>, + 6, + kernel_config<256, 17>, typename std::conditional<1, - WarpSortConfig<16, 8, 256, 5, 32, 16, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -3212,7 +3527,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = short, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -3220,10 +3535,28 @@ struct default_segmented_radix_sort_config< && (std::is_same::value))>> : segmented_radix_sort_config< 7, - 2, - kernel_config<256, 16>, + 6, + kernel_config<256, 17>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -3231,7 +3564,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int8_t, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -3249,52 +3582,54 @@ struct default_segmented_radix_sort_config< // Based on key_type = int8_t, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : segmented_radix_sort_config<7, - 4, - kernel_config<128, 8>, - typename std::conditional<1, - WarpSortConfig<4, 4, 128, 5, 8, 8, 128>, - DisabledWarpSortConfig>::type, - 1> + : segmented_radix_sort_config< + 7, + 6, + kernel_config<256, 17>, + typename std::conditional<1, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + DisabledWarpSortConfig>::type, + 1> {}; // Based on key_type = int8_t, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : segmented_radix_sort_config<6, - 4, - kernel_config<128, 8>, - typename std::conditional<1, - WarpSortConfig<4, 4, 128, 5, 8, 8, 128>, - DisabledWarpSortConfig>::type, - 1> + : segmented_radix_sort_config< + 7, + 6, + kernel_config<256, 17>, + typename std::conditional<1, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + DisabledWarpSortConfig>::type, + 1> {}; // Based on key_type = int8_t, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> : segmented_radix_sort_config< + 7, 6, - 4, - kernel_config<256, 16>, + kernel_config<256, 17>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -3302,17 +3637,36 @@ struct default_segmented_radix_sort_config< // Based on key_type = int8_t, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx908), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (std::is_same::value))>> : segmented_radix_sort_config< - 4, - 3, - kernel_config<256, 16>, + 7, + 6, + kernel_config<256, 17>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 5, 16, 16, 256>, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -3320,7 +3674,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = double, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -3339,7 +3693,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = double, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -3358,7 +3712,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = double, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -3377,7 +3731,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = double, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -3396,7 +3750,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = double, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -3412,10 +3766,29 @@ struct default_segmented_radix_sort_config< 1> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + // Based on key_type = float, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -3434,7 +3807,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = float, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -3453,7 +3826,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = float, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -3472,7 +3845,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = float, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -3491,7 +3864,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = float, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -3507,20 +3880,38 @@ struct default_segmented_radix_sort_config< 1> {}; +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + // Based on key_type = rocprim::half, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : segmented_radix_sort_config< - 7, - 6, - kernel_config<256, 17>, + 8, + 8, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -3528,17 +3919,17 @@ struct default_segmented_radix_sort_config< // Based on key_type = rocprim::half, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : segmented_radix_sort_config< - 7, - 6, - kernel_config<256, 17>, + 8, + 8, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -3546,7 +3937,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = rocprim::half, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -3564,12 +3955,30 @@ struct default_segmented_radix_sort_config< // Based on key_type = rocprim::half, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::half, value_type = empty_type +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (std::is_same::value))>> : segmented_radix_sort_config< 7, 6, @@ -3580,18 +3989,133 @@ struct default_segmented_radix_sort_config< 1> {}; -// Based on key_type = rocprim::half, value_type = empty_type +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (std::is_same::value))>> : segmented_radix_sort_config< 7, 6, - kernel_config<256, 17>, + kernel_config<128, 17>, + typename std::conditional<1, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 7, + 6, + kernel_config<128, 17>, typename std::conditional<1, WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, @@ -3601,7 +4125,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int64_t, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -3620,7 +4144,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int64_t, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -3639,7 +4163,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int64_t, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -3658,7 +4182,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int64_t, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -3677,7 +4201,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int64_t, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -3693,10 +4217,29 @@ struct default_segmented_radix_sort_config< 1> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + // Based on key_type = int, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -3715,7 +4258,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -3734,7 +4277,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -3753,7 +4296,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -3772,7 +4315,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -3788,10 +4331,29 @@ struct default_segmented_radix_sort_config< 1> {}; +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + // Based on key_type = short, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -3810,7 +4372,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = short, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -3829,7 +4391,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = short, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -3848,7 +4410,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = short, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -3867,7 +4429,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = short, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -3883,10 +4445,28 @@ struct default_segmented_radix_sort_config< 1> {}; +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 4>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + // Based on key_type = int8_t, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -3904,7 +4484,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int8_t, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -3922,7 +4502,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int8_t, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -3940,7 +4520,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int8_t, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -3959,7 +4539,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int8_t, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -3974,10 +4554,29 @@ struct default_segmented_radix_sort_config< 1> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + // Based on key_type = double, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -3996,7 +4595,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = double, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -4015,7 +4614,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = double, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -4034,7 +4633,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = double, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -4053,7 +4652,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = double, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -4069,10 +4668,29 @@ struct default_segmented_radix_sort_config< 1> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + // Based on key_type = float, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -4091,7 +4709,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = float, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -4110,7 +4728,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = float, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -4129,7 +4747,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = float, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -4148,7 +4766,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = float, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -4164,20 +4782,38 @@ struct default_segmented_radix_sort_config< 1> {}; +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + // Based on key_type = rocprim::half, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> : segmented_radix_sort_config< - 7, - 6, - kernel_config<256, 17>, + 8, + 8, + kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -4185,17 +4821,17 @@ struct default_segmented_radix_sort_config< // Based on key_type = rocprim::half, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> : segmented_radix_sort_config< - 7, - 6, - kernel_config<256, 17>, + 8, + 8, + kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -4203,7 +4839,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = rocprim::half, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -4221,12 +4857,30 @@ struct default_segmented_radix_sort_config< // Based on key_type = rocprim::half, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 1) && (!std::is_same::value))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 16>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::half, value_type = empty_type +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (std::is_same::value))>> : segmented_radix_sort_config< 7, 6, @@ -4237,18 +4891,133 @@ struct default_segmented_radix_sort_config< 1> {}; -// Based on key_type = rocprim::half, value_type = empty_type +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 7, + 6, + kernel_config<128, 17>, + typename std::conditional<1, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (std::is_same::value))>> : segmented_radix_sort_config< 7, 6, - kernel_config<256, 17>, + kernel_config<128, 17>, + typename std::conditional<1, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 7, + 6, + kernel_config<128, 17>, typename std::conditional<1, WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, DisabledWarpSortConfig>::type, @@ -4258,7 +5027,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int64_t, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -4277,7 +5046,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int64_t, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -4296,7 +5065,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int64_t, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -4315,7 +5084,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int64_t, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -4334,7 +5103,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int64_t, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) @@ -4350,10 +5119,29 @@ struct default_segmented_radix_sort_config< 1> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + // Based on key_type = int, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -4372,7 +5160,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -4391,7 +5179,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -4410,7 +5198,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -4429,7 +5217,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) @@ -4445,10 +5233,29 @@ struct default_segmented_radix_sort_config< 1> {}; +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + // Based on key_type = short, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -4467,7 +5274,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = short, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -4486,7 +5293,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = short, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -4505,7 +5312,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = short, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) @@ -4524,18 +5331,36 @@ struct default_segmented_radix_sort_config< // Based on key_type = short, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (std::is_same::value))>> : segmented_radix_sort_config< - 7, - 6, - kernel_config<256, 17>, + 7, + 6, + kernel_config<256, 17>, + typename std::conditional<1, + WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<32, 4, 256, 3000, 32, 4, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -4543,7 +5368,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int8_t, value_type = int64_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -4561,7 +5386,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int8_t, value_type = int template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -4579,7 +5404,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int8_t, value_type = short template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -4597,7 +5422,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int8_t, value_type = int8_t template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -4616,7 +5441,7 @@ struct default_segmented_radix_sort_config< // Based on key_type = int8_t, value_type = empty_type template struct default_segmented_radix_sort_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) @@ -4631,6 +5456,25 @@ struct default_segmented_radix_sort_config< 1> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + // Based on key_type = double, value_type = int64_t template struct default_segmented_radix_sort_config< @@ -4645,7 +5489,7 @@ struct default_segmented_radix_sort_config< 8, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 1024, 64, 8, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -4664,7 +5508,7 @@ struct default_segmented_radix_sort_config< 8, kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<32, 2, 256, 2048, 64, 8, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -4681,9 +5525,9 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 8, - kernel_config<256, 17>, + kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 256, 64, 8, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -4700,9 +5544,9 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 8, - kernel_config<256, 13>, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<16, 4, 256, 2048, 64, 8, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -4719,9 +5563,28 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 0, - kernel_config<256, 13>, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 4>, typename std::conditional<1, - WarpSortConfig<32, 8, 256, 4096, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -4738,9 +5601,9 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 8, - kernel_config<256, 16>, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 1024, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -4757,9 +5620,9 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 8, - kernel_config<256, 16>, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 2048, 64, 8, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -4776,9 +5639,9 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 8, - kernel_config<256, 17>, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<32, 2, 256, 2048, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -4795,9 +5658,9 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 8, - kernel_config<256, 16>, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 128, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -4814,9 +5677,27 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 0, - kernel_config<256, 17>, + kernel_config<256, 16>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 4>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 4096, 64, 8, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -4834,7 +5715,7 @@ struct default_segmented_radix_sort_config< 8, kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<32, 2, 256, 2048, 64, 4, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -4852,7 +5733,7 @@ struct default_segmented_radix_sort_config< 8, kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<32, 2, 256, 4096, 64, 8, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -4868,9 +5749,9 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 8, - kernel_config<256, 17>, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<32, 2, 256, 2048, 64, 8, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -4887,9 +5768,9 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 8, - kernel_config<256, 16>, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 1024, 64, 8, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -4907,7 +5788,140 @@ struct default_segmented_radix_sort_config< 0, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<32, 8, 256, 4096, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 4>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1) + && (!std::is_same::value))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 4>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = rocprim::int128_t, value_type = empty_type +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) + && (std::is_same::value))>> + : segmented_radix_sort_config< + 8, + 0, + kernel_config<256, 4>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -4924,9 +5938,9 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 8, - kernel_config<256, 17>, + kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<16, 4, 256, 1024, 64, 8, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -4943,9 +5957,9 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 8, - kernel_config<256, 13>, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<32, 2, 256, 1024, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -4962,9 +5976,9 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 8, - kernel_config<256, 17>, + kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<32, 2, 256, 64, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -4981,9 +5995,9 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 8, - kernel_config<256, 13>, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<32, 2, 256, 256, 64, 8, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -5000,9 +6014,28 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 0, - kernel_config<256, 13>, + kernel_config<256, 8>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<16, 4, 256, 1024, 64, 8, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -5019,9 +6052,9 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 8, - kernel_config<256, 16>, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<16, 4, 256, 1024, 64, 8, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -5038,9 +6071,9 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 8, - kernel_config<256, 17>, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 2048, 64, 8, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -5057,9 +6090,9 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 8, - kernel_config<256, 17>, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<16, 4, 256, 256, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -5076,9 +6109,9 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 8, - kernel_config<256, 16>, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<16, 4, 256, 256, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -5097,7 +6130,26 @@ struct default_segmented_radix_sort_config< 0, kernel_config<256, 16>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 1024, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 4>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -5116,7 +6168,7 @@ struct default_segmented_radix_sort_config< 8, kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<32, 2, 256, 4096, 32, 8, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -5133,9 +6185,9 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 8, - kernel_config<256, 17>, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<16, 4, 256, 2048, 64, 8, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -5152,9 +6204,9 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 8, - kernel_config<256, 17>, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 2048, 64, 8, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -5171,9 +6223,9 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 8, - kernel_config<256, 17>, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 1024, 64, 8, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -5190,9 +6242,27 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 0, - kernel_config<256, 17>, + kernel_config<256, 16>, + typename std::conditional<1, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, + DisabledWarpSortConfig>::type, + 1> +{}; + +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_segmented_radix_sort_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : segmented_radix_sort_config< + 8, + 8, + kernel_config<256, 4>, typename std::conditional<1, - WarpSortConfig<16, 4, 256, 4096, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -5210,7 +6280,7 @@ struct default_segmented_radix_sort_config< 8, kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<32, 2, 256, 2048, 16, 4, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -5228,7 +6298,7 @@ struct default_segmented_radix_sort_config< 8, kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<16, 4, 256, 4096, 64, 4, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -5246,7 +6316,7 @@ struct default_segmented_radix_sort_config< 8, kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 4096, 64, 8, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -5265,7 +6335,7 @@ struct default_segmented_radix_sort_config< 8, kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<16, 4, 256, 4096, 64, 8, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; @@ -5281,9 +6351,9 @@ struct default_segmented_radix_sort_config< : segmented_radix_sort_config< 8, 0, - kernel_config<256, 17>, + kernel_config<256, 8>, typename std::conditional<1, - WarpSortConfig<8, 8, 256, 4096, 32, 16, 256>, + WarpSortConfig<8, 4, 256, 64, 16, 8, 256>, DisabledWarpSortConfig>::type, 1> {}; diff --git a/rocprim/include/rocprim/device/detail/config/device_segmented_reduce.hpp b/rocprim/include/rocprim/device/detail/config/device_segmented_reduce.hpp new file mode 100644 index 000000000..6c96529a4 --- /dev/null +++ b/rocprim/include/rocprim/device/detail/config/device_segmented_reduce.hpp @@ -0,0 +1,533 @@ +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_SEGMENTED_REDUCE_HPP_ +#define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_SEGMENTED_REDUCE_HPP_ + +#include "../../../type_traits.hpp" +#include "../device_config_helper.hpp" + +#include + +/* DO NOT EDIT THIS FILE + * This file is automatically generated by `/scripts/autotune/create_optimization.py`. + * so most likely you want to edit rocprim/device/device_(algo)_config.hpp + */ + +/// \addtogroup primitivesmodule_deviceconfigs +/// @{ + +BEGIN_ROCPRIM_NAMESPACE + +namespace detail +{ + +template +struct default_segmented_reduce_config : default_reduce_config_base::type +{}; + +// Based on key_type = double +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx1030), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = float +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx1030), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = rocprim::half +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx1030), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 2))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = rocprim::int128_t +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx1030), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = int64_t +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx1030), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = int +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx1030), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = short +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx1030), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1))>> + : reduce_config<256, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = int8_t +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx1030), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 1))>> + : reduce_config<256, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = double +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx1100), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = float +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx1100), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = rocprim::half +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx1100), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 2))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = rocprim::int128_t +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx1100), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = int64_t +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx1100), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = int +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx1100), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = short +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx1100), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = int8_t +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx1100), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 1))>> + : reduce_config<256, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = double +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx906), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_config<256, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = float +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx906), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = rocprim::half +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx906), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 2))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = rocprim::int128_t +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx906), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = int64_t +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx906), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = int +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx906), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = short +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx906), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = int8_t +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx906), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 1))>> + : reduce_config<256, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = double +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx908), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_config<256, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = float +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx908), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = rocprim::half +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx908), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 2))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = rocprim::int128_t +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx908), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : reduce_config<128, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = int64_t +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx908), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = int +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx908), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = short +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx908), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = int8_t +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx908), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 1))>> + : reduce_config<256, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = double +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx90a), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = float +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx90a), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = rocprim::half +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx90a), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 2))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = rocprim::int128_t +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx90a), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : reduce_config<256, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = int64_t +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx90a), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = int +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx90a), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_config<256, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = short +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx90a), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1))>> + : reduce_config<256, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = int8_t +template +struct default_segmented_reduce_config< + static_cast(target_arch::gfx90a), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 1))>> + : reduce_config<256, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = double +template +struct default_segmented_reduce_config< + static_cast(target_arch::unknown), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_config<256, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = float +template +struct default_segmented_reduce_config< + static_cast(target_arch::unknown), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = rocprim::half +template +struct default_segmented_reduce_config< + static_cast(target_arch::unknown), + key_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 2))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = rocprim::int128_t +template +struct default_segmented_reduce_config< + static_cast(target_arch::unknown), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8))>> + : reduce_config<128, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = int64_t +template +struct default_segmented_reduce_config< + static_cast(target_arch::unknown), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = int +template +struct default_segmented_reduce_config< + static_cast(target_arch::unknown), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = short +template +struct default_segmented_reduce_config< + static_cast(target_arch::unknown), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1))>> + : reduce_config<256, 16, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +// Based on key_type = int8_t +template +struct default_segmented_reduce_config< + static_cast(target_arch::unknown), + key_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(key_type) <= 1))>> + : reduce_config<256, 8, ::rocprim::block_reduce_algorithm::using_warp_reduce> +{}; + +} // end namespace detail + +END_ROCPRIM_NAMESPACE + +/// @} +// end of group primitivesmodule_deviceconfigs + +#endif // ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_SEGMENTED_REDUCE_HPP_ \ No newline at end of file diff --git a/rocprim/include/rocprim/device/detail/config/device_select_flag.hpp b/rocprim/include/rocprim/device/detail/config/device_select_flag.hpp index 3ffd1217c..f17a92893 100644 --- a/rocprim/include/rocprim/device/detail/config/device_select_flag.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_select_flag.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,7 +21,9 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_SELECT_FLAG_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_SELECT_FLAG_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" #include @@ -67,7 +69,17 @@ struct default_select_flag_config< static_cast(target_arch::gfx1030), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 24> + && (sizeof(data_type) <= 2))>> : select_config<512, 16> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_select_flag_config< + static_cast(target_arch::gfx1030), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 4> {}; // Based on data_type = int64_t @@ -133,7 +145,17 @@ struct default_select_flag_config< static_cast(target_arch::gfx1100), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<128, 24> + && (sizeof(data_type) <= 2))>> : select_config<512, 20> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_select_flag_config< + static_cast(target_arch::gfx1100), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<512, 4> {}; // Based on data_type = int64_t @@ -199,7 +221,17 @@ struct default_select_flag_config< static_cast(target_arch::gfx906), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 24> + && (sizeof(data_type) <= 2))>> : select_config<256, 16> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_select_flag_config< + static_cast(target_arch::gfx906), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 3> {}; // Based on data_type = int64_t @@ -265,7 +297,17 @@ struct default_select_flag_config< static_cast(target_arch::gfx908), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 24> + && (sizeof(data_type) <= 2))>> : select_config<256, 16> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_select_flag_config< + static_cast(target_arch::gfx908), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<128, 4> {}; // Based on data_type = int64_t @@ -310,64 +352,74 @@ struct default_select_flag_config< // Based on data_type = double template struct default_select_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<128, 7> + && (sizeof(data_type) > 4))>> : select_config<192, 4> {}; // Based on data_type = float template struct default_select_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<512, 12> + && (sizeof(data_type) > 2))>> : select_config<256, 7> {}; // Based on data_type = rocprim::half template struct default_select_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 24> + && (sizeof(data_type) <= 2))>> : select_config<256, 20> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_select_flag_config< + static_cast(target_arch::gfx90a), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 3> {}; // Based on data_type = int64_t template struct default_select_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<128, 7> + : select_config<192, 4> {}; // Based on data_type = int template struct default_select_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<512, 12> + : select_config<256, 7> {}; // Based on data_type = short template struct default_select_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 24> + : select_config<256, 14> {}; // Based on data_type = int8_t template struct default_select_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 1))>> : select_config<256, 24> @@ -376,64 +428,74 @@ struct default_select_flag_config< // Based on data_type = double template struct default_select_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<192, 4> + && (sizeof(data_type) > 4))>> : select_config<128, 7> {}; // Based on data_type = float template struct default_select_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<256, 7> + && (sizeof(data_type) > 2))>> : select_config<512, 12> {}; // Based on data_type = rocprim::half template struct default_select_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 20> + && (sizeof(data_type) <= 2))>> : select_config<256, 16> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_select_flag_config< + static_cast(target_arch::unknown), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<128, 4> {}; // Based on data_type = int64_t template struct default_select_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<192, 4> + : select_config<128, 7> {}; // Based on data_type = int template struct default_select_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<256, 7> + : select_config<512, 12> {}; // Based on data_type = short template struct default_select_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 14> + : select_config<256, 24> {}; // Based on data_type = int8_t template struct default_select_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 1))>> : select_config<256, 24> @@ -466,6 +528,16 @@ struct default_select_flag_config< && (sizeof(data_type) <= 2))>> : select_config<512, 28> {}; +// Based on data_type = rocprim::int128_t +template +struct default_select_flag_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<384, 4> +{}; + // Based on data_type = int64_t template struct default_select_flag_config< @@ -493,7 +565,7 @@ struct default_select_flag_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<512, 30> + : select_config<512, 28> {}; // Based on data_type = int8_t diff --git a/rocprim/include/rocprim/device/detail/config/device_select_predicate.hpp b/rocprim/include/rocprim/device/detail/config/device_select_predicate.hpp index 84c2e635c..2d7c464cd 100644 --- a/rocprim/include/rocprim/device/detail/config/device_select_predicate.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_select_predicate.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,7 +21,9 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_SELECT_PREDICATE_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_SELECT_PREDICATE_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" #include @@ -70,6 +72,16 @@ struct default_select_predicate_config< && (sizeof(data_type) <= 2))>> : select_config<256, 30> {}; +// Based on data_type = rocprim::int128_t +template +struct default_select_predicate_config< + static_cast(target_arch::gfx1030), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 4> +{}; + // Based on data_type = int64_t template struct default_select_predicate_config< @@ -133,7 +145,17 @@ struct default_select_predicate_config< static_cast(target_arch::gfx1100), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 30> + && (sizeof(data_type) <= 2))>> : select_config<192, 22> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_select_predicate_config< + static_cast(target_arch::gfx1100), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<512, 4> {}; // Based on data_type = int64_t @@ -199,7 +221,17 @@ struct default_select_predicate_config< static_cast(target_arch::gfx906), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 26> + && (sizeof(data_type) <= 2))>> : select_config<256, 24> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_select_predicate_config< + static_cast(target_arch::gfx906), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 3> {}; // Based on data_type = int64_t @@ -265,7 +297,17 @@ struct default_select_predicate_config< static_cast(target_arch::gfx908), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 26> + && (sizeof(data_type) <= 2))>> : select_config<256, 24> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_select_predicate_config< + static_cast(target_arch::gfx908), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 3> {}; // Based on data_type = int64_t @@ -310,64 +352,74 @@ struct default_select_predicate_config< // Based on data_type = double template struct default_select_predicate_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<128, 7> + && (sizeof(data_type) > 4))>> : select_config<128, 6> {}; // Based on data_type = float template struct default_select_predicate_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<256, 15> + && (sizeof(data_type) > 2))>> : select_config<512, 10> {}; // Based on data_type = rocprim::half template struct default_select_predicate_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 26> + && (sizeof(data_type) <= 2))>> : select_config<256, 20> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_select_predicate_config< + static_cast(target_arch::gfx90a), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 3> {}; // Based on data_type = int64_t template struct default_select_predicate_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<128, 7> + : select_config<512, 5> {}; // Based on data_type = int template struct default_select_predicate_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<256, 14> + : select_config<512, 10> {}; // Based on data_type = short template struct default_select_predicate_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 26> + : select_config<256, 20> {}; // Based on data_type = int8_t template struct default_select_predicate_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 1))>> : select_config<256, 24> @@ -376,64 +428,74 @@ struct default_select_predicate_config< // Based on data_type = double template struct default_select_predicate_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<128, 6> + && (sizeof(data_type) > 4))>> : select_config<128, 7> {}; // Based on data_type = float template struct default_select_predicate_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<512, 10> + && (sizeof(data_type) > 2))>> : select_config<256, 15> {}; // Based on data_type = rocprim::half template struct default_select_predicate_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 20> + && (sizeof(data_type) <= 2))>> : select_config<256, 24> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_select_predicate_config< + static_cast(target_arch::unknown), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 3> {}; // Based on data_type = int64_t template struct default_select_predicate_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<512, 5> + : select_config<128, 7> {}; // Based on data_type = int template struct default_select_predicate_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<512, 10> + : select_config<256, 14> {}; // Based on data_type = short template struct default_select_predicate_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 20> + : select_config<256, 26> {}; // Based on data_type = int8_t template struct default_select_predicate_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 1))>> : select_config<256, 24> @@ -466,6 +528,16 @@ struct default_select_predicate_config< && (sizeof(data_type) <= 2))>> : select_config<512, 30> {}; +// Based on data_type = rocprim::int128_t +template +struct default_select_predicate_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<384, 4> +{}; + // Based on data_type = int64_t template struct default_select_predicate_config< @@ -493,7 +565,7 @@ struct default_select_predicate_config< data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<512, 28> + : select_config<512, 30> {}; // Based on data_type = int8_t diff --git a/rocprim/include/rocprim/device/detail/config/device_select_predicated_flag.hpp b/rocprim/include/rocprim/device/detail/config/device_select_predicated_flag.hpp index 79acf6b67..d61a3433e 100644 --- a/rocprim/include/rocprim/device/detail/config/device_select_predicated_flag.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_select_predicated_flag.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,7 +21,9 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_SELECT_PREDICATED_FLAG_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_SELECT_PREDICATED_FLAG_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" #include @@ -43,6 +45,17 @@ template::type {}; +// Based on data_type = double, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1030), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) + && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 16) + && (sizeof(flag_type) > 8))>> : select_config<256, 4> +{}; + // Based on data_type = double, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -87,6 +100,17 @@ struct default_select_predicated_flag_config< : select_config<512, 4> {}; +// Based on data_type = float, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1030), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) + && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 16) + && (sizeof(flag_type) > 8))>> : select_config<256, 4> +{}; + // Based on data_type = float, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -131,6 +155,17 @@ struct default_select_predicated_flag_config< : select_config<512, 4> {}; +// Based on data_type = rocprim::half, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1030), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<256, 4> +{}; + // Based on data_type = rocprim::half, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -139,7 +174,7 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(flag_type) <= 8) && (sizeof(flag_type) > 4))>> - : select_config<512, 4> + : select_config<512, 8> {}; // Based on data_type = rocprim::half, flag_type = int @@ -161,7 +196,7 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> - : select_config<512, 8> + : select_config<192, 25> {}; // Based on data_type = rocprim::half, flag_type = int8_t @@ -174,6 +209,77 @@ struct default_select_predicated_flag_config< && (sizeof(flag_type) <= 1))>> : select_config<512, 16> {}; +// Based on data_type = rocprim::int128_t, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1030), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<256, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = int64_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1030), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 8) && (sizeof(flag_type) > 4))>> + : select_config<256, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = int +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1030), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> + : select_config<256, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = short +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1030), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> + : select_config<256, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = int8_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1030), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 1))>> : select_config<256, 4> +{}; + +// Based on data_type = int64_t, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1030), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<256, 4> +{}; + // Based on data_type = int64_t, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -221,6 +327,18 @@ struct default_select_predicated_flag_config< && (sizeof(flag_type) <= 1))>> : select_config<512, 4> {}; +// Based on data_type = int, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1030), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<256, 4> +{}; + // Based on data_type = int, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -268,6 +386,18 @@ struct default_select_predicated_flag_config< && (sizeof(flag_type) <= 1))>> : select_config<512, 4> {}; +// Based on data_type = short, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1030), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<256, 4> +{}; + // Based on data_type = short, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -315,6 +445,17 @@ struct default_select_predicated_flag_config< && (sizeof(flag_type) <= 1))>> : select_config<512, 16> {}; +// Based on data_type = int8_t, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1030), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 1) && (sizeof(flag_type) <= 16) + && (sizeof(flag_type) > 8))>> : select_config<256, 4> +{}; + // Based on data_type = int8_t, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -359,6 +500,17 @@ struct default_select_predicated_flag_config< : select_config<512, 16> {}; +// Based on data_type = double, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1100), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) + && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 16) + && (sizeof(flag_type) > 8))>> : select_config<512, 4> +{}; + // Based on data_type = double, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -403,6 +555,17 @@ struct default_select_predicated_flag_config< : select_config<512, 8> {}; +// Based on data_type = float, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1100), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) + && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 16) + && (sizeof(flag_type) > 8))>> : select_config<512, 4> +{}; + // Based on data_type = float, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -447,6 +610,17 @@ struct default_select_predicated_flag_config< : select_config<192, 12> {}; +// Based on data_type = rocprim::half, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1100), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<512, 4> +{}; + // Based on data_type = rocprim::half, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -477,7 +651,7 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> - : select_config<384, 18> + : select_config<512, 16> {}; // Based on data_type = rocprim::half, flag_type = int8_t @@ -487,7 +661,78 @@ struct default_select_predicated_flag_config< data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) - && (sizeof(flag_type) <= 1))>> : select_config<512, 20> + && (sizeof(flag_type) <= 1))>> : select_config<192, 22> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1100), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<512, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = int64_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1100), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 8) && (sizeof(flag_type) > 4))>> + : select_config<512, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = int +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1100), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> + : select_config<512, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = short +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1100), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> + : select_config<512, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = int8_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1100), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 1))>> : select_config<512, 4> +{}; + +// Based on data_type = int64_t, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1100), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<512, 4> {}; // Based on data_type = int64_t, flag_type = int64_t @@ -537,6 +782,18 @@ struct default_select_predicated_flag_config< && (sizeof(flag_type) <= 1))>> : select_config<512, 8> {}; +// Based on data_type = int, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1100), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<512, 4> +{}; + // Based on data_type = int, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -584,6 +841,18 @@ struct default_select_predicated_flag_config< && (sizeof(flag_type) <= 1))>> : select_config<192, 12> {}; +// Based on data_type = short, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1100), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<512, 4> +{}; + // Based on data_type = short, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -631,6 +900,17 @@ struct default_select_predicated_flag_config< && (sizeof(flag_type) <= 1))>> : select_config<512, 20> {}; +// Based on data_type = int8_t, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx1100), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 1) && (sizeof(flag_type) <= 16) + && (sizeof(flag_type) > 8))>> : select_config<512, 4> +{}; + // Based on data_type = int8_t, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -675,6 +955,17 @@ struct default_select_predicated_flag_config< : select_config<512, 20> {}; +// Based on data_type = double, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx906), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) + && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 16) + && (sizeof(flag_type) > 8))>> : select_config<192, 4> +{}; + // Based on data_type = double, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -719,6 +1010,17 @@ struct default_select_predicated_flag_config< : select_config<256, 7> {}; +// Based on data_type = float, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx906), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) + && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 16) + && (sizeof(flag_type) > 8))>> : select_config<192, 4> +{}; + // Based on data_type = float, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -763,6 +1065,17 @@ struct default_select_predicated_flag_config< : select_config<256, 11> {}; +// Based on data_type = rocprim::half, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx906), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<192, 4> +{}; + // Based on data_type = rocprim::half, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -806,39 +1119,110 @@ struct default_select_predicated_flag_config< && (sizeof(flag_type) <= 1))>> : select_config<256, 18> {}; -// Based on data_type = int64_t, flag_type = int64_t +// Based on data_type = rocprim::int128_t, flag_type = rocprim::int128_t template struct default_select_predicated_flag_config< static_cast(target_arch::gfx906), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) - && (sizeof(flag_type) <= 8) && (sizeof(flag_type) > 4))>> - : select_config<192, 7> + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<256, 4> {}; -// Based on data_type = int64_t, flag_type = int +// Based on data_type = rocprim::int128_t, flag_type = int64_t template struct default_select_predicated_flag_config< static_cast(target_arch::gfx906), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) - && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> - : select_config<192, 7> + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 8) && (sizeof(flag_type) > 4))>> + : select_config<128, 4> {}; -// Based on data_type = int64_t, flag_type = short +// Based on data_type = rocprim::int128_t, flag_type = int template struct default_select_predicated_flag_config< static_cast(target_arch::gfx906), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) - && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> + : select_config<192, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = short +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx906), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> + : select_config<192, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = int8_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx906), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 1))>> : select_config<256, 3> +{}; + +// Based on data_type = int64_t, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx906), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<192, 4> +{}; + +// Based on data_type = int64_t, flag_type = int64_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx906), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) + && (sizeof(flag_type) <= 8) && (sizeof(flag_type) > 4))>> + : select_config<192, 7> +{}; + +// Based on data_type = int64_t, flag_type = int +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx906), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) + && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> + : select_config<192, 7> +{}; + +// Based on data_type = int64_t, flag_type = short +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx906), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) + && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> : select_config<256, 7> {}; @@ -853,6 +1237,18 @@ struct default_select_predicated_flag_config< && (sizeof(flag_type) <= 1))>> : select_config<256, 7> {}; +// Based on data_type = int, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx906), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<192, 4> +{}; + // Based on data_type = int, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -900,6 +1296,18 @@ struct default_select_predicated_flag_config< && (sizeof(flag_type) <= 1))>> : select_config<256, 11> {}; +// Based on data_type = short, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx906), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<192, 4> +{}; + // Based on data_type = short, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -947,6 +1355,17 @@ struct default_select_predicated_flag_config< && (sizeof(flag_type) <= 1))>> : select_config<256, 18> {}; +// Based on data_type = int8_t, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx906), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 1) && (sizeof(flag_type) <= 16) + && (sizeof(flag_type) > 8))>> : select_config<192, 4> +{}; + // Based on data_type = int8_t, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -991,6 +1410,17 @@ struct default_select_predicated_flag_config< : select_config<192, 24> {}; +// Based on data_type = double, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx908), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) + && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 16) + && (sizeof(flag_type) > 8))>> : select_config<192, 4> +{}; + // Based on data_type = double, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -1035,6 +1465,17 @@ struct default_select_predicated_flag_config< : select_config<128, 7> {}; +// Based on data_type = float, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx908), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) + && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 16) + && (sizeof(flag_type) > 8))>> : select_config<192, 4> +{}; + // Based on data_type = float, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -1079,6 +1520,17 @@ struct default_select_predicated_flag_config< : select_config<256, 11> {}; +// Based on data_type = rocprim::half, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx908), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<192, 4> +{}; + // Based on data_type = rocprim::half, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -1119,7 +1571,78 @@ struct default_select_predicated_flag_config< data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) - && (sizeof(flag_type) <= 1))>> : select_config<256, 18> + && (sizeof(flag_type) <= 1))>> : select_config<256, 24> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx908), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<256, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = int64_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx908), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 8) && (sizeof(flag_type) > 4))>> + : select_config<128, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = int +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx908), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> + : select_config<128, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = short +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx908), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> + : select_config<128, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = int8_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx908), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 1))>> : select_config<128, 4> +{}; + +// Based on data_type = int64_t, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx908), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<192, 4> {}; // Based on data_type = int64_t, flag_type = int64_t @@ -1169,6 +1692,18 @@ struct default_select_predicated_flag_config< && (sizeof(flag_type) <= 1))>> : select_config<128, 7> {}; +// Based on data_type = int, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx908), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<192, 4> +{}; + // Based on data_type = int, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -1216,6 +1751,18 @@ struct default_select_predicated_flag_config< && (sizeof(flag_type) <= 1))>> : select_config<256, 11> {}; +// Based on data_type = short, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx908), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<192, 4> +{}; + // Based on data_type = short, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -1263,6 +1810,17 @@ struct default_select_predicated_flag_config< && (sizeof(flag_type) <= 1))>> : select_config<256, 18> {}; +// Based on data_type = int8_t, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx908), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 1) && (sizeof(flag_type) <= 16) + && (sizeof(flag_type) > 8))>> : select_config<192, 4> +{}; + // Based on data_type = int8_t, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -1307,235 +1865,363 @@ struct default_select_predicated_flag_config< : select_config<256, 24> {}; +// Based on data_type = double, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx90a), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) + && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 16) + && (sizeof(flag_type) > 8))>> : select_config<128, 4> +{}; + // Based on data_type = double, flag_type = int64_t template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 8) - && (sizeof(flag_type) > 4))>> : select_config<128, 6> + && (sizeof(flag_type) > 4))>> : select_config<192, 4> {}; // Based on data_type = double, flag_type = int template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 4) - && (sizeof(flag_type) > 2))>> : select_config<128, 6> + && (sizeof(flag_type) > 2))>> : select_config<512, 5> {}; // Based on data_type = double, flag_type = short template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 2) - && (sizeof(flag_type) > 1))>> : select_config<128, 6> + && (sizeof(flag_type) > 1))>> : select_config<192, 4> {}; // Based on data_type = double, flag_type = int8_t template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 1))>> - : select_config<128, 7> + : select_config<192, 4> +{}; + +// Based on data_type = float, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx90a), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) + && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 16) + && (sizeof(flag_type) > 8))>> : select_config<192, 4> {}; // Based on data_type = float, flag_type = int64_t template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 8) - && (sizeof(flag_type) > 4))>> : select_config<256, 6> + && (sizeof(flag_type) > 4))>> : select_config<192, 5> {}; // Based on data_type = float, flag_type = int template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 4) - && (sizeof(flag_type) > 2))>> : select_config<128, 11> + && (sizeof(flag_type) > 2))>> : select_config<256, 6> {}; // Based on data_type = float, flag_type = short template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 2) - && (sizeof(flag_type) > 1))>> : select_config<256, 10> + && (sizeof(flag_type) > 1))>> : select_config<192, 10> {}; // Based on data_type = float, flag_type = int8_t template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 1))>> - : select_config<256, 11> + : select_config<256, 7> {}; -// Based on data_type = rocprim::half, flag_type = int64_t +// Based on data_type = rocprim::half, flag_type = rocprim::int128_t template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) - && (sizeof(flag_type) <= 8) && (sizeof(flag_type) > 4))>> - : select_config<256, 6> + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<192, 4> {}; -// Based on data_type = rocprim::half, flag_type = int +// Based on data_type = rocprim::half, flag_type = int64_t template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) + && (sizeof(flag_type) <= 8) && (sizeof(flag_type) > 4))>> + : select_config<256, 6> +{}; + +// Based on data_type = rocprim::half, flag_type = int +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> - : select_config<256, 12> + : select_config<256, 10> {}; // Based on data_type = rocprim::half, flag_type = short template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> - : select_config<512, 18> + : select_config<256, 20> {}; // Based on data_type = rocprim::half, flag_type = int8_t template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) - && (sizeof(flag_type) <= 1))>> : select_config<256, 18> + && (sizeof(flag_type) <= 1))>> : select_config<256, 20> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx90a), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<128, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = int64_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx90a), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 8) && (sizeof(flag_type) > 4))>> + : select_config<128, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = int +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx90a), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> + : select_config<128, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = short +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx90a), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> + : select_config<128, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = int8_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx90a), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 1))>> : select_config<128, 4> +{}; + +// Based on data_type = int64_t, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx90a), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<128, 4> {}; // Based on data_type = int64_t, flag_type = int64_t template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 8) && (sizeof(flag_type) > 4))>> - : select_config<128, 6> + : select_config<192, 4> {}; // Based on data_type = int64_t, flag_type = int template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> - : select_config<128, 6> + : select_config<192, 4> {}; // Based on data_type = int64_t, flag_type = short template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> - : select_config<128, 6> + : select_config<192, 4> {}; // Based on data_type = int64_t, flag_type = int8_t template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) - && (sizeof(flag_type) <= 1))>> : select_config<128, 7> + && (sizeof(flag_type) <= 1))>> : select_config<192, 5> +{}; + +// Based on data_type = int, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx90a), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<192, 4> {}; // Based on data_type = int, flag_type = int64_t template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 8) && (sizeof(flag_type) > 4))>> - : select_config<256, 6> + : select_config<192, 5> {}; // Based on data_type = int, flag_type = int template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> - : select_config<128, 11> + : select_config<256, 6> {}; // Based on data_type = int, flag_type = short template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> - : select_config<256, 10> + : select_config<192, 10> {}; // Based on data_type = int, flag_type = int8_t template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) - && (sizeof(flag_type) <= 1))>> : select_config<256, 11> + && (sizeof(flag_type) <= 1))>> : select_config<192, 10> +{}; + +// Based on data_type = short, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx90a), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<192, 4> {}; // Based on data_type = short, flag_type = int64_t template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -1547,75 +2233,86 @@ struct default_select_predicated_flag_config< // Based on data_type = short, flag_type = int template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1) && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> - : select_config<256, 12> + : select_config<256, 10> {}; // Based on data_type = short, flag_type = short template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1) && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> - : select_config<512, 18> + : select_config<256, 20> {}; // Based on data_type = short, flag_type = int8_t template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1) - && (sizeof(flag_type) <= 1))>> : select_config<256, 18> + && (sizeof(flag_type) <= 1))>> : select_config<256, 20> +{}; + +// Based on data_type = int8_t, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx90a), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 1) && (sizeof(flag_type) <= 16) + && (sizeof(flag_type) > 8))>> : select_config<192, 4> {}; // Based on data_type = int8_t, flag_type = int64_t template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 1) && (sizeof(flag_type) <= 8) - && (sizeof(flag_type) > 4))>> : select_config<256, 6> + && (sizeof(flag_type) > 4))>> : select_config<192, 5> {}; // Based on data_type = int8_t, flag_type = int template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 1) && (sizeof(flag_type) <= 4) - && (sizeof(flag_type) > 2))>> : select_config<256, 11> + && (sizeof(flag_type) > 2))>> : select_config<256, 12> {}; // Based on data_type = int8_t, flag_type = short template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 1) && (sizeof(flag_type) <= 2) - && (sizeof(flag_type) > 1))>> : select_config<256, 24> + && (sizeof(flag_type) > 1))>> : select_config<192, 20> {}; // Based on data_type = int8_t, flag_type = int8_t template struct default_select_predicated_flag_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -1623,98 +2320,131 @@ struct default_select_predicated_flag_config< : select_config<256, 24> {}; +// Based on data_type = double, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::unknown), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) + && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 16) + && (sizeof(flag_type) > 8))>> : select_config<192, 4> +{}; + // Based on data_type = double, flag_type = int64_t template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 8) - && (sizeof(flag_type) > 4))>> : select_config<192, 4> + && (sizeof(flag_type) > 4))>> : select_config<128, 6> {}; // Based on data_type = double, flag_type = int template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 4) - && (sizeof(flag_type) > 2))>> : select_config<512, 5> + && (sizeof(flag_type) > 2))>> : select_config<128, 6> {}; // Based on data_type = double, flag_type = short template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 2) - && (sizeof(flag_type) > 1))>> : select_config<192, 4> + && (sizeof(flag_type) > 1))>> : select_config<128, 6> {}; // Based on data_type = double, flag_type = int8_t template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 1))>> - : select_config<192, 4> + : select_config<128, 7> +{}; + +// Based on data_type = float, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::unknown), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) + && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 16) + && (sizeof(flag_type) > 8))>> : select_config<192, 4> {}; // Based on data_type = float, flag_type = int64_t template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 8) - && (sizeof(flag_type) > 4))>> : select_config<192, 5> + && (sizeof(flag_type) > 4))>> : select_config<256, 6> {}; // Based on data_type = float, flag_type = int template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 4) - && (sizeof(flag_type) > 2))>> : select_config<256, 6> + && (sizeof(flag_type) > 2))>> : select_config<128, 11> {}; // Based on data_type = float, flag_type = short template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 2) - && (sizeof(flag_type) > 1))>> : select_config<192, 10> + && (sizeof(flag_type) > 1))>> : select_config<256, 10> {}; // Based on data_type = float, flag_type = int8_t template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 1))>> - : select_config<256, 7> + : select_config<256, 11> +{}; + +// Based on data_type = rocprim::half, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::unknown), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<192, 4> {}; // Based on data_type = rocprim::half, flag_type = int64_t template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) @@ -1725,133 +2455,228 @@ struct default_select_predicated_flag_config< // Based on data_type = rocprim::half, flag_type = int template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> - : select_config<256, 10> + : select_config<256, 12> {}; // Based on data_type = rocprim::half, flag_type = short template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> - : select_config<256, 18> + : select_config<512, 18> {}; // Based on data_type = rocprim::half, flag_type = int8_t template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) - && (sizeof(flag_type) <= 1))>> : select_config<256, 20> + && (sizeof(flag_type) <= 1))>> : select_config<256, 24> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::unknown), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<256, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = int64_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::unknown), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 8) && (sizeof(flag_type) > 4))>> + : select_config<128, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = int +template +struct default_select_predicated_flag_config< + static_cast(target_arch::unknown), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> + : select_config<128, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = short +template +struct default_select_predicated_flag_config< + static_cast(target_arch::unknown), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> + : select_config<128, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = int8_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::unknown), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 1))>> : select_config<128, 4> +{}; + +// Based on data_type = int64_t, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::unknown), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<192, 4> {}; // Based on data_type = int64_t, flag_type = int64_t template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 8) && (sizeof(flag_type) > 4))>> - : select_config<192, 4> + : select_config<128, 6> {}; // Based on data_type = int64_t, flag_type = int template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> - : select_config<192, 4> + : select_config<128, 6> {}; // Based on data_type = int64_t, flag_type = short template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> - : select_config<192, 4> + : select_config<128, 6> {}; // Based on data_type = int64_t, flag_type = int8_t template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) - && (sizeof(flag_type) <= 1))>> : select_config<192, 5> + && (sizeof(flag_type) <= 1))>> : select_config<128, 7> +{}; + +// Based on data_type = int, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::unknown), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<192, 4> {}; // Based on data_type = int, flag_type = int64_t template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 8) && (sizeof(flag_type) > 4))>> - : select_config<192, 5> + : select_config<256, 6> {}; // Based on data_type = int, flag_type = int template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> - : select_config<256, 6> + : select_config<128, 11> {}; // Based on data_type = int, flag_type = short template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> - : select_config<192, 10> + : select_config<256, 10> {}; // Based on data_type = int, flag_type = int8_t template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) - && (sizeof(flag_type) <= 1))>> : select_config<192, 10> + && (sizeof(flag_type) <= 1))>> : select_config<256, 11> +{}; + +// Based on data_type = short, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::unknown), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<192, 4> {}; // Based on data_type = short, flag_type = int64_t template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -1863,75 +2688,86 @@ struct default_select_predicated_flag_config< // Based on data_type = short, flag_type = int template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1) && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> - : select_config<256, 10> + : select_config<256, 12> {}; // Based on data_type = short, flag_type = short template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1) && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> - : select_config<256, 20> + : select_config<512, 18> {}; // Based on data_type = short, flag_type = int8_t template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1) - && (sizeof(flag_type) <= 1))>> : select_config<256, 20> + && (sizeof(flag_type) <= 1))>> : select_config<256, 18> +{}; + +// Based on data_type = int8_t, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::unknown), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 1) && (sizeof(flag_type) <= 16) + && (sizeof(flag_type) > 8))>> : select_config<192, 4> {}; // Based on data_type = int8_t, flag_type = int64_t template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 1) && (sizeof(flag_type) <= 8) - && (sizeof(flag_type) > 4))>> : select_config<192, 5> + && (sizeof(flag_type) > 4))>> : select_config<256, 6> {}; // Based on data_type = int8_t, flag_type = int template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 1) && (sizeof(flag_type) <= 4) - && (sizeof(flag_type) > 2))>> : select_config<256, 12> + && (sizeof(flag_type) > 2))>> : select_config<256, 11> {}; // Based on data_type = int8_t, flag_type = short template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 1) && (sizeof(flag_type) <= 2) - && (sizeof(flag_type) > 1))>> : select_config<192, 20> + && (sizeof(flag_type) > 1))>> : select_config<256, 24> {}; // Based on data_type = int8_t, flag_type = int8_t template struct default_select_predicated_flag_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -1939,6 +2775,17 @@ struct default_select_predicated_flag_config< : select_config<256, 24> {}; +// Based on data_type = double, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) + && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 16) + && (sizeof(flag_type) > 8))>> : select_config<384, 4> +{}; + // Based on data_type = double, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -1947,7 +2794,7 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 8) - && (sizeof(flag_type) > 4))>> : select_config<512, 5> + && (sizeof(flag_type) > 4))>> : select_config<512, 7> {}; // Based on data_type = double, flag_type = int @@ -1983,6 +2830,17 @@ struct default_select_predicated_flag_config< : select_config<512, 7> {}; +// Based on data_type = float, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) + && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 16) + && (sizeof(flag_type) > 8))>> : select_config<384, 4> +{}; + // Based on data_type = float, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -2013,7 +2871,7 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) && (sizeof(flag_type) <= 2) - && (sizeof(flag_type) > 1))>> : select_config<512, 14> + && (sizeof(flag_type) > 1))>> : select_config<512, 15> {}; // Based on data_type = float, flag_type = int8_t @@ -2027,6 +2885,17 @@ struct default_select_predicated_flag_config< : select_config<512, 15> {}; +// Based on data_type = rocprim::half, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<384, 4> +{}; + // Based on data_type = rocprim::half, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -2067,7 +2936,78 @@ struct default_select_predicated_flag_config< data_type, flag_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) - && (sizeof(flag_type) <= 1))>> : select_config<512, 20> + && (sizeof(flag_type) <= 1))>> : select_config<512, 28> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<384, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = int64_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 8) && (sizeof(flag_type) > 4))>> + : select_config<384, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = int +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 4) && (sizeof(flag_type) > 2))>> + : select_config<384, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = short +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 2) && (sizeof(flag_type) > 1))>> + : select_config<384, 4> +{}; + +// Based on data_type = rocprim::int128_t, flag_type = int8_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8) + && (sizeof(flag_type) <= 1))>> : select_config<384, 4> +{}; + +// Based on data_type = int64_t, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<384, 4> {}; // Based on data_type = int64_t, flag_type = int64_t @@ -2079,7 +3019,7 @@ struct default_select_predicated_flag_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) && (sizeof(flag_type) <= 8) && (sizeof(flag_type) > 4))>> - : select_config<512, 5> + : select_config<512, 7> {}; // Based on data_type = int64_t, flag_type = int @@ -2114,7 +3054,19 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4) - && (sizeof(flag_type) <= 1))>> : select_config<512, 5> + && (sizeof(flag_type) <= 1))>> : select_config<512, 7> +{}; + +// Based on data_type = int, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<384, 4> {}; // Based on data_type = int, flag_type = int64_t @@ -2164,6 +3116,18 @@ struct default_select_predicated_flag_config< && (sizeof(flag_type) <= 1))>> : select_config<512, 15> {}; +// Based on data_type = short, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1) + && (sizeof(flag_type) <= 16) && (sizeof(flag_type) > 8))>> + : select_config<384, 4> +{}; + // Based on data_type = short, flag_type = int64_t template struct default_select_predicated_flag_config< @@ -2208,7 +3172,18 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1) - && (sizeof(flag_type) <= 1))>> : select_config<512, 20> + && (sizeof(flag_type) <= 1))>> : select_config<512, 28> +{}; + +// Based on data_type = int8_t, flag_type = rocprim::int128_t +template +struct default_select_predicated_flag_config< + static_cast(target_arch::gfx942), + data_type, + flag_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 1) && (sizeof(flag_type) <= 16) + && (sizeof(flag_type) > 8))>> : select_config<384, 4> {}; // Based on data_type = int8_t, flag_type = int64_t @@ -2241,7 +3216,7 @@ struct default_select_predicated_flag_config< flag_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 1) && (sizeof(flag_type) <= 2) - && (sizeof(flag_type) > 1))>> : select_config<512, 20> + && (sizeof(flag_type) > 1))>> : select_config<512, 28> {}; // Based on data_type = int8_t, flag_type = int8_t diff --git a/rocprim/include/rocprim/device/detail/config/device_select_unique.hpp b/rocprim/include/rocprim/device/detail/config/device_select_unique.hpp index 2ed70b13c..6653818e7 100644 --- a/rocprim/include/rocprim/device/detail/config/device_select_unique.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_select_unique.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,7 +21,9 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_SELECT_UNIQUE_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_SELECT_UNIQUE_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" #include @@ -67,7 +69,17 @@ struct default_select_unique_config< static_cast(target_arch::gfx1030), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<384, 18> + && (sizeof(data_type) <= 2))>> : select_config<512, 12> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_select_unique_config< + static_cast(target_arch::gfx1030), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 4> {}; // Based on data_type = int64_t @@ -133,7 +145,17 @@ struct default_select_unique_config< static_cast(target_arch::gfx1100), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<128, 20> + && (sizeof(data_type) <= 2))>> : select_config<384, 22> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_select_unique_config< + static_cast(target_arch::gfx1100), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<512, 4> {}; // Based on data_type = int64_t @@ -199,7 +221,17 @@ struct default_select_unique_config< static_cast(target_arch::gfx906), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<192, 18> + && (sizeof(data_type) <= 2))>> : select_config<256, 18> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_select_unique_config< + static_cast(target_arch::gfx906), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 4> {}; // Based on data_type = int64_t @@ -268,6 +300,16 @@ struct default_select_unique_config< && (sizeof(data_type) <= 2))>> : select_config<256, 30> {}; +// Based on data_type = rocprim::int128_t +template +struct default_select_unique_config< + static_cast(target_arch::gfx908), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<128, 4> +{}; + // Based on data_type = int64_t template struct default_select_unique_config< @@ -310,133 +352,153 @@ struct default_select_unique_config< // Based on data_type = double template struct default_select_unique_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<512, 7> + && (sizeof(data_type) > 4))>> : select_config<256, 5> {}; // Based on data_type = float template struct default_select_unique_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<512, 11> + && (sizeof(data_type) > 2))>> : select_config<256, 10> {}; // Based on data_type = rocprim::half template struct default_select_unique_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 30> + && (sizeof(data_type) <= 2))>> : select_config<256, 22> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_select_unique_config< + static_cast(target_arch::gfx90a), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<256, 3> {}; // Based on data_type = int64_t template struct default_select_unique_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<512, 7> + : select_config<256, 5> {}; // Based on data_type = int template struct default_select_unique_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<512, 11> + : select_config<256, 10> {}; // Based on data_type = short template struct default_select_unique_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 30> + : select_config<256, 22> {}; // Based on data_type = int8_t template struct default_select_unique_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 1))>> : select_config<256, 28> + && (sizeof(data_type) <= 1))>> : select_config<128, 32> {}; // Based on data_type = double template struct default_select_unique_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<256, 5> + && (sizeof(data_type) > 4))>> : select_config<512, 7> {}; // Based on data_type = float template struct default_select_unique_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) - && (sizeof(data_type) > 2))>> : select_config<256, 10> + && (sizeof(data_type) > 2))>> : select_config<512, 11> {}; // Based on data_type = rocprim::half template struct default_select_unique_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 2))>> : select_config<256, 22> + && (sizeof(data_type) <= 2))>> : select_config<256, 30> +{}; + +// Based on data_type = rocprim::int128_t +template +struct default_select_unique_config< + static_cast(target_arch::unknown), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<128, 4> {}; // Based on data_type = int64_t template struct default_select_unique_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) && (sizeof(data_type) > 4))>> - : select_config<256, 5> + : select_config<512, 7> {}; // Based on data_type = int template struct default_select_unique_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 4) && (sizeof(data_type) > 2))>> - : select_config<256, 10> + : select_config<512, 11> {}; // Based on data_type = short template struct default_select_unique_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 2) && (sizeof(data_type) > 1))>> - : select_config<256, 22> + : select_config<256, 30> {}; // Based on data_type = int8_t template struct default_select_unique_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), data_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(data_type) <= 1))>> : select_config<128, 32> + && (sizeof(data_type) <= 1))>> : select_config<256, 28> {}; // Based on data_type = double @@ -445,7 +507,7 @@ struct default_select_unique_config< static_cast(target_arch::gfx942), data_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(data_type) <= 8) - && (sizeof(data_type) > 4))>> : select_config<512, 5> + && (sizeof(data_type) > 4))>> : select_config<512, 7> {}; // Based on data_type = float @@ -466,6 +528,16 @@ struct default_select_unique_config< && (sizeof(data_type) <= 2))>> : select_config<512, 30> {}; +// Based on data_type = rocprim::int128_t +template +struct default_select_unique_config< + static_cast(target_arch::gfx942), + data_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(data_type) <= 16) && (sizeof(data_type) > 8))>> + : select_config<384, 4> +{}; + // Based on data_type = int64_t template struct default_select_unique_config< diff --git a/rocprim/include/rocprim/device/detail/config/device_select_unique_by_key.hpp b/rocprim/include/rocprim/device/detail/config/device_select_unique_by_key.hpp index 423e88f5d..36c641f00 100644 --- a/rocprim/include/rocprim/device/detail/config/device_select_unique_by_key.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_select_unique_by_key.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,7 +21,9 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_SELECT_UNIQUE_BY_KEY_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_SELECT_UNIQUE_BY_KEY_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" #include @@ -43,6 +45,17 @@ template::type {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<256, 4> +{}; + // Based on key_type = double, value_type = int64_t template struct default_select_unique_by_key_config< @@ -87,6 +100,17 @@ struct default_select_unique_by_key_config< : select_config<512, 6> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<256, 4> +{}; + // Based on key_type = float, value_type = int64_t template struct default_select_unique_by_key_config< @@ -131,6 +155,17 @@ struct default_select_unique_by_key_config< : select_config<512, 8> {}; +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : select_config<256, 4> +{}; + // Based on key_type = rocprim::half, value_type = int64_t template struct default_select_unique_by_key_config< @@ -139,7 +174,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : select_config<512, 6> + : select_config<512, 8> {}; // Based on key_type = rocprim::half, value_type = int @@ -174,6 +209,72 @@ struct default_select_unique_by_key_config< && (sizeof(value_type) <= 1))>> : select_config<384, 14> {}; +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : select_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : select_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : select_config<256, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1))>> + : select_config<256, 4> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<256, 4> +{}; + // Based on key_type = int64_t, value_type = int64_t template struct default_select_unique_by_key_config< @@ -218,6 +319,17 @@ struct default_select_unique_by_key_config< : select_config<512, 6> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<256, 4> +{}; + // Based on key_type = int, value_type = int64_t template struct default_select_unique_by_key_config< @@ -262,6 +374,17 @@ struct default_select_unique_by_key_config< : select_config<512, 8> {}; +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<256, 4> +{}; + // Based on key_type = short, value_type = int64_t template struct default_select_unique_by_key_config< @@ -306,6 +429,17 @@ struct default_select_unique_by_key_config< : select_config<384, 14> {}; +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1030), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : select_config<256, 4> +{}; + // Based on key_type = int8_t, value_type = int64_t template struct default_select_unique_by_key_config< @@ -349,6 +483,17 @@ struct default_select_unique_by_key_config< && (sizeof(value_type) <= 1))>> : select_config<384, 12> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<512, 4> +{}; + // Based on key_type = double, value_type = int64_t template struct default_select_unique_by_key_config< @@ -393,6 +538,17 @@ struct default_select_unique_by_key_config< : select_config<128, 7> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<512, 4> +{}; + // Based on key_type = float, value_type = int64_t template struct default_select_unique_by_key_config< @@ -437,6 +593,17 @@ struct default_select_unique_by_key_config< : select_config<128, 12> {}; +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : select_config<192, 4> +{}; + // Based on key_type = rocprim::half, value_type = int64_t template struct default_select_unique_by_key_config< @@ -456,7 +623,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : select_config<128, 11> + : select_config<512, 16> {}; // Based on key_type = rocprim::half, value_type = short @@ -467,7 +634,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : select_config<128, 12> + : select_config<512, 16> {}; // Based on key_type = rocprim::half, value_type = int8_t @@ -477,7 +644,73 @@ struct default_select_unique_by_key_config< key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(value_type) <= 1))>> : select_config<512, 20> + && (sizeof(value_type) <= 1))>> : select_config<192, 16> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<512, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : select_config<512, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : select_config<512, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : select_config<512, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1))>> + : select_config<512, 4> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<512, 4> {}; // Based on key_type = int64_t, value_type = int64_t @@ -524,6 +757,17 @@ struct default_select_unique_by_key_config< : select_config<128, 7> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<512, 4> +{}; + // Based on key_type = int, value_type = int64_t template struct default_select_unique_by_key_config< @@ -568,6 +812,17 @@ struct default_select_unique_by_key_config< : select_config<128, 12> {}; +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<192, 4> +{}; + // Based on key_type = short, value_type = int64_t template struct default_select_unique_by_key_config< @@ -612,6 +867,17 @@ struct default_select_unique_by_key_config< : select_config<512, 20> {}; +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx1100), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : select_config<192, 4> +{}; + // Based on key_type = int8_t, value_type = int64_t template struct default_select_unique_by_key_config< @@ -655,6 +921,17 @@ struct default_select_unique_by_key_config< && (sizeof(value_type) <= 1))>> : select_config<384, 20> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<192, 4> +{}; + // Based on key_type = double, value_type = int64_t template struct default_select_unique_by_key_config< @@ -699,6 +976,17 @@ struct default_select_unique_by_key_config< : select_config<192, 8> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<192, 4> +{}; + // Based on key_type = float, value_type = int64_t template struct default_select_unique_by_key_config< @@ -743,6 +1031,17 @@ struct default_select_unique_by_key_config< : select_config<192, 16> {}; +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : select_config<192, 4> +{}; + // Based on key_type = rocprim::half, value_type = int64_t template struct default_select_unique_by_key_config< @@ -773,7 +1072,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : select_config<256, 14> + : select_config<192, 14> {}; // Based on key_type = rocprim::half, value_type = int8_t @@ -783,95 +1082,183 @@ struct default_select_unique_by_key_config< key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(value_type) <= 1))>> : select_config<192, 16> + && (sizeof(value_type) <= 1))>> : select_config<256, 16> {}; -// Based on key_type = int64_t, value_type = int64_t +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t template struct default_select_unique_by_key_config< static_cast(target_arch::gfx906), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : select_config<192, 8> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<256, 3> {}; -// Based on key_type = int64_t, value_type = int +// Based on key_type = rocprim::int128_t, value_type = int64_t template struct default_select_unique_by_key_config< static_cast(target_arch::gfx906), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : select_config<256, 6> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : select_config<192, 4> {}; -// Based on key_type = int64_t, value_type = short +// Based on key_type = rocprim::int128_t, value_type = int template struct default_select_unique_by_key_config< static_cast(target_arch::gfx906), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : select_config<192, 7> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : select_config<192, 4> {}; -// Based on key_type = int64_t, value_type = int8_t +// Based on key_type = rocprim::int128_t, value_type = short template struct default_select_unique_by_key_config< static_cast(target_arch::gfx906), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) - && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1))>> - : select_config<192, 8> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : select_config<192, 4> {}; -// Based on key_type = int, value_type = int64_t +// Based on key_type = rocprim::int128_t, value_type = int8_t template struct default_select_unique_by_key_config< static_cast(target_arch::gfx906), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : select_config<256, 6> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1))>> + : select_config<192, 4> {}; -// Based on key_type = int, value_type = int +// Based on key_type = int64_t, value_type = rocprim::int128_t template struct default_select_unique_by_key_config< static_cast(target_arch::gfx906), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : select_config<256, 13> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<192, 4> {}; -// Based on key_type = int, value_type = short +// Based on key_type = int64_t, value_type = int64_t template struct default_select_unique_by_key_config< static_cast(target_arch::gfx906), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : select_config<192, 14> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : select_config<192, 8> {}; -// Based on key_type = int, value_type = int8_t +// Based on key_type = int64_t, value_type = int template struct default_select_unique_by_key_config< static_cast(target_arch::gfx906), key_type, value_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) - && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1))>> - : select_config<192, 16> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : select_config<256, 6> +{}; + +// Based on key_type = int64_t, value_type = short +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : select_config<192, 7> +{}; + +// Based on key_type = int64_t, value_type = int8_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1))>> + : select_config<192, 8> +{}; + +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<192, 4> +{}; + +// Based on key_type = int, value_type = int64_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : select_config<256, 6> +{}; + +// Based on key_type = int, value_type = int +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : select_config<256, 13> +{}; + +// Based on key_type = int, value_type = short +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : select_config<192, 14> +{}; + +// Based on key_type = int, value_type = int8_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1))>> + : select_config<192, 16> +{}; + +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<192, 4> {}; // Based on key_type = short, value_type = int64_t @@ -918,6 +1305,17 @@ struct default_select_unique_by_key_config< : select_config<192, 16> {}; +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx906), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : select_config<192, 4> +{}; + // Based on key_type = int8_t, value_type = int64_t template struct default_select_unique_by_key_config< @@ -961,6 +1359,17 @@ struct default_select_unique_by_key_config< && (sizeof(value_type) <= 1))>> : select_config<192, 17> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<192, 4> +{}; + // Based on key_type = double, value_type = int64_t template struct default_select_unique_by_key_config< @@ -1005,6 +1414,17 @@ struct default_select_unique_by_key_config< : select_config<192, 8> {}; +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<192, 4> +{}; + // Based on key_type = float, value_type = int64_t template struct default_select_unique_by_key_config< @@ -1049,6 +1469,17 @@ struct default_select_unique_by_key_config< : select_config<256, 16> {}; +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : select_config<192, 4> +{}; + // Based on key_type = rocprim::half, value_type = int64_t template struct default_select_unique_by_key_config< @@ -1092,6 +1523,72 @@ struct default_select_unique_by_key_config< && (sizeof(value_type) <= 1))>> : select_config<256, 16> {}; +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<256, 3> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : select_config<192, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : select_config<192, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : select_config<192, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1))>> + : select_config<192, 4> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<192, 4> +{}; + // Based on key_type = int64_t, value_type = int64_t template struct default_select_unique_by_key_config< @@ -1136,6 +1633,17 @@ struct default_select_unique_by_key_config< : select_config<192, 8> {}; +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<192, 4> +{}; + // Based on key_type = int, value_type = int64_t template struct default_select_unique_by_key_config< @@ -1180,6 +1688,17 @@ struct default_select_unique_by_key_config< : select_config<192, 16> {}; +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<192, 4> +{}; + // Based on key_type = short, value_type = int64_t template struct default_select_unique_by_key_config< @@ -1224,6 +1743,17 @@ struct default_select_unique_by_key_config< : select_config<256, 14> {}; +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx908), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : select_config<192, 4> +{}; + // Based on key_type = int8_t, value_type = int64_t template struct default_select_unique_by_key_config< @@ -1267,616 +1797,891 @@ struct default_select_unique_by_key_config< && (sizeof(value_type) <= 1))>> : select_config<192, 17> {}; +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<192, 4> +{}; + // Based on key_type = double, value_type = int64_t template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : select_config<128, 5> + && (sizeof(value_type) > 4))>> : select_config<128, 4> {}; // Based on key_type = double, value_type = int template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : select_config<128, 6> + && (sizeof(value_type) > 2))>> : select_config<192, 4> {}; // Based on key_type = double, value_type = short template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : select_config<128, 6> + && (sizeof(value_type) > 1))>> : select_config<256, 4> {}; // Based on key_type = double, value_type = int8_t template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1))>> - : select_config<192, 8> + : select_config<256, 5> +{}; + +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<192, 4> {}; // Based on key_type = float, value_type = int64_t template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : select_config<128, 6> + && (sizeof(value_type) > 4))>> : select_config<192, 5> {}; // Based on key_type = float, value_type = int template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : select_config<256, 13> + && (sizeof(value_type) > 2))>> : select_config<192, 7> {}; // Based on key_type = float, value_type = short template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : select_config<256, 14> + && (sizeof(value_type) > 1))>> : select_config<192, 10> {}; // Based on key_type = float, value_type = int8_t template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1))>> - : select_config<256, 16> + : select_config<192, 9> +{}; + +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : select_config<192, 4> {}; // Based on key_type = rocprim::half, value_type = int64_t template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : select_config<256, 6> + : select_config<192, 5> {}; // Based on key_type = rocprim::half, value_type = int template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : select_config<256, 12> + : select_config<192, 10> {}; // Based on key_type = rocprim::half, value_type = short template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : select_config<256, 14> + : select_config<256, 24> {}; // Based on key_type = rocprim::half, value_type = int8_t template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(value_type) <= 1))>> : select_config<256, 16> + && (sizeof(value_type) <= 1))>> : select_config<256, 24> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<256, 3> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : select_config<128, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : select_config<192, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : select_config<192, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1))>> + : select_config<192, 4> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<192, 4> {}; // Based on key_type = int64_t, value_type = int64_t template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : select_config<128, 5> + && (sizeof(value_type) > 4))>> : select_config<128, 4> {}; // Based on key_type = int64_t, value_type = int template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : select_config<128, 6> + && (sizeof(value_type) > 2))>> : select_config<192, 4> {}; // Based on key_type = int64_t, value_type = short template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : select_config<192, 7> + && (sizeof(value_type) > 1))>> : select_config<256, 4> {}; // Based on key_type = int64_t, value_type = int8_t template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1))>> - : select_config<192, 8> + : select_config<256, 5> +{}; + +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<192, 4> {}; // Based on key_type = int, value_type = int64_t template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : select_config<128, 6> + && (sizeof(value_type) > 4))>> : select_config<192, 5> {}; // Based on key_type = int, value_type = int template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : select_config<256, 13> + && (sizeof(value_type) > 2))>> : select_config<192, 7> {}; // Based on key_type = int, value_type = short template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : select_config<256, 14> + && (sizeof(value_type) > 1))>> : select_config<192, 10> {}; // Based on key_type = int, value_type = int8_t template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1))>> - : select_config<192, 16> + : select_config<192, 9> +{}; + +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<192, 4> {}; // Based on key_type = short, value_type = int64_t template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : select_config<256, 6> + && (sizeof(value_type) > 4))>> : select_config<192, 5> {}; // Based on key_type = short, value_type = int template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : select_config<256, 12> + && (sizeof(value_type) > 2))>> : select_config<192, 10> {}; // Based on key_type = short, value_type = short template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : select_config<256, 14> + && (sizeof(value_type) > 1))>> : select_config<256, 24> {}; // Based on key_type = short, value_type = int8_t template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1))>> - : select_config<256, 14> + : select_config<256, 24> +{}; + +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx90a), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : select_config<192, 4> {}; // Based on key_type = int8_t, value_type = int64_t template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : select_config<192, 7> + : select_config<256, 6> {}; // Based on key_type = int8_t, value_type = int template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : select_config<256, 13> + : select_config<192, 10> {}; // Based on key_type = int8_t, value_type = short template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : select_config<256, 14> + : select_config<256, 28> {}; // Based on key_type = int8_t, value_type = int8_t template struct default_select_unique_by_key_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) - && (sizeof(value_type) <= 1))>> : select_config<192, 17> + && (sizeof(value_type) <= 1))>> : select_config<192, 28> +{}; + +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<192, 4> {}; // Based on key_type = double, value_type = int64_t template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : select_config<128, 4> + && (sizeof(value_type) > 4))>> : select_config<128, 5> {}; // Based on key_type = double, value_type = int template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : select_config<192, 4> + && (sizeof(value_type) > 2))>> : select_config<128, 6> {}; // Based on key_type = double, value_type = short template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : select_config<256, 4> + && (sizeof(value_type) > 1))>> : select_config<128, 6> {}; // Based on key_type = double, value_type = int8_t template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1))>> - : select_config<256, 5> + : select_config<192, 8> +{}; + +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<192, 4> {}; // Based on key_type = float, value_type = int64_t template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : select_config<192, 5> + && (sizeof(value_type) > 4))>> : select_config<128, 6> {}; // Based on key_type = float, value_type = int template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : select_config<192, 7> + && (sizeof(value_type) > 2))>> : select_config<256, 13> {}; // Based on key_type = float, value_type = short template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : select_config<192, 10> + && (sizeof(value_type) > 1))>> : select_config<256, 14> {}; // Based on key_type = float, value_type = int8_t template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1))>> - : select_config<192, 9> + : select_config<256, 16> +{}; + +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : select_config<192, 4> {}; // Based on key_type = rocprim::half, value_type = int64_t template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : select_config<192, 5> + : select_config<256, 6> {}; // Based on key_type = rocprim::half, value_type = int template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : select_config<192, 10> + : select_config<256, 12> {}; // Based on key_type = rocprim::half, value_type = short template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : select_config<256, 24> + : select_config<256, 14> {}; // Based on key_type = rocprim::half, value_type = int8_t template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) - && (sizeof(value_type) <= 1))>> : select_config<256, 24> + && (sizeof(value_type) <= 1))>> : select_config<256, 16> +{}; + +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<256, 3> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : select_config<192, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_select_unique_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : select_config<192, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_select_unique_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : select_config<192, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1))>> + : select_config<192, 4> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<192, 4> {}; // Based on key_type = int64_t, value_type = int64_t template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : select_config<128, 4> + && (sizeof(value_type) > 4))>> : select_config<128, 5> {}; // Based on key_type = int64_t, value_type = int template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : select_config<192, 4> + && (sizeof(value_type) > 2))>> : select_config<128, 6> {}; // Based on key_type = int64_t, value_type = short template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : select_config<256, 4> + && (sizeof(value_type) > 1))>> : select_config<192, 7> {}; // Based on key_type = int64_t, value_type = int8_t template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1))>> - : select_config<256, 5> + : select_config<192, 8> +{}; + +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<192, 4> {}; // Based on key_type = int, value_type = int64_t template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : select_config<192, 5> + && (sizeof(value_type) > 4))>> : select_config<128, 6> {}; // Based on key_type = int, value_type = int template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : select_config<192, 7> + && (sizeof(value_type) > 2))>> : select_config<256, 13> {}; // Based on key_type = int, value_type = short template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : select_config<192, 10> + && (sizeof(value_type) > 1))>> : select_config<256, 14> {}; // Based on key_type = int, value_type = int8_t template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) && (sizeof(key_type) > 2) && (sizeof(value_type) <= 1))>> - : select_config<192, 9> + : select_config<192, 16> +{}; + +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<192, 4> {}; // Based on key_type = short, value_type = int64_t template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : select_config<192, 5> + && (sizeof(value_type) > 4))>> : select_config<256, 6> {}; // Based on key_type = short, value_type = int template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 4) - && (sizeof(value_type) > 2))>> : select_config<192, 10> + && (sizeof(value_type) > 2))>> : select_config<256, 12> {}; // Based on key_type = short, value_type = short template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 2) - && (sizeof(value_type) > 1))>> : select_config<256, 24> + && (sizeof(value_type) > 1))>> : select_config<256, 14> {}; // Based on key_type = short, value_type = int8_t template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) && (sizeof(key_type) > 1) && (sizeof(value_type) <= 1))>> - : select_config<256, 24> + : select_config<256, 14> +{}; + +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::unknown), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : select_config<192, 4> {}; // Based on key_type = int8_t, value_type = int64_t template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : select_config<256, 6> + : select_config<192, 7> {}; // Based on key_type = int8_t, value_type = int template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : select_config<192, 10> + : select_config<256, 13> {}; // Based on key_type = int8_t, value_type = short template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : select_config<256, 28> + : select_config<256, 14> {}; // Based on key_type = int8_t, value_type = int8_t template struct default_select_unique_by_key_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), key_type, value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) - && (sizeof(value_type) <= 1))>> : select_config<192, 28> + && (sizeof(value_type) <= 1))>> : select_config<192, 17> +{}; + +// Based on key_type = double, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<384, 4> {}; // Based on key_type = double, value_type = int64_t @@ -1887,7 +2692,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : select_config<512, 5> + && (sizeof(value_type) > 4))>> : select_config<512, 7> {}; // Based on key_type = double, value_type = int @@ -1920,7 +2725,18 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1))>> - : select_config<512, 5> + : select_config<512, 7> +{}; + +// Based on key_type = float, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<384, 4> {}; // Based on key_type = float, value_type = int64_t @@ -1967,6 +2783,17 @@ struct default_select_unique_by_key_config< : select_config<512, 15> {}; +// Based on key_type = rocprim::half, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : select_config<384, 4> +{}; + // Based on key_type = rocprim::half, value_type = int64_t template struct default_select_unique_by_key_config< @@ -2010,6 +2837,72 @@ struct default_select_unique_by_key_config< && (sizeof(value_type) <= 1))>> : select_config<512, 24> {}; +// Based on key_type = rocprim::int128_t, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<384, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int64_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 8) + && (sizeof(value_type) > 4))>> : select_config<384, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 4) + && (sizeof(value_type) > 2))>> : select_config<384, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = short +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 2) + && (sizeof(value_type) > 1))>> : select_config<384, 4> +{}; + +// Based on key_type = rocprim::int128_t, value_type = int8_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 16) + && (sizeof(key_type) > 8) && (sizeof(value_type) <= 1))>> + : select_config<384, 4> +{}; + +// Based on key_type = int64_t, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) + && (sizeof(key_type) > 4) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<384, 4> +{}; + // Based on key_type = int64_t, value_type = int64_t template struct default_select_unique_by_key_config< @@ -2018,7 +2911,7 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 8) - && (sizeof(value_type) > 4))>> : select_config<512, 5> + && (sizeof(value_type) > 4))>> : select_config<512, 7> {}; // Based on key_type = int64_t, value_type = int @@ -2051,7 +2944,18 @@ struct default_select_unique_by_key_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 8) && (sizeof(key_type) > 4) && (sizeof(value_type) <= 1))>> - : select_config<512, 5> + : select_config<512, 7> +{}; + +// Based on key_type = int, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 4) + && (sizeof(key_type) > 2) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<384, 4> {}; // Based on key_type = int, value_type = int64_t @@ -2098,6 +3002,17 @@ struct default_select_unique_by_key_config< : select_config<512, 15> {}; +// Based on key_type = short, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 2) + && (sizeof(key_type) > 1) && (sizeof(value_type) <= 16) + && (sizeof(value_type) > 8))>> : select_config<384, 4> +{}; + // Based on key_type = short, value_type = int64_t template struct default_select_unique_by_key_config< @@ -2142,6 +3057,17 @@ struct default_select_unique_by_key_config< : select_config<512, 24> {}; +// Based on key_type = int8_t, value_type = rocprim::int128_t +template +struct default_select_unique_by_key_config< + static_cast(target_arch::gfx942), + key_type, + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(key_type) <= 1) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : select_config<384, 4> +{}; + // Based on key_type = int8_t, value_type = int64_t template struct default_select_unique_by_key_config< diff --git a/rocprim/include/rocprim/device/detail/config/device_transform.hpp b/rocprim/include/rocprim/device/detail/config/device_transform.hpp index 3eeaf3efb..7a60e3116 100644 --- a/rocprim/include/rocprim/device/detail/config/device_transform.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_transform.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,8 +21,11 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_TRANSFORM_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_TRANSFORM_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" + #include /* DO NOT EDIT THIS FILE @@ -68,7 +71,17 @@ struct default_transform_config< static_cast(target_arch::gfx1030), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2))>> : transform_config<256, 1> + && (sizeof(value_type) <= 2))>> : transform_config<1024, 1> +{}; + +// Based on value_type = rocprim::int128_t +template +struct default_transform_config< + static_cast(target_arch::gfx1030), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : transform_config<1024, 1> {}; // Based on value_type = int64_t @@ -113,17 +126,17 @@ struct default_transform_config< // Based on value_type = double template struct default_transform_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : transform_config<1024, 1> + : transform_config<512, 1> {}; // Based on value_type = float template struct default_transform_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> @@ -133,49 +146,59 @@ struct default_transform_config< // Based on value_type = rocprim::half template struct default_transform_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2))>> : transform_config<1024, 2> {}; +// Based on value_type = rocprim::int128_t +template +struct default_transform_config< + static_cast(target_arch::gfx1100), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : transform_config<1024, 1> +{}; + // Based on value_type = int64_t template struct default_transform_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : transform_config<1024, 1> + : transform_config<512, 1> {}; // Based on value_type = int template struct default_transform_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : transform_config<1024, 1> + : transform_config<256, 16> {}; // Based on value_type = short template struct default_transform_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : transform_config<512, 2> + : transform_config<1024, 2> {}; // Based on value_type = int8_t template struct default_transform_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1))>> : transform_config<256, 4> + && (sizeof(value_type) <= 1))>> : transform_config<1024, 4> {}; // Based on value_type = double @@ -207,6 +230,16 @@ struct default_transform_config< && (sizeof(value_type) <= 2))>> : transform_config<512, 4> {}; +// Based on value_type = rocprim::int128_t +template +struct default_transform_config< + static_cast(target_arch::gfx906), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : transform_config<1024, 1> +{}; + // Based on value_type = int64_t template struct default_transform_config< @@ -275,6 +308,16 @@ struct default_transform_config< && (sizeof(value_type) <= 2))>> : transform_config<128, 4> {}; +// Based on value_type = rocprim::int128_t +template +struct default_transform_config< + static_cast(target_arch::gfx908), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : transform_config<128, 1> +{}; + // Based on value_type = int64_t template struct default_transform_config< @@ -317,137 +360,167 @@ struct default_transform_config< // Based on value_type = double template struct default_transform_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : transform_config<128, 1> + : transform_config<256, 2> {}; // Based on value_type = float template struct default_transform_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : transform_config<128, 2> + : transform_config<1024, 2> {}; // Based on value_type = rocprim::half template struct default_transform_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2))>> : transform_config<128, 4> + && (sizeof(value_type) <= 2))>> : transform_config<64, 8> +{}; + +// Based on value_type = rocprim::int128_t +template +struct default_transform_config< + static_cast(target_arch::gfx90a), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : transform_config<1024, 1> {}; // Based on value_type = int64_t template struct default_transform_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : transform_config<128, 1> + : transform_config<256, 2> {}; // Based on value_type = int template struct default_transform_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : transform_config<128, 2> + : transform_config<1024, 2> {}; // Based on value_type = short template struct default_transform_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : transform_config<128, 4> + : transform_config<64, 8> {}; // Based on value_type = int8_t template struct default_transform_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1))>> : transform_config<128, 8> + && (sizeof(value_type) <= 1))>> : transform_config<64, 16> {}; // Based on value_type = double template struct default_transform_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : transform_config<256, 2> + : transform_config<128, 1> {}; // Based on value_type = float template struct default_transform_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : transform_config<1024, 2> + : transform_config<128, 2> {}; // Based on value_type = rocprim::half template struct default_transform_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2))>> : transform_config<64, 8> + && (sizeof(value_type) <= 2))>> : transform_config<128, 4> +{}; + +// Based on value_type = rocprim::int128_t +template +struct default_transform_config< + static_cast(target_arch::unknown), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : transform_config<128, 1> {}; // Based on value_type = int64_t template struct default_transform_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : transform_config<256, 2> + : transform_config<128, 1> {}; // Based on value_type = int template struct default_transform_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : transform_config<1024, 2> + : transform_config<128, 2> {}; // Based on value_type = short template struct default_transform_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : transform_config<64, 8> + : transform_config<128, 4> {}; // Based on value_type = int8_t template struct default_transform_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1))>> : transform_config<64, 16> + && (sizeof(value_type) <= 1))>> : transform_config<128, 8> +{}; + +// Based on value_type = double +template +struct default_transform_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> + : transform_config<1024, 4> {}; // Based on value_type = float @@ -469,6 +542,16 @@ struct default_transform_config< && (sizeof(value_type) <= 2))>> : transform_config<128, 16> {}; +// Based on value_type = rocprim::int128_t +template +struct default_transform_config< + static_cast(target_arch::gfx942), + value_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8))>> + : transform_config<256, 1> +{}; + // Based on value_type = int64_t template struct default_transform_config< @@ -476,7 +559,7 @@ struct default_transform_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4))>> - : transform_config<1024, 2> + : transform_config<256, 2> {}; // Based on value_type = int @@ -486,7 +569,7 @@ struct default_transform_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2))>> - : transform_config<1024, 4> + : transform_config<256, 4> {}; // Based on value_type = short @@ -496,7 +579,7 @@ struct default_transform_config< value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1))>> - : transform_config<512, 8> + : transform_config<256, 8> {}; // Based on value_type = int8_t @@ -505,7 +588,7 @@ struct default_transform_config< static_cast(target_arch::gfx942), value_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1))>> : transform_config<512, 16> + && (sizeof(value_type) <= 1))>> : transform_config<256, 16> {}; } // end namespace detail diff --git a/rocprim/include/rocprim/device/detail/config/device_upper_bound.hpp b/rocprim/include/rocprim/device/detail/config/device_upper_bound.hpp index b1b0096ab..ff1ad05d4 100644 --- a/rocprim/include/rocprim/device/detail/config/device_upper_bound.hpp +++ b/rocprim/include/rocprim/device/detail/config/device_upper_bound.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2023-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,8 +21,11 @@ #ifndef ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_UPPER_BOUND_HPP_ #define ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_UPPER_BOUND_HPP_ -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" + #include /* DO NOT EDIT THIS FILE @@ -89,6 +92,18 @@ struct default_upper_bound_config< && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> {}; +// Based on value_type = double, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<128, 1> +{}; + // Based on value_type = float, output_type = int64_t template struct default_upper_bound_config< @@ -136,6 +151,18 @@ struct default_upper_bound_config< && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> {}; +// Based on value_type = float, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<128, 1> +{}; + // Based on value_type = rocprim::half, output_type = int64_t template struct default_upper_bound_config< @@ -144,7 +171,7 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : upper_bound_config<64, 4> + && (sizeof(output_type) > 4))>> : upper_bound_config<64, 8> {}; // Based on value_type = rocprim::half, output_type = int @@ -155,7 +182,7 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : upper_bound_config<64, 4> + && (sizeof(output_type) > 2))>> : upper_bound_config<64, 8> {}; // Based on value_type = rocprim::half, output_type = short @@ -166,7 +193,7 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : upper_bound_config<64, 4> + && (sizeof(output_type) > 1))>> : upper_bound_config<64, 8> {}; // Based on value_type = rocprim::half, output_type = int8_t @@ -177,7 +204,18 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> - : upper_bound_config<64, 4> + : upper_bound_config<64, 8> +{}; + +// Based on value_type = rocprim::half, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : upper_bound_config<64, 8> {}; // Based on value_type = int64_t, output_type = int64_t @@ -227,6 +265,18 @@ struct default_upper_bound_config< && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> {}; +// Based on value_type = int64_t, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<128, 1> +{}; + // Based on value_type = int, output_type = int64_t template struct default_upper_bound_config< @@ -274,6 +324,18 @@ struct default_upper_bound_config< && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> {}; +// Based on value_type = int, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<128, 1> +{}; + // Based on value_type = short, output_type = int64_t template struct default_upper_bound_config< @@ -321,6 +383,18 @@ struct default_upper_bound_config< && (sizeof(output_type) <= 1))>> : upper_bound_config<128, 4> {}; +// Based on value_type = short, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 1> +{}; + // Based on value_type = int8_t, output_type = int64_t template struct default_upper_bound_config< @@ -365,57 +439,139 @@ struct default_upper_bound_config< : upper_bound_config<128, 4> {}; +// Based on value_type = int8_t, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : upper_bound_config<256, 8> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int64_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : upper_bound_config<256, 1> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int +template +struct default_upper_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : upper_bound_config<128, 1> +{}; + +// Based on value_type = rocprim::int128_t, output_type = short +template +struct default_upper_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : upper_bound_config<128, 1> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int8_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 1))>> : upper_bound_config<128, 1> +{}; + +// Based on value_type = rocprim::int128_t, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx1030), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<128, 1> +{}; + // Based on value_type = double, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : upper_bound_config<256, 2> + : upper_bound_config<256, 1> {}; // Based on value_type = double, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : upper_bound_config<256, 1> + : upper_bound_config<64, 16> {}; // Based on value_type = double, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : upper_bound_config<256, 1> + : upper_bound_config<256, 16> {}; // Based on value_type = double, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> + && (sizeof(output_type) <= 1))>> : upper_bound_config<64, 16> +{}; + +// Based on value_type = double, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<64, 16> {}; // Based on value_type = float, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -427,7 +583,7 @@ struct default_upper_bound_config< // Based on value_type = float, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -439,19 +595,19 @@ struct default_upper_bound_config< // Based on value_type = float, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : upper_bound_config<256, 1> + : upper_bound_config<128, 1> {}; // Based on value_type = float, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -459,54 +615,77 @@ struct default_upper_bound_config< && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> {}; +// Based on value_type = float, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<128, 16> +{}; + // Based on value_type = rocprim::half, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : upper_bound_config<128, 8> + && (sizeof(output_type) > 4))>> : upper_bound_config<64, 8> {}; // Based on value_type = rocprim::half, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : upper_bound_config<256, 4> + && (sizeof(output_type) > 2))>> : upper_bound_config<64, 8> {}; // Based on value_type = rocprim::half, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : upper_bound_config<64, 4> + && (sizeof(output_type) > 1))>> : upper_bound_config<64, 8> {}; // Based on value_type = rocprim::half, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> - : upper_bound_config<64, 4> + : upper_bound_config<64, 8> +{}; + +// Based on value_type = rocprim::half, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : upper_bound_config<64, 8> {}; // Based on value_type = int64_t, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -518,42 +697,54 @@ struct default_upper_bound_config< // Based on value_type = int64_t, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : upper_bound_config<256, 1> + : upper_bound_config<64, 16> {}; // Based on value_type = int64_t, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : upper_bound_config<256, 1> + : upper_bound_config<128, 16> {}; // Based on value_type = int64_t, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> + && (sizeof(output_type) <= 1))>> : upper_bound_config<64, 16> +{}; + +// Based on value_type = int64_t, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<128, 16> {}; // Based on value_type = int, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -565,7 +756,7 @@ struct default_upper_bound_config< // Based on value_type = int, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -577,7 +768,7 @@ struct default_upper_bound_config< // Based on value_type = int, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -589,7 +780,7 @@ struct default_upper_bound_config< // Based on value_type = int, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -597,68 +788,92 @@ struct default_upper_bound_config< && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> {}; +// Based on value_type = int, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<64, 16> +{}; + // Based on value_type = short, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : upper_bound_config<64, 8> + : upper_bound_config<64, 16> {}; // Based on value_type = short, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : upper_bound_config<128, 8> + : upper_bound_config<256, 8> {}; // Based on value_type = short, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : upper_bound_config<128, 4> + : upper_bound_config<64, 8> {}; // Based on value_type = short, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 1))>> : upper_bound_config<64, 4> + && (sizeof(output_type) <= 1))>> : upper_bound_config<64, 8> +{}; + +// Based on value_type = short, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 8> {}; // Based on value_type = int8_t, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : upper_bound_config<256, 4> + && (sizeof(output_type) > 4))>> : upper_bound_config<64, 8> {}; // Based on value_type = int8_t, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -669,18 +884,18 @@ struct default_upper_bound_config< // Based on value_type = int8_t, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : upper_bound_config<64, 4> + && (sizeof(output_type) > 1))>> : upper_bound_config<64, 8> {}; // Based on value_type = int8_t, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::gfx1102), + static_cast(target_arch::gfx1100), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -688,650 +903,538 @@ struct default_upper_bound_config< : upper_bound_config<64, 8> {}; -// Based on value_type = double, output_type = int64_t +// Based on value_type = int8_t, output_type = rocprim::int128_t template struct default_upper_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : upper_bound_config<256, 8> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int64_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> : upper_bound_config<256, 1> {}; -// Based on value_type = double, output_type = int +// Based on value_type = rocprim::int128_t, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : upper_bound_config<256, 1> + : upper_bound_config<128, 16> {}; -// Based on value_type = double, output_type = short +// Based on value_type = rocprim::int128_t, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : upper_bound_config<256, 1> + : upper_bound_config<128, 16> {}; -// Based on value_type = double, output_type = int8_t +// Based on value_type = rocprim::int128_t, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx1100), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 1))>> : upper_bound_config<64, 16> {}; -// Based on value_type = float, output_type = int64_t +// Based on value_type = rocprim::int128_t, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx1100), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<128, 16> +{}; + +// Based on value_type = double, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> : upper_bound_config<256, 1> {}; -// Based on value_type = float, output_type = int +// Based on value_type = double, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> : upper_bound_config<256, 1> {}; -// Based on value_type = float, output_type = short +// Based on value_type = double, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> : upper_bound_config<256, 1> {}; -// Based on value_type = float, output_type = int8_t +// Based on value_type = double, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> {}; -// Based on value_type = rocprim::half, output_type = int64_t +// Based on value_type = double, output_type = rocprim::int128_t template struct default_upper_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : upper_bound_config<256, 1> + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 16> {}; -// Based on value_type = rocprim::half, output_type = int +// Based on value_type = float, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : upper_bound_config<256, 1> -{}; - -// Based on value_type = rocprim::half, output_type = short -template -struct default_upper_bound_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : upper_bound_config<256, 1> -{}; - -// Based on value_type = rocprim::half, output_type = int8_t -template -struct default_upper_bound_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> - : upper_bound_config<256, 1> -{}; - -// Based on value_type = int64_t, output_type = int64_t -template -struct default_upper_bound_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : upper_bound_config<256, 1> -{}; - -// Based on value_type = int64_t, output_type = int -template -struct default_upper_bound_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : upper_bound_config<256, 1> -{}; - -// Based on value_type = int64_t, output_type = short -template -struct default_upper_bound_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : upper_bound_config<256, 1> -{}; - -// Based on value_type = int64_t, output_type = int8_t -template -struct default_upper_bound_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> -{}; - -// Based on value_type = int, output_type = int64_t -template -struct default_upper_bound_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> : upper_bound_config<256, 1> {}; -// Based on value_type = int, output_type = int +// Based on value_type = float, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> : upper_bound_config<256, 1> {}; -// Based on value_type = int, output_type = short +// Based on value_type = float, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> : upper_bound_config<256, 1> {}; -// Based on value_type = int, output_type = int8_t +// Based on value_type = float, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) + std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> {}; -// Based on value_type = short, output_type = int64_t -template -struct default_upper_bound_config< - static_cast(target_arch::gfx900), - value_type, - output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : upper_bound_config<256, 1> -{}; - -// Based on value_type = short, output_type = int +// Based on value_type = float, output_type = rocprim::int128_t template struct default_upper_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> : upper_bound_config<256, 1> {}; -// Based on value_type = short, output_type = short +// Based on value_type = rocprim::half, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : upper_bound_config<256, 1> + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) + && (sizeof(output_type) > 4))>> : upper_bound_config<64, 4> {}; -// Based on value_type = short, output_type = int8_t +// Based on value_type = rocprim::half, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) + && (sizeof(output_type) > 2))>> : upper_bound_config<64, 4> {}; -// Based on value_type = int8_t, output_type = int64_t +// Based on value_type = rocprim::half, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : upper_bound_config<256, 1> + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) + && (sizeof(output_type) > 1))>> : upper_bound_config<256, 4> {}; -// Based on value_type = int8_t, output_type = int +// Based on value_type = rocprim::half, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : upper_bound_config<256, 1> + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> + : upper_bound_config<64, 4> {}; -// Based on value_type = int8_t, output_type = short +// Based on value_type = rocprim::half, output_type = rocprim::int128_t template struct default_upper_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : upper_bound_config<256, 1> + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : upper_bound_config<256, 1> {}; -// Based on value_type = int8_t, output_type = int8_t +// Based on value_type = int64_t, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::gfx900), + static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> : upper_bound_config<256, 1> {}; -// Based on value_type = double, output_type = int64_t +// Based on value_type = int64_t, output_type = int template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> : upper_bound_config<256, 1> {}; -// Based on value_type = double, output_type = int +// Based on value_type = int64_t, output_type = short template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> : upper_bound_config<256, 1> {}; -// Based on value_type = double, output_type = short +// Based on value_type = int64_t, output_type = int8_t template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : upper_bound_config<256, 1> + && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> {}; -// Based on value_type = double, output_type = int8_t +// Based on value_type = int64_t, output_type = rocprim::int128_t template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 16> {}; -// Based on value_type = float, output_type = int64_t +// Based on value_type = int, output_type = int64_t template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> : upper_bound_config<256, 1> {}; -// Based on value_type = float, output_type = int +// Based on value_type = int, output_type = int template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> : upper_bound_config<256, 1> {}; -// Based on value_type = float, output_type = short +// Based on value_type = int, output_type = short template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> : upper_bound_config<256, 1> {}; -// Based on value_type = float, output_type = int8_t +// Based on value_type = int, output_type = int8_t template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) + std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> {}; -// Based on value_type = rocprim::half, output_type = int64_t -template -struct default_upper_bound_config< - static_cast(target_arch::gfx906), - value_type, - output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : upper_bound_config<256, 4> -{}; - -// Based on value_type = rocprim::half, output_type = int -template -struct default_upper_bound_config< - static_cast(target_arch::gfx906), - value_type, - output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : upper_bound_config<256, 4> -{}; - -// Based on value_type = rocprim::half, output_type = short +// Based on value_type = int, output_type = rocprim::int128_t template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : upper_bound_config<256, 4> -{}; - -// Based on value_type = rocprim::half, output_type = int8_t -template -struct default_upper_bound_config< - static_cast(target_arch::gfx906), - value_type, - output_type, - std::enable_if_t<(bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> - : upper_bound_config<256, 4> + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 1> {}; -// Based on value_type = int64_t, output_type = int64_t +// Based on value_type = short, output_type = int64_t template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : upper_bound_config<256, 1> + : upper_bound_config<256, 4> {}; -// Based on value_type = int64_t, output_type = int +// Based on value_type = short, output_type = int template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : upper_bound_config<256, 1> + : upper_bound_config<256, 4> {}; -// Based on value_type = int64_t, output_type = short +// Based on value_type = short, output_type = short template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : upper_bound_config<256, 1> -{}; - -// Based on value_type = int64_t, output_type = int8_t -template -struct default_upper_bound_config< - static_cast(target_arch::gfx906), - value_type, - output_type, - std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> + : upper_bound_config<256, 4> {}; -// Based on value_type = int, output_type = int64_t +// Based on value_type = short, output_type = int8_t template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) - && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : upper_bound_config<256, 1> + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 4> {}; -// Based on value_type = int, output_type = int +// Based on value_type = short, output_type = rocprim::int128_t template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) - && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : upper_bound_config<256, 1> + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<128, 16> {}; -// Based on value_type = int, output_type = short +// Based on value_type = int8_t, output_type = int64_t template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) - && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : upper_bound_config<256, 1> + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) + && (sizeof(output_type) > 4))>> : upper_bound_config<256, 1> {}; -// Based on value_type = int, output_type = int8_t +// Based on value_type = int8_t, output_type = int template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) - && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) + && (sizeof(output_type) > 2))>> : upper_bound_config<256, 4> {}; -// Based on value_type = short, output_type = int64_t +// Based on value_type = int8_t, output_type = short template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : upper_bound_config<256, 4> + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) + && (sizeof(output_type) > 1))>> : upper_bound_config<256, 4> {}; -// Based on value_type = short, output_type = int +// Based on value_type = int8_t, output_type = int8_t template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 4> {}; -// Based on value_type = short, output_type = short +// Based on value_type = int8_t, output_type = rocprim::int128_t template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : upper_bound_config<256, 4> + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : upper_bound_config<64, 4> {}; -// Based on value_type = short, output_type = int8_t +// Based on value_type = rocprim::int128_t, output_type = int64_t template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 4> + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : upper_bound_config<256, 1> {}; -// Based on value_type = int8_t, output_type = int64_t +// Based on value_type = rocprim::int128_t, output_type = int template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : upper_bound_config<256, 4> + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : upper_bound_config<256, 8> {}; -// Based on value_type = int8_t, output_type = int +// Based on value_type = rocprim::int128_t, output_type = short template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : upper_bound_config<256, 4> + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : upper_bound_config<256, 16> {}; -// Based on value_type = int8_t, output_type = short +// Based on value_type = rocprim::int128_t, output_type = int8_t template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : upper_bound_config<256, 4> + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 16> {}; -// Based on value_type = int8_t, output_type = int8_t +// Based on value_type = rocprim::int128_t, output_type = rocprim::int128_t template struct default_upper_bound_config< static_cast(target_arch::gfx906), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) - && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> - : upper_bound_config<256, 4> + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 16> {}; // Based on value_type = double, output_type = int64_t @@ -1381,6 +1484,18 @@ struct default_upper_bound_config< && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> {}; +// Based on value_type = double, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 16> +{}; + // Based on value_type = float, output_type = int64_t template struct default_upper_bound_config< @@ -1428,6 +1543,18 @@ struct default_upper_bound_config< && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> {}; +// Based on value_type = float, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 16> +{}; + // Based on value_type = rocprim::half, output_type = int64_t template struct default_upper_bound_config< @@ -1436,7 +1563,7 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : upper_bound_config<64, 4> + && (sizeof(output_type) > 4))>> : upper_bound_config<256, 1> {}; // Based on value_type = rocprim::half, output_type = int @@ -1447,7 +1574,7 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : upper_bound_config<64, 4> + && (sizeof(output_type) > 2))>> : upper_bound_config<256, 4> {}; // Based on value_type = rocprim::half, output_type = short @@ -1469,7 +1596,18 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> - : upper_bound_config<64, 4> + : upper_bound_config<256, 4> +{}; + +// Based on value_type = rocprim::half, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : upper_bound_config<64, 4> {}; // Based on value_type = int64_t, output_type = int64_t @@ -1519,6 +1657,18 @@ struct default_upper_bound_config< && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> {}; +// Based on value_type = int64_t, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 16> +{}; + // Based on value_type = int, output_type = int64_t template struct default_upper_bound_config< @@ -1566,6 +1716,18 @@ struct default_upper_bound_config< && (sizeof(output_type) <= 1))>> : upper_bound_config<128, 1> {}; +// Based on value_type = int, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 16> +{}; + // Based on value_type = short, output_type = int64_t template struct default_upper_bound_config< @@ -1613,6 +1775,18 @@ struct default_upper_bound_config< && (sizeof(output_type) <= 1))>> : upper_bound_config<64, 4> {}; +// Based on value_type = short, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 16> +{}; + // Based on value_type = int8_t, output_type = int64_t template struct default_upper_bound_config< @@ -1657,57 +1831,139 @@ struct default_upper_bound_config< : upper_bound_config<256, 4> {}; +// Based on value_type = int8_t, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : upper_bound_config<256, 4> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int64_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : upper_bound_config<256, 1> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int +template +struct default_upper_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : upper_bound_config<256, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = short +template +struct default_upper_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : upper_bound_config<256, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int8_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx908), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 16> +{}; + // Based on value_type = double, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : upper_bound_config<256, 8> + : upper_bound_config<64, 4> {}; // Based on value_type = double, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : upper_bound_config<64, 8> + : upper_bound_config<64, 4> {}; // Based on value_type = double, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : upper_bound_config<256, 16> + : upper_bound_config<64, 4> {}; // Based on value_type = double, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> + && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 4> +{}; + +// Based on value_type = double, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<64, 4> {}; // Based on value_type = float, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -1719,7 +1975,7 @@ struct default_upper_bound_config< // Based on value_type = float, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -1731,7 +1987,7 @@ struct default_upper_bound_config< // Based on value_type = float, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -1743,51 +1999,63 @@ struct default_upper_bound_config< // Based on value_type = float, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> +{}; + +// Based on value_type = float, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) - && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 1> {}; // Based on value_type = rocprim::half, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : upper_bound_config<64, 4> + && (sizeof(output_type) > 4))>> : upper_bound_config<128, 4> {}; // Based on value_type = rocprim::half, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : upper_bound_config<64, 4> + && (sizeof(output_type) > 2))>> : upper_bound_config<64, 2> {}; // Based on value_type = rocprim::half, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : upper_bound_config<64, 4> + && (sizeof(output_type) > 1))>> : upper_bound_config<64, 2> {}; // Based on value_type = rocprim::half, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -1795,57 +2063,80 @@ struct default_upper_bound_config< : upper_bound_config<64, 4> {}; +// Based on value_type = rocprim::half, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : upper_bound_config<256, 4> +{}; + // Based on value_type = int64_t, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : upper_bound_config<128, 1> + : upper_bound_config<64, 4> {}; // Based on value_type = int64_t, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : upper_bound_config<64, 1> + : upper_bound_config<64, 4> {}; // Based on value_type = int64_t, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : upper_bound_config<256, 16> + : upper_bound_config<64, 4> {}; // Based on value_type = int64_t, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> + && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 4> +{}; + +// Based on value_type = int64_t, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<64, 4> {}; // Based on value_type = int, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -1857,7 +2148,7 @@ struct default_upper_bound_config< // Based on value_type = int, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -1869,7 +2160,7 @@ struct default_upper_bound_config< // Based on value_type = int, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -1881,156 +2172,262 @@ struct default_upper_bound_config< // Based on value_type = int, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) - && (sizeof(output_type) <= 1))>> : upper_bound_config<128, 1> + && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> +{}; + +// Based on value_type = int, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 1> {}; // Based on value_type = short, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : upper_bound_config<128, 1> + : upper_bound_config<256, 1> {}; // Based on value_type = short, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : upper_bound_config<128, 1> + : upper_bound_config<256, 1> {}; // Based on value_type = short, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : upper_bound_config<64, 4> + : upper_bound_config<256, 1> {}; // Based on value_type = short, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 1))>> : upper_bound_config<64, 4> + && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> +{}; + +// Based on value_type = short, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 1> {}; // Based on value_type = int8_t, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : upper_bound_config<128, 1> + && (sizeof(output_type) > 4))>> : upper_bound_config<256, 2> {}; // Based on value_type = int8_t, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : upper_bound_config<128, 1> + && (sizeof(output_type) > 2))>> : upper_bound_config<256, 2> {}; // Based on value_type = int8_t, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : upper_bound_config<128, 1> + && (sizeof(output_type) > 1))>> : upper_bound_config<256, 2> {}; // Based on value_type = int8_t, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::unknown), + static_cast(target_arch::gfx90a), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> - : upper_bound_config<256, 4> + : upper_bound_config<256, 2> {}; -// Based on value_type = double, output_type = int64_t +// Based on value_type = int8_t, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : upper_bound_config<256, 2> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int64_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : upper_bound_config<256, 1> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int +template +struct default_upper_bound_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : upper_bound_config<64, 4> +{}; + +// Based on value_type = rocprim::int128_t, output_type = short +template +struct default_upper_bound_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : upper_bound_config<128, 4> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int8_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx90a), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 1))>> : upper_bound_config<64, 4> +{}; + +// Based on value_type = rocprim::int128_t, output_type = rocprim::int128_t template struct default_upper_bound_config< static_cast(target_arch::gfx90a), value_type, output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<64, 4> +{}; + +// Based on value_type = double, output_type = int64_t +template +struct default_upper_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : upper_bound_config<64, 4> + : upper_bound_config<256, 8> {}; // Based on value_type = double, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : upper_bound_config<64, 4> + : upper_bound_config<64, 8> {}; // Based on value_type = double, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : upper_bound_config<64, 4> + : upper_bound_config<256, 16> {}; // Based on value_type = double, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 4> + && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> +{}; + +// Based on value_type = double, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 16> {}; // Based on value_type = float, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -2042,7 +2439,7 @@ struct default_upper_bound_config< // Based on value_type = float, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -2054,7 +2451,7 @@ struct default_upper_bound_config< // Based on value_type = float, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -2066,7 +2463,7 @@ struct default_upper_bound_config< // Based on value_type = float, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) @@ -2074,101 +2471,136 @@ struct default_upper_bound_config< && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> {}; +// Based on value_type = float, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 16> +{}; + // Based on value_type = rocprim::half, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : upper_bound_config<128, 4> + && (sizeof(output_type) > 4))>> : upper_bound_config<256, 1> {}; // Based on value_type = rocprim::half, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : upper_bound_config<64, 2> + && (sizeof(output_type) > 2))>> : upper_bound_config<256, 4> {}; // Based on value_type = rocprim::half, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : upper_bound_config<64, 2> + && (sizeof(output_type) > 1))>> : upper_bound_config<64, 4> {}; // Based on value_type = rocprim::half, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 1))>> - : upper_bound_config<64, 4> + : upper_bound_config<256, 4> +{}; + +// Based on value_type = rocprim::half, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : upper_bound_config<64, 4> {}; // Based on value_type = int64_t, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : upper_bound_config<64, 4> + : upper_bound_config<128, 1> {}; // Based on value_type = int64_t, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : upper_bound_config<64, 4> + : upper_bound_config<64, 1> {}; // Based on value_type = int64_t, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : upper_bound_config<256, 16> +{}; + +// Based on value_type = int64_t, output_type = int8_t +template +struct default_upper_bound_config< + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : upper_bound_config<64, 4> + && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> {}; -// Based on value_type = int64_t, output_type = int8_t +// Based on value_type = int64_t, output_type = rocprim::int128_t template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 4> + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 16> {}; // Based on value_type = int, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -2180,7 +2612,7 @@ struct default_upper_bound_config< // Based on value_type = int, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -2192,7 +2624,7 @@ struct default_upper_bound_config< // Based on value_type = int, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) @@ -2204,103 +2636,209 @@ struct default_upper_bound_config< // Based on value_type = int, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) - && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> + && (sizeof(output_type) <= 1))>> : upper_bound_config<128, 1> +{}; + +// Based on value_type = int, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 16> {}; // Based on value_type = short, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : upper_bound_config<256, 1> + : upper_bound_config<128, 1> {}; // Based on value_type = short, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : upper_bound_config<256, 1> + : upper_bound_config<128, 1> {}; // Based on value_type = short, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : upper_bound_config<256, 1> + : upper_bound_config<64, 4> {}; // Based on value_type = short, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) - && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> + && (sizeof(output_type) <= 1))>> : upper_bound_config<64, 4> +{}; + +// Based on value_type = short, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 16> {}; // Based on value_type = int8_t, output_type = int64_t template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : upper_bound_config<256, 2> + && (sizeof(output_type) > 4))>> : upper_bound_config<128, 1> {}; // Based on value_type = int8_t, output_type = int template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : upper_bound_config<256, 2> + && (sizeof(output_type) > 2))>> : upper_bound_config<128, 1> {}; // Based on value_type = int8_t, output_type = short template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : upper_bound_config<256, 2> + && (sizeof(output_type) > 1))>> : upper_bound_config<128, 1> {}; // Based on value_type = int8_t, output_type = int8_t template struct default_upper_bound_config< - static_cast(target_arch::gfx90a), + static_cast(target_arch::unknown), value_type, output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> - : upper_bound_config<256, 2> + : upper_bound_config<256, 4> +{}; + +// Based on value_type = int8_t, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : upper_bound_config<256, 4> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int64_t +template +struct default_upper_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : upper_bound_config<256, 1> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int +template +struct default_upper_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : upper_bound_config<256, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = short +template +struct default_upper_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : upper_bound_config<256, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int8_t +template +struct default_upper_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::unknown), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 16> +{}; + +// Based on value_type = double, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<64, 16> {}; // Based on value_type = double, output_type = int64_t @@ -2312,7 +2850,7 @@ struct default_upper_bound_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> - : upper_bound_config<256, 1> + : upper_bound_config<64, 16> {}; // Based on value_type = double, output_type = int @@ -2324,7 +2862,7 @@ struct default_upper_bound_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : upper_bound_config<256, 1> + : upper_bound_config<64, 16> {}; // Based on value_type = double, output_type = short @@ -2336,7 +2874,7 @@ struct default_upper_bound_config< std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : upper_bound_config<256, 1> + : upper_bound_config<64, 16> {}; // Based on value_type = double, output_type = int8_t @@ -2347,7 +2885,19 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) - && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> + && (sizeof(output_type) <= 1))>> : upper_bound_config<64, 16> +{}; + +// Based on value_type = float, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 1> {}; // Based on value_type = float, output_type = int64_t @@ -2397,6 +2947,17 @@ struct default_upper_bound_config< && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> {}; +// Based on value_type = rocprim::half, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : upper_bound_config<128, 8> +{}; + // Based on value_type = rocprim::half, output_type = int64_t template struct default_upper_bound_config< @@ -2427,7 +2988,7 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 2) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : upper_bound_config<256, 4> + && (sizeof(output_type) > 1))>> : upper_bound_config<64, 8> {}; // Based on value_type = rocprim::half, output_type = int8_t @@ -2441,6 +3002,77 @@ struct default_upper_bound_config< : upper_bound_config<256, 4> {}; +// Based on value_type = rocprim::int128_t, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<64, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int64_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 8) && (sizeof(output_type) > 4))>> + : upper_bound_config<128, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> + : upper_bound_config<128, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = short +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> + : upper_bound_config<64, 16> +{}; + +// Based on value_type = rocprim::int128_t, output_type = int8_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 16) && (sizeof(value_type) > 8) + && (sizeof(output_type) <= 1))>> : upper_bound_config<128, 16> +{}; + +// Based on value_type = int64_t, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<128, 16> +{}; + // Based on value_type = int64_t, output_type = int64_t template struct default_upper_bound_config< @@ -2462,7 +3094,7 @@ struct default_upper_bound_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 8) && (sizeof(value_type) > 4) && (sizeof(output_type) <= 4) && (sizeof(output_type) > 2))>> - : upper_bound_config<256, 1> + : upper_bound_config<64, 16> {}; // Based on value_type = int64_t, output_type = short @@ -2488,6 +3120,18 @@ struct default_upper_bound_config< && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> {}; +// Based on value_type = int, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 1> +{}; + // Based on value_type = int, output_type = int64_t template struct default_upper_bound_config< @@ -2521,7 +3165,7 @@ struct default_upper_bound_config< std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 4) && (sizeof(value_type) > 2) && (sizeof(output_type) <= 2) && (sizeof(output_type) > 1))>> - : upper_bound_config<256, 1> + : upper_bound_config<128, 1> {}; // Based on value_type = int, output_type = int8_t @@ -2535,6 +3179,18 @@ struct default_upper_bound_config< && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> {}; +// Based on value_type = short, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 2) && (sizeof(value_type) > 1) + && (sizeof(output_type) <= 16) && (sizeof(output_type) > 8))>> + : upper_bound_config<256, 1> +{}; + // Based on value_type = short, output_type = int64_t template struct default_upper_bound_config< @@ -2582,6 +3238,17 @@ struct default_upper_bound_config< && (sizeof(output_type) <= 1))>> : upper_bound_config<256, 1> {}; +// Based on value_type = int8_t, output_type = rocprim::int128_t +template +struct default_upper_bound_config< + static_cast(target_arch::gfx942), + value_type, + output_type, + std::enable_if_t<(!bool(rocprim::is_floating_point::value) + && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 16) + && (sizeof(output_type) > 8))>> : upper_bound_config<128, 4> +{}; + // Based on value_type = int8_t, output_type = int64_t template struct default_upper_bound_config< @@ -2590,7 +3257,7 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 8) - && (sizeof(output_type) > 4))>> : upper_bound_config<256, 16> + && (sizeof(output_type) > 4))>> : upper_bound_config<128, 4> {}; // Based on value_type = int8_t, output_type = int @@ -2601,7 +3268,7 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 4) - && (sizeof(output_type) > 2))>> : upper_bound_config<256, 4> + && (sizeof(output_type) > 2))>> : upper_bound_config<128, 1> {}; // Based on value_type = int8_t, output_type = short @@ -2612,7 +3279,7 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 2) - && (sizeof(output_type) > 1))>> : upper_bound_config<64, 16> + && (sizeof(output_type) > 1))>> : upper_bound_config<128, 4> {}; // Based on value_type = int8_t, output_type = int8_t @@ -2623,7 +3290,7 @@ struct default_upper_bound_config< output_type, std::enable_if_t<(!bool(rocprim::is_floating_point::value) && (sizeof(value_type) <= 1) && (sizeof(output_type) <= 1))>> - : upper_bound_config<64, 16> + : upper_bound_config<256, 4> {}; } // end namespace detail diff --git a/rocprim/include/rocprim/device/detail/device_config_helper.hpp b/rocprim/include/rocprim/device/detail/device_config_helper.hpp index 553d290b9..dcdd71984 100644 --- a/rocprim/include/rocprim/device/detail/device_config_helper.hpp +++ b/rocprim/include/rocprim/device/detail/device_config_helper.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -972,7 +972,8 @@ struct reduce_by_key_config_params * \tparam LoadKeysMethod method of loading keys * \tparam LoadValuesMethod method of loading values * \tparam ScanAlgorithm block level scan algorithm to use - * \tparam TilesPerBlock number of tiles (`BlockSize` * `ItemsPerThread` items) to process per block + * \tparam TilesPerBlock number of tiles (`BlockSize` * `ItemsPerThread` items) to process per block. + * This parameter is only here for legacy purposes. Its no longer used. * \tparam SizeLimit limit on the number of items for a single reduce_by_key kernel launch. */ template - ROCPRIM_DEVICE void count_digits_at_place(const unsigned int flat_id, - const unsigned int stripe, - const KeyType (&keys)[ItemsPerThread], - const unsigned int place, - Decomposer decomposer, - const unsigned int start_bit, - const unsigned int current_radix_bits, - const unsigned int valid_count, - storage_type& storage) - { - ROCPRIM_UNROLL - for(unsigned int i = 0; i < ItemsPerThread; ++i) - { - const unsigned int pos = i * BlockSize + flat_id; - if(IsFull || pos < valid_count) - { - const unsigned int digit - = key_codec::extract_digit(keys[i], start_bit, current_radix_bits, decomposer); - ::rocprim::detail::atomic_add(&get_counter(stripe, place, digit, storage), 1); - } - } - } - - template - ROCPRIM_DEVICE void count_digits(KeysInputIterator keys_input, - Offset* global_digit_counts, - const unsigned int valid_count, - Decomposer decomposer, - const unsigned int begin_bit, - const unsigned int end_bit, - storage_type& storage) + template + ROCPRIM_DEVICE + void count_digits(KeysInputIterator keys_input, + Offset* global_digit_counts, + const unsigned int valid_count, + Decomposer decomposer, + const unsigned int begin_bit, + const unsigned int end_bit, + storage_type& storage) { const unsigned int flat_id = ::rocprim::detail::block_thread_id<0>(); const unsigned int stripe = flat_id % atomic_stripes; @@ -850,17 +826,48 @@ struct onesweep_histograms_helper key_codec::encode_inplace(keys[i], decomposer); } - for(unsigned int bit = begin_bit, place = 0; bit < end_bit; bit += RadixBits, ++place) + if ROCPRIM_IF_CONSTEXPR(AllBits) { - count_digits_at_place(flat_id, - stripe, - keys, - place, - decomposer, - bit, - min(RadixBits, end_bit - bit), - valid_count, - storage); + ROCPRIM_UNROLL + for(unsigned int i = 0; i < ItemsPerThread; ++i) + { + ROCPRIM_UNROLL + for(unsigned int bit = 0, place = 0; bit < sizeof(KeyType) * 8; + bit += RadixBits, ++place) + { + const unsigned int pos = i * BlockSize + flat_id; + if(IsFull || pos < valid_count) + { + const unsigned int digit + = key_codec::extract_digit(keys[i], + bit, + min(RadixBits, end_bit - bit), + decomposer); + atomic_add(&storage.histogram[get_counter(stripe, place, digit)], 1); + } + } + } + } + else + { + ROCPRIM_UNROLL + for(unsigned int i = 0; i < ItemsPerThread; ++i) + { + for(unsigned int bit = begin_bit, place = 0; bit < end_bit; + bit += RadixBits, ++place) + { + const unsigned int pos = i * BlockSize + flat_id; + if(IsFull || pos < valid_count) + { + const unsigned int digit + = key_codec::extract_digit(keys[i], + bit, + min(RadixBits, end_bit - bit), + decomposer); + atomic_add(&storage.histogram[get_counter(stripe, place, digit)], 1); + } + } + } } ::rocprim::syncthreads(); @@ -877,7 +884,7 @@ struct onesweep_histograms_helper ROCPRIM_UNROLL for(unsigned int stripe = 0; stripe < atomic_stripes; ++stripe) { - total += get_counter(stripe, place, digit, storage); + total += storage.histogram[get_counter(stripe, place, digit)]; } ::rocprim::detail::atomic_add(&global_digit_counts[place * radix_size + digit], @@ -920,24 +927,37 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void onesweep_histograms(KeysInputIterator if(block_id < full_blocks) { - count_helper_type{}.template count_digits(keys_input + block_offset, - global_digit_counts, - items_per_block, - decomposer, - begin_bit, - end_bit, - storage); + if(begin_bit == 0 && end_bit == count_helper_type::total_bits) + { + count_helper_type{}.template count_digits(keys_input + block_offset, + global_digit_counts, + items_per_block, + decomposer, + begin_bit, + end_bit, + storage); + } + else + { + count_helper_type{}.template count_digits(keys_input + block_offset, + global_digit_counts, + items_per_block, + decomposer, + begin_bit, + end_bit, + storage); + } } else { const unsigned int valid_in_last_block = size - items_per_block * full_blocks; - count_helper_type{}.template count_digits(keys_input + block_offset, - global_digit_counts, - valid_in_last_block, - decomposer, - begin_bit, - end_bit, - storage); + count_helper_type{}.template count_digits(keys_input + block_offset, + global_digit_counts, + valid_in_last_block, + decomposer, + begin_bit, + end_bit, + storage); } } diff --git a/rocprim/include/rocprim/device/detail/device_reduce_by_key.hpp b/rocprim/include/rocprim/device/detail/device_reduce_by_key.hpp index a4e37d2db..1fb80c62d 100644 --- a/rocprim/include/rocprim/device/detail/device_reduce_by_key.hpp +++ b/rocprim/include/rocprim/device/detail/device_reduce_by_key.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -23,7 +23,6 @@ #include "device_config_helper.hpp" #include "lookback_scan_state.hpp" -#include "ordered_block_id.hpp" #include "../../block/block_discontinuity.hpp" #include "../../block/block_load.hpp" @@ -71,6 +70,14 @@ struct load_helper using block_load_keys = block_load; using block_load_values = block_load; + + /// We only need to sync between loading keys & values if BOTH the key and value + /// loading method require shared memory. + constexpr static bool requires_inner_sync + = !std::is_same::value + && !std::is_same::value; + union storage_type { typename block_load_keys::storage_type keys; @@ -86,16 +93,23 @@ struct load_helper AccumulatorType (&values)[ItemsPerThread], storage_type& storage) { + if(!is_global_last_tile) { block_load_keys{}.load(tile_keys, keys, storage.keys); - ::rocprim::syncthreads(); + if ROCPRIM_IF_CONSTEXPR(requires_inner_sync) + { + ::rocprim::syncthreads(); + } block_load_values{}.load(tile_values, values, storage.values); } else { block_load_keys{}.load(tile_keys, keys, valid_in_global_last_tile, storage.keys); - ::rocprim::syncthreads(); + if ROCPRIM_IF_CONSTEXPR(requires_inner_sync) + { + ::rocprim::syncthreads(); + } block_load_values{}.load(tile_values, values, valid_in_global_last_tile, @@ -464,13 +478,11 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE auto kernel_impl(KeyIterator, const BinaryOp, const CompareFunction, const LookbackScanState, - ordered_block_id, const std::size_t, const std::size_t, const std::size_t, const std::size_t* const, - const AccumulatorType* const, - const std::size_t) + const AccumulatorType* const) -> std::enable_if_t()> { // No need to build the kernel with sleep on a device that does not require it @@ -496,24 +508,21 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE auto const BinaryOp reduce_op, const CompareFunction compare, const LookbackScanState scan_state, - ordered_block_id ordered_tile_id, - const std::size_t starting_tile, - const std::size_t total_number_of_tiles, + const std::size_t starting_block, + const std::size_t total_number_of_blocks, const std::size_t size, const std::size_t* const global_head_count, - const AccumulatorType* const previous_accumulated, - const std::size_t number_of_tiles_launch) + const AccumulatorType* const previous_accumulated) -> std::enable_if_t()> { static constexpr reduce_by_key_config_params params = device_params(); static constexpr unsigned int block_size = params.kernel_config.block_size; static constexpr unsigned int items_per_thread = params.kernel_config.items_per_thread; - static constexpr unsigned int tiles_per_block = params.tiles_per_block; static constexpr block_load_method load_keys_method = params.load_keys_method; static constexpr block_load_method load_values_method = params.load_values_method; static constexpr block_scan_algorithm scan_algorithm = params.scan_algorithm; - static constexpr unsigned int items_per_tile = block_size * items_per_thread; + static constexpr unsigned int items_per_block = block_size * items_per_thread; using key_type = ::rocprim::detail::value_type_t; @@ -528,40 +537,29 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE auto ROCPRIM_SHARED_MEMORY union { - typename decltype(ordered_tile_id)::storage_type tile_id; - typename tile_processor::storage_type tile; + typename tile_processor::storage_type tile; } storage; - for(unsigned int i = 0; i < tiles_per_block; ++i) - { - rocprim::syncthreads(); - const std::size_t tile_id = ordered_tile_id.get(threadIdx.x, storage.tile_id); - if(tile_id >= number_of_tiles_launch) - { - return; - } - - const std::size_t tile_offset = tile_id * items_per_tile; - const KeyIterator tile_keys = keys_input + tile_offset; - const ValueIterator tile_values = values_input + tile_offset; - - rocprim::syncthreads(); - tile_processor{}.process_tile(tile_keys, - tile_values, - unique_keys, - reductions, - unique_count, - reduce_op, - compare, - scan_state, - tile_id, - starting_tile, - total_number_of_tiles, - size, - storage.tile, - global_head_count, - previous_accumulated); - } + const unsigned int block_id = rocprim::flat_block_id(); + const unsigned int block_offset = block_id * items_per_block; + const KeyIterator block_keys = keys_input + block_offset; + const ValueIterator block_values = values_input + block_offset; + + tile_processor{}.process_tile(block_keys, + block_values, + unique_keys, + reductions, + unique_count, + reduce_op, + compare, + scan_state, + block_id, + starting_block, + total_number_of_blocks, + size, + storage.tile, + global_head_count, + previous_accumulated); } } // namespace reduce_by_key diff --git a/rocprim/include/rocprim/device/detail/lookback_scan_state.hpp b/rocprim/include/rocprim/device/detail/lookback_scan_state.hpp index 25dc36c61..dddab29c9 100644 --- a/rocprim/include/rocprim/device/detail/lookback_scan_state.hpp +++ b/rocprim/include/rocprim/device/detail/lookback_scan_state.hpp @@ -86,6 +86,7 @@ struct match_prefix_underlying_type = select_type, select_type_case, select_type_case, + select_type_case, void>; }; @@ -111,7 +112,7 @@ enum class lookback_scan_determinism // a look-back prefix scan. Initially every prefix can be either // invalid (padding values) or empty. One thread in a block should // later set it to partial, and later to complete. -template +template struct lookback_scan_state; /// Reduce lanes `0-valid_items` and return the result in lane 0. @@ -186,13 +187,13 @@ struct lookback_scan_state using prefix_underlying_type = typename match_prefix_underlying_type::type; // Helper struct - struct alignas(sizeof(prefix_underlying_type)) prefix_type + struct prefix_type { T value; prefix_flag flag; }; - static_assert(sizeof(prefix_underlying_type) == sizeof(prefix_type), ""); + static_assert(sizeof(prefix_underlying_type) >= sizeof(prefix_type), ""); public: // Type used for flag/flag of block prefix @@ -846,10 +847,10 @@ class lookback_scan_prefix_op // to select the corresponding kernel (with or without sleep). However, since the check is runtime, // both versions of the kernel must be compiled for all architectures. // is_lookback_kernel_runnable() can be used in device code to prevent compilation of the version -// with sleep on all device arhitectures except gfx908. +// with sleep on all device architectures except gfx908. -ROCPRIM_HOST ROCPRIM_INLINE hipError_t is_sleep_scan_state_used(const hipStream_t stream, - bool& use_sleep) +ROCPRIM_HOST ROCPRIM_INLINE +hipError_t is_sleep_scan_state_used(const hipStream_t stream, bool& use_sleep) { hipDeviceProp_t prop; int device_id; diff --git a/rocprim/include/rocprim/device/device_reduce_by_key.hpp b/rocprim/include/rocprim/device/device_reduce_by_key.hpp index a5caa68fe..190d19f28 100644 --- a/rocprim/include/rocprim/device/device_reduce_by_key.hpp +++ b/rocprim/include/rocprim/device/device_reduce_by_key.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -53,13 +53,12 @@ namespace detail template ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) void - reduce_by_key_init_kernel(LookBackScanState lookback_scan_state, - const unsigned int number_of_tiles, - ordered_block_id ordered_bid, - const bool is_first_launch, - const unsigned int tile_save_idx, - std::size_t* const global_head_count, - AccumulatorType* const previous_accumulated) + reduce_by_key_init_kernel(LookBackScanState lookback_scan_state, + const unsigned int number_of_blocks, + const bool is_first_launch, + const unsigned int block_save_idx, + std::size_t* const global_head_count, + AccumulatorType* const previous_accumulated) { const unsigned int block_id = ::rocprim::detail::block_id<0>(); const unsigned int block_size = ::rocprim::detail::block_size<0>(); @@ -84,13 +83,13 @@ ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) void *previous_accumulated = ::rocprim::get<1>(value); }; access_indexed_lookback_value(lookback_scan_state, - number_of_tiles, - tile_save_idx, + number_of_blocks, + block_save_idx, flat_thread_id, update_func); } - init_lookback_scan_state(lookback_scan_state, number_of_tiles, ordered_bid, flat_thread_id); + init_lookback_scan_state(lookback_scan_state, number_of_blocks, flat_thread_id); } template -ROCPRIM_KERNEL __launch_bounds__(device_params().kernel_config.block_size) -void reduce_by_key_kernel(const KeyIterator keys_input, - const ValueIterator values_input, - const UniqueIterator unique_keys, - const ReductionIterator reductions, - const UniqueCountIterator unique_count, - const BinaryOp reduce_op, - const CompareFunction compare, - const LookbackScanState scan_state, - const ordered_block_id ordered_tile_id, - const std::size_t starting_tile, - const std::size_t total_number_of_tiles, - const std::size_t size, - const std::size_t* const global_head_count, - const AccumulatorType* const previous_accumulated, - const std::size_t number_of_tiles_launch) +ROCPRIM_KERNEL __launch_bounds__(device_params().kernel_config.block_size) void + reduce_by_key_kernel(const KeyIterator keys_input, + const ValueIterator values_input, + const UniqueIterator unique_keys, + const ReductionIterator reductions, + const UniqueCountIterator unique_count, + const BinaryOp reduce_op, + const CompareFunction compare, + const LookbackScanState scan_state, + const std::size_t starting_block, + const std::size_t total_number_of_blocks, + const std::size_t size, + const std::size_t* const global_head_count, + const AccumulatorType* const previous_accumulated) { reduce_by_key::kernel_impl(keys_input, values_input, @@ -129,13 +126,11 @@ void reduce_by_key_kernel(const KeyIterator keys_input, reduce_op, compare, scan_state, - ordered_tile_id, - starting_tile, - total_number_of_tiles, + starting_block, + total_number_of_blocks, size, global_head_count, - previous_accumulated, - number_of_tiles_launch); + previous_accumulated); } template; - using ordered_tile_id_type = detail::ordered_block_id; - const unsigned int block_size = params.kernel_config.block_size; - const unsigned int tiles_per_block = params.tiles_per_block; - const unsigned int items_per_tile = block_size * params.kernel_config.items_per_thread; - const unsigned int items_per_block = items_per_tile * tiles_per_block; + const unsigned int items_per_block = block_size * params.kernel_config.items_per_thread; const size_t size_limit = params.kernel_config.size_limit; const size_t aligned_size_limit @@ -188,20 +179,18 @@ hipError_t reduce_by_key_impl_wrapped_config(void* temporary const size_t limited_size = std::min(size, aligned_size_limit); const bool use_limited_size = limited_size == aligned_size_limit; - // Number of tiles in a single launch (or the only launch if it fits) - const std::size_t number_of_tiles = detail::ceiling_div(limited_size, items_per_tile); - const std::size_t number_of_blocks = detail::ceiling_div(number_of_tiles, tiles_per_block); + // Number of blocks in a single launch (or the only launch if it fits) + const std::size_t number_of_blocks = detail::ceiling_div(limited_size, items_per_block); // Calculate required temporary storage - void* scan_state_storage; - ordered_tile_id_type::id_type* ordered_bid_storage; + void* scan_state_storage; // The number of segment heads in previous launches. std::size_t* d_global_head_count = nullptr; // The running accumulation across the launch boundary. accumulator_type* d_previous_accumulated = nullptr; detail::temp_storage::layout layout{}; - result = scan_state_type::get_temp_storage_layout(number_of_tiles, stream, layout); + result = scan_state_type::get_temp_storage_layout(number_of_blocks, stream, layout); if(result != hipSuccess) { return result; @@ -213,8 +202,6 @@ hipError_t reduce_by_key_impl_wrapped_config(void* temporary detail::temp_storage::make_linear_partition( // This is valid even with scan_state_with_sleep_type detail::temp_storage::make_partition(&scan_state_storage, layout), - detail::temp_storage::make_partition(&ordered_bid_storage, - ordered_tile_id_type::get_temp_storage_layout()), detail::temp_storage::ptr_aligned_array(&d_global_head_count, use_limited_size ? 1 : 0), detail::temp_storage::ptr_aligned_array(&d_previous_accumulated, use_limited_size ? 1 : 0))); @@ -230,7 +217,7 @@ hipError_t reduce_by_key_impl_wrapped_config(void* temporary return result; } scan_state_type scan_state{}; - result = scan_state_type::create(scan_state, scan_state_storage, number_of_tiles, stream); + result = scan_state_type::create(scan_state, scan_state_storage, number_of_blocks, stream); if(result != hipSuccess) { return result; @@ -238,7 +225,7 @@ hipError_t reduce_by_key_impl_wrapped_config(void* temporary scan_state_with_sleep_type scan_state_with_sleep{}; result = scan_state_with_sleep_type::create(scan_state_with_sleep, scan_state_storage, - number_of_tiles, + number_of_blocks, stream); if(result != hipSuccess) { @@ -258,8 +245,6 @@ hipError_t reduce_by_key_impl_wrapped_config(void* temporary } }; - auto ordered_bid = ordered_tile_id_type::create(ordered_bid_storage); - if(size == 0) { // Fill out unique_count_output with zero @@ -271,9 +256,9 @@ hipError_t reduce_by_key_impl_wrapped_config(void* temporary debug_synchronous); } - // Total number of tiles in all launches - const std::size_t total_number_of_tiles = ceiling_div(size, items_per_tile); - const std::size_t number_of_launch = ceiling_div(size, limited_size); + // Total number of blocks in all launches + const std::size_t total_number_of_blocks = ceiling_div(size, items_per_block); + const std::size_t number_of_launch = ceiling_div(size, limited_size); if(debug_synchronous) { @@ -282,18 +267,14 @@ hipError_t reduce_by_key_impl_wrapped_config(void* temporary std::cout << "use_limited_size: " << std::boolalpha << use_limited_size << '\n'; std::cout << "number_of_launch: " << number_of_launch << '\n'; std::cout << "block_size: " << block_size << '\n'; - std::cout << "tiles_per_block: " << tiles_per_block << '\n'; - std::cout << "number_of_tiles: " << number_of_tiles << '\n'; std::cout << "number_of_blocks: " << number_of_blocks << '\n'; - std::cout << "items_per_tile: " << items_per_tile << '\n'; + std::cout << "items_per_block: " << items_per_block << '\n'; } for(size_t i = 0, offset = 0; i < number_of_launch; ++i, offset += limited_size) { const std::size_t current_size = std::min(size - offset, limited_size); - const std::size_t number_of_tiles_launch = ceiling_div(current_size, items_per_tile); - const std::size_t number_of_blocks_launch - = ceiling_div(number_of_tiles_launch, tiles_per_block); + const std::size_t number_of_blocks_launch = ceiling_div(current_size, items_per_block); // Start point for time measurements std::chrono::steady_clock::time_point start; @@ -301,7 +282,6 @@ hipError_t reduce_by_key_impl_wrapped_config(void* temporary { std::cout << "index: " << i << '\n'; std::cout << "current_size: " << current_size << '\n'; - std::cout << "number of tiles: " << number_of_tiles_launch << '\n'; std::cout << "number of blocks: " << number_of_blocks_launch << '\n'; start = std::chrono::steady_clock::now(); @@ -312,19 +292,18 @@ hipError_t reduce_by_key_impl_wrapped_config(void* temporary { const unsigned int block_size = ROCPRIM_DEFAULT_MAX_BLOCK_SIZE; const std::size_t grid_size - = detail::ceiling_div(number_of_tiles_launch, block_size); + = detail::ceiling_div(number_of_blocks_launch, block_size); reduce_by_key_init_kernel<<>>( scan_state, - number_of_tiles_launch, - ordered_bid, + number_of_blocks_launch, i == 0, - number_of_tiles - 1, + number_of_blocks - 1, d_global_head_count, d_previous_accumulated); }); ROCPRIM_DETAIL_HIP_SYNC_AND_RETURN_ON_ERROR("reduce_by_key_init_kernel", - number_of_tiles_launch, + number_of_blocks_launch, start); with_scan_state( @@ -340,13 +319,11 @@ hipError_t reduce_by_key_impl_wrapped_config(void* temporary reduce_op, key_compare_op, scan_state, - ordered_bid, - i * number_of_tiles, - total_number_of_tiles, + i * number_of_blocks, + total_number_of_blocks, size, i > 0 ? d_global_head_count : nullptr, - i > 0 ? d_previous_accumulated : nullptr, - number_of_tiles_launch); + i > 0 ? d_previous_accumulated : nullptr); }); ROCPRIM_DETAIL_HIP_SYNC_AND_RETURN_ON_ERROR("reduce_by_key_kernel", current_size, start); } diff --git a/rocprim/include/rocprim/device/device_run_length_encode.hpp b/rocprim/include/rocprim/device/device_run_length_encode.hpp index 95abaad31..e96b57a98 100644 --- a/rocprim/include/rocprim/device/device_run_length_encode.hpp +++ b/rocprim/include/rocprim/device/device_run_length_encode.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -51,8 +51,7 @@ namespace detail namespace run_length_encode { -template; return detail::reduce_by_key_impl_wrapped_config< - detail::lookback_scan_determinism::default_determinism, + detail::lookback_scan_determinism::nondeterministic, config>(temporary_storage, storage_size, keys_input, @@ -360,20 +359,19 @@ inline hipError_t run_length_encode(void* temporary_storage, using input_type = typename std::iterator_traits::value_type; using count_type = unsigned int; - return detail::run_length_encode:: - run_length_encode_impl( - temporary_storage, - storage_size, - input, - make_constant_iterator(1), - size, - unique_output, - counts_output, - runs_count_output, - ::rocprim::plus(), - ::rocprim::equal_to(), - stream, - debug_synchronous); + return detail::run_length_encode::run_length_encode_impl( + temporary_storage, + storage_size, + input, + make_constant_iterator(1), + size, + unique_output, + counts_output, + runs_count_output, + ::rocprim::plus(), + ::rocprim::equal_to(), + stream, + debug_synchronous); } /// \brief Parallel run-length encoding of non-trivial runs for device level. diff --git a/rocprim/include/rocprim/intrinsics/atomic.hpp b/rocprim/include/rocprim/intrinsics/atomic.hpp index 995993fdb..602785fe0 100644 --- a/rocprim/include/rocprim/intrinsics/atomic.hpp +++ b/rocprim/include/rocprim/intrinsics/atomic.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -162,6 +162,91 @@ namespace detail return __hip_atomic_load(address, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); } + ROCPRIM_DEVICE ROCPRIM_INLINE + __uint128_t atomic_load(const __uint128_t* address) + { + __uint128_t result; + +#define ROCPRIM_ATOMIC_LOAD(inst, mod, wait, ptr) \ + asm volatile(inst " %0, %1 " mod "\t\n" wait : "=v"(result) : "v"(ptr) : "memory") + +#if ROCPRIM_TARGET_CDNA3 + #define ROCPRIM_ATOMIC_LOAD_FLAT(ptr) \ + ROCPRIM_ATOMIC_LOAD("flat_load_dwordx4", "sc1", "s_waitcnt vmcnt(0)", ptr) + #define ROCPRIM_ATOMIC_LOAD_SHARED(ptr) \ + ROCPRIM_ATOMIC_LOAD("ds_read_b128", "", "s_waitcnt lgkmcnt(0)", ptr) + #define ROCPRIM_ATOMIC_LOAD_GLOBAL(ptr) \ + ROCPRIM_ATOMIC_LOAD("global_load_dwordx4", "off sc1", "s_waitcnt vmcnt(0)", ptr) +#elif ROCPRIM_TARGET_RDNA4 + #define ROCPRIM_ATOMIC_LOAD_FLAT(ptr) \ + ROCPRIM_ATOMIC_LOAD("flat_load_b128", "scope:SCOPE_DEV", "s_wait_loadcnt_dscnt 0x0", ptr) + #define ROCPRIM_ATOMIC_LOAD_SHARED(ptr) \ + ROCPRIM_ATOMIC_LOAD("ds_load_b128", "", "s_wait_dscnt 0x0", ptr) + #define ROCPRIM_ATOMIC_LOAD_GLOBAL(ptr) \ + ROCPRIM_ATOMIC_LOAD("global_load_b128", "off scope:SCOPE_DEV", "s_wait_loadcnt 0x0", ptr) +#elif ROCPRIM_TARGET_RDNA2 || ROCPRIM_TARGET_RDNA1 + #define ROCPRIM_ATOMIC_LOAD_FLAT(ptr) \ + ROCPRIM_ATOMIC_LOAD("flat_load_dwordx4", "glc dlc", "s_waitcnt vmcnt(0)", ptr) + #define ROCPRIM_ATOMIC_LOAD_SHARED(ptr) \ + ROCPRIM_ATOMIC_LOAD("ds_read_b128", "", "s_waitcnt lgkmcnt(0)", ptr) + #define ROCPRIM_ATOMIC_LOAD_GLOBAL(ptr) \ + ROCPRIM_ATOMIC_LOAD("global_load_dwordx4", "off glc dlc", "s_waitcnt vmcnt(0)", ptr) +#elif ROCPRIM_TARGET_GCN3 + #define ROCPRIM_ATOMIC_LOAD_FLAT(ptr) \ + ROCPRIM_ATOMIC_LOAD("flat_load_dwordx4", "glc", "s_waitcnt vmcnt(0)", ptr) + #define ROCPRIM_ATOMIC_LOAD_SHARED(ptr) \ + ROCPRIM_ATOMIC_LOAD("ds_read_b128", "", "s_waitcnt lgkmcnt(0)", ptr) + // This architecture doesn't support atomics on the global AS. + #define ROCPRIM_ATOMIC_LOAD_GLOBAL(ptr) ROCPRIM_ATOMIC_LOAD_FLAT(ptr) +#elif ROCPRIM_TARGET_RDNA3 || ROCPRIM_TARGET_CDNA2 || ROCPRIM_TARGET_CDNA1 || ROCPRIM_TARGET_GCN5 + #define ROCPRIM_ATOMIC_LOAD_FLAT(ptr) \ + ROCPRIM_ATOMIC_LOAD("flat_load_dwordx4", "glc", "s_waitcnt vmcnt(0)", ptr) + #define ROCPRIM_ATOMIC_LOAD_SHARED(ptr) \ + ROCPRIM_ATOMIC_LOAD("ds_read_b128", "", "s_waitcnt lgkmcnt(0)", ptr) + #define ROCPRIM_ATOMIC_LOAD_GLOBAL(ptr) \ + ROCPRIM_ATOMIC_LOAD("global_load_dwordx4", "off glc", "s_waitcnt vmcnt(0)", ptr) +#elif defined(__HIP_DEVICE_COMPILE__) + // Please submit an issue or pull request! + #error support for 128-bit atomics not implemented for current architecture +#endif + +#ifdef __HIP_DEVICE_COMPILE__ + #if defined(__has_builtin) && __has_builtin(__builtin_amdgcn_is_shared) \ + && __has_builtin(__builtin_amdgcn_is_private) + + auto* ptr = (const __attribute__((address_space(0 /*flat*/))) __uint128_t*)address; + if(__builtin_amdgcn_is_shared(ptr)) + { + auto* shared_ptr + = (const __attribute__((address_space(3 /*lds*/))) __uint128_t*)address; + ROCPRIM_ATOMIC_LOAD_SHARED(shared_ptr); + } + else if(__builtin_amdgcn_is_private(ptr)) + { + ROCPRIM_ATOMIC_LOAD_FLAT(address); + } + else + { + auto* global_ptr + = (const __attribute__((address_space(1 /*global*/))) __uint128_t*)address; + ROCPRIM_ATOMIC_LOAD_GLOBAL(global_ptr); + } + #else + ROCPRIM_ATOMIC_LOAD_FLAT(address); + #endif +#else + (void)address; + result = 0; +#endif + + return result; + +#undef ROCPRIM_ATOMIC_LOAD +#undef ROCPRIM_ATOMIC_LOAD_FLAT +#undef ROCPRIM_ATOMIC_LOAD_SHARED +#undef ROCPRIM_ATOMIC_LOAD_GLOBAL + } + ROCPRIM_DEVICE ROCPRIM_INLINE void atomic_store(unsigned char* address, unsigned char value) { @@ -189,6 +274,79 @@ namespace detail __hip_atomic_store(address, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); } + ROCPRIM_DEVICE ROCPRIM_INLINE + void atomic_store(__uint128_t* address, const __uint128_t value) + { +#define ROCPRIM_ATOMIC_STORE(inst, mod, wait, ptr) \ + asm volatile(inst " %0, %1 " mod "\t\n" wait : : "v"(ptr), "v"(value) : "memory") + +#if ROCPRIM_TARGET_CDNA3 + #define ROCPRIM_ATOMIC_STORE_FLAT(ptr) \ + ROCPRIM_ATOMIC_STORE("flat_store_dwordx4", "sc1", "s_waitcnt vmcnt(0)", ptr) + #define ROCPRIM_ATOMIC_STORE_SHARED(ptr) \ + ROCPRIM_ATOMIC_STORE("ds_write_b128", "", "s_waitcnt lgkmcnt(0)", ptr) + #define ROCPRIM_ATOMIC_STORE_GLOBAL(ptr) \ + ROCPRIM_ATOMIC_STORE("global_store_dwordx4", "off sc1", "s_waitcnt vmcnt(0)", ptr) +#elif ROCPRIM_TARGET_RDNA4 + #define ROCPRIM_ATOMIC_STORE_FLAT(ptr) \ + ROCPRIM_ATOMIC_STORE("flat_store_b128", "scope:SCOPE_DEV", "s_wait_storecnt_dscnt 0x0", ptr) + #define ROCPRIM_ATOMIC_STORE_SHARED(ptr) \ + ROCPRIM_ATOMIC_STORE("ds_store_b128", "", "s_wait_dscnt 0x0", ptr) + #define ROCPRIM_ATOMIC_STORE_GLOBAL(ptr) \ + ROCPRIM_ATOMIC_STORE("global_store_b128", "off scope:SCOPE_DEV", "s_wait_storecnt 0x0", ptr) +#elif ROCPRIM_TARGET_GCN3 + #define ROCPRIM_ATOMIC_STORE_FLAT(ptr) \ + ROCPRIM_ATOMIC_STORE("flat_store_dwordx4", "", "s_waitcnt vmcnt(0)", ptr) + #define ROCPRIM_ATOMIC_STORE_SHARED(ptr) \ + ROCPRIM_ATOMIC_STORE("ds_write_b128", "", "s_waitcnt lgkmcnt(0)", ptr) + // This architecture doesn't support atomics on the global AS. + #define ROCPRIM_ATOMIC_STORE_GLOBAL(ptr) ROCPRIM_ATOMIC_STORE_FLAT(ptr) +#elif ROCPRIM_TARGET_RDNA3 || ROCPRIM_TARGET_RDNA2 || ROCPRIM_TARGET_RDNA1 || ROCPRIM_TARGET_CDNA2 \ + || ROCPRIM_TARGET_CDNA1 || ROCPRIM_TARGET_GCN5 + #define ROCPRIM_ATOMIC_STORE_FLAT(ptr) \ + ROCPRIM_ATOMIC_STORE("flat_store_dwordx4", "", "s_waitcnt vmcnt(0)", ptr) + #define ROCPRIM_ATOMIC_STORE_SHARED(ptr) \ + ROCPRIM_ATOMIC_STORE("ds_write_b128", "", "s_waitcnt lgkmcnt(0)", ptr) + #define ROCPRIM_ATOMIC_STORE_GLOBAL(ptr) \ + ROCPRIM_ATOMIC_STORE("global_store_dwordx4", "off", "s_waitcnt vmcnt(0)", ptr) +#elif defined(__HIP_DEVICE_COMPILE__) + // Please submit an issue or pull request! + #error support for 128-bit atomics not implemented for current architecture +#endif + +#ifdef __HIP_DEVICE_COMPILE__ + #if defined(__has_builtin) && __has_builtin(__builtin_amdgcn_is_shared) \ + && __has_builtin(__builtin_amdgcn_is_private) + + auto* ptr = (__attribute__((address_space(0 /*flat*/))) __uint128_t*)address; + if(__builtin_amdgcn_is_shared(ptr)) + { + auto* shared_ptr = (__attribute__((address_space(3 /*lds*/))) __uint128_t*)address; + ROCPRIM_ATOMIC_STORE_SHARED(shared_ptr); + } + else if(__builtin_amdgcn_is_private(ptr)) + { + ROCPRIM_ATOMIC_STORE_FLAT(address); + } + else + { + auto* global_ptr = (__attribute__((address_space(1 /*global*/))) __uint128_t*)address; + ROCPRIM_ATOMIC_STORE_GLOBAL(global_ptr); + } + #else + ROCPRIM_ATOMIC_STORE_FLAT(address); + #endif +#else + (void)address; + (void)value; +#endif + +#undef ROCPRIM_ATOMIC_STORE +#undef ROCPRIM_ATOMIC_STORE_FLAT +#undef ROCPRIM_ATOMIC_STORE_SHARED +#undef ROCPRIM_ATOMIC_STORE_GLOBAL + } + /// \brief Wait for all vector memory operations to complete /// /// This ensures that previous visible writes to vector memory have completed before the function diff --git a/scripts/autotune/templates/config_template b/scripts/autotune/templates/config_template index 3adbb88a5..e93bf26ae 100644 --- a/scripts/autotune/templates/config_template +++ b/scripts/autotune/templates/config_template @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal diff --git a/scripts/autotune/templates/reduce_by_key_config_template b/scripts/autotune/templates/reduce_by_key_config_template index 1bb5b7003..e13ea2e59 100644 --- a/scripts/autotune/templates/reduce_by_key_config_template +++ b/scripts/autotune/templates/reduce_by_key_config_template @@ -6,12 +6,11 @@ ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_REDUCE_BY_KEY_HPP_ {% macro kernel_configuration(measurement) -%} reduce_by_key_config< - {{ measurement['cfg']['bs'] }}, - {{ measurement['cfg']['ipt'] }}, + {{ measurement['cfg']['bs'] }}, + {{ measurement['cfg']['ipt'] }}, block_load_method::block_load_transpose, block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - {{ measurement['cfg']['tpb'] }}> { }; + block_scan_algorithm::using_warp_scan> { }; {%- endmacro %} {% macro general_case() -%} @@ -24,8 +23,8 @@ struct default_reduce_by_key_config : default_reduce_by_key_config_base struct default_reduce_by_key_config< - static_cast({{ benchmark_of_architecture.name }}), - key_type, - value_type, + static_cast({{ benchmark_of_architecture.name }}), + key_type, + value_type, {{ fallback_selection_criteria }}> : {%- endmacro %} diff --git a/scripts/autotune/templates/run_length_encode_config_template b/scripts/autotune/templates/run_length_encode_config_template index 1800f9909..17b2003d7 100644 --- a/scripts/autotune/templates/run_length_encode_config_template +++ b/scripts/autotune/templates/run_length_encode_config_template @@ -6,12 +6,11 @@ ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_RUN_LENGTH_ENCODE_HPP_ {% macro kernel_configuration(measurement) -%} reduce_by_key_config< - {{ measurement['cfg']['bs'] }}, - {{ measurement['cfg']['ipt'] }}, + {{ measurement['cfg']['bs'] }}, + {{ measurement['cfg']['ipt'] }}, block_load_method::block_load_transpose, - block_load_method::block_load_transpose, - block_scan_algorithm::using_warp_scan, - {{ measurement['cfg']['tpb'] }}> { }; + block_load_method::block_load_direct, + block_scan_algorithm::using_warp_scan> { }; {%- endmacro %} {% macro general_case() -%} @@ -24,8 +23,8 @@ struct default_trivial_runs_config : default_reduce_by_key_config_base struct default_trivial_runs_config< - static_cast({{ benchmark_of_architecture.name }}), - key_type, - value_type, + static_cast({{ benchmark_of_architecture.name }}), + key_type, + value_type, {{ fallback_selection_criteria }}> : {%- endmacro %} diff --git a/test/rocprim/CMakeLists.txt b/test/rocprim/CMakeLists.txt index 4d210c203..c6d5cf258 100644 --- a/test/rocprim/CMakeLists.txt +++ b/test/rocprim/CMakeLists.txt @@ -297,6 +297,7 @@ add_rocprim_test("rocprim.transform_iterator" test_transform_iterator.cpp) add_rocprim_test("rocprim.type_traits_interface" test_type_traits_interface.cpp) add_rocprim_test("rocprim.no_half_operators" test_no_half_operators.cpp) add_rocprim_test("rocprim.intrinsics" test_intrinsics.cpp) +add_rocprim_test("rocprim.intrinsics_atomic" test_intrinsics_atomic.cpp) add_rocprim_test("rocprim.invoke_result" test_invoke_result.cpp) add_rocprim_test("rocprim.warp_exchange" test_warp_exchange.cpp) add_rocprim_test("rocprim.warp_load" test_warp_load.cpp) diff --git a/test/rocprim/test_device_reduce_by_key.cpp b/test/rocprim/test_device_reduce_by_key.cpp index c7ff2936e..ae9dc7a56 100644 --- a/test/rocprim/test_device_reduce_by_key.cpp +++ b/test/rocprim/test_device_reduce_by_key.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -524,13 +524,11 @@ TEST(RocprimDeviceReduceByKey, LargeIndicesReduceByKeySmallValueType) TEST(RocprimDeviceReduceByKey, LargeIndicesReduceByKeyLargeValueType) { - // large value type to test TilesPerBlock > 1 large_indices_reduce_by_key>(); } TEST(RocprimDeviceReduceByKey, LargeIndicesReduceByKeyLargeValueTypeWithGraphs) { - // large value type to test TilesPerBlock > 1 large_indices_reduce_by_key, true>(); } @@ -650,7 +648,6 @@ TEST(RocprimDeviceReduceByKey, LargeSegmentCountReduceByKeySmallValueType) TEST(RocprimDeviceReduceByKey, LargeSegmentCountReduceByKeyLargeValueType) { - // large value type to test TilesPerBlock > 1 large_segment_count_reduce_by_key>(); } diff --git a/test/rocprim/test_intrinsics_atomic.cpp b/test/rocprim/test_intrinsics_atomic.cpp new file mode 100644 index 000000000..7614053ba --- /dev/null +++ b/test/rocprim/test_intrinsics_atomic.cpp @@ -0,0 +1,178 @@ +// Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "rocprim/intrinsics/atomic.hpp" +#include "rocprim/types.hpp" + +#include "../common_test_header.hpp" +#include "test_utils.hpp" +#include "test_utils_device_ptr.hpp" + +#include + +template +ROCPRIM_INLINE ROCPRIM_DEVICE +void test_atomics( + GetPtr get_ptr, bool* error, const uint32_t* test_data, size_t random_size, bool delay) +{ + // This test attempts to test some atomicity properties of the operation by repeatedly writing + // some random data to the value, distributed over the atomic bits. This way, we can attempt to + // test that the write really is atomic: If it happens in multiple parts, there is a chance + // that another write will happen at the same time and mess up the parts. Manually replacing + // rocprim's 128-bit atomic store with 2 64-bit atomic stores seems to trigger this as expected. + // + // Unfortunately, this test cannot really check caching mechanisms, reordering, etc, for now. + + for(int i = 0; i < 1000; ++i) + { + const rocprim::uint128_t new_value = rocprim::detail::atomic_load(get_ptr(i * 2)); + const unsigned int j = new_value >> 96; + const unsigned int a = (new_value >> 64) & 0xFFFF'FFFF; + const unsigned int b = (new_value >> 32) & 0xFFFF'FFFF; + const unsigned int c = new_value & 0xFFFF'FFFF; + + if(a != test_data[j * 3] || b != test_data[j * 3 + 1] || c != test_data[j * 3 + 2]) + { + *error = true; + } + + if(delay) + { + __builtin_amdgcn_s_sleep(63); + } + + const unsigned int index = (threadIdx.x + blockIdx.x * blockDim.x + i) % random_size; + rocprim::uint128_t value = rocprim::uint128_t{index} << 96; + value |= rocprim::uint128_t{test_data[index * 3]} << 64; + value |= rocprim::uint128_t{test_data[index * 3 + 1]} << 32; + value |= rocprim::uint128_t{test_data[index * 3 + 2]}; + rocprim::detail::atomic_store(get_ptr(i * 2 + 1), value); + } +} + +__global__ +void test_global(rocprim::uint128_t* ptr, + bool* error, + const uint32_t* test_data, + size_t random_size) +{ + test_atomics([=](int /* i */) { return ptr; }, error, test_data, random_size, true); +} + +__global__ +void test_shared(bool* error, const uint32_t* test_data, size_t random_size) +{ + __shared__ rocprim::uint128_t shared_data; + if(threadIdx.x == 0) + { + shared_data = 0; + } + + __syncthreads(); + + test_atomics([&](int /* i */) { return &shared_data; }, error, test_data, random_size, false); +} + +__global__ +void test_flat(rocprim::uint128_t* global_ptr, + bool* error, + const uint32_t* test_data, + size_t random_size) +{ + __shared__ rocprim::uint128_t shared_data; + if(threadIdx.x == 0) + { + shared_data = 0; + } + + __syncthreads(); + + test_atomics([&](int i) { return test_data[i % random_size] & 1 ? &shared_data : global_ptr; }, + error, + test_data, + random_size, + false); +} + +template +void generic_atomic_test(F cbk) +{ + static constexpr uint32_t block_size = 1024; + static constexpr uint32_t grid_size = 1024; + static constexpr uint32_t size = block_size * grid_size; + + test_utils::device_ptr d_error(1); + + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + { + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); + + // Generate data + auto input = test_utils::get_random_data(size * 3, + rocprim::numeric_limits::min(), + rocprim::numeric_limits::max(), + seed_value); + // Set the correct initial value if the initial atomic value is 0. + input[0] = 0; + input[1] = 0; + input[2] = 0; + + HIP_CHECK(hipMemset(d_error.get(), 0, sizeof(bool))); + test_utils::device_ptr d_input(input); + + cbk(block_size, grid_size, d_error.get(), d_input.get(), size); + HIP_CHECK(hipGetLastError()); + + bool result; + HIP_CHECK(hipMemcpy(&result, d_error.get(), sizeof(bool), hipMemcpyDeviceToHost)); + ASSERT_EQ(result, false); + } +} + +TEST(RocprimAtomicTests, Global128Bits) +{ + test_utils::device_ptr d_ptr(1); + generic_atomic_test( + [&](auto block_size, auto grid_size, auto* d_error, auto* d_input, auto size) + { + HIP_CHECK(hipMemset(d_ptr.get(), 0, sizeof(rocprim::uint128_t))); + test_global<<>>(d_ptr.get(), d_error, d_input, size); + }); +} + +TEST(RocprimAtomicTests, Shared128Bits) +{ + generic_atomic_test( + [&](auto block_size, auto grid_size, auto* d_error, auto* d_input, auto size) + { test_shared<<>>(d_error, d_input, size); }); +} + +TEST(RocprimAtomicTests, Flat128Bits) +{ + test_utils::device_ptr d_ptr(1); + generic_atomic_test( + [&](auto block_size, auto grid_size, auto* d_error, auto* d_input, auto size) + { + HIP_CHECK(hipMemset(d_ptr.get(), 0, sizeof(rocprim::uint128_t))); + test_flat<<>>(d_ptr.get(), d_error, d_input, size); + }); +} From 84c6d8d26f954abf8a9f8feda6a3e7da3f2a4fdd Mon Sep 17 00:00:00 2001 From: Robin Voetter Date: Mon, 20 Jan 2025 18:45:53 +0100 Subject: [PATCH 10/84] improve type traits interface docs --- docs/concepts/type_traits.rst | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/concepts/type_traits.rst b/docs/concepts/type_traits.rst index 836c8761a..2fe17362f 100644 --- a/docs/concepts/type_traits.rst +++ b/docs/concepts/type_traits.rst @@ -1,11 +1,11 @@ .. meta:: - :description: rocPRIM documentation and API reference library - :keywords: rocPRIM, ROCm, API, documentation + :description: Implementing traits for custom types in rocPRIM + :keywords: rocPRIM, ROCm, custom types, type traits .. _type_traits: ******************************************************************** - Implement traits for custom types + Implementing traits for custom types in rocPRIM ******************************************************************** Overview @@ -13,10 +13,15 @@ Overview This interface is designed to enable users to provide additional type trait information to rocPRIM, facilitating better compatibility with custom types. -Custom types that implement arithmetic operators can behave like built-in arithmetic types; however, algorithms in rocPRIM may still interpret them as generic `struct` or `class` types. For some algorithms, processing methods differ between floating-point and integral types. Accurately describing custom types, therefore, is essential not only for performance optimization but also for ensuring correctness. The rocPRIM type traits interface allows users to supply custom trait information for their types, greatly enhancing compatibility between these types and rocPRIM algorithms. +Accurately describing custom types is important for performance optimization and computational correctness. -This interface is strict but flexible, similar to operator overloading. Users should only implement traits as required by specific algorithms, and some traits cannot be defined if they can be inferred from others. Adhering to these rules is crucial for successful compilation. Fortunately, the interface provides clear error messages to guide users when issues arise. +Custom types that implement arithmetic operators can behave like built-in arithmetic types but might still be interpreted by rocPRIM algorithms as generic ``struct`` or ``class`` types. +The rocPRIM type traits interface lets users add custom trait information for their types, improving compatibility between these types and rocPRIM algorithms. + +This interface is similar to operator overloading. + +Traits should be implemented as required by specific algorithms. Some traits can't be defined if they can be inferred from others. Interface ========= @@ -25,7 +30,6 @@ Interface :content-only: :members: - Available traits ================ @@ -47,5 +51,5 @@ Types with predefined traits :members: :outline: :no-link: - - \ No newline at end of file + + From 4bcdd4b23132ca8bdb312c427811db1343f8deb2 Mon Sep 17 00:00:00 2001 From: Jaap Blok Date: Thu, 21 Nov 2024 12:58:55 +0100 Subject: [PATCH 11/84] Fix static asserts and remove template specialization --- rocprim/include/rocprim/warp/warp_reduce.hpp | 103 ++------ rocprim/include/rocprim/warp/warp_scan.hpp | 239 +++++-------------- rocprim/include/rocprim/warp/warp_sort.hpp | 182 +++----------- 3 files changed, 115 insertions(+), 409 deletions(-) diff --git a/rocprim/include/rocprim/warp/warp_reduce.hpp b/rocprim/include/rocprim/warp/warp_reduce.hpp index 56b8d6fd4..725a27f32 100644 --- a/rocprim/include/rocprim/warp/warp_reduce.hpp +++ b/rocprim/include/rocprim/warp/warp_reduce.hpp @@ -119,7 +119,8 @@ class warp_reduce using base_type = typename detail::select_warp_reduce_impl::type; // Check if WarpSize is valid for the targets - static_assert(WarpSize <= ROCPRIM_MAX_WARP_SIZE, "WarpSize can't be greater than hardware warp size."); + ROCPRIM_DETAIL_DEVICE_STATIC_ASSERT(WarpSize <= ::rocprim::device_warp_size(), + "WarpSize can't be greater than hardware warp size."); public: /// \brief Struct used to allocate a temporary memory that is required for thread @@ -177,27 +178,15 @@ class warp_reduce /// \endcode /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto reduce(T input, - T& output, - storage_type& storage, - BinaryFunction reduce_op = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type + ROCPRIM_DEVICE ROCPRIM_INLINE + auto reduce(T input, + T& output, + storage_type& storage, + BinaryFunction reduce_op = BinaryFunction()) { base_type::reduce(input, output, storage, reduce_op); } - /// \brief Performs reduction across threads in a logical warp. - /// Invalid Warp Size - template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto - reduce(T, T&, storage_type&, BinaryFunction reduce_op = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type - { - (void) reduce_op; - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); - return; - } - /// \brief Performs reduction across threads in a logical warp. /// /// \tparam BinaryFunction - type of binary function used for reduce. Default type @@ -245,28 +234,16 @@ class warp_reduce /// \endcode /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto reduce(T input, - T& output, - int valid_items, - storage_type& storage, - BinaryFunction reduce_op = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type + ROCPRIM_DEVICE ROCPRIM_INLINE + auto reduce(T input, + T& output, + int valid_items, + storage_type& storage, + BinaryFunction reduce_op = BinaryFunction()) { base_type::reduce(input, output, valid_items, storage, reduce_op); } - /// \brief Performs reduction across threads in a logical warp. - /// Invalid Warp Size - template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto - reduce(T, T&, int, storage_type&, BinaryFunction reduce_op = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type - { - (void) reduce_op; - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); - return; - } - /// \brief Performs head-segmented reduction across threads in a logical warp. /// /// \tparam Flag - type of head flags. Must be contextually convertible to \p bool. @@ -288,31 +265,16 @@ class warp_reduce template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto head_segmented_reduce(T input, - T& output, - Flag flag, - storage_type& storage, - BinaryFunction reduce_op - = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type + ROCPRIM_DEVICE ROCPRIM_INLINE + auto head_segmented_reduce(T input, + T& output, + Flag flag, + storage_type& storage, + BinaryFunction reduce_op = BinaryFunction()) { base_type::head_segmented_reduce(input, output, flag, storage, reduce_op); } - /// \brief Performs head-segmented reduction across threads in a logical warp. - /// Invalid Warp Size - template, - unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto head_segmented_reduce( - T, T&, Flag, storage_type&, BinaryFunction reduce_op = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type - { - (void) reduce_op; - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); - return; - } - /// \brief Performs tail-segmented reduction across threads in a logical warp. /// /// \tparam Flag - type of tail flags. Must be contextually convertible to \p bool. @@ -334,30 +296,15 @@ class warp_reduce template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto tail_segmented_reduce(T input, - T& output, - Flag flag, - storage_type& storage, - BinaryFunction reduce_op - = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type + ROCPRIM_DEVICE ROCPRIM_INLINE + auto tail_segmented_reduce(T input, + T& output, + Flag flag, + storage_type& storage, + BinaryFunction reduce_op = BinaryFunction()) { base_type::tail_segmented_reduce(input, output, flag, storage, reduce_op); } - - /// \brief Performs tail-segmented reduction across threads in a logical warp. - /// Invalid Warp Size - template, - unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto tail_segmented_reduce( - T, T&, Flag, storage_type&, BinaryFunction reduce_op = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type - { - (void) reduce_op; - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); - return; - } }; END_ROCPRIM_NAMESPACE diff --git a/rocprim/include/rocprim/warp/warp_scan.hpp b/rocprim/include/rocprim/warp/warp_scan.hpp index dfc2985db..6c5c6195f 100644 --- a/rocprim/include/rocprim/warp/warp_scan.hpp +++ b/rocprim/include/rocprim/warp/warp_scan.hpp @@ -116,7 +116,8 @@ class warp_scan using base_type = typename detail::select_warp_scan_impl::type; // Check if WarpSize is valid for the targets - static_assert(WarpSize <= ROCPRIM_MAX_WARP_SIZE, "WarpSize can't be greater than hardware warp size."); + ROCPRIM_DETAIL_DEVICE_STATIC_ASSERT(WarpSize <= ::rocprim::device_warp_size(), + "WarpSize can't be greater than hardware warp size."); public: /// \brief Struct used to allocate a temporary memory that is required for thread @@ -178,27 +179,15 @@ class warp_scan /// {33, -34, -34, -36, ..., -64} etc. /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto inclusive_scan(T input, - T& output, - storage_type& storage, - BinaryFunction scan_op = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type + ROCPRIM_DEVICE ROCPRIM_INLINE + auto inclusive_scan(T input, + T& output, + storage_type& storage, + BinaryFunction scan_op = BinaryFunction()) { base_type::inclusive_scan(input, output, storage, scan_op); } - /// \brief Performs inclusive scan across threads in a logical warp. - /// Invalid Warp Size - template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto - inclusive_scan(T, T&, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type - { - (void) scan_op; - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size . Aborting warp sort."); - return; - } - /// \brief Performs inclusive scan and reduction across threads in a logical warp. /// /// \tparam BinaryFunction - type of binary function used for scan. Default type @@ -249,28 +238,16 @@ class warp_scan /// The \p reduction will be equal \p 64. /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto inclusive_scan(T input, - T& output, - T& reduction, - storage_type& storage, - BinaryFunction scan_op = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type + ROCPRIM_DEVICE ROCPRIM_INLINE + auto inclusive_scan(T input, + T& output, + T& reduction, + storage_type& storage, + BinaryFunction scan_op = BinaryFunction()) { base_type::inclusive_scan(input, output, reduction, storage, scan_op); } - /// \brief Performs inclusive scan and reduction across threads in a logical warp. - /// Invalid Warp Size - template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto - inclusive_scan(T, T&, T&, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type - { - (void) scan_op; - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size . Aborting warp sort."); - return; - } - /// \brief Performs exclusive scan across threads in a logical warp. /// /// \tparam BinaryFunction - type of binary function used for scan. Default type @@ -324,28 +301,16 @@ class warp_scan /// {100, 33, -34, -34, -36, ..., -62} etc. /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto exclusive_scan(T input, - T& output, - T init, - storage_type& storage, - BinaryFunction scan_op = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type + ROCPRIM_DEVICE ROCPRIM_INLINE + auto exclusive_scan(T input, + T& output, + T init, + storage_type& storage, + BinaryFunction scan_op = BinaryFunction()) { base_type::exclusive_scan(input, output, init, storage, scan_op); } - /// \brief Performs exclusive scan across threads in a logical warp. - /// Invalid Warp Size - template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto - exclusive_scan(T, T&, T, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type - { - (void) scan_op; - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size . Aborting warp sort."); - return; - } - /// \brief Performs exclusive scan and reduction across threads in a logical warp. /// /// \tparam BinaryFunction - type of binary function used for scan. Default type @@ -400,29 +365,17 @@ class warp_scan /// {10, 11, 12, 13, ..., 73}. The \p reduction will be 64. /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto exclusive_scan(T input, - T& output, - T init, - T& reduction, - storage_type& storage, - BinaryFunction scan_op = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type + ROCPRIM_DEVICE ROCPRIM_INLINE + auto exclusive_scan(T input, + T& output, + T init, + T& reduction, + storage_type& storage, + BinaryFunction scan_op = BinaryFunction()) { base_type::exclusive_scan(input, output, init, reduction, storage, scan_op); } - /// \brief Performs exclusive scan and reduction across threads in a logical warp. - /// Invalid Warp Size - template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto - exclusive_scan(T, T&, T, T&, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type - { - (void) scan_op; - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size . Aborting warp sort."); - return; - } - /// \brief Performs exclusive scan without an initial value across threads in a logical warp /// \tparam BinaryFunction binary function used for scan /// \param input Thread input value @@ -432,32 +385,15 @@ class warp_scan /// \param [in] storage Reference to a temporary storage object of type storage_type. /// \param scan_op The function object used to combine elements used for the scan template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto exclusive_scan(T input, - T& output, - storage_type& storage, - BinaryFunction scan_op = BinaryFunction()) -#ifndef DOXYGEN_DOCUMENTATION_BUILD - -> std::enable_if_t -#else - -> void -#endif + ROCPRIM_DEVICE ROCPRIM_INLINE + auto exclusive_scan(T input, + T& output, + storage_type& storage, + BinaryFunction scan_op = BinaryFunction()) { base_type::exclusive_scan(input, output, storage, scan_op); } - /// \cond - template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto exclusive_scan(T /*input*/, - T& /*output*/, - storage_type& /*storage*/, - BinaryFunction /*scan_op*/ = BinaryFunction()) - -> std::enable_if_t<(FunctionWarpSize > device_warp_size())> - { - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size." - " Aborting warp scan."); - } - /// \endcond - /// \brief Performs exclusive scan and reduction without an initial value across threads in /// a logical warp /// \tparam BinaryFunction binary function used for scan @@ -469,34 +405,16 @@ class warp_scan /// \param [in] storage Reference to a temporary storage object of type storage_type. /// \param scan_op The function object used to combine elements used for the scan template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto exclusive_scan(T input, - T& output, - storage_type& storage, - T& reduction, - BinaryFunction scan_op = BinaryFunction()) -#ifndef DOXYGEN_DOCUMENTATION_BUILD - -> std::enable_if_t -#else - -> void -#endif + ROCPRIM_DEVICE ROCPRIM_INLINE + auto exclusive_scan(T input, + T& output, + storage_type& storage, + T& reduction, + BinaryFunction scan_op = BinaryFunction()) { base_type::exclusive_scan(input, output, storage, reduction, scan_op); } - /// \cond - template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto exclusive_scan(T /*input*/, - T& /*output*/, - storage_type& /*storage*/, - T& /*reduction*/, - BinaryFunction /*scan_op*/ = BinaryFunction()) - -> std::enable_if_t<(FunctionWarpSize > device_warp_size())> - { - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size." - " Aborting warp scan."); - } - /// \endcond - /// \brief Performs inclusive and exclusive scan operations across threads /// in a logical warp. /// @@ -556,29 +474,17 @@ class warp_scan /// {100, 33, -34, -34, -36, ..., -62} etc. /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto scan(T input, - T& inclusive_output, - T& exclusive_output, - T init, - storage_type& storage, - BinaryFunction scan_op = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type + ROCPRIM_DEVICE ROCPRIM_INLINE + auto scan(T input, + T& inclusive_output, + T& exclusive_output, + T init, + storage_type& storage, + BinaryFunction scan_op = BinaryFunction()) { base_type::scan(input, inclusive_output, exclusive_output, init, storage, scan_op); } - /// \brief Performs inclusive and exclusive scan operations across threads - /// Invalid Warp Size - template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto - scan(T, T&, T&, T, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type - { - (void) scan_op; - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size . Aborting warp sort."); - return; - } - /// \brief Performs inclusive and exclusive scan operations, and reduction across /// threads in a logical warp. /// @@ -638,14 +544,14 @@ class warp_scan /// be {10, 11, 12, 13, ..., 73}. The \p reduction will be 64. /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto scan(T input, - T& inclusive_output, - T& exclusive_output, - T init, - T& reduction, - storage_type& storage, - BinaryFunction scan_op = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type + ROCPRIM_DEVICE ROCPRIM_INLINE + auto scan(T input, + T& inclusive_output, + T& exclusive_output, + T init, + T& reduction, + storage_type& storage, + BinaryFunction scan_op = BinaryFunction()) { base_type::scan( input, inclusive_output, exclusive_output, init, reduction, @@ -653,18 +559,6 @@ class warp_scan ); } - /// \brief Performs inclusive and exclusive scan operations across threads - /// Invalid Warp Size - template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto - scan(T, T&, T&, T, T&, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type - { - (void) scan_op; - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size . Aborting warp sort."); - return; - } - /// \brief Broadcasts value from one thread to all threads in logical warp. /// /// \param [in] input - value to broadcast. @@ -675,41 +569,22 @@ class warp_scan /// Synchronization barrier should be placed before \p storage is reused /// or repurposed: \p __syncthreads() or \p rocprim::syncthreads(). template - ROCPRIM_DEVICE ROCPRIM_INLINE auto - broadcast(T input, const unsigned int src_lane, storage_type& storage) -> - typename std::enable_if<(FunctionWarpSize <= device_warp_size()), T>::type + ROCPRIM_DEVICE ROCPRIM_INLINE + auto broadcast(T input, const unsigned int src_lane, storage_type& storage) { return base_type::broadcast(input, src_lane, storage); } - /// \brief Broadcasts value from one thread to all threads in logical warp. - /// Invalid Warp Size - template - ROCPRIM_DEVICE ROCPRIM_INLINE auto broadcast(T, const unsigned int, storage_type&) -> - typename std::enable_if<(FunctionWarpSize > device_warp_size()), T>::type - { - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); - return T(); - } - #ifndef DOXYGEN_SHOULD_SKIP_THIS protected: // These undocumented functions are used by hipCUB prior to version 3.1 template - [[deprecated]] ROCPRIM_DEVICE ROCPRIM_INLINE auto - to_exclusive(T inclusive_input, T& exclusive_output, storage_type& storage) -> - typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type + [[deprecated]] + ROCPRIM_DEVICE ROCPRIM_INLINE + auto to_exclusive(T inclusive_input, T& exclusive_output, storage_type& storage) { return base_type::to_exclusive(inclusive_input, exclusive_output, storage); } - - template - [[deprecated]] ROCPRIM_DEVICE ROCPRIM_INLINE auto to_exclusive(T, T&, storage_type&) -> - typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type - { - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); - return; - } #endif }; diff --git a/rocprim/include/rocprim/warp/warp_sort.hpp b/rocprim/include/rocprim/warp/warp_sort.hpp index 59ccdccb2..fd570aa18 100644 --- a/rocprim/include/rocprim/warp/warp_sort.hpp +++ b/rocprim/include/rocprim/warp/warp_sort.hpp @@ -110,7 +110,8 @@ class warp_sort : detail::warp_sort_shuffle typedef typename detail::warp_sort_shuffle base_type; // Check if WarpSize is valid for the targets - static_assert(WarpSize <= ROCPRIM_MAX_WARP_SIZE, "WarpSize can't be greater than hardware warp size."); + ROCPRIM_DETAIL_DEVICE_STATIC_ASSERT(WarpSize <= ::rocprim::device_warp_size(), + "WarpSize can't be greater than hardware warp size."); public: /// \brief Struct used to allocate a temporary memory that is required for thread @@ -134,25 +135,12 @@ class warp_sort : detail::warp_sort_shuffle /// bool f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key& thread_key, - BinaryFunction compare_function = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key& thread_key, BinaryFunction compare_function = BinaryFunction()) { base_type::sort(thread_key, compare_function); } - /// \brief Warp sort for any data type. - /// Invalid Warp Size - template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key&, - BinaryFunction compare_function = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type - { - (void) compare_function; // disables unused parameter warning - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); - return; - } - /// \brief Warp sort for any data type. /// /// \tparam BinaryFunction - type of binary function used for sort. Default type @@ -166,28 +154,13 @@ class warp_sort : detail::warp_sort_shuffle template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], - BinaryFunction compare_function = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key (&thread_keys)[ItemsPerThread], + BinaryFunction compare_function = BinaryFunction()) { base_type::sort(thread_keys, compare_function); } - /// \brief Warp sort for any data type. - /// Invalid Warp Size - template, - unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], - BinaryFunction compare_function = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type - { - (void) thread_keys; // disables unused parameter warning - (void) compare_function; // disables unused parameter warning - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); - return; - } - /// \brief Warp sort for any data type using temporary storage. /// /// \tparam BinaryFunction - type of binary function used for sort. Default type @@ -217,29 +190,16 @@ class warp_sort : detail::warp_sort_shuffle /// } /// \endcode template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key& thread_key, - storage_type& storage, - BinaryFunction compare_function = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key& thread_key, + storage_type& storage, + BinaryFunction compare_function = BinaryFunction()) { base_type::sort( thread_key, storage, compare_function ); } - /// \brief Warp sort for any data type using temporary storage. - /// Invalid Warp Size - template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto - sort(Key&, storage_type&, BinaryFunction compare_function = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type - { - (void) compare_function; // disables unused parameter warning - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); - return; - } - - /// \brief Warp sort for any data type using temporary storage. /// /// \tparam BinaryFunction - type of binary function used for sort. Default type @@ -271,32 +231,16 @@ class warp_sort : detail::warp_sort_shuffle template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], - storage_type& storage, - BinaryFunction compare_function = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key (&thread_keys)[ItemsPerThread], + storage_type& storage, + BinaryFunction compare_function = BinaryFunction()) { base_type::sort( thread_keys, storage, compare_function ); } - /// \brief Warp sort for any data type using temporary storage. - /// Invalid Warp Size - template, - unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], - storage_type&, - BinaryFunction compare_function = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type - { - (void) thread_keys; // disables unused parameter warning - (void) compare_function; // disables unused parameter warning - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); - return; - } - /// \brief Warp sort by key for any data type. /// /// \tparam BinaryFunction - type of binary function used for sort. Default type @@ -309,28 +253,16 @@ class warp_sort : detail::warp_sort_shuffle /// bool f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key& thread_key, - Value& thread_value, - BinaryFunction compare_function = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key& thread_key, + Value& thread_value, + BinaryFunction compare_function = BinaryFunction()) { base_type::sort( thread_key, thread_value, compare_function ); } - /// \brief Warp sort by key for any data type. - /// Invalid Warp Size - template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto - sort(Key&, Value&, BinaryFunction compare_function = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type - { - (void) compare_function; // disables unused parameter warning - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); - return; - } - /// \brief Warp sort by key for any data type. /// /// \tparam BinaryFunction - type of binary function used for sort. Default type @@ -345,33 +277,16 @@ class warp_sort : detail::warp_sort_shuffle template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], - Value (&thread_values)[ItemsPerThread], - BinaryFunction compare_function = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key (&thread_keys)[ItemsPerThread], + Value (&thread_values)[ItemsPerThread], + BinaryFunction compare_function = BinaryFunction()) { base_type::sort( thread_keys, thread_values, compare_function ); } - /// \brief Warp sort by key for any data type. - /// Invalid Warp Size - template, - unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], - Value (&thread_values)[ItemsPerThread], - BinaryFunction compare_function = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type - { - (void) thread_keys; // disables unused parameter warning - (void) thread_values; // disables unused parameter warning - (void) compare_function; // disables unused parameter warning - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); - return; - } - /// \brief Warp sort by key for any data type using temporary storage. /// /// \tparam BinaryFunction - type of binary function used for sort. Default type @@ -402,30 +317,17 @@ class warp_sort : detail::warp_sort_shuffle /// } /// \endcode template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key& thread_key, - Value& thread_value, - storage_type& storage, - BinaryFunction compare_function = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key& thread_key, + Value& thread_value, + storage_type& storage, + BinaryFunction compare_function = BinaryFunction()) { base_type::sort( thread_key, thread_value, storage, compare_function ); } - /// \brief Warp sort by key for any data type using temporary storage. - /// Invalid Warp Size - template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto - sort(Key&, Value&, storage_type&, BinaryFunction compare_function = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type - { - (void) compare_function; // disables unused parameter warning - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); - return; - } - - /// \brief Warp sort by key for any data type using temporary storage. /// /// \tparam BinaryFunction - type of binary function used for sort. Default type @@ -458,34 +360,16 @@ class warp_sort : detail::warp_sort_shuffle template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], - Value (&thread_values)[ItemsPerThread], - storage_type& storage, - BinaryFunction compare_function = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key (&thread_keys)[ItemsPerThread], + Value (&thread_values)[ItemsPerThread], + storage_type& storage, + BinaryFunction compare_function = BinaryFunction()) { base_type::sort( thread_keys, thread_values, storage, compare_function ); } - - /// \brief Warp sort by key for any data type using temporary storage. - /// Invalid Warp Size - template, - unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], - Value (&thread_values)[ItemsPerThread], - storage_type&, - BinaryFunction compare_function = BinaryFunction()) -> - typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type - { - (void) thread_keys; // disables unused parameter warning - (void) thread_values; // disables unused parameter warning - (void) compare_function; // disables unused parameter warning - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); - return; - } }; END_ROCPRIM_NAMESPACE From 19bd48aa0e79f0aee78c71b5d5ec54e779f1be71 Mon Sep 17 00:00:00 2001 From: Jaap Blok Date: Thu, 21 Nov 2024 12:58:55 +0100 Subject: [PATCH 12/84] Fix tests triggering static asserts --- test/rocprim/test_warp_reduce.kernels.hpp | 158 +++++++++---- test/rocprim/test_warp_scan.kernels.hpp | 268 +++++++++++++++++----- test/rocprim/test_warp_sort.hpp | 15 +- test/rocprim/test_warp_sort.kernels.hpp | 178 +++++++------- 4 files changed, 421 insertions(+), 198 deletions(-) diff --git a/test/rocprim/test_warp_reduce.kernels.hpp b/test/rocprim/test_warp_reduce.kernels.hpp index 846d11a32..48a34c54f 100644 --- a/test/rocprim/test_warp_reduce.kernels.hpp +++ b/test/rocprim/test_warp_reduce.kernels.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2019-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,14 +21,19 @@ #ifndef TEST_WARP_REDUCE_KERNELS_HPP_ #define TEST_WARP_REDUCE_KERNELS_HPP_ -template< - class T, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> -__global__ -__launch_bounds__(BlockSize) -void warp_reduce_sum_kernel(T* device_input, T* device_output) +template +__device__ +auto warp_reduce_sum_test(T* /*device_input*/, T* /*device_output*/) + -> std::enable_if_t> +{ + // This kernel should never be actually called; tests are filtered out at runtime + // if the device does not support the LogicalWarpSize +} + +template +__device__ +auto warp_reduce_sum_test(T* device_input, T* device_output) + -> std::enable_if_t> { static constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -46,14 +51,27 @@ void warp_reduce_sum_kernel(T* device_input, T* device_output) } } -template< - class T, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> +template __global__ __launch_bounds__(BlockSize) -void warp_allreduce_sum_kernel(T* device_input, T* device_output) +void warp_reduce_sum_kernel(T* device_input, T* device_output) +{ + warp_reduce_sum_test(device_input, device_output); +} + +template +__device__ +auto warp_allreduce_sum_test(T* /*device_input*/, T* /*device_output*/) + -> std::enable_if_t> +{ + // This kernel should never be actually called; tests are filtered out at runtime + // if the device does not support the LogicalWarpSize +} + +template +__device__ +auto warp_allreduce_sum_test(T* device_input, T* device_output) + -> std::enable_if_t> { static constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -68,14 +86,27 @@ void warp_allreduce_sum_kernel(T* device_input, T* device_output) device_output[index] = value; } -template< - class T, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> +template __global__ __launch_bounds__(BlockSize) -void warp_reduce_sum_kernel(T* device_input, T* device_output, size_t valid) +void warp_allreduce_sum_kernel(T* device_input, T* device_output) +{ + warp_allreduce_sum_test(device_input, device_output); +} + +template +__device__ +auto warp_reduce_sum_test(T* /*device_input*/, T* /*device_output*/, size_t /*valid*/) + -> std::enable_if_t> +{ + // This kernel should never be actually called; tests are filtered out at runtime + // if the device does not support the LogicalWarpSize +} + +template +__device__ +auto warp_reduce_sum_test(T* device_input, T* device_output, size_t valid) + -> std::enable_if_t> { static constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -93,14 +124,27 @@ void warp_reduce_sum_kernel(T* device_input, T* device_output, size_t valid) } } -template< - class T, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> +template __global__ __launch_bounds__(BlockSize) -void warp_allreduce_sum_kernel(T* device_input, T* device_output, size_t valid) +void warp_reduce_sum_kernel(T* device_input, T* device_output, size_t valid) +{ + warp_reduce_sum_test(device_input, device_output, valid); +} + +template +__device__ +auto warp_allreduce_sum_test(T* /*device_input*/, T* /*device_output*/, size_t /*valid*/) + -> std::enable_if_t> +{ + // This kernel should never be actually called; tests are filtered out at runtime + // if the device does not support the LogicalWarpSize +} + +template +__device__ +auto warp_allreduce_sum_test(T* device_input, T* device_output, size_t valid) + -> std::enable_if_t> { constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -115,15 +159,27 @@ void warp_allreduce_sum_kernel(T* device_input, T* device_output, size_t valid) device_output[index] = value; } -template< - class T, - class Flag, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> +template __global__ __launch_bounds__(BlockSize) -void head_segmented_warp_reduce_kernel(T* input, Flag* flags, T* output) +void warp_allreduce_sum_kernel(T* device_input, T* device_output, size_t valid) +{ + warp_allreduce_sum_test(device_input, device_output, valid); +} + +template +__device__ +auto head_segmented_warp_reduce_test(T* /*input*/, Flag* /*flags*/, T* /*output*/) + -> std::enable_if_t> +{ + // This kernel should never be actually called; tests are filtered out at runtime + // if the device does not support the LogicalWarpSize +} + +template +__device__ +auto head_segmented_warp_reduce_test(T* input, Flag* flags, T* output) + -> std::enable_if_t> { constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -139,15 +195,27 @@ void head_segmented_warp_reduce_kernel(T* input, Flag* flags, T* output) output[index] = value; } -template< - class T, - class Flag, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> +template __global__ __launch_bounds__(BlockSize) -void tail_segmented_warp_reduce_kernel(T* input, Flag* flags, T* output) +void head_segmented_warp_reduce_kernel(T* input, Flag* flags, T* output) +{ + head_segmented_warp_reduce_test(input, flags, output); +} + +template +__device__ +auto tail_segmented_warp_reduce_test(T* /*input*/, Flag* /*flags*/, T* /*output*/) + -> std::enable_if_t> +{ + // This kernel should never be actually called; tests are filtered out at runtime + // if the device does not support the LogicalWarpSize +} + +template +__device__ +auto tail_segmented_warp_reduce_test(T* input, Flag* flags, T* output) + -> std::enable_if_t> { constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -163,4 +231,12 @@ void tail_segmented_warp_reduce_kernel(T* input, Flag* flags, T* output) output[index] = value; } +template +__global__ +__launch_bounds__(BlockSize) +void tail_segmented_warp_reduce_kernel(T* input, Flag* flags, T* output) +{ + tail_segmented_warp_reduce_test(input, flags, output); +} + #endif // TEST_WARP_REDUCE_KERNELS_HPP_ diff --git a/test/rocprim/test_warp_scan.kernels.hpp b/test/rocprim/test_warp_scan.kernels.hpp index 875389058..88e02b89d 100644 --- a/test/rocprim/test_warp_scan.kernels.hpp +++ b/test/rocprim/test_warp_scan.kernels.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -23,14 +23,19 @@ #ifndef TEST_SCAN_REDUCE_KERNELS_HPP_ #define TEST_SCAN_REDUCE_KERNELS_HPP_ -template< - class T, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> -__global__ -__launch_bounds__(BlockSize) -void warp_inclusive_scan_kernel(T* device_input, T* device_output) +template +__device__ +auto warp_inclusive_scan_test(T* /*device_input*/, T* /*device_output*/) + -> std::enable_if_t> +{ + // This kernel should never be actually called; tests are filtered out at runtime + // if the device does not support the LogicalWarpSize +} + +template +__device__ +auto warp_inclusive_scan_test(T* device_input, T* device_output) + -> std::enable_if_t> { constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -45,17 +50,29 @@ void warp_inclusive_scan_kernel(T* device_input, T* device_output) device_output[index] = value; } -template< - class T, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> +template __global__ __launch_bounds__(BlockSize) -void warp_inclusive_scan_reduce_kernel( - T* device_input, - T* device_output, - T* device_output_reductions) +void warp_inclusive_scan_kernel(T* device_input, T* device_output) +{ + warp_inclusive_scan_test(device_input, device_output); +} + +template +__device__ +auto warp_inclusive_scan_reduce_test(T* /*device_input*/, + T* /*device_output*/, + T* /*device_output_reductions*/) + -> std::enable_if_t> +{ + // This kernel should never be actually called; tests are filtered out at runtime + // if the device does not support the LogicalWarpSize +} + +template +__device__ +auto warp_inclusive_scan_reduce_test(T* device_input, T* device_output, T* device_output_reductions) + -> std::enable_if_t> { constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -75,14 +92,31 @@ void warp_inclusive_scan_reduce_kernel( } } -template< - class T, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> +template __global__ __launch_bounds__(BlockSize) -void warp_exclusive_scan_kernel(T* device_input, T* device_output, T init) +void warp_inclusive_scan_reduce_kernel(T* device_input, + T* device_output, + T* device_output_reductions) +{ + warp_inclusive_scan_reduce_test(device_input, + device_output, + device_output_reductions); +} + +template +__device__ +auto warp_exclusive_scan_test(T* /*device_input*/, T* /*device_output*/, T /*init*/) + -> std::enable_if_t> +{ + // This kernel should never be actually called; tests are filtered out at runtime + // if the device does not support the LogicalWarpSize +} + +template +__device__ +auto warp_exclusive_scan_test(T* device_input, T* device_output, T init) + -> std::enable_if_t> { constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -97,18 +131,33 @@ void warp_exclusive_scan_kernel(T* device_input, T* device_output, T init) device_output[index] = value; } -template< - class T, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> +template __global__ __launch_bounds__(BlockSize) -void warp_exclusive_scan_reduce_kernel( - T* device_input, - T* device_output, - T* device_output_reductions, - T init) +void warp_exclusive_scan_kernel(T* device_input, T* device_output, T init) +{ + warp_exclusive_scan_test(device_input, device_output, init); +} + +template +__device__ +auto warp_exclusive_scan_reduce_test(T* /*device_input*/, + T* /*device_output*/, + T* /*device_output_reductions*/, + T /*init*/) + -> std::enable_if_t> +{ + // This kernel should never be actually called; tests are filtered out at runtime + // if the device does not support the LogicalWarpSize +} + +template +__device__ +auto warp_exclusive_scan_reduce_test(T* device_input, + T* device_output, + T* device_output_reductions, + T init) + -> std::enable_if_t> { constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -129,8 +178,32 @@ void warp_exclusive_scan_reduce_kernel( } template -__global__ __launch_bounds__(BlockSize) void warp_exclusive_scan_wo_init_kernel(T* device_input, - T* device_output) +__global__ +__launch_bounds__(BlockSize) +void warp_exclusive_scan_reduce_kernel(T* device_input, + T* device_output, + T* device_output_reductions, + T init) +{ + warp_exclusive_scan_reduce_test(device_input, + device_output, + device_output_reductions, + init); +} + +template +__device__ +auto warp_exclusive_scan_wo_init_test(T* /*device_input*/, T* /*device_output*/) + -> std::enable_if_t> +{ + // This kernel should never be actually called; tests are filtered out at runtime + // if the device does not support the LogicalWarpSize +} + +template +__device__ +auto warp_exclusive_scan_wo_init_test(T* device_input, T* device_output) + -> std::enable_if_t> { static constexpr unsigned int block_warps_no = BlockSize / LogicalWarpSize; @@ -147,8 +220,30 @@ __global__ __launch_bounds__(BlockSize) void warp_exclusive_scan_wo_init_kernel( } template -__global__ __launch_bounds__(BlockSize) void warp_exclusive_scan_reduce_wo_init_kernel( - T* device_input, T* device_output, T* device_output_reductions) +__global__ +__launch_bounds__(BlockSize) +void warp_exclusive_scan_wo_init_kernel(T* device_input, T* device_output) +{ + warp_exclusive_scan_wo_init_test(device_input, device_output); +} + +template +__device__ +auto warp_exclusive_scan_reduce_wo_init_test(T* /*device_input*/, + T* /*device_output*/, + T* /*device_output_reductions*/) + -> std::enable_if_t> +{ + // This kernel should never be actually called; tests are filtered out at runtime + // if the device does not support the LogicalWarpSize +} + +template +__device__ +auto warp_exclusive_scan_reduce_wo_init_test(T* device_input, + T* device_output, + T* device_output_reductions) + -> std::enable_if_t> { static constexpr unsigned int block_warps_no = BlockSize / LogicalWarpSize; @@ -171,18 +266,35 @@ __global__ __launch_bounds__(BlockSize) void warp_exclusive_scan_reduce_wo_init_ } } -template< - class T, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> +template __global__ __launch_bounds__(BlockSize) -void warp_scan_kernel( - T* device_input, - T* device_inclusive_output, - T* device_exclusive_output, - T init) +void warp_exclusive_scan_reduce_wo_init_kernel(T* device_input, + T* device_output, + T* device_output_reductions) +{ + warp_exclusive_scan_reduce_wo_init_test( + device_input, + device_output, + device_output_reductions); +} + +template +__device__ +auto warp_scan_test(T* /*device_input*/, + T* /*device_inclusive_output*/, + T* /*device_exclusive_output*/, + T /*init*/) + -> std::enable_if_t> +{ + // This kernel should never be actually called; tests are filtered out at runtime + // if the device does not support the LogicalWarpSize +} + +template +__device__ +auto warp_scan_test(T* device_input, T* device_inclusive_output, T* device_exclusive_output, T init) + -> std::enable_if_t> { constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -199,19 +311,41 @@ void warp_scan_kernel( device_exclusive_output[index] = exclusive_output; } -template< - class T, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> +template __global__ __launch_bounds__(BlockSize) -void warp_scan_reduce_kernel( - T* device_input, - T* device_inclusive_output, - T* device_exclusive_output, - T* device_output_reductions, - T init) +void warp_scan_kernel(T* device_input, + T* device_inclusive_output, + T* device_exclusive_output, + T init) +{ + warp_scan_test(device_input, + device_inclusive_output, + device_exclusive_output, + init); +} + +template +__device__ +auto warp_scan_reduce_test(T* /*device_input*/, + T* /*device_inclusive_output*/, + T* /*device_exclusive_output*/, + T* /*device_output_reductions*/, + T /*init*/) + -> std::enable_if_t> +{ + // This kernel should never be actually called; tests are filtered out at runtime + // if the device does not support the LogicalWarpSize +} + +template +__device__ +auto warp_scan_reduce_test(T* device_input, + T* device_inclusive_output, + T* device_exclusive_output, + T* device_output_reductions, + T init) + -> std::enable_if_t> { constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -232,4 +366,20 @@ void warp_scan_reduce_kernel( } } +template +__global__ +__launch_bounds__(BlockSize) +void warp_scan_reduce_kernel(T* device_input, + T* device_inclusive_output, + T* device_exclusive_output, + T* device_output_reductions, + T init) +{ + warp_scan_reduce_test(device_input, + device_inclusive_output, + device_exclusive_output, + device_output_reductions, + init); +} + #endif // TEST_SCAN_REDUCE_KERNELS_HPP_ diff --git a/test/rocprim/test_warp_sort.hpp b/test/rocprim/test_warp_sort.hpp index 85addaa4e..b34b35bae 100644 --- a/test/rocprim/test_warp_sort.hpp +++ b/test/rocprim/test_warp_sort.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -94,12 +94,13 @@ typed_test_def(RocprimWarpSortShuffleBasedTests, name_suffix, Sort) // Launching kernel hipLaunchKernelGGL( - HIP_KERNEL_NAME(test_hip_warp_sort< - items_per_thread, block_size, logical_warp_size, T - >), - dim3(grid_size), dim3(block_size), 0, 0, - d_output - ); + HIP_KERNEL_NAME( + test_hip_warp_sort_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + d_output); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); diff --git a/test/rocprim/test_warp_sort.kernels.hpp b/test/rocprim/test_warp_sort.kernels.hpp index d00ea2380..ddf6f8233 100644 --- a/test/rocprim/test_warp_sort.kernels.hpp +++ b/test/rocprim/test_warp_sort.kernels.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2021 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -23,18 +23,20 @@ #ifndef TEST_WARP_SORT_KERNELS_HPP_ #define TEST_WARP_SORT_KERNELS_HPP_ -template< - unsigned int ItemsPerThread, - unsigned int BlockSize, - unsigned int LogicalWarpSize, - class T -> -__global__ -__launch_bounds__(BlockSize) -auto test_hip_warp_sort(T* d_output) - -> typename std::enable_if< - ItemsPerThread == 1 , void - >::type +template +__device__ +auto test_hip_warp_sort_test(T* /*d_output*/) -> + typename std::enable_if_t> +{ + // This kernel should never be actually called; tests are filtered out at runtime + // if the device does not support the LogicalWarpSize +} + +template +__device__ +auto test_hip_warp_sort_test(T* d_output) -> + typename std::enable_if_t<(ItemsPerThread == 1 + && test_utils::device_test_enabled_for_warp_size_v)> { unsigned int i = threadIdx.x + (blockIdx.x * blockDim.x); T value = d_output[i]; @@ -43,41 +45,14 @@ auto test_hip_warp_sort(T* d_output) d_output[i] = value; } -template< - unsigned int ItemsPerThread, - unsigned int BlockSize, - unsigned int LogicalWarpSize, - class KeyType, - class ValueType -> -__global__ -__launch_bounds__(BlockSize) -auto test_hip_sort_key_value_kernel(KeyType* d_output_key, ValueType* d_output_value) - -> typename std::enable_if< - ItemsPerThread == 1 , void - >::type -{ - unsigned int i = threadIdx.x + (blockIdx.x * blockDim.x); - KeyType key = d_output_key[i]; - ValueType value = d_output_value[i]; - rocprim::warp_sort wsort; - wsort.sort(key, value); - d_output_key[i] = key; - d_output_value[i] = value; -} - -template< - unsigned int ItemsPerThread, - unsigned int BlockSize, - unsigned int LogicalWarpSize, - class KeyType -> -__global__ -__launch_bounds__(BlockSize) -auto test_hip_warp_sort(KeyType * device_key_output) - -> typename std::enable_if< - (ItemsPerThread != 1 && LogicalWarpSize <= ::rocprim::device_warp_size()) , void - >::type +template +__device__ +auto test_hip_warp_sort_test(KeyType* device_key_output) -> + typename std::enable_if_t<(ItemsPerThread != 1 + && test_utils::device_test_enabled_for_warp_size_v)> { const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; @@ -91,69 +66,90 @@ auto test_hip_warp_sort(KeyType * device_key_output) ::rocprim::block_store_direct_blocked(lid, device_key_output + block_offset, keys); } -template< - unsigned int ItemsPerThread, - unsigned int BlockSize, - unsigned int LogicalWarpSize, - class KeyType -> +template __global__ __launch_bounds__(BlockSize) -auto test_hip_warp_sort(KeyType* /*device_key_output*/) - -> typename std::enable_if< - (ItemsPerThread != 1 && LogicalWarpSize > ::rocprim::device_warp_size()), void - >::type +void test_hip_warp_sort_kernel(KeyType* device_key_output) { - // This kernel will never be actually called, the tests are filtered at runtime if the warp size - // of the current device is less than the tested LogicalWarpSize + test_hip_warp_sort_test(device_key_output); } -template< - unsigned int ItemsPerThread, - unsigned int BlockSize, - unsigned int LogicalWarpSize, - class KeyType, - class ValueType -> -__global__ -__launch_bounds__(BlockSize) -auto test_hip_sort_key_value_kernel(KeyType* device_key_output, ValueType* device_value_output) - -> typename std::enable_if< - (ItemsPerThread != 1 && LogicalWarpSize <= ::rocprim::device_warp_size()), void - >::type +template +__device__ +auto test_hip_sort_key_value_test(KeyType* /*d_output_key*/, ValueType* /*d_output_value*/) -> + typename std::enable_if_t> { - const unsigned int lid = threadIdx.x; + // This kernel should never be actually called; tests are filtered out at runtime + // if the device does not support the LogicalWarpSize +} + +template +__device__ +auto test_hip_sort_key_value_test(KeyType* d_output_key, ValueType* d_output_value) -> + typename std::enable_if_t<(ItemsPerThread == 1 + && test_utils::device_test_enabled_for_warp_size_v)> +{ + unsigned int i = threadIdx.x + (blockIdx.x * blockDim.x); + KeyType key = d_output_key[i]; + ValueType value = d_output_value[i]; + rocprim::warp_sort wsort; + wsort.sort(key, value); + d_output_key[i] = key; + d_output_value[i] = value; +} + +template +__device__ +auto test_hip_sort_key_value_test(KeyType* device_key_output, ValueType* device_value_output) -> + typename std::enable_if_t<(ItemsPerThread != 1 + && test_utils::device_test_enabled_for_warp_size_v)> +{ + const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; - KeyType keys[ItemsPerThread]; + KeyType keys[ItemsPerThread]; ValueType values[ItemsPerThread]; - ::rocprim::block_load_direct_warp_striped(lid, device_key_output + block_offset, keys); - ::rocprim::block_load_direct_warp_striped(lid, device_value_output + block_offset, values); + ::rocprim::block_load_direct_warp_striped(lid, + device_key_output + block_offset, + keys); + ::rocprim::block_load_direct_warp_striped(lid, + device_value_output + block_offset, + values); rocprim::warp_sort wsort; wsort.sort(keys, values); ::rocprim::block_store_direct_blocked(lid, device_key_output + block_offset, keys); ::rocprim::block_store_direct_blocked(lid, device_value_output + block_offset, values); - } -template< - unsigned int ItemsPerThread, - unsigned int BlockSize, - unsigned int LogicalWarpSize, - class KeyType, - class ValueType -> +template __global__ __launch_bounds__(BlockSize) -auto test_hip_sort_key_value_kernel(KeyType* /*device_key_output*/, ValueType* /*device_value_output*/) - -> typename std::enable_if< - (ItemsPerThread != 1 && LogicalWarpSize > ::rocprim::device_warp_size()), void - >::type +void test_hip_sort_key_value_kernel(KeyType* device_key_output, ValueType* device_value_output) { - // This kernel will never be actually called, the tests are filtered at runtime if the warp size - // of the current device is less than the tested LogicalWarpSize + test_hip_sort_key_value_test( + device_key_output, + device_value_output); } #endif // TEST_WARP_SORT_KERNELS_HPP_ From a4769a7a35c964332326ae704f33b7dbff0f3524 Mon Sep 17 00:00:00 2001 From: Jaap Blok Date: Thu, 21 Nov 2024 12:58:55 +0100 Subject: [PATCH 13/84] Fix benchmarks triggering static asserts --- benchmark/benchmark_warp_reduce.cpp | 142 +++++++++++++++---------- benchmark/benchmark_warp_scan.cpp | 94 ++++++++++++----- benchmark/benchmark_warp_sort.cpp | 158 +++++++++++++++++++--------- 3 files changed, 266 insertions(+), 128 deletions(-) diff --git a/benchmark/benchmark_warp_reduce.cpp b/benchmark/benchmark_warp_reduce.cpp index 1cddacbfd..7198021b1 100644 --- a/benchmark/benchmark_warp_reduce.cpp +++ b/benchmark/benchmark_warp_reduce.cpp @@ -45,15 +45,19 @@ const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif -template< - bool AllReduce, - class T, - unsigned int WarpSize, - unsigned int Trials -> -__global__ -__launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) -void warp_reduce_kernel(const T * d_input, T * d_output) +template +__device__ +auto warp_reduce_benchmark(const T* /*d_input*/, T* /*d_output*/) + -> std::enable_if_t> +{ + // This kernel should never be actually called; benchmarks are selected at runtime only + // if the current device supports WarpSize +} + +template +__device__ +auto warp_reduce_benchmark(const T* d_input, T* d_output) + -> std::enable_if_t> { const unsigned int i = blockIdx.x * blockDim.x + threadIdx.x; @@ -70,15 +74,27 @@ void warp_reduce_kernel(const T * d_input, T * d_output) d_output[i] = value; } -template< - class T, - class Flag, - unsigned int WarpSize, - unsigned int Trials -> +template __global__ __launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) -void segmented_warp_reduce_kernel(const T* d_input, Flag* d_flags, T* d_output) +void warp_reduce_kernel(const T* d_input, T* d_output) +{ + warp_reduce_benchmark(d_input, d_output); +} + +template +__device__ +auto segmented_warp_reduce_benchmark(const T* /*d_input*/, Flag* /*d_flags*/, T* /*d_output*/) + -> std::enable_if_t> +{ + // This kernel should never be actually called; benchmarks are selected at runtime only + // if the current device supports WarpSize +} + +template +__device__ +auto segmented_warp_reduce_benchmark(const T* d_input, Flag* d_flags, T* d_output) + -> std::enable_if_t> { const unsigned int i = blockIdx.x * blockDim.x + threadIdx.x; @@ -96,19 +112,24 @@ void segmented_warp_reduce_kernel(const T* d_input, Flag* d_flags, T* d_output) d_output[i] = value; } -template< - bool AllReduce, - bool Segmented, - unsigned int WarpSize, - unsigned int BlockSize, - unsigned int Trials, - class T, - class Flag -> -inline -auto execute_warp_reduce_kernel(T* input, T* output, Flag* /* flags */, - size_t size, hipStream_t stream) - -> typename std::enable_if::type +template +__global__ +__launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) +void segmented_warp_reduce_kernel(const T* d_input, Flag* d_flags, T* d_output) +{ + segmented_warp_reduce_benchmark(d_input, d_flags, d_output); +} + +template +inline auto execute_warp_reduce_kernel( + T* input, T* output, Flag* /* flags */, size_t size, hipStream_t stream) -> + typename std::enable_if_t { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_reduce_kernel), @@ -118,19 +139,16 @@ auto execute_warp_reduce_kernel(T* input, T* output, Flag* /* flags */, HIP_CHECK(hipGetLastError()); } -template< - bool AllReduce, - bool Segmented, - unsigned int WarpSize, - unsigned int BlockSize, - unsigned int Trials, - class T, - class Flag -> -inline -auto execute_warp_reduce_kernel(T* input, T* output, Flag* flags, - size_t size, hipStream_t stream) - -> typename std::enable_if::type +template +inline auto + execute_warp_reduce_kernel(T* input, T* output, Flag* flags, size_t size, hipStream_t stream) -> + typename std::enable_if_t { hipLaunchKernelGGL( HIP_KERNEL_NAME(segmented_warp_reduce_kernel), @@ -230,11 +248,12 @@ void run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& se seed, \ stream) -#define BENCHMARK_TYPE(type) \ - CREATE_BENCHMARK(type, 32, 64), \ - CREATE_BENCHMARK(type, 37, 64), \ - CREATE_BENCHMARK(type, 61, 64), \ - CREATE_BENCHMARK(type, 64, 64) +// For devices with a physical warp size of at least 32 +#define BENCHMARK_TYPE_WS32(type) CREATE_BENCHMARK(type, 32, 64) + +// For devices with a physical warp size of at least 64 +#define BENCHMARK_TYPE_WS64(type) \ + CREATE_BENCHMARK(type, 37, 64), CREATE_BENCHMARK(type, 61, 64), CREATE_BENCHMARK(type, 64, 64) template void add_benchmarks(std::vector& benchmarks, @@ -242,17 +261,26 @@ void add_benchmarks(std::vector& benchmarks, const managed_seed& seed, hipStream_t stream) { - std::vector bs = - { - BENCHMARK_TYPE(int), - BENCHMARK_TYPE(float), - BENCHMARK_TYPE(double), - BENCHMARK_TYPE(int8_t), - BENCHMARK_TYPE(uint8_t), - BENCHMARK_TYPE(rocprim::half) - }; - + std::vector bs = {BENCHMARK_TYPE_WS32(int), + BENCHMARK_TYPE_WS32(float), + BENCHMARK_TYPE_WS32(double), + BENCHMARK_TYPE_WS32(int8_t), + BENCHMARK_TYPE_WS32(uint8_t), + BENCHMARK_TYPE_WS32(rocprim::half)}; benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); + + int hip_device = 0; + HIP_CHECK(::rocprim::detail::get_device_from_stream(stream, hip_device)); + if(is_warp_size_supported(64, hip_device)) + { + std::vector bs = {BENCHMARK_TYPE_WS64(int), + BENCHMARK_TYPE_WS64(float), + BENCHMARK_TYPE_WS64(double), + BENCHMARK_TYPE_WS64(int8_t), + BENCHMARK_TYPE_WS64(uint8_t), + BENCHMARK_TYPE_WS64(rocprim::half)}; + benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); + } } int main(int argc, char *argv[]) diff --git a/benchmark/benchmark_warp_scan.cpp b/benchmark/benchmark_warp_scan.cpp index f7983761a..eca1a4c78 100644 --- a/benchmark/benchmark_warp_scan.cpp +++ b/benchmark/benchmark_warp_scan.cpp @@ -46,9 +46,18 @@ const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; namespace rp = rocprim; template -__global__ -__launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) -void warp_inclusive_scan_kernel(const T* input, T* output) +__device__ +auto warp_inclusive_scan_benchmark(const T* /*input*/, T* /*output*/) + -> std::enable_if_t> +{ + // This kernel should never be actually called; benchmarks are selected at runtime only + // if the current device supports WarpSize +} + +template +__device__ +auto warp_inclusive_scan_benchmark(const T* input, T* output) + -> std::enable_if_t> { const unsigned int i = blockIdx.x * blockDim.x + threadIdx.x; auto value = input[i]; @@ -67,7 +76,24 @@ void warp_inclusive_scan_kernel(const T* input, T* output) template __global__ __launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) -void warp_exclusive_scan_kernel(const T* input, T* output, const T init) +void warp_inclusive_scan_kernel(const T* input, T* output) +{ + warp_inclusive_scan_benchmark(input, output); +} + +template +__device__ +auto warp_exclusive_scan_benchmark(const T* /*input*/, T* /*output*/, const T /*init*/) + -> std::enable_if_t> +{ + // This kernel should never be actually called; benchmarks are selected at runtime only + // if the current device supports WarpSize +} + +template +__device__ +auto warp_exclusive_scan_benchmark(const T* input, T* output, const T init) + -> std::enable_if_t> { const unsigned int i = blockIdx.x * blockDim.x + threadIdx.x; auto value = input[i]; @@ -83,6 +109,14 @@ void warp_exclusive_scan_kernel(const T* input, T* output, const T init) output[i] = value; } +template +__global__ +__launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) +void warp_exclusive_scan_kernel(const T* input, T* output, const T init) +{ + warp_exclusive_scan_benchmark(input, output, init); +} + template< class T, unsigned int BlockSize, @@ -170,15 +204,15 @@ void run_benchmark(benchmark::State& state, hipStream_t stream, size_t bytes) stream, \ bytes) -#define BENCHMARK_TYPE(type) \ - CREATE_BENCHMARK(type, 64, 64, Inclusive), \ - CREATE_BENCHMARK(type, 128, 64, Inclusive), \ - CREATE_BENCHMARK(type, 256, 64, Inclusive), \ - CREATE_BENCHMARK(type, 256, 32, Inclusive), \ - CREATE_BENCHMARK(type, 256, 16, Inclusive), \ - CREATE_BENCHMARK(type, 63, 63, Inclusive), \ - CREATE_BENCHMARK(type, 62, 31, Inclusive), \ - CREATE_BENCHMARK(type, 60, 15, Inclusive) +// for devices with a physical warp size of at least 32 +#define BENCHMARK_TYPE_WS32(type) \ + CREATE_BENCHMARK(type, 60, 15, Inclusive), CREATE_BENCHMARK(type, 256, 16, Inclusive), \ + CREATE_BENCHMARK(type, 62, 31, Inclusive), CREATE_BENCHMARK(type, 256, 32, Inclusive) + +// for devices with a physical warp size of at least 64 +#define BENCHMARK_TYPE_WS64(type) \ + CREATE_BENCHMARK(type, 63, 63, Inclusive), CREATE_BENCHMARK(type, 64, 64, Inclusive), \ + CREATE_BENCHMARK(type, 128, 64, Inclusive), CREATE_BENCHMARK(type, 256, 64, Inclusive) template void add_benchmarks(std::vector& benchmarks, @@ -188,18 +222,30 @@ void add_benchmarks(std::vector& benchmarks, using custom_double2 = custom_type; using custom_int_double = custom_type; - std::vector new_benchmarks = + std::vector bs = {BENCHMARK_TYPE_WS32(int), + BENCHMARK_TYPE_WS32(float), + BENCHMARK_TYPE_WS32(double), + BENCHMARK_TYPE_WS32(int8_t), + BENCHMARK_TYPE_WS32(uint8_t), + BENCHMARK_TYPE_WS32(rocprim::half), + BENCHMARK_TYPE_WS32(custom_double2), + BENCHMARK_TYPE_WS32(custom_int_double)}; + benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); + + int hip_device = 0; + HIP_CHECK(::rocprim::detail::get_device_from_stream(stream, hip_device)); + if(is_warp_size_supported(64, hip_device)) { - BENCHMARK_TYPE(int), - BENCHMARK_TYPE(float), - BENCHMARK_TYPE(double), - BENCHMARK_TYPE(int8_t), - BENCHMARK_TYPE(uint8_t), - BENCHMARK_TYPE(rocprim::half), - BENCHMARK_TYPE(custom_double2), - BENCHMARK_TYPE(custom_int_double) - }; - benchmarks.insert(benchmarks.end(), new_benchmarks.begin(), new_benchmarks.end()); + std::vector bs = {BENCHMARK_TYPE_WS64(int), + BENCHMARK_TYPE_WS64(float), + BENCHMARK_TYPE_WS64(double), + BENCHMARK_TYPE_WS64(int8_t), + BENCHMARK_TYPE_WS64(uint8_t), + BENCHMARK_TYPE_WS64(rocprim::half), + BENCHMARK_TYPE_WS64(custom_double2), + BENCHMARK_TYPE_WS64(custom_int_double)}; + benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); + } } int main(int argc, char *argv[]) diff --git a/benchmark/benchmark_warp_sort.cpp b/benchmark/benchmark_warp_sort.cpp index a12f1e7b1..f6c97b002 100644 --- a/benchmark/benchmark_warp_sort.cpp +++ b/benchmark/benchmark_warp_sort.cpp @@ -48,9 +48,18 @@ const size_t DEFAULT_BYTES = 1024 * 1024 * 32 *4; namespace rp = rocprim; template -__global__ -__launch_bounds__(BlockSize) -void warp_sort_kernel(K* input_keys, K* output_keys) +__device__ +auto warp_sort_benchmark(K* /*input_keys*/, K* /*output_keys*/) + -> std::enable_if_t> +{ + // This kernel should never be actually called; benchmarks are selected at runtime only + // if the current device supports WarpSize +} + +template +__device__ +auto warp_sort_benchmark(K* input_keys, K* output_keys) + -> std::enable_if_t> { const unsigned int flat_tid = threadIdx.x; const unsigned int items_per_block = BlockSize * ItemsPerThread; @@ -65,10 +74,38 @@ void warp_sort_kernel(K* input_keys, K* output_keys) rp::block_store_direct_blocked(flat_tid, output_keys + block_offset, keys); } -template +template __global__ __launch_bounds__(BlockSize) -void warp_sort_by_key_kernel(K* input_keys, V* input_values, K* output_keys, V* output_values) +void warp_sort_kernel(K* input_keys, K* output_keys) +{ + warp_sort_benchmark(input_keys, output_keys); +} + +template +__device__ +auto warp_sort_by_key_benchmark(K* /*input_keys*/, + V* /*input_values*/, + K* /*output_keys*/, + V* /*output_values*/) + -> std::enable_if_t> +{ + // This kernel should never be actually called; benchmarks are selected at runtime only + // if the current device supports WarpSize +} + +template +__device__ +auto warp_sort_by_key_benchmark(K* input_keys, V* input_values, K* output_keys, V* output_values) + -> std::enable_if_t> { const unsigned int flat_tid = threadIdx.x; const unsigned int items_per_block = BlockSize * ItemsPerThread; @@ -87,6 +124,21 @@ void warp_sort_by_key_kernel(K* input_keys, V* input_values, K* output_keys, V* rp::block_store_direct_blocked(flat_tid, output_values + block_offset, values); } +template +__global__ +__launch_bounds__(BlockSize) +void warp_sort_by_key_kernel(K* input_keys, V* input_values, K* output_keys, V* output_values) +{ + warp_sort_by_key_benchmark(input_keys, + input_values, + output_keys, + output_values); +} + template; using custom_longlong_double = custom_type; - std::vector benchmarks = + std::vector benchmarks = {BENCHMARK_TYPE_WS32(int), + BENCHMARK_TYPE_WS32(float), + BENCHMARK_TYPE_WS32(double), + BENCHMARK_TYPE_WS32(int8_t), + BENCHMARK_TYPE_WS32(uint8_t), + BENCHMARK_TYPE_WS32(rocprim::half)}; + + int hip_device = 0; + HIP_CHECK(::rocprim::detail::get_device_from_stream(stream, hip_device)); + if(is_warp_size_supported(64, hip_device)) { - BENCHMARK_TYPE(int), - BENCHMARK_TYPE(float), - BENCHMARK_TYPE(double), - BENCHMARK_TYPE(int8_t), - BENCHMARK_TYPE(uint8_t), - BENCHMARK_TYPE(rocprim::half), - - BENCHMARK_KEY_TYPE(float, float), - BENCHMARK_KEY_TYPE(unsigned int, int), - BENCHMARK_KEY_TYPE(int, custom_double2), - BENCHMARK_KEY_TYPE(int, custom_int_double), - BENCHMARK_KEY_TYPE(custom_int2, custom_double2), - BENCHMARK_KEY_TYPE(custom_int2, custom_char_double), - BENCHMARK_KEY_TYPE(custom_int2, custom_longlong_double), - BENCHMARK_KEY_TYPE(int8_t, int8_t), - BENCHMARK_KEY_TYPE(uint8_t, uint8_t), - BENCHMARK_KEY_TYPE(rocprim::half, rocprim::half) - }; + std::vector additional_benchmarks + = {BENCHMARK_TYPE_WS64(int), + BENCHMARK_TYPE_WS64(float), + BENCHMARK_TYPE_WS64(double), + BENCHMARK_TYPE_WS64(int8_t), + BENCHMARK_TYPE_WS64(uint8_t), + BENCHMARK_TYPE_WS64(rocprim::half), + + BENCHMARK_KEY_TYPE_WS64(float, float), + BENCHMARK_KEY_TYPE_WS64(unsigned int, int), + BENCHMARK_KEY_TYPE_WS64(int, custom_double2), + BENCHMARK_KEY_TYPE_WS64(int, custom_int_double), + BENCHMARK_KEY_TYPE_WS64(custom_int2, custom_double2), + BENCHMARK_KEY_TYPE_WS64(custom_int2, custom_char_double), + BENCHMARK_KEY_TYPE_WS64(custom_int2, custom_longlong_double), + BENCHMARK_KEY_TYPE_WS64(int8_t, int8_t), + BENCHMARK_KEY_TYPE_WS64(uint8_t, uint8_t), + BENCHMARK_KEY_TYPE_WS64(rocprim::half, rocprim::half)}; + benchmarks.insert(benchmarks.end(), + additional_benchmarks.begin(), + additional_benchmarks.end()); + } // Use manual timing for(auto& b : benchmarks) From b11414de79d89afe6b4cbf933a4abb93830b698a Mon Sep 17 00:00:00 2001 From: Jaap Blok Date: Thu, 21 Nov 2024 12:58:55 +0100 Subject: [PATCH 14/84] Set explicit return types --- rocprim/include/rocprim/warp/warp_reduce.hpp | 8 ++++---- rocprim/include/rocprim/warp/warp_scan.hpp | 20 ++++++++++---------- rocprim/include/rocprim/warp/warp_sort.hpp | 16 ++++++++-------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/rocprim/include/rocprim/warp/warp_reduce.hpp b/rocprim/include/rocprim/warp/warp_reduce.hpp index 725a27f32..8dea60d51 100644 --- a/rocprim/include/rocprim/warp/warp_reduce.hpp +++ b/rocprim/include/rocprim/warp/warp_reduce.hpp @@ -179,7 +179,7 @@ class warp_reduce /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - auto reduce(T input, + void reduce(T input, T& output, storage_type& storage, BinaryFunction reduce_op = BinaryFunction()) @@ -235,7 +235,7 @@ class warp_reduce /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - auto reduce(T input, + void reduce(T input, T& output, int valid_items, storage_type& storage, @@ -266,7 +266,7 @@ class warp_reduce class BinaryFunction = ::rocprim::plus, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - auto head_segmented_reduce(T input, + void head_segmented_reduce(T input, T& output, Flag flag, storage_type& storage, @@ -297,7 +297,7 @@ class warp_reduce class BinaryFunction = ::rocprim::plus, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - auto tail_segmented_reduce(T input, + void tail_segmented_reduce(T input, T& output, Flag flag, storage_type& storage, diff --git a/rocprim/include/rocprim/warp/warp_scan.hpp b/rocprim/include/rocprim/warp/warp_scan.hpp index 6c5c6195f..35d3624c0 100644 --- a/rocprim/include/rocprim/warp/warp_scan.hpp +++ b/rocprim/include/rocprim/warp/warp_scan.hpp @@ -180,7 +180,7 @@ class warp_scan /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - auto inclusive_scan(T input, + void inclusive_scan(T input, T& output, storage_type& storage, BinaryFunction scan_op = BinaryFunction()) @@ -239,7 +239,7 @@ class warp_scan /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - auto inclusive_scan(T input, + void inclusive_scan(T input, T& output, T& reduction, storage_type& storage, @@ -302,7 +302,7 @@ class warp_scan /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - auto exclusive_scan(T input, + void exclusive_scan(T input, T& output, T init, storage_type& storage, @@ -366,7 +366,7 @@ class warp_scan /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - auto exclusive_scan(T input, + void exclusive_scan(T input, T& output, T init, T& reduction, @@ -386,7 +386,7 @@ class warp_scan /// \param scan_op The function object used to combine elements used for the scan template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - auto exclusive_scan(T input, + void exclusive_scan(T input, T& output, storage_type& storage, BinaryFunction scan_op = BinaryFunction()) @@ -406,7 +406,7 @@ class warp_scan /// \param scan_op The function object used to combine elements used for the scan template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - auto exclusive_scan(T input, + void exclusive_scan(T input, T& output, storage_type& storage, T& reduction, @@ -475,7 +475,7 @@ class warp_scan /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - auto scan(T input, + void scan(T input, T& inclusive_output, T& exclusive_output, T init, @@ -545,7 +545,7 @@ class warp_scan /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - auto scan(T input, + void scan(T input, T& inclusive_output, T& exclusive_output, T init, @@ -570,7 +570,7 @@ class warp_scan /// or repurposed: \p __syncthreads() or \p rocprim::syncthreads(). template ROCPRIM_DEVICE ROCPRIM_INLINE - auto broadcast(T input, const unsigned int src_lane, storage_type& storage) + T broadcast(T input, const unsigned int src_lane, storage_type& storage) { return base_type::broadcast(input, src_lane, storage); } @@ -581,7 +581,7 @@ class warp_scan template [[deprecated]] ROCPRIM_DEVICE ROCPRIM_INLINE - auto to_exclusive(T inclusive_input, T& exclusive_output, storage_type& storage) + void to_exclusive(T inclusive_input, T& exclusive_output, storage_type& storage) { return base_type::to_exclusive(inclusive_input, exclusive_output, storage); } diff --git a/rocprim/include/rocprim/warp/warp_sort.hpp b/rocprim/include/rocprim/warp/warp_sort.hpp index fd570aa18..665bcf5a7 100644 --- a/rocprim/include/rocprim/warp/warp_sort.hpp +++ b/rocprim/include/rocprim/warp/warp_sort.hpp @@ -136,7 +136,7 @@ class warp_sort : detail::warp_sort_shuffle /// const &, but function object must not modify the objects passed to it. template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - auto sort(Key& thread_key, BinaryFunction compare_function = BinaryFunction()) + void sort(Key& thread_key, BinaryFunction compare_function = BinaryFunction()) { base_type::sort(thread_key, compare_function); } @@ -155,7 +155,7 @@ class warp_sort : detail::warp_sort_shuffle class BinaryFunction = ::rocprim::less, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - auto sort(Key (&thread_keys)[ItemsPerThread], + void sort(Key (&thread_keys)[ItemsPerThread], BinaryFunction compare_function = BinaryFunction()) { base_type::sort(thread_keys, compare_function); @@ -191,7 +191,7 @@ class warp_sort : detail::warp_sort_shuffle /// \endcode template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - auto sort(Key& thread_key, + void sort(Key& thread_key, storage_type& storage, BinaryFunction compare_function = BinaryFunction()) { @@ -232,7 +232,7 @@ class warp_sort : detail::warp_sort_shuffle class BinaryFunction = ::rocprim::less, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - auto sort(Key (&thread_keys)[ItemsPerThread], + void sort(Key (&thread_keys)[ItemsPerThread], storage_type& storage, BinaryFunction compare_function = BinaryFunction()) { @@ -254,7 +254,7 @@ class warp_sort : detail::warp_sort_shuffle /// const &, but function object must not modify the objects passed to it. template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - auto sort(Key& thread_key, + void sort(Key& thread_key, Value& thread_value, BinaryFunction compare_function = BinaryFunction()) { @@ -278,7 +278,7 @@ class warp_sort : detail::warp_sort_shuffle class BinaryFunction = ::rocprim::less, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - auto sort(Key (&thread_keys)[ItemsPerThread], + void sort(Key (&thread_keys)[ItemsPerThread], Value (&thread_values)[ItemsPerThread], BinaryFunction compare_function = BinaryFunction()) { @@ -318,7 +318,7 @@ class warp_sort : detail::warp_sort_shuffle /// \endcode template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - auto sort(Key& thread_key, + void sort(Key& thread_key, Value& thread_value, storage_type& storage, BinaryFunction compare_function = BinaryFunction()) @@ -361,7 +361,7 @@ class warp_sort : detail::warp_sort_shuffle class BinaryFunction = ::rocprim::less, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - auto sort(Key (&thread_keys)[ItemsPerThread], + void sort(Key (&thread_keys)[ItemsPerThread], Value (&thread_values)[ItemsPerThread], storage_type& storage, BinaryFunction compare_function = BinaryFunction()) From 70a321a0761d9ad5d0a55dd7b202f3ee3b20c421 Mon Sep 17 00:00:00 2001 From: Beatriz Navidad Vilches Date: Thu, 21 Nov 2024 12:58:59 +0100 Subject: [PATCH 15/84] Resolve "Investigate RLE (non trivial runs) failure" --- rocprim/include/rocprim/warp/warp_reduce.hpp | 8 ++++---- rocprim/include/rocprim/warp/warp_scan.hpp | 20 ++++++++++---------- rocprim/include/rocprim/warp/warp_sort.hpp | 16 ++++++++-------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/rocprim/include/rocprim/warp/warp_reduce.hpp b/rocprim/include/rocprim/warp/warp_reduce.hpp index 8dea60d51..725a27f32 100644 --- a/rocprim/include/rocprim/warp/warp_reduce.hpp +++ b/rocprim/include/rocprim/warp/warp_reduce.hpp @@ -179,7 +179,7 @@ class warp_reduce /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - void reduce(T input, + auto reduce(T input, T& output, storage_type& storage, BinaryFunction reduce_op = BinaryFunction()) @@ -235,7 +235,7 @@ class warp_reduce /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - void reduce(T input, + auto reduce(T input, T& output, int valid_items, storage_type& storage, @@ -266,7 +266,7 @@ class warp_reduce class BinaryFunction = ::rocprim::plus, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - void head_segmented_reduce(T input, + auto head_segmented_reduce(T input, T& output, Flag flag, storage_type& storage, @@ -297,7 +297,7 @@ class warp_reduce class BinaryFunction = ::rocprim::plus, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - void tail_segmented_reduce(T input, + auto tail_segmented_reduce(T input, T& output, Flag flag, storage_type& storage, diff --git a/rocprim/include/rocprim/warp/warp_scan.hpp b/rocprim/include/rocprim/warp/warp_scan.hpp index 35d3624c0..6c5c6195f 100644 --- a/rocprim/include/rocprim/warp/warp_scan.hpp +++ b/rocprim/include/rocprim/warp/warp_scan.hpp @@ -180,7 +180,7 @@ class warp_scan /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - void inclusive_scan(T input, + auto inclusive_scan(T input, T& output, storage_type& storage, BinaryFunction scan_op = BinaryFunction()) @@ -239,7 +239,7 @@ class warp_scan /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - void inclusive_scan(T input, + auto inclusive_scan(T input, T& output, T& reduction, storage_type& storage, @@ -302,7 +302,7 @@ class warp_scan /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - void exclusive_scan(T input, + auto exclusive_scan(T input, T& output, T init, storage_type& storage, @@ -366,7 +366,7 @@ class warp_scan /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - void exclusive_scan(T input, + auto exclusive_scan(T input, T& output, T init, T& reduction, @@ -386,7 +386,7 @@ class warp_scan /// \param scan_op The function object used to combine elements used for the scan template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - void exclusive_scan(T input, + auto exclusive_scan(T input, T& output, storage_type& storage, BinaryFunction scan_op = BinaryFunction()) @@ -406,7 +406,7 @@ class warp_scan /// \param scan_op The function object used to combine elements used for the scan template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - void exclusive_scan(T input, + auto exclusive_scan(T input, T& output, storage_type& storage, T& reduction, @@ -475,7 +475,7 @@ class warp_scan /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - void scan(T input, + auto scan(T input, T& inclusive_output, T& exclusive_output, T init, @@ -545,7 +545,7 @@ class warp_scan /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - void scan(T input, + auto scan(T input, T& inclusive_output, T& exclusive_output, T init, @@ -570,7 +570,7 @@ class warp_scan /// or repurposed: \p __syncthreads() or \p rocprim::syncthreads(). template ROCPRIM_DEVICE ROCPRIM_INLINE - T broadcast(T input, const unsigned int src_lane, storage_type& storage) + auto broadcast(T input, const unsigned int src_lane, storage_type& storage) { return base_type::broadcast(input, src_lane, storage); } @@ -581,7 +581,7 @@ class warp_scan template [[deprecated]] ROCPRIM_DEVICE ROCPRIM_INLINE - void to_exclusive(T inclusive_input, T& exclusive_output, storage_type& storage) + auto to_exclusive(T inclusive_input, T& exclusive_output, storage_type& storage) { return base_type::to_exclusive(inclusive_input, exclusive_output, storage); } diff --git a/rocprim/include/rocprim/warp/warp_sort.hpp b/rocprim/include/rocprim/warp/warp_sort.hpp index 665bcf5a7..fd570aa18 100644 --- a/rocprim/include/rocprim/warp/warp_sort.hpp +++ b/rocprim/include/rocprim/warp/warp_sort.hpp @@ -136,7 +136,7 @@ class warp_sort : detail::warp_sort_shuffle /// const &, but function object must not modify the objects passed to it. template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - void sort(Key& thread_key, BinaryFunction compare_function = BinaryFunction()) + auto sort(Key& thread_key, BinaryFunction compare_function = BinaryFunction()) { base_type::sort(thread_key, compare_function); } @@ -155,7 +155,7 @@ class warp_sort : detail::warp_sort_shuffle class BinaryFunction = ::rocprim::less, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - void sort(Key (&thread_keys)[ItemsPerThread], + auto sort(Key (&thread_keys)[ItemsPerThread], BinaryFunction compare_function = BinaryFunction()) { base_type::sort(thread_keys, compare_function); @@ -191,7 +191,7 @@ class warp_sort : detail::warp_sort_shuffle /// \endcode template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - void sort(Key& thread_key, + auto sort(Key& thread_key, storage_type& storage, BinaryFunction compare_function = BinaryFunction()) { @@ -232,7 +232,7 @@ class warp_sort : detail::warp_sort_shuffle class BinaryFunction = ::rocprim::less, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - void sort(Key (&thread_keys)[ItemsPerThread], + auto sort(Key (&thread_keys)[ItemsPerThread], storage_type& storage, BinaryFunction compare_function = BinaryFunction()) { @@ -254,7 +254,7 @@ class warp_sort : detail::warp_sort_shuffle /// const &, but function object must not modify the objects passed to it. template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - void sort(Key& thread_key, + auto sort(Key& thread_key, Value& thread_value, BinaryFunction compare_function = BinaryFunction()) { @@ -278,7 +278,7 @@ class warp_sort : detail::warp_sort_shuffle class BinaryFunction = ::rocprim::less, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - void sort(Key (&thread_keys)[ItemsPerThread], + auto sort(Key (&thread_keys)[ItemsPerThread], Value (&thread_values)[ItemsPerThread], BinaryFunction compare_function = BinaryFunction()) { @@ -318,7 +318,7 @@ class warp_sort : detail::warp_sort_shuffle /// \endcode template, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - void sort(Key& thread_key, + auto sort(Key& thread_key, Value& thread_value, storage_type& storage, BinaryFunction compare_function = BinaryFunction()) @@ -361,7 +361,7 @@ class warp_sort : detail::warp_sort_shuffle class BinaryFunction = ::rocprim::less, unsigned int FunctionWarpSize = WarpSize> ROCPRIM_DEVICE ROCPRIM_INLINE - void sort(Key (&thread_keys)[ItemsPerThread], + auto sort(Key (&thread_keys)[ItemsPerThread], Value (&thread_values)[ItemsPerThread], storage_type& storage, BinaryFunction compare_function = BinaryFunction()) From b6c63e826d05b98efc8337ad8f9c5fe8293b30d9 Mon Sep 17 00:00:00 2001 From: Jaap Blok Date: Thu, 21 Nov 2024 12:59:00 +0100 Subject: [PATCH 16/84] Revert "Fix benchmarks triggering static asserts" This reverts commit a1634a9382e268285c1e21c0743b6a98e67a69ef. --- benchmark/benchmark_warp_reduce.cpp | 142 ++++++++++--------------- benchmark/benchmark_warp_scan.cpp | 94 +++++------------ benchmark/benchmark_warp_sort.cpp | 158 +++++++++------------------- 3 files changed, 128 insertions(+), 266 deletions(-) diff --git a/benchmark/benchmark_warp_reduce.cpp b/benchmark/benchmark_warp_reduce.cpp index 7198021b1..1cddacbfd 100644 --- a/benchmark/benchmark_warp_reduce.cpp +++ b/benchmark/benchmark_warp_reduce.cpp @@ -45,19 +45,15 @@ const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif -template -__device__ -auto warp_reduce_benchmark(const T* /*d_input*/, T* /*d_output*/) - -> std::enable_if_t> -{ - // This kernel should never be actually called; benchmarks are selected at runtime only - // if the current device supports WarpSize -} - -template -__device__ -auto warp_reduce_benchmark(const T* d_input, T* d_output) - -> std::enable_if_t> +template< + bool AllReduce, + class T, + unsigned int WarpSize, + unsigned int Trials +> +__global__ +__launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) +void warp_reduce_kernel(const T * d_input, T * d_output) { const unsigned int i = blockIdx.x * blockDim.x + threadIdx.x; @@ -74,27 +70,15 @@ auto warp_reduce_benchmark(const T* d_input, T* d_output) d_output[i] = value; } -template +template< + class T, + class Flag, + unsigned int WarpSize, + unsigned int Trials +> __global__ __launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) -void warp_reduce_kernel(const T* d_input, T* d_output) -{ - warp_reduce_benchmark(d_input, d_output); -} - -template -__device__ -auto segmented_warp_reduce_benchmark(const T* /*d_input*/, Flag* /*d_flags*/, T* /*d_output*/) - -> std::enable_if_t> -{ - // This kernel should never be actually called; benchmarks are selected at runtime only - // if the current device supports WarpSize -} - -template -__device__ -auto segmented_warp_reduce_benchmark(const T* d_input, Flag* d_flags, T* d_output) - -> std::enable_if_t> +void segmented_warp_reduce_kernel(const T* d_input, Flag* d_flags, T* d_output) { const unsigned int i = blockIdx.x * blockDim.x + threadIdx.x; @@ -112,24 +96,19 @@ auto segmented_warp_reduce_benchmark(const T* d_input, Flag* d_flags, T* d_outpu d_output[i] = value; } -template -__global__ -__launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) -void segmented_warp_reduce_kernel(const T* d_input, Flag* d_flags, T* d_output) -{ - segmented_warp_reduce_benchmark(d_input, d_flags, d_output); -} - -template -inline auto execute_warp_reduce_kernel( - T* input, T* output, Flag* /* flags */, size_t size, hipStream_t stream) -> - typename std::enable_if_t +template< + bool AllReduce, + bool Segmented, + unsigned int WarpSize, + unsigned int BlockSize, + unsigned int Trials, + class T, + class Flag +> +inline +auto execute_warp_reduce_kernel(T* input, T* output, Flag* /* flags */, + size_t size, hipStream_t stream) + -> typename std::enable_if::type { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_reduce_kernel), @@ -139,16 +118,19 @@ inline auto execute_warp_reduce_kernel( HIP_CHECK(hipGetLastError()); } -template -inline auto - execute_warp_reduce_kernel(T* input, T* output, Flag* flags, size_t size, hipStream_t stream) -> - typename std::enable_if_t +template< + bool AllReduce, + bool Segmented, + unsigned int WarpSize, + unsigned int BlockSize, + unsigned int Trials, + class T, + class Flag +> +inline +auto execute_warp_reduce_kernel(T* input, T* output, Flag* flags, + size_t size, hipStream_t stream) + -> typename std::enable_if::type { hipLaunchKernelGGL( HIP_KERNEL_NAME(segmented_warp_reduce_kernel), @@ -248,12 +230,11 @@ void run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& se seed, \ stream) -// For devices with a physical warp size of at least 32 -#define BENCHMARK_TYPE_WS32(type) CREATE_BENCHMARK(type, 32, 64) - -// For devices with a physical warp size of at least 64 -#define BENCHMARK_TYPE_WS64(type) \ - CREATE_BENCHMARK(type, 37, 64), CREATE_BENCHMARK(type, 61, 64), CREATE_BENCHMARK(type, 64, 64) +#define BENCHMARK_TYPE(type) \ + CREATE_BENCHMARK(type, 32, 64), \ + CREATE_BENCHMARK(type, 37, 64), \ + CREATE_BENCHMARK(type, 61, 64), \ + CREATE_BENCHMARK(type, 64, 64) template void add_benchmarks(std::vector& benchmarks, @@ -261,26 +242,17 @@ void add_benchmarks(std::vector& benchmarks, const managed_seed& seed, hipStream_t stream) { - std::vector bs = {BENCHMARK_TYPE_WS32(int), - BENCHMARK_TYPE_WS32(float), - BENCHMARK_TYPE_WS32(double), - BENCHMARK_TYPE_WS32(int8_t), - BENCHMARK_TYPE_WS32(uint8_t), - BENCHMARK_TYPE_WS32(rocprim::half)}; - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); - - int hip_device = 0; - HIP_CHECK(::rocprim::detail::get_device_from_stream(stream, hip_device)); - if(is_warp_size_supported(64, hip_device)) + std::vector bs = { - std::vector bs = {BENCHMARK_TYPE_WS64(int), - BENCHMARK_TYPE_WS64(float), - BENCHMARK_TYPE_WS64(double), - BENCHMARK_TYPE_WS64(int8_t), - BENCHMARK_TYPE_WS64(uint8_t), - BENCHMARK_TYPE_WS64(rocprim::half)}; - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); - } + BENCHMARK_TYPE(int), + BENCHMARK_TYPE(float), + BENCHMARK_TYPE(double), + BENCHMARK_TYPE(int8_t), + BENCHMARK_TYPE(uint8_t), + BENCHMARK_TYPE(rocprim::half) + }; + + benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); } int main(int argc, char *argv[]) diff --git a/benchmark/benchmark_warp_scan.cpp b/benchmark/benchmark_warp_scan.cpp index eca1a4c78..f7983761a 100644 --- a/benchmark/benchmark_warp_scan.cpp +++ b/benchmark/benchmark_warp_scan.cpp @@ -46,18 +46,9 @@ const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; namespace rp = rocprim; template -__device__ -auto warp_inclusive_scan_benchmark(const T* /*input*/, T* /*output*/) - -> std::enable_if_t> -{ - // This kernel should never be actually called; benchmarks are selected at runtime only - // if the current device supports WarpSize -} - -template -__device__ -auto warp_inclusive_scan_benchmark(const T* input, T* output) - -> std::enable_if_t> +__global__ +__launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) +void warp_inclusive_scan_kernel(const T* input, T* output) { const unsigned int i = blockIdx.x * blockDim.x + threadIdx.x; auto value = input[i]; @@ -76,24 +67,7 @@ auto warp_inclusive_scan_benchmark(const T* input, T* output) template __global__ __launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) -void warp_inclusive_scan_kernel(const T* input, T* output) -{ - warp_inclusive_scan_benchmark(input, output); -} - -template -__device__ -auto warp_exclusive_scan_benchmark(const T* /*input*/, T* /*output*/, const T /*init*/) - -> std::enable_if_t> -{ - // This kernel should never be actually called; benchmarks are selected at runtime only - // if the current device supports WarpSize -} - -template -__device__ -auto warp_exclusive_scan_benchmark(const T* input, T* output, const T init) - -> std::enable_if_t> +void warp_exclusive_scan_kernel(const T* input, T* output, const T init) { const unsigned int i = blockIdx.x * blockDim.x + threadIdx.x; auto value = input[i]; @@ -109,14 +83,6 @@ auto warp_exclusive_scan_benchmark(const T* input, T* output, const T init) output[i] = value; } -template -__global__ -__launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) -void warp_exclusive_scan_kernel(const T* input, T* output, const T init) -{ - warp_exclusive_scan_benchmark(input, output, init); -} - template< class T, unsigned int BlockSize, @@ -204,15 +170,15 @@ void run_benchmark(benchmark::State& state, hipStream_t stream, size_t bytes) stream, \ bytes) -// for devices with a physical warp size of at least 32 -#define BENCHMARK_TYPE_WS32(type) \ - CREATE_BENCHMARK(type, 60, 15, Inclusive), CREATE_BENCHMARK(type, 256, 16, Inclusive), \ - CREATE_BENCHMARK(type, 62, 31, Inclusive), CREATE_BENCHMARK(type, 256, 32, Inclusive) - -// for devices with a physical warp size of at least 64 -#define BENCHMARK_TYPE_WS64(type) \ - CREATE_BENCHMARK(type, 63, 63, Inclusive), CREATE_BENCHMARK(type, 64, 64, Inclusive), \ - CREATE_BENCHMARK(type, 128, 64, Inclusive), CREATE_BENCHMARK(type, 256, 64, Inclusive) +#define BENCHMARK_TYPE(type) \ + CREATE_BENCHMARK(type, 64, 64, Inclusive), \ + CREATE_BENCHMARK(type, 128, 64, Inclusive), \ + CREATE_BENCHMARK(type, 256, 64, Inclusive), \ + CREATE_BENCHMARK(type, 256, 32, Inclusive), \ + CREATE_BENCHMARK(type, 256, 16, Inclusive), \ + CREATE_BENCHMARK(type, 63, 63, Inclusive), \ + CREATE_BENCHMARK(type, 62, 31, Inclusive), \ + CREATE_BENCHMARK(type, 60, 15, Inclusive) template void add_benchmarks(std::vector& benchmarks, @@ -222,30 +188,18 @@ void add_benchmarks(std::vector& benchmarks, using custom_double2 = custom_type; using custom_int_double = custom_type; - std::vector bs = {BENCHMARK_TYPE_WS32(int), - BENCHMARK_TYPE_WS32(float), - BENCHMARK_TYPE_WS32(double), - BENCHMARK_TYPE_WS32(int8_t), - BENCHMARK_TYPE_WS32(uint8_t), - BENCHMARK_TYPE_WS32(rocprim::half), - BENCHMARK_TYPE_WS32(custom_double2), - BENCHMARK_TYPE_WS32(custom_int_double)}; - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); - - int hip_device = 0; - HIP_CHECK(::rocprim::detail::get_device_from_stream(stream, hip_device)); - if(is_warp_size_supported(64, hip_device)) + std::vector new_benchmarks = { - std::vector bs = {BENCHMARK_TYPE_WS64(int), - BENCHMARK_TYPE_WS64(float), - BENCHMARK_TYPE_WS64(double), - BENCHMARK_TYPE_WS64(int8_t), - BENCHMARK_TYPE_WS64(uint8_t), - BENCHMARK_TYPE_WS64(rocprim::half), - BENCHMARK_TYPE_WS64(custom_double2), - BENCHMARK_TYPE_WS64(custom_int_double)}; - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); - } + BENCHMARK_TYPE(int), + BENCHMARK_TYPE(float), + BENCHMARK_TYPE(double), + BENCHMARK_TYPE(int8_t), + BENCHMARK_TYPE(uint8_t), + BENCHMARK_TYPE(rocprim::half), + BENCHMARK_TYPE(custom_double2), + BENCHMARK_TYPE(custom_int_double) + }; + benchmarks.insert(benchmarks.end(), new_benchmarks.begin(), new_benchmarks.end()); } int main(int argc, char *argv[]) diff --git a/benchmark/benchmark_warp_sort.cpp b/benchmark/benchmark_warp_sort.cpp index f6c97b002..a12f1e7b1 100644 --- a/benchmark/benchmark_warp_sort.cpp +++ b/benchmark/benchmark_warp_sort.cpp @@ -48,18 +48,9 @@ const size_t DEFAULT_BYTES = 1024 * 1024 * 32 *4; namespace rp = rocprim; template -__device__ -auto warp_sort_benchmark(K* /*input_keys*/, K* /*output_keys*/) - -> std::enable_if_t> -{ - // This kernel should never be actually called; benchmarks are selected at runtime only - // if the current device supports WarpSize -} - -template -__device__ -auto warp_sort_benchmark(K* input_keys, K* output_keys) - -> std::enable_if_t> +__global__ +__launch_bounds__(BlockSize) +void warp_sort_kernel(K* input_keys, K* output_keys) { const unsigned int flat_tid = threadIdx.x; const unsigned int items_per_block = BlockSize * ItemsPerThread; @@ -74,38 +65,10 @@ auto warp_sort_benchmark(K* input_keys, K* output_keys) rp::block_store_direct_blocked(flat_tid, output_keys + block_offset, keys); } -template +template __global__ __launch_bounds__(BlockSize) -void warp_sort_kernel(K* input_keys, K* output_keys) -{ - warp_sort_benchmark(input_keys, output_keys); -} - -template -__device__ -auto warp_sort_by_key_benchmark(K* /*input_keys*/, - V* /*input_values*/, - K* /*output_keys*/, - V* /*output_values*/) - -> std::enable_if_t> -{ - // This kernel should never be actually called; benchmarks are selected at runtime only - // if the current device supports WarpSize -} - -template -__device__ -auto warp_sort_by_key_benchmark(K* input_keys, V* input_values, K* output_keys, V* output_values) - -> std::enable_if_t> +void warp_sort_by_key_kernel(K* input_keys, V* input_values, K* output_keys, V* output_values) { const unsigned int flat_tid = threadIdx.x; const unsigned int items_per_block = BlockSize * ItemsPerThread; @@ -124,21 +87,6 @@ auto warp_sort_by_key_benchmark(K* input_keys, V* input_values, K* output_keys, rp::block_store_direct_blocked(flat_tid, output_values + block_offset, values); } -template -__global__ -__launch_bounds__(BlockSize) -void warp_sort_by_key_kernel(K* input_keys, V* input_values, K* output_keys, V* output_values) -{ - warp_sort_by_key_benchmark(input_keys, - input_values, - output_keys, - output_values); -} - template; using custom_longlong_double = custom_type; - std::vector benchmarks = {BENCHMARK_TYPE_WS32(int), - BENCHMARK_TYPE_WS32(float), - BENCHMARK_TYPE_WS32(double), - BENCHMARK_TYPE_WS32(int8_t), - BENCHMARK_TYPE_WS32(uint8_t), - BENCHMARK_TYPE_WS32(rocprim::half)}; - - int hip_device = 0; - HIP_CHECK(::rocprim::detail::get_device_from_stream(stream, hip_device)); - if(is_warp_size_supported(64, hip_device)) + std::vector benchmarks = { - std::vector additional_benchmarks - = {BENCHMARK_TYPE_WS64(int), - BENCHMARK_TYPE_WS64(float), - BENCHMARK_TYPE_WS64(double), - BENCHMARK_TYPE_WS64(int8_t), - BENCHMARK_TYPE_WS64(uint8_t), - BENCHMARK_TYPE_WS64(rocprim::half), - - BENCHMARK_KEY_TYPE_WS64(float, float), - BENCHMARK_KEY_TYPE_WS64(unsigned int, int), - BENCHMARK_KEY_TYPE_WS64(int, custom_double2), - BENCHMARK_KEY_TYPE_WS64(int, custom_int_double), - BENCHMARK_KEY_TYPE_WS64(custom_int2, custom_double2), - BENCHMARK_KEY_TYPE_WS64(custom_int2, custom_char_double), - BENCHMARK_KEY_TYPE_WS64(custom_int2, custom_longlong_double), - BENCHMARK_KEY_TYPE_WS64(int8_t, int8_t), - BENCHMARK_KEY_TYPE_WS64(uint8_t, uint8_t), - BENCHMARK_KEY_TYPE_WS64(rocprim::half, rocprim::half)}; - benchmarks.insert(benchmarks.end(), - additional_benchmarks.begin(), - additional_benchmarks.end()); - } + BENCHMARK_TYPE(int), + BENCHMARK_TYPE(float), + BENCHMARK_TYPE(double), + BENCHMARK_TYPE(int8_t), + BENCHMARK_TYPE(uint8_t), + BENCHMARK_TYPE(rocprim::half), + + BENCHMARK_KEY_TYPE(float, float), + BENCHMARK_KEY_TYPE(unsigned int, int), + BENCHMARK_KEY_TYPE(int, custom_double2), + BENCHMARK_KEY_TYPE(int, custom_int_double), + BENCHMARK_KEY_TYPE(custom_int2, custom_double2), + BENCHMARK_KEY_TYPE(custom_int2, custom_char_double), + BENCHMARK_KEY_TYPE(custom_int2, custom_longlong_double), + BENCHMARK_KEY_TYPE(int8_t, int8_t), + BENCHMARK_KEY_TYPE(uint8_t, uint8_t), + BENCHMARK_KEY_TYPE(rocprim::half, rocprim::half) + }; // Use manual timing for(auto& b : benchmarks) From c1f2fc44689dceca45a222b3e0aadc6a0000d7de Mon Sep 17 00:00:00 2001 From: Jaap Blok Date: Thu, 21 Nov 2024 12:59:00 +0100 Subject: [PATCH 17/84] Revert "Fix tests triggering static asserts" This reverts commit 24a36cb86283098036a5931562d0b4d3b42ca675. --- test/rocprim/test_warp_reduce.kernels.hpp | 158 ++++--------- test/rocprim/test_warp_scan.kernels.hpp | 268 +++++----------------- test/rocprim/test_warp_sort.hpp | 15 +- test/rocprim/test_warp_sort.kernels.hpp | 178 +++++++------- 4 files changed, 198 insertions(+), 421 deletions(-) diff --git a/test/rocprim/test_warp_reduce.kernels.hpp b/test/rocprim/test_warp_reduce.kernels.hpp index 48a34c54f..846d11a32 100644 --- a/test/rocprim/test_warp_reduce.kernels.hpp +++ b/test/rocprim/test_warp_reduce.kernels.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,19 +21,14 @@ #ifndef TEST_WARP_REDUCE_KERNELS_HPP_ #define TEST_WARP_REDUCE_KERNELS_HPP_ -template -__device__ -auto warp_reduce_sum_test(T* /*device_input*/, T* /*device_output*/) - -> std::enable_if_t> -{ - // This kernel should never be actually called; tests are filtered out at runtime - // if the device does not support the LogicalWarpSize -} - -template -__device__ -auto warp_reduce_sum_test(T* device_input, T* device_output) - -> std::enable_if_t> +template< + class T, + unsigned int BlockSize, + unsigned int LogicalWarpSize +> +__global__ +__launch_bounds__(BlockSize) +void warp_reduce_sum_kernel(T* device_input, T* device_output) { static constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -51,27 +46,14 @@ auto warp_reduce_sum_test(T* device_input, T* device_output) } } -template +template< + class T, + unsigned int BlockSize, + unsigned int LogicalWarpSize +> __global__ __launch_bounds__(BlockSize) -void warp_reduce_sum_kernel(T* device_input, T* device_output) -{ - warp_reduce_sum_test(device_input, device_output); -} - -template -__device__ -auto warp_allreduce_sum_test(T* /*device_input*/, T* /*device_output*/) - -> std::enable_if_t> -{ - // This kernel should never be actually called; tests are filtered out at runtime - // if the device does not support the LogicalWarpSize -} - -template -__device__ -auto warp_allreduce_sum_test(T* device_input, T* device_output) - -> std::enable_if_t> +void warp_allreduce_sum_kernel(T* device_input, T* device_output) { static constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -86,27 +68,14 @@ auto warp_allreduce_sum_test(T* device_input, T* device_output) device_output[index] = value; } -template +template< + class T, + unsigned int BlockSize, + unsigned int LogicalWarpSize +> __global__ __launch_bounds__(BlockSize) -void warp_allreduce_sum_kernel(T* device_input, T* device_output) -{ - warp_allreduce_sum_test(device_input, device_output); -} - -template -__device__ -auto warp_reduce_sum_test(T* /*device_input*/, T* /*device_output*/, size_t /*valid*/) - -> std::enable_if_t> -{ - // This kernel should never be actually called; tests are filtered out at runtime - // if the device does not support the LogicalWarpSize -} - -template -__device__ -auto warp_reduce_sum_test(T* device_input, T* device_output, size_t valid) - -> std::enable_if_t> +void warp_reduce_sum_kernel(T* device_input, T* device_output, size_t valid) { static constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -124,27 +93,14 @@ auto warp_reduce_sum_test(T* device_input, T* device_output, size_t valid) } } -template +template< + class T, + unsigned int BlockSize, + unsigned int LogicalWarpSize +> __global__ __launch_bounds__(BlockSize) -void warp_reduce_sum_kernel(T* device_input, T* device_output, size_t valid) -{ - warp_reduce_sum_test(device_input, device_output, valid); -} - -template -__device__ -auto warp_allreduce_sum_test(T* /*device_input*/, T* /*device_output*/, size_t /*valid*/) - -> std::enable_if_t> -{ - // This kernel should never be actually called; tests are filtered out at runtime - // if the device does not support the LogicalWarpSize -} - -template -__device__ -auto warp_allreduce_sum_test(T* device_input, T* device_output, size_t valid) - -> std::enable_if_t> +void warp_allreduce_sum_kernel(T* device_input, T* device_output, size_t valid) { constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -159,27 +115,15 @@ auto warp_allreduce_sum_test(T* device_input, T* device_output, size_t valid) device_output[index] = value; } -template +template< + class T, + class Flag, + unsigned int BlockSize, + unsigned int LogicalWarpSize +> __global__ __launch_bounds__(BlockSize) -void warp_allreduce_sum_kernel(T* device_input, T* device_output, size_t valid) -{ - warp_allreduce_sum_test(device_input, device_output, valid); -} - -template -__device__ -auto head_segmented_warp_reduce_test(T* /*input*/, Flag* /*flags*/, T* /*output*/) - -> std::enable_if_t> -{ - // This kernel should never be actually called; tests are filtered out at runtime - // if the device does not support the LogicalWarpSize -} - -template -__device__ -auto head_segmented_warp_reduce_test(T* input, Flag* flags, T* output) - -> std::enable_if_t> +void head_segmented_warp_reduce_kernel(T* input, Flag* flags, T* output) { constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -195,27 +139,15 @@ auto head_segmented_warp_reduce_test(T* input, Flag* flags, T* output) output[index] = value; } -template +template< + class T, + class Flag, + unsigned int BlockSize, + unsigned int LogicalWarpSize +> __global__ __launch_bounds__(BlockSize) -void head_segmented_warp_reduce_kernel(T* input, Flag* flags, T* output) -{ - head_segmented_warp_reduce_test(input, flags, output); -} - -template -__device__ -auto tail_segmented_warp_reduce_test(T* /*input*/, Flag* /*flags*/, T* /*output*/) - -> std::enable_if_t> -{ - // This kernel should never be actually called; tests are filtered out at runtime - // if the device does not support the LogicalWarpSize -} - -template -__device__ -auto tail_segmented_warp_reduce_test(T* input, Flag* flags, T* output) - -> std::enable_if_t> +void tail_segmented_warp_reduce_kernel(T* input, Flag* flags, T* output) { constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -231,12 +163,4 @@ auto tail_segmented_warp_reduce_test(T* input, Flag* flags, T* output) output[index] = value; } -template -__global__ -__launch_bounds__(BlockSize) -void tail_segmented_warp_reduce_kernel(T* input, Flag* flags, T* output) -{ - tail_segmented_warp_reduce_test(input, flags, output); -} - #endif // TEST_WARP_REDUCE_KERNELS_HPP_ diff --git a/test/rocprim/test_warp_scan.kernels.hpp b/test/rocprim/test_warp_scan.kernels.hpp index 88e02b89d..875389058 100644 --- a/test/rocprim/test_warp_scan.kernels.hpp +++ b/test/rocprim/test_warp_scan.kernels.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2023 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -23,19 +23,14 @@ #ifndef TEST_SCAN_REDUCE_KERNELS_HPP_ #define TEST_SCAN_REDUCE_KERNELS_HPP_ -template -__device__ -auto warp_inclusive_scan_test(T* /*device_input*/, T* /*device_output*/) - -> std::enable_if_t> -{ - // This kernel should never be actually called; tests are filtered out at runtime - // if the device does not support the LogicalWarpSize -} - -template -__device__ -auto warp_inclusive_scan_test(T* device_input, T* device_output) - -> std::enable_if_t> +template< + class T, + unsigned int BlockSize, + unsigned int LogicalWarpSize +> +__global__ +__launch_bounds__(BlockSize) +void warp_inclusive_scan_kernel(T* device_input, T* device_output) { constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -50,29 +45,17 @@ auto warp_inclusive_scan_test(T* device_input, T* device_output) device_output[index] = value; } -template +template< + class T, + unsigned int BlockSize, + unsigned int LogicalWarpSize +> __global__ __launch_bounds__(BlockSize) -void warp_inclusive_scan_kernel(T* device_input, T* device_output) -{ - warp_inclusive_scan_test(device_input, device_output); -} - -template -__device__ -auto warp_inclusive_scan_reduce_test(T* /*device_input*/, - T* /*device_output*/, - T* /*device_output_reductions*/) - -> std::enable_if_t> -{ - // This kernel should never be actually called; tests are filtered out at runtime - // if the device does not support the LogicalWarpSize -} - -template -__device__ -auto warp_inclusive_scan_reduce_test(T* device_input, T* device_output, T* device_output_reductions) - -> std::enable_if_t> +void warp_inclusive_scan_reduce_kernel( + T* device_input, + T* device_output, + T* device_output_reductions) { constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -92,31 +75,14 @@ auto warp_inclusive_scan_reduce_test(T* device_input, T* device_output, T* devic } } -template +template< + class T, + unsigned int BlockSize, + unsigned int LogicalWarpSize +> __global__ __launch_bounds__(BlockSize) -void warp_inclusive_scan_reduce_kernel(T* device_input, - T* device_output, - T* device_output_reductions) -{ - warp_inclusive_scan_reduce_test(device_input, - device_output, - device_output_reductions); -} - -template -__device__ -auto warp_exclusive_scan_test(T* /*device_input*/, T* /*device_output*/, T /*init*/) - -> std::enable_if_t> -{ - // This kernel should never be actually called; tests are filtered out at runtime - // if the device does not support the LogicalWarpSize -} - -template -__device__ -auto warp_exclusive_scan_test(T* device_input, T* device_output, T init) - -> std::enable_if_t> +void warp_exclusive_scan_kernel(T* device_input, T* device_output, T init) { constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -131,33 +97,18 @@ auto warp_exclusive_scan_test(T* device_input, T* device_output, T init) device_output[index] = value; } -template +template< + class T, + unsigned int BlockSize, + unsigned int LogicalWarpSize +> __global__ __launch_bounds__(BlockSize) -void warp_exclusive_scan_kernel(T* device_input, T* device_output, T init) -{ - warp_exclusive_scan_test(device_input, device_output, init); -} - -template -__device__ -auto warp_exclusive_scan_reduce_test(T* /*device_input*/, - T* /*device_output*/, - T* /*device_output_reductions*/, - T /*init*/) - -> std::enable_if_t> -{ - // This kernel should never be actually called; tests are filtered out at runtime - // if the device does not support the LogicalWarpSize -} - -template -__device__ -auto warp_exclusive_scan_reduce_test(T* device_input, - T* device_output, - T* device_output_reductions, - T init) - -> std::enable_if_t> +void warp_exclusive_scan_reduce_kernel( + T* device_input, + T* device_output, + T* device_output_reductions, + T init) { constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -178,32 +129,8 @@ auto warp_exclusive_scan_reduce_test(T* device_input, } template -__global__ -__launch_bounds__(BlockSize) -void warp_exclusive_scan_reduce_kernel(T* device_input, - T* device_output, - T* device_output_reductions, - T init) -{ - warp_exclusive_scan_reduce_test(device_input, - device_output, - device_output_reductions, - init); -} - -template -__device__ -auto warp_exclusive_scan_wo_init_test(T* /*device_input*/, T* /*device_output*/) - -> std::enable_if_t> -{ - // This kernel should never be actually called; tests are filtered out at runtime - // if the device does not support the LogicalWarpSize -} - -template -__device__ -auto warp_exclusive_scan_wo_init_test(T* device_input, T* device_output) - -> std::enable_if_t> +__global__ __launch_bounds__(BlockSize) void warp_exclusive_scan_wo_init_kernel(T* device_input, + T* device_output) { static constexpr unsigned int block_warps_no = BlockSize / LogicalWarpSize; @@ -220,30 +147,8 @@ auto warp_exclusive_scan_wo_init_test(T* device_input, T* device_output) } template -__global__ -__launch_bounds__(BlockSize) -void warp_exclusive_scan_wo_init_kernel(T* device_input, T* device_output) -{ - warp_exclusive_scan_wo_init_test(device_input, device_output); -} - -template -__device__ -auto warp_exclusive_scan_reduce_wo_init_test(T* /*device_input*/, - T* /*device_output*/, - T* /*device_output_reductions*/) - -> std::enable_if_t> -{ - // This kernel should never be actually called; tests are filtered out at runtime - // if the device does not support the LogicalWarpSize -} - -template -__device__ -auto warp_exclusive_scan_reduce_wo_init_test(T* device_input, - T* device_output, - T* device_output_reductions) - -> std::enable_if_t> +__global__ __launch_bounds__(BlockSize) void warp_exclusive_scan_reduce_wo_init_kernel( + T* device_input, T* device_output, T* device_output_reductions) { static constexpr unsigned int block_warps_no = BlockSize / LogicalWarpSize; @@ -266,35 +171,18 @@ auto warp_exclusive_scan_reduce_wo_init_test(T* device_input, } } -template +template< + class T, + unsigned int BlockSize, + unsigned int LogicalWarpSize +> __global__ __launch_bounds__(BlockSize) -void warp_exclusive_scan_reduce_wo_init_kernel(T* device_input, - T* device_output, - T* device_output_reductions) -{ - warp_exclusive_scan_reduce_wo_init_test( - device_input, - device_output, - device_output_reductions); -} - -template -__device__ -auto warp_scan_test(T* /*device_input*/, - T* /*device_inclusive_output*/, - T* /*device_exclusive_output*/, - T /*init*/) - -> std::enable_if_t> -{ - // This kernel should never be actually called; tests are filtered out at runtime - // if the device does not support the LogicalWarpSize -} - -template -__device__ -auto warp_scan_test(T* device_input, T* device_inclusive_output, T* device_exclusive_output, T init) - -> std::enable_if_t> +void warp_scan_kernel( + T* device_input, + T* device_inclusive_output, + T* device_exclusive_output, + T init) { constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -311,41 +199,19 @@ auto warp_scan_test(T* device_input, T* device_inclusive_output, T* device_exclu device_exclusive_output[index] = exclusive_output; } -template +template< + class T, + unsigned int BlockSize, + unsigned int LogicalWarpSize +> __global__ __launch_bounds__(BlockSize) -void warp_scan_kernel(T* device_input, - T* device_inclusive_output, - T* device_exclusive_output, - T init) -{ - warp_scan_test(device_input, - device_inclusive_output, - device_exclusive_output, - init); -} - -template -__device__ -auto warp_scan_reduce_test(T* /*device_input*/, - T* /*device_inclusive_output*/, - T* /*device_exclusive_output*/, - T* /*device_output_reductions*/, - T /*init*/) - -> std::enable_if_t> -{ - // This kernel should never be actually called; tests are filtered out at runtime - // if the device does not support the LogicalWarpSize -} - -template -__device__ -auto warp_scan_reduce_test(T* device_input, - T* device_inclusive_output, - T* device_exclusive_output, - T* device_output_reductions, - T init) - -> std::enable_if_t> +void warp_scan_reduce_kernel( + T* device_input, + T* device_inclusive_output, + T* device_exclusive_output, + T* device_output_reductions, + T init) { constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -366,20 +232,4 @@ auto warp_scan_reduce_test(T* device_input, } } -template -__global__ -__launch_bounds__(BlockSize) -void warp_scan_reduce_kernel(T* device_input, - T* device_inclusive_output, - T* device_exclusive_output, - T* device_output_reductions, - T init) -{ - warp_scan_reduce_test(device_input, - device_inclusive_output, - device_exclusive_output, - device_output_reductions, - init); -} - #endif // TEST_SCAN_REDUCE_KERNELS_HPP_ diff --git a/test/rocprim/test_warp_sort.hpp b/test/rocprim/test_warp_sort.hpp index b34b35bae..85addaa4e 100644 --- a/test/rocprim/test_warp_sort.hpp +++ b/test/rocprim/test_warp_sort.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2023 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -94,13 +94,12 @@ typed_test_def(RocprimWarpSortShuffleBasedTests, name_suffix, Sort) // Launching kernel hipLaunchKernelGGL( - HIP_KERNEL_NAME( - test_hip_warp_sort_kernel), - dim3(grid_size), - dim3(block_size), - 0, - 0, - d_output); + HIP_KERNEL_NAME(test_hip_warp_sort< + items_per_thread, block_size, logical_warp_size, T + >), + dim3(grid_size), dim3(block_size), 0, 0, + d_output + ); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); diff --git a/test/rocprim/test_warp_sort.kernels.hpp b/test/rocprim/test_warp_sort.kernels.hpp index ddf6f8233..d00ea2380 100644 --- a/test/rocprim/test_warp_sort.kernels.hpp +++ b/test/rocprim/test_warp_sort.kernels.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2021 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -23,20 +23,18 @@ #ifndef TEST_WARP_SORT_KERNELS_HPP_ #define TEST_WARP_SORT_KERNELS_HPP_ -template -__device__ -auto test_hip_warp_sort_test(T* /*d_output*/) -> - typename std::enable_if_t> -{ - // This kernel should never be actually called; tests are filtered out at runtime - // if the device does not support the LogicalWarpSize -} - -template -__device__ -auto test_hip_warp_sort_test(T* d_output) -> - typename std::enable_if_t<(ItemsPerThread == 1 - && test_utils::device_test_enabled_for_warp_size_v)> +template< + unsigned int ItemsPerThread, + unsigned int BlockSize, + unsigned int LogicalWarpSize, + class T +> +__global__ +__launch_bounds__(BlockSize) +auto test_hip_warp_sort(T* d_output) + -> typename std::enable_if< + ItemsPerThread == 1 , void + >::type { unsigned int i = threadIdx.x + (blockIdx.x * blockDim.x); T value = d_output[i]; @@ -45,14 +43,41 @@ auto test_hip_warp_sort_test(T* d_output) -> d_output[i] = value; } -template -__device__ -auto test_hip_warp_sort_test(KeyType* device_key_output) -> - typename std::enable_if_t<(ItemsPerThread != 1 - && test_utils::device_test_enabled_for_warp_size_v)> +template< + unsigned int ItemsPerThread, + unsigned int BlockSize, + unsigned int LogicalWarpSize, + class KeyType, + class ValueType +> +__global__ +__launch_bounds__(BlockSize) +auto test_hip_sort_key_value_kernel(KeyType* d_output_key, ValueType* d_output_value) + -> typename std::enable_if< + ItemsPerThread == 1 , void + >::type +{ + unsigned int i = threadIdx.x + (blockIdx.x * blockDim.x); + KeyType key = d_output_key[i]; + ValueType value = d_output_value[i]; + rocprim::warp_sort wsort; + wsort.sort(key, value); + d_output_key[i] = key; + d_output_value[i] = value; +} + +template< + unsigned int ItemsPerThread, + unsigned int BlockSize, + unsigned int LogicalWarpSize, + class KeyType +> +__global__ +__launch_bounds__(BlockSize) +auto test_hip_warp_sort(KeyType * device_key_output) + -> typename std::enable_if< + (ItemsPerThread != 1 && LogicalWarpSize <= ::rocprim::device_warp_size()) , void + >::type { const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; @@ -66,90 +91,69 @@ auto test_hip_warp_sort_test(KeyType* device_key_output) -> ::rocprim::block_store_direct_blocked(lid, device_key_output + block_offset, keys); } -template +template< + unsigned int ItemsPerThread, + unsigned int BlockSize, + unsigned int LogicalWarpSize, + class KeyType +> __global__ __launch_bounds__(BlockSize) -void test_hip_warp_sort_kernel(KeyType* device_key_output) +auto test_hip_warp_sort(KeyType* /*device_key_output*/) + -> typename std::enable_if< + (ItemsPerThread != 1 && LogicalWarpSize > ::rocprim::device_warp_size()), void + >::type { - test_hip_warp_sort_test(device_key_output); + // This kernel will never be actually called, the tests are filtered at runtime if the warp size + // of the current device is less than the tested LogicalWarpSize } -template -__device__ -auto test_hip_sort_key_value_test(KeyType* /*d_output_key*/, ValueType* /*d_output_value*/) -> - typename std::enable_if_t> -{ - // This kernel should never be actually called; tests are filtered out at runtime - // if the device does not support the LogicalWarpSize -} - -template -__device__ -auto test_hip_sort_key_value_test(KeyType* d_output_key, ValueType* d_output_value) -> - typename std::enable_if_t<(ItemsPerThread == 1 - && test_utils::device_test_enabled_for_warp_size_v)> -{ - unsigned int i = threadIdx.x + (blockIdx.x * blockDim.x); - KeyType key = d_output_key[i]; - ValueType value = d_output_value[i]; - rocprim::warp_sort wsort; - wsort.sort(key, value); - d_output_key[i] = key; - d_output_value[i] = value; -} - -template -__device__ -auto test_hip_sort_key_value_test(KeyType* device_key_output, ValueType* device_value_output) -> - typename std::enable_if_t<(ItemsPerThread != 1 - && test_utils::device_test_enabled_for_warp_size_v)> +template< + unsigned int ItemsPerThread, + unsigned int BlockSize, + unsigned int LogicalWarpSize, + class KeyType, + class ValueType +> +__global__ +__launch_bounds__(BlockSize) +auto test_hip_sort_key_value_kernel(KeyType* device_key_output, ValueType* device_value_output) + -> typename std::enable_if< + (ItemsPerThread != 1 && LogicalWarpSize <= ::rocprim::device_warp_size()), void + >::type { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; - KeyType keys[ItemsPerThread]; + KeyType keys[ItemsPerThread]; ValueType values[ItemsPerThread]; - ::rocprim::block_load_direct_warp_striped(lid, - device_key_output + block_offset, - keys); - ::rocprim::block_load_direct_warp_striped(lid, - device_value_output + block_offset, - values); + ::rocprim::block_load_direct_warp_striped(lid, device_key_output + block_offset, keys); + ::rocprim::block_load_direct_warp_striped(lid, device_value_output + block_offset, values); rocprim::warp_sort wsort; wsort.sort(keys, values); ::rocprim::block_store_direct_blocked(lid, device_key_output + block_offset, keys); ::rocprim::block_store_direct_blocked(lid, device_value_output + block_offset, values); + } -template +template< + unsigned int ItemsPerThread, + unsigned int BlockSize, + unsigned int LogicalWarpSize, + class KeyType, + class ValueType +> __global__ __launch_bounds__(BlockSize) -void test_hip_sort_key_value_kernel(KeyType* device_key_output, ValueType* device_value_output) +auto test_hip_sort_key_value_kernel(KeyType* /*device_key_output*/, ValueType* /*device_value_output*/) + -> typename std::enable_if< + (ItemsPerThread != 1 && LogicalWarpSize > ::rocprim::device_warp_size()), void + >::type { - test_hip_sort_key_value_test( - device_key_output, - device_value_output); + // This kernel will never be actually called, the tests are filtered at runtime if the warp size + // of the current device is less than the tested LogicalWarpSize } #endif // TEST_WARP_SORT_KERNELS_HPP_ From 13aeb1df7fd78d16a462e38fd96cd47fff6aa705 Mon Sep 17 00:00:00 2001 From: Jaap Blok Date: Thu, 21 Nov 2024 12:59:00 +0100 Subject: [PATCH 18/84] Revert "Fix static asserts and remove template specialization" This reverts commit b2089eeedec6b32415f27d0e3c0860442b0ba5e0. --- rocprim/include/rocprim/warp/warp_reduce.hpp | 103 ++++++-- rocprim/include/rocprim/warp/warp_scan.hpp | 239 ++++++++++++++----- rocprim/include/rocprim/warp/warp_sort.hpp | 182 +++++++++++--- 3 files changed, 409 insertions(+), 115 deletions(-) diff --git a/rocprim/include/rocprim/warp/warp_reduce.hpp b/rocprim/include/rocprim/warp/warp_reduce.hpp index 725a27f32..56b8d6fd4 100644 --- a/rocprim/include/rocprim/warp/warp_reduce.hpp +++ b/rocprim/include/rocprim/warp/warp_reduce.hpp @@ -119,8 +119,7 @@ class warp_reduce using base_type = typename detail::select_warp_reduce_impl::type; // Check if WarpSize is valid for the targets - ROCPRIM_DETAIL_DEVICE_STATIC_ASSERT(WarpSize <= ::rocprim::device_warp_size(), - "WarpSize can't be greater than hardware warp size."); + static_assert(WarpSize <= ROCPRIM_MAX_WARP_SIZE, "WarpSize can't be greater than hardware warp size."); public: /// \brief Struct used to allocate a temporary memory that is required for thread @@ -178,15 +177,27 @@ class warp_reduce /// \endcode /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE - auto reduce(T input, - T& output, - storage_type& storage, - BinaryFunction reduce_op = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE auto reduce(T input, + T& output, + storage_type& storage, + BinaryFunction reduce_op = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::reduce(input, output, storage, reduce_op); } + /// \brief Performs reduction across threads in a logical warp. + /// Invalid Warp Size + template, unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE auto + reduce(T, T&, storage_type&, BinaryFunction reduce_op = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type + { + (void) reduce_op; + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + return; + } + /// \brief Performs reduction across threads in a logical warp. /// /// \tparam BinaryFunction - type of binary function used for reduce. Default type @@ -234,16 +245,28 @@ class warp_reduce /// \endcode /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE - auto reduce(T input, - T& output, - int valid_items, - storage_type& storage, - BinaryFunction reduce_op = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE auto reduce(T input, + T& output, + int valid_items, + storage_type& storage, + BinaryFunction reduce_op = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::reduce(input, output, valid_items, storage, reduce_op); } + /// \brief Performs reduction across threads in a logical warp. + /// Invalid Warp Size + template, unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE auto + reduce(T, T&, int, storage_type&, BinaryFunction reduce_op = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type + { + (void) reduce_op; + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + return; + } + /// \brief Performs head-segmented reduction across threads in a logical warp. /// /// \tparam Flag - type of head flags. Must be contextually convertible to \p bool. @@ -265,16 +288,31 @@ class warp_reduce template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE - auto head_segmented_reduce(T input, - T& output, - Flag flag, - storage_type& storage, - BinaryFunction reduce_op = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE auto head_segmented_reduce(T input, + T& output, + Flag flag, + storage_type& storage, + BinaryFunction reduce_op + = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::head_segmented_reduce(input, output, flag, storage, reduce_op); } + /// \brief Performs head-segmented reduction across threads in a logical warp. + /// Invalid Warp Size + template, + unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE auto head_segmented_reduce( + T, T&, Flag, storage_type&, BinaryFunction reduce_op = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type + { + (void) reduce_op; + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + return; + } + /// \brief Performs tail-segmented reduction across threads in a logical warp. /// /// \tparam Flag - type of tail flags. Must be contextually convertible to \p bool. @@ -296,15 +334,30 @@ class warp_reduce template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE - auto tail_segmented_reduce(T input, - T& output, - Flag flag, - storage_type& storage, - BinaryFunction reduce_op = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE auto tail_segmented_reduce(T input, + T& output, + Flag flag, + storage_type& storage, + BinaryFunction reduce_op + = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::tail_segmented_reduce(input, output, flag, storage, reduce_op); } + + /// \brief Performs tail-segmented reduction across threads in a logical warp. + /// Invalid Warp Size + template, + unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE auto tail_segmented_reduce( + T, T&, Flag, storage_type&, BinaryFunction reduce_op = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type + { + (void) reduce_op; + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + return; + } }; END_ROCPRIM_NAMESPACE diff --git a/rocprim/include/rocprim/warp/warp_scan.hpp b/rocprim/include/rocprim/warp/warp_scan.hpp index 6c5c6195f..dfc2985db 100644 --- a/rocprim/include/rocprim/warp/warp_scan.hpp +++ b/rocprim/include/rocprim/warp/warp_scan.hpp @@ -116,8 +116,7 @@ class warp_scan using base_type = typename detail::select_warp_scan_impl::type; // Check if WarpSize is valid for the targets - ROCPRIM_DETAIL_DEVICE_STATIC_ASSERT(WarpSize <= ::rocprim::device_warp_size(), - "WarpSize can't be greater than hardware warp size."); + static_assert(WarpSize <= ROCPRIM_MAX_WARP_SIZE, "WarpSize can't be greater than hardware warp size."); public: /// \brief Struct used to allocate a temporary memory that is required for thread @@ -179,15 +178,27 @@ class warp_scan /// {33, -34, -34, -36, ..., -64} etc. /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE - auto inclusive_scan(T input, - T& output, - storage_type& storage, - BinaryFunction scan_op = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE auto inclusive_scan(T input, + T& output, + storage_type& storage, + BinaryFunction scan_op = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::inclusive_scan(input, output, storage, scan_op); } + /// \brief Performs inclusive scan across threads in a logical warp. + /// Invalid Warp Size + template, unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE auto + inclusive_scan(T, T&, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type + { + (void) scan_op; + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size . Aborting warp sort."); + return; + } + /// \brief Performs inclusive scan and reduction across threads in a logical warp. /// /// \tparam BinaryFunction - type of binary function used for scan. Default type @@ -238,16 +249,28 @@ class warp_scan /// The \p reduction will be equal \p 64. /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE - auto inclusive_scan(T input, - T& output, - T& reduction, - storage_type& storage, - BinaryFunction scan_op = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE auto inclusive_scan(T input, + T& output, + T& reduction, + storage_type& storage, + BinaryFunction scan_op = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::inclusive_scan(input, output, reduction, storage, scan_op); } + /// \brief Performs inclusive scan and reduction across threads in a logical warp. + /// Invalid Warp Size + template, unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE auto + inclusive_scan(T, T&, T&, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type + { + (void) scan_op; + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size . Aborting warp sort."); + return; + } + /// \brief Performs exclusive scan across threads in a logical warp. /// /// \tparam BinaryFunction - type of binary function used for scan. Default type @@ -301,16 +324,28 @@ class warp_scan /// {100, 33, -34, -34, -36, ..., -62} etc. /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE - auto exclusive_scan(T input, - T& output, - T init, - storage_type& storage, - BinaryFunction scan_op = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE auto exclusive_scan(T input, + T& output, + T init, + storage_type& storage, + BinaryFunction scan_op = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::exclusive_scan(input, output, init, storage, scan_op); } + /// \brief Performs exclusive scan across threads in a logical warp. + /// Invalid Warp Size + template, unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE auto + exclusive_scan(T, T&, T, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type + { + (void) scan_op; + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size . Aborting warp sort."); + return; + } + /// \brief Performs exclusive scan and reduction across threads in a logical warp. /// /// \tparam BinaryFunction - type of binary function used for scan. Default type @@ -365,17 +400,29 @@ class warp_scan /// {10, 11, 12, 13, ..., 73}. The \p reduction will be 64. /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE - auto exclusive_scan(T input, - T& output, - T init, - T& reduction, - storage_type& storage, - BinaryFunction scan_op = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE auto exclusive_scan(T input, + T& output, + T init, + T& reduction, + storage_type& storage, + BinaryFunction scan_op = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::exclusive_scan(input, output, init, reduction, storage, scan_op); } + /// \brief Performs exclusive scan and reduction across threads in a logical warp. + /// Invalid Warp Size + template, unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE auto + exclusive_scan(T, T&, T, T&, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type + { + (void) scan_op; + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size . Aborting warp sort."); + return; + } + /// \brief Performs exclusive scan without an initial value across threads in a logical warp /// \tparam BinaryFunction binary function used for scan /// \param input Thread input value @@ -385,15 +432,32 @@ class warp_scan /// \param [in] storage Reference to a temporary storage object of type storage_type. /// \param scan_op The function object used to combine elements used for the scan template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE - auto exclusive_scan(T input, - T& output, - storage_type& storage, - BinaryFunction scan_op = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE auto exclusive_scan(T input, + T& output, + storage_type& storage, + BinaryFunction scan_op = BinaryFunction()) +#ifndef DOXYGEN_DOCUMENTATION_BUILD + -> std::enable_if_t +#else + -> void +#endif { base_type::exclusive_scan(input, output, storage, scan_op); } + /// \cond + template, unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE auto exclusive_scan(T /*input*/, + T& /*output*/, + storage_type& /*storage*/, + BinaryFunction /*scan_op*/ = BinaryFunction()) + -> std::enable_if_t<(FunctionWarpSize > device_warp_size())> + { + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size." + " Aborting warp scan."); + } + /// \endcond + /// \brief Performs exclusive scan and reduction without an initial value across threads in /// a logical warp /// \tparam BinaryFunction binary function used for scan @@ -405,16 +469,34 @@ class warp_scan /// \param [in] storage Reference to a temporary storage object of type storage_type. /// \param scan_op The function object used to combine elements used for the scan template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE - auto exclusive_scan(T input, - T& output, - storage_type& storage, - T& reduction, - BinaryFunction scan_op = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE auto exclusive_scan(T input, + T& output, + storage_type& storage, + T& reduction, + BinaryFunction scan_op = BinaryFunction()) +#ifndef DOXYGEN_DOCUMENTATION_BUILD + -> std::enable_if_t +#else + -> void +#endif { base_type::exclusive_scan(input, output, storage, reduction, scan_op); } + /// \cond + template, unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE auto exclusive_scan(T /*input*/, + T& /*output*/, + storage_type& /*storage*/, + T& /*reduction*/, + BinaryFunction /*scan_op*/ = BinaryFunction()) + -> std::enable_if_t<(FunctionWarpSize > device_warp_size())> + { + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size." + " Aborting warp scan."); + } + /// \endcond + /// \brief Performs inclusive and exclusive scan operations across threads /// in a logical warp. /// @@ -474,17 +556,29 @@ class warp_scan /// {100, 33, -34, -34, -36, ..., -62} etc. /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE - auto scan(T input, - T& inclusive_output, - T& exclusive_output, - T init, - storage_type& storage, - BinaryFunction scan_op = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE auto scan(T input, + T& inclusive_output, + T& exclusive_output, + T init, + storage_type& storage, + BinaryFunction scan_op = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::scan(input, inclusive_output, exclusive_output, init, storage, scan_op); } + /// \brief Performs inclusive and exclusive scan operations across threads + /// Invalid Warp Size + template, unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE auto + scan(T, T&, T&, T, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type + { + (void) scan_op; + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size . Aborting warp sort."); + return; + } + /// \brief Performs inclusive and exclusive scan operations, and reduction across /// threads in a logical warp. /// @@ -544,14 +638,14 @@ class warp_scan /// be {10, 11, 12, 13, ..., 73}. The \p reduction will be 64. /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE - auto scan(T input, - T& inclusive_output, - T& exclusive_output, - T init, - T& reduction, - storage_type& storage, - BinaryFunction scan_op = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE auto scan(T input, + T& inclusive_output, + T& exclusive_output, + T init, + T& reduction, + storage_type& storage, + BinaryFunction scan_op = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::scan( input, inclusive_output, exclusive_output, init, reduction, @@ -559,6 +653,18 @@ class warp_scan ); } + /// \brief Performs inclusive and exclusive scan operations across threads + /// Invalid Warp Size + template, unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE auto + scan(T, T&, T&, T, T&, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type + { + (void) scan_op; + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size . Aborting warp sort."); + return; + } + /// \brief Broadcasts value from one thread to all threads in logical warp. /// /// \param [in] input - value to broadcast. @@ -569,22 +675,41 @@ class warp_scan /// Synchronization barrier should be placed before \p storage is reused /// or repurposed: \p __syncthreads() or \p rocprim::syncthreads(). template - ROCPRIM_DEVICE ROCPRIM_INLINE - auto broadcast(T input, const unsigned int src_lane, storage_type& storage) + ROCPRIM_DEVICE ROCPRIM_INLINE auto + broadcast(T input, const unsigned int src_lane, storage_type& storage) -> + typename std::enable_if<(FunctionWarpSize <= device_warp_size()), T>::type { return base_type::broadcast(input, src_lane, storage); } + /// \brief Broadcasts value from one thread to all threads in logical warp. + /// Invalid Warp Size + template + ROCPRIM_DEVICE ROCPRIM_INLINE auto broadcast(T, const unsigned int, storage_type&) -> + typename std::enable_if<(FunctionWarpSize > device_warp_size()), T>::type + { + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + return T(); + } + #ifndef DOXYGEN_SHOULD_SKIP_THIS protected: // These undocumented functions are used by hipCUB prior to version 3.1 template - [[deprecated]] - ROCPRIM_DEVICE ROCPRIM_INLINE - auto to_exclusive(T inclusive_input, T& exclusive_output, storage_type& storage) + [[deprecated]] ROCPRIM_DEVICE ROCPRIM_INLINE auto + to_exclusive(T inclusive_input, T& exclusive_output, storage_type& storage) -> + typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { return base_type::to_exclusive(inclusive_input, exclusive_output, storage); } + + template + [[deprecated]] ROCPRIM_DEVICE ROCPRIM_INLINE auto to_exclusive(T, T&, storage_type&) -> + typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type + { + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + return; + } #endif }; diff --git a/rocprim/include/rocprim/warp/warp_sort.hpp b/rocprim/include/rocprim/warp/warp_sort.hpp index fd570aa18..59ccdccb2 100644 --- a/rocprim/include/rocprim/warp/warp_sort.hpp +++ b/rocprim/include/rocprim/warp/warp_sort.hpp @@ -110,8 +110,7 @@ class warp_sort : detail::warp_sort_shuffle typedef typename detail::warp_sort_shuffle base_type; // Check if WarpSize is valid for the targets - ROCPRIM_DETAIL_DEVICE_STATIC_ASSERT(WarpSize <= ::rocprim::device_warp_size(), - "WarpSize can't be greater than hardware warp size."); + static_assert(WarpSize <= ROCPRIM_MAX_WARP_SIZE, "WarpSize can't be greater than hardware warp size."); public: /// \brief Struct used to allocate a temporary memory that is required for thread @@ -135,12 +134,25 @@ class warp_sort : detail::warp_sort_shuffle /// bool f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE - auto sort(Key& thread_key, BinaryFunction compare_function = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key& thread_key, + BinaryFunction compare_function = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::sort(thread_key, compare_function); } + /// \brief Warp sort for any data type. + /// Invalid Warp Size + template, unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key&, + BinaryFunction compare_function = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type + { + (void) compare_function; // disables unused parameter warning + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + return; + } + /// \brief Warp sort for any data type. /// /// \tparam BinaryFunction - type of binary function used for sort. Default type @@ -154,13 +166,28 @@ class warp_sort : detail::warp_sort_shuffle template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE - auto sort(Key (&thread_keys)[ItemsPerThread], - BinaryFunction compare_function = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], + BinaryFunction compare_function = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::sort(thread_keys, compare_function); } + /// \brief Warp sort for any data type. + /// Invalid Warp Size + template, + unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], + BinaryFunction compare_function = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type + { + (void) thread_keys; // disables unused parameter warning + (void) compare_function; // disables unused parameter warning + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + return; + } + /// \brief Warp sort for any data type using temporary storage. /// /// \tparam BinaryFunction - type of binary function used for sort. Default type @@ -190,16 +217,29 @@ class warp_sort : detail::warp_sort_shuffle /// } /// \endcode template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE - auto sort(Key& thread_key, - storage_type& storage, - BinaryFunction compare_function = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key& thread_key, + storage_type& storage, + BinaryFunction compare_function = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::sort( thread_key, storage, compare_function ); } + /// \brief Warp sort for any data type using temporary storage. + /// Invalid Warp Size + template, unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE auto + sort(Key&, storage_type&, BinaryFunction compare_function = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type + { + (void) compare_function; // disables unused parameter warning + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + return; + } + + /// \brief Warp sort for any data type using temporary storage. /// /// \tparam BinaryFunction - type of binary function used for sort. Default type @@ -231,16 +271,32 @@ class warp_sort : detail::warp_sort_shuffle template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE - auto sort(Key (&thread_keys)[ItemsPerThread], - storage_type& storage, - BinaryFunction compare_function = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], + storage_type& storage, + BinaryFunction compare_function = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::sort( thread_keys, storage, compare_function ); } + /// \brief Warp sort for any data type using temporary storage. + /// Invalid Warp Size + template, + unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], + storage_type&, + BinaryFunction compare_function = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type + { + (void) thread_keys; // disables unused parameter warning + (void) compare_function; // disables unused parameter warning + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + return; + } + /// \brief Warp sort by key for any data type. /// /// \tparam BinaryFunction - type of binary function used for sort. Default type @@ -253,16 +309,28 @@ class warp_sort : detail::warp_sort_shuffle /// bool f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE - auto sort(Key& thread_key, - Value& thread_value, - BinaryFunction compare_function = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key& thread_key, + Value& thread_value, + BinaryFunction compare_function = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::sort( thread_key, thread_value, compare_function ); } + /// \brief Warp sort by key for any data type. + /// Invalid Warp Size + template, unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE auto + sort(Key&, Value&, BinaryFunction compare_function = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type + { + (void) compare_function; // disables unused parameter warning + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + return; + } + /// \brief Warp sort by key for any data type. /// /// \tparam BinaryFunction - type of binary function used for sort. Default type @@ -277,16 +345,33 @@ class warp_sort : detail::warp_sort_shuffle template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE - auto sort(Key (&thread_keys)[ItemsPerThread], - Value (&thread_values)[ItemsPerThread], - BinaryFunction compare_function = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], + Value (&thread_values)[ItemsPerThread], + BinaryFunction compare_function = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::sort( thread_keys, thread_values, compare_function ); } + /// \brief Warp sort by key for any data type. + /// Invalid Warp Size + template, + unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], + Value (&thread_values)[ItemsPerThread], + BinaryFunction compare_function = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type + { + (void) thread_keys; // disables unused parameter warning + (void) thread_values; // disables unused parameter warning + (void) compare_function; // disables unused parameter warning + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + return; + } + /// \brief Warp sort by key for any data type using temporary storage. /// /// \tparam BinaryFunction - type of binary function used for sort. Default type @@ -317,17 +402,30 @@ class warp_sort : detail::warp_sort_shuffle /// } /// \endcode template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE - auto sort(Key& thread_key, - Value& thread_value, - storage_type& storage, - BinaryFunction compare_function = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key& thread_key, + Value& thread_value, + storage_type& storage, + BinaryFunction compare_function = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::sort( thread_key, thread_value, storage, compare_function ); } + /// \brief Warp sort by key for any data type using temporary storage. + /// Invalid Warp Size + template, unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE auto + sort(Key&, Value&, storage_type&, BinaryFunction compare_function = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type + { + (void) compare_function; // disables unused parameter warning + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + return; + } + + /// \brief Warp sort by key for any data type using temporary storage. /// /// \tparam BinaryFunction - type of binary function used for sort. Default type @@ -360,16 +458,34 @@ class warp_sort : detail::warp_sort_shuffle template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE - auto sort(Key (&thread_keys)[ItemsPerThread], - Value (&thread_values)[ItemsPerThread], - storage_type& storage, - BinaryFunction compare_function = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], + Value (&thread_values)[ItemsPerThread], + storage_type& storage, + BinaryFunction compare_function = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::sort( thread_keys, thread_values, storage, compare_function ); } + + /// \brief Warp sort by key for any data type using temporary storage. + /// Invalid Warp Size + template, + unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], + Value (&thread_values)[ItemsPerThread], + storage_type&, + BinaryFunction compare_function = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type + { + (void) thread_keys; // disables unused parameter warning + (void) thread_values; // disables unused parameter warning + (void) compare_function; // disables unused parameter warning + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + return; + } }; END_ROCPRIM_NAMESPACE From 098b500413ca5c4fcb33d7f3f17a104ca24e8ede Mon Sep 17 00:00:00 2001 From: Jaap Blok Date: Thu, 21 Nov 2024 12:59:01 +0100 Subject: [PATCH 19/84] Fix formatting and copyright date --- benchmark/benchmark_warp_reduce.cpp | 85 ++++----- benchmark/benchmark_warp_scan.cpp | 34 ++-- benchmark/benchmark_warp_sort.cpp | 83 ++++---- rocprim/include/rocprim/warp/warp_reduce.hpp | 81 ++++---- rocprim/include/rocprim/warp/warp_scan.hpp | 189 ++++++++++--------- rocprim/include/rocprim/warp/warp_sort.hpp | 156 ++++++++------- test/rocprim/test_warp_reduce.kernels.hpp | 40 +--- test/rocprim/test_warp_scan.kernels.hpp | 84 +++------ test/rocprim/test_warp_sort.hpp | 14 +- test/rocprim/test_warp_sort.kernels.hpp | 123 +++++------- 10 files changed, 415 insertions(+), 474 deletions(-) diff --git a/benchmark/benchmark_warp_reduce.cpp b/benchmark/benchmark_warp_reduce.cpp index 1cddacbfd..6c63e9af4 100644 --- a/benchmark/benchmark_warp_reduce.cpp +++ b/benchmark/benchmark_warp_reduce.cpp @@ -45,15 +45,10 @@ const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif -template< - bool AllReduce, - class T, - unsigned int WarpSize, - unsigned int Trials -> +template __global__ __launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) -void warp_reduce_kernel(const T * d_input, T * d_output) +void warp_reduce_kernel(const T* d_input, T* d_output) { const unsigned int i = blockIdx.x * blockDim.x + threadIdx.x; @@ -70,12 +65,7 @@ void warp_reduce_kernel(const T * d_input, T * d_output) d_output[i] = value; } -template< - class T, - class Flag, - unsigned int WarpSize, - unsigned int Trials -> +template __global__ __launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) void segmented_warp_reduce_kernel(const T* d_input, Flag* d_flags, T* d_output) @@ -96,19 +86,16 @@ void segmented_warp_reduce_kernel(const T* d_input, Flag* d_flags, T* d_output) d_output[i] = value; } -template< - bool AllReduce, - bool Segmented, - unsigned int WarpSize, - unsigned int BlockSize, - unsigned int Trials, - class T, - class Flag -> -inline -auto execute_warp_reduce_kernel(T* input, T* output, Flag* /* flags */, - size_t size, hipStream_t stream) - -> typename std::enable_if::type +template +inline auto execute_warp_reduce_kernel( + T* input, T* output, Flag* /* flags */, size_t size, hipStream_t stream) -> + typename std::enable_if::type { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_reduce_kernel), @@ -118,19 +105,16 @@ auto execute_warp_reduce_kernel(T* input, T* output, Flag* /* flags */, HIP_CHECK(hipGetLastError()); } -template< - bool AllReduce, - bool Segmented, - unsigned int WarpSize, - unsigned int BlockSize, - unsigned int Trials, - class T, - class Flag -> -inline -auto execute_warp_reduce_kernel(T* input, T* output, Flag* flags, - size_t size, hipStream_t stream) - -> typename std::enable_if::type +template +inline auto + execute_warp_reduce_kernel(T* input, T* output, Flag* flags, size_t size, hipStream_t stream) -> + typename std::enable_if::type { hipLaunchKernelGGL( HIP_KERNEL_NAME(segmented_warp_reduce_kernel), @@ -230,11 +214,9 @@ void run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& se seed, \ stream) -#define BENCHMARK_TYPE(type) \ - CREATE_BENCHMARK(type, 32, 64), \ - CREATE_BENCHMARK(type, 37, 64), \ - CREATE_BENCHMARK(type, 61, 64), \ - CREATE_BENCHMARK(type, 64, 64) +#define BENCHMARK_TYPE(type) \ + CREATE_BENCHMARK(type, 32, 64), CREATE_BENCHMARK(type, 37, 64), \ + CREATE_BENCHMARK(type, 61, 64), CREATE_BENCHMARK(type, 64, 64) template void add_benchmarks(std::vector& benchmarks, @@ -242,15 +224,12 @@ void add_benchmarks(std::vector& benchmarks, const managed_seed& seed, hipStream_t stream) { - std::vector bs = - { - BENCHMARK_TYPE(int), - BENCHMARK_TYPE(float), - BENCHMARK_TYPE(double), - BENCHMARK_TYPE(int8_t), - BENCHMARK_TYPE(uint8_t), - BENCHMARK_TYPE(rocprim::half) - }; + std::vector bs = {BENCHMARK_TYPE(int), + BENCHMARK_TYPE(float), + BENCHMARK_TYPE(double), + BENCHMARK_TYPE(int8_t), + BENCHMARK_TYPE(uint8_t), + BENCHMARK_TYPE(rocprim::half)}; benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); } diff --git a/benchmark/benchmark_warp_scan.cpp b/benchmark/benchmark_warp_scan.cpp index f7983761a..bfa87924a 100644 --- a/benchmark/benchmark_warp_scan.cpp +++ b/benchmark/benchmark_warp_scan.cpp @@ -170,15 +170,11 @@ void run_benchmark(benchmark::State& state, hipStream_t stream, size_t bytes) stream, \ bytes) -#define BENCHMARK_TYPE(type) \ - CREATE_BENCHMARK(type, 64, 64, Inclusive), \ - CREATE_BENCHMARK(type, 128, 64, Inclusive), \ - CREATE_BENCHMARK(type, 256, 64, Inclusive), \ - CREATE_BENCHMARK(type, 256, 32, Inclusive), \ - CREATE_BENCHMARK(type, 256, 16, Inclusive), \ - CREATE_BENCHMARK(type, 63, 63, Inclusive), \ - CREATE_BENCHMARK(type, 62, 31, Inclusive), \ - CREATE_BENCHMARK(type, 60, 15, Inclusive) +#define BENCHMARK_TYPE(type) \ + CREATE_BENCHMARK(type, 64, 64, Inclusive), CREATE_BENCHMARK(type, 128, 64, Inclusive), \ + CREATE_BENCHMARK(type, 256, 64, Inclusive), CREATE_BENCHMARK(type, 256, 32, Inclusive), \ + CREATE_BENCHMARK(type, 256, 16, Inclusive), CREATE_BENCHMARK(type, 63, 63, Inclusive), \ + CREATE_BENCHMARK(type, 62, 31, Inclusive), CREATE_BENCHMARK(type, 60, 15, Inclusive) template void add_benchmarks(std::vector& benchmarks, @@ -188,17 +184,15 @@ void add_benchmarks(std::vector& benchmarks, using custom_double2 = custom_type; using custom_int_double = custom_type; - std::vector new_benchmarks = - { - BENCHMARK_TYPE(int), - BENCHMARK_TYPE(float), - BENCHMARK_TYPE(double), - BENCHMARK_TYPE(int8_t), - BENCHMARK_TYPE(uint8_t), - BENCHMARK_TYPE(rocprim::half), - BENCHMARK_TYPE(custom_double2), - BENCHMARK_TYPE(custom_int_double) - }; + std::vector new_benchmarks + = {BENCHMARK_TYPE(int), + BENCHMARK_TYPE(float), + BENCHMARK_TYPE(double), + BENCHMARK_TYPE(int8_t), + BENCHMARK_TYPE(uint8_t), + BENCHMARK_TYPE(rocprim::half), + BENCHMARK_TYPE(custom_double2), + BENCHMARK_TYPE(custom_int_double)}; benchmarks.insert(benchmarks.end(), new_benchmarks.begin(), new_benchmarks.end()); } diff --git a/benchmark/benchmark_warp_sort.cpp b/benchmark/benchmark_warp_sort.cpp index a12f1e7b1..c99ebc9d6 100644 --- a/benchmark/benchmark_warp_sort.cpp +++ b/benchmark/benchmark_warp_sort.cpp @@ -65,7 +65,11 @@ void warp_sort_kernel(K* input_keys, K* output_keys) rp::block_store_direct_blocked(flat_tid, output_keys + block_offset, keys); } -template +template __global__ __launch_bounds__(BlockSize) void warp_sort_by_key_kernel(K* input_keys, V* input_values, K* output_keys, V* output_values) @@ -222,29 +226,22 @@ void run_benchmark(benchmark::State& state, seed, \ stream) -#define BENCHMARK_TYPE(type) \ - CREATE_SORT_BENCHMARK(type, 64, 64, 1), \ - CREATE_SORT_BENCHMARK(type, 64, 64, 2), \ - CREATE_SORT_BENCHMARK(type, 64, 64, 4), \ - CREATE_SORT_BENCHMARK(type, 128, 64, 1), \ - CREATE_SORT_BENCHMARK(type, 128, 64, 2), \ - CREATE_SORT_BENCHMARK(type, 128, 64, 4), \ - CREATE_SORT_BENCHMARK(type, 256, 64, 1), \ - CREATE_SORT_BENCHMARK(type, 256, 64, 2), \ - CREATE_SORT_BENCHMARK(type, 256, 64, 4), \ - CREATE_SORT_BENCHMARK(type, 64, 32, 1), \ - CREATE_SORT_BENCHMARK(type, 64, 32, 2), \ - CREATE_SORT_BENCHMARK(type, 64, 16, 1), \ - CREATE_SORT_BENCHMARK(type, 64, 16, 2), \ - CREATE_SORT_BENCHMARK(type, 64, 16, 4) - -#define BENCHMARK_KEY_TYPE(type, value) \ - CREATE_SORTBYKEY_BENCHMARK(type, value, 64, 64, 1), \ - CREATE_SORTBYKEY_BENCHMARK(type, value, 64, 64, 2), \ - CREATE_SORTBYKEY_BENCHMARK(type, value, 64, 64, 4), \ - CREATE_SORTBYKEY_BENCHMARK(type, value, 256, 64, 1), \ - CREATE_SORTBYKEY_BENCHMARK(type, value, 256, 64, 2), \ - CREATE_SORTBYKEY_BENCHMARK(type, value, 256, 64, 4) +#define BENCHMARK_TYPE(type) \ + CREATE_SORT_BENCHMARK(type, 64, 64, 1), CREATE_SORT_BENCHMARK(type, 64, 64, 2), \ + CREATE_SORT_BENCHMARK(type, 64, 64, 4), CREATE_SORT_BENCHMARK(type, 128, 64, 1), \ + CREATE_SORT_BENCHMARK(type, 128, 64, 2), CREATE_SORT_BENCHMARK(type, 128, 64, 4), \ + CREATE_SORT_BENCHMARK(type, 256, 64, 1), CREATE_SORT_BENCHMARK(type, 256, 64, 2), \ + CREATE_SORT_BENCHMARK(type, 256, 64, 4), CREATE_SORT_BENCHMARK(type, 64, 32, 1), \ + CREATE_SORT_BENCHMARK(type, 64, 32, 2), CREATE_SORT_BENCHMARK(type, 64, 16, 1), \ + CREATE_SORT_BENCHMARK(type, 64, 16, 2), CREATE_SORT_BENCHMARK(type, 64, 16, 4) + +#define BENCHMARK_KEY_TYPE(type, value) \ + CREATE_SORTBYKEY_BENCHMARK(type, value, 64, 64, 1), \ + CREATE_SORTBYKEY_BENCHMARK(type, value, 64, 64, 2), \ + CREATE_SORTBYKEY_BENCHMARK(type, value, 64, 64, 4), \ + CREATE_SORTBYKEY_BENCHMARK(type, value, 256, 64, 1), \ + CREATE_SORTBYKEY_BENCHMARK(type, value, 256, 64, 2), \ + CREATE_SORTBYKEY_BENCHMARK(type, value, 256, 64, 4) int main(int argc, char *argv[]) { @@ -281,26 +278,24 @@ int main(int argc, char *argv[]) using custom_char_double = custom_type; using custom_longlong_double = custom_type; - std::vector benchmarks = - { - BENCHMARK_TYPE(int), - BENCHMARK_TYPE(float), - BENCHMARK_TYPE(double), - BENCHMARK_TYPE(int8_t), - BENCHMARK_TYPE(uint8_t), - BENCHMARK_TYPE(rocprim::half), - - BENCHMARK_KEY_TYPE(float, float), - BENCHMARK_KEY_TYPE(unsigned int, int), - BENCHMARK_KEY_TYPE(int, custom_double2), - BENCHMARK_KEY_TYPE(int, custom_int_double), - BENCHMARK_KEY_TYPE(custom_int2, custom_double2), - BENCHMARK_KEY_TYPE(custom_int2, custom_char_double), - BENCHMARK_KEY_TYPE(custom_int2, custom_longlong_double), - BENCHMARK_KEY_TYPE(int8_t, int8_t), - BENCHMARK_KEY_TYPE(uint8_t, uint8_t), - BENCHMARK_KEY_TYPE(rocprim::half, rocprim::half) - }; + std::vector benchmarks + = {BENCHMARK_TYPE(int), + BENCHMARK_TYPE(float), + BENCHMARK_TYPE(double), + BENCHMARK_TYPE(int8_t), + BENCHMARK_TYPE(uint8_t), + BENCHMARK_TYPE(rocprim::half), + + BENCHMARK_KEY_TYPE(float, float), + BENCHMARK_KEY_TYPE(unsigned int, int), + BENCHMARK_KEY_TYPE(int, custom_double2), + BENCHMARK_KEY_TYPE(int, custom_int_double), + BENCHMARK_KEY_TYPE(custom_int2, custom_double2), + BENCHMARK_KEY_TYPE(custom_int2, custom_char_double), + BENCHMARK_KEY_TYPE(custom_int2, custom_longlong_double), + BENCHMARK_KEY_TYPE(int8_t, int8_t), + BENCHMARK_KEY_TYPE(uint8_t, uint8_t), + BENCHMARK_KEY_TYPE(rocprim::half, rocprim::half)}; // Use manual timing for(auto& b : benchmarks) diff --git a/rocprim/include/rocprim/warp/warp_reduce.hpp b/rocprim/include/rocprim/warp/warp_reduce.hpp index 56b8d6fd4..34b7df901 100644 --- a/rocprim/include/rocprim/warp/warp_reduce.hpp +++ b/rocprim/include/rocprim/warp/warp_reduce.hpp @@ -119,7 +119,8 @@ class warp_reduce using base_type = typename detail::select_warp_reduce_impl::type; // Check if WarpSize is valid for the targets - static_assert(WarpSize <= ROCPRIM_MAX_WARP_SIZE, "WarpSize can't be greater than hardware warp size."); + static_assert(WarpSize <= ROCPRIM_MAX_WARP_SIZE, + "WarpSize can't be greater than hardware warp size."); public: /// \brief Struct used to allocate a temporary memory that is required for thread @@ -177,10 +178,11 @@ class warp_reduce /// \endcode /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto reduce(T input, - T& output, - storage_type& storage, - BinaryFunction reduce_op = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto reduce(T input, + T& output, + storage_type& storage, + BinaryFunction reduce_op = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::reduce(input, output, storage, reduce_op); @@ -189,12 +191,13 @@ class warp_reduce /// \brief Performs reduction across threads in a logical warp. /// Invalid Warp Size template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto - reduce(T, T&, storage_type&, BinaryFunction reduce_op = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto reduce(T, T&, storage_type&, BinaryFunction reduce_op = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type { - (void) reduce_op; - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + (void)reduce_op; + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp " + "size. Aborting warp sort."); return; } @@ -245,11 +248,12 @@ class warp_reduce /// \endcode /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto reduce(T input, - T& output, - int valid_items, - storage_type& storage, - BinaryFunction reduce_op = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto reduce(T input, + T& output, + int valid_items, + storage_type& storage, + BinaryFunction reduce_op = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::reduce(input, output, valid_items, storage, reduce_op); @@ -258,12 +262,13 @@ class warp_reduce /// \brief Performs reduction across threads in a logical warp. /// Invalid Warp Size template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto - reduce(T, T&, int, storage_type&, BinaryFunction reduce_op = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto reduce(T, T&, int, storage_type&, BinaryFunction reduce_op = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type { - (void) reduce_op; - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + (void)reduce_op; + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp " + "size. Aborting warp sort."); return; } @@ -288,12 +293,12 @@ class warp_reduce template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto head_segmented_reduce(T input, - T& output, - Flag flag, - storage_type& storage, - BinaryFunction reduce_op - = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto head_segmented_reduce(T input, + T& output, + Flag flag, + storage_type& storage, + BinaryFunction reduce_op = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::head_segmented_reduce(input, output, flag, storage, reduce_op); @@ -304,12 +309,14 @@ class warp_reduce template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto head_segmented_reduce( + ROCPRIM_DEVICE ROCPRIM_INLINE + auto head_segmented_reduce( T, T&, Flag, storage_type&, BinaryFunction reduce_op = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type { - (void) reduce_op; - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + (void)reduce_op; + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp " + "size. Aborting warp sort."); return; } @@ -334,12 +341,12 @@ class warp_reduce template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto tail_segmented_reduce(T input, - T& output, - Flag flag, - storage_type& storage, - BinaryFunction reduce_op - = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto tail_segmented_reduce(T input, + T& output, + Flag flag, + storage_type& storage, + BinaryFunction reduce_op = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::tail_segmented_reduce(input, output, flag, storage, reduce_op); @@ -350,12 +357,14 @@ class warp_reduce template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto tail_segmented_reduce( + ROCPRIM_DEVICE ROCPRIM_INLINE + auto tail_segmented_reduce( T, T&, Flag, storage_type&, BinaryFunction reduce_op = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type { - (void) reduce_op; - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + (void)reduce_op; + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp " + "size. Aborting warp sort."); return; } }; diff --git a/rocprim/include/rocprim/warp/warp_scan.hpp b/rocprim/include/rocprim/warp/warp_scan.hpp index dfc2985db..22daed51f 100644 --- a/rocprim/include/rocprim/warp/warp_scan.hpp +++ b/rocprim/include/rocprim/warp/warp_scan.hpp @@ -116,7 +116,8 @@ class warp_scan using base_type = typename detail::select_warp_scan_impl::type; // Check if WarpSize is valid for the targets - static_assert(WarpSize <= ROCPRIM_MAX_WARP_SIZE, "WarpSize can't be greater than hardware warp size."); + static_assert(WarpSize <= ROCPRIM_MAX_WARP_SIZE, + "WarpSize can't be greater than hardware warp size."); public: /// \brief Struct used to allocate a temporary memory that is required for thread @@ -178,10 +179,11 @@ class warp_scan /// {33, -34, -34, -36, ..., -64} etc. /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto inclusive_scan(T input, - T& output, - storage_type& storage, - BinaryFunction scan_op = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto inclusive_scan(T input, + T& output, + storage_type& storage, + BinaryFunction scan_op = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::inclusive_scan(input, output, storage, scan_op); @@ -190,12 +192,13 @@ class warp_scan /// \brief Performs inclusive scan across threads in a logical warp. /// Invalid Warp Size template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto - inclusive_scan(T, T&, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto inclusive_scan(T, T&, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type { - (void) scan_op; - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size . Aborting warp sort."); + (void)scan_op; + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size " + ". Aborting warp sort."); return; } @@ -249,11 +252,12 @@ class warp_scan /// The \p reduction will be equal \p 64. /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto inclusive_scan(T input, - T& output, - T& reduction, - storage_type& storage, - BinaryFunction scan_op = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto inclusive_scan(T input, + T& output, + T& reduction, + storage_type& storage, + BinaryFunction scan_op = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::inclusive_scan(input, output, reduction, storage, scan_op); @@ -262,12 +266,13 @@ class warp_scan /// \brief Performs inclusive scan and reduction across threads in a logical warp. /// Invalid Warp Size template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto - inclusive_scan(T, T&, T&, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto inclusive_scan(T, T&, T&, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type { - (void) scan_op; - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size . Aborting warp sort."); + (void)scan_op; + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size " + ". Aborting warp sort."); return; } @@ -324,11 +329,12 @@ class warp_scan /// {100, 33, -34, -34, -36, ..., -62} etc. /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto exclusive_scan(T input, - T& output, - T init, - storage_type& storage, - BinaryFunction scan_op = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto exclusive_scan(T input, + T& output, + T init, + storage_type& storage, + BinaryFunction scan_op = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::exclusive_scan(input, output, init, storage, scan_op); @@ -337,12 +343,13 @@ class warp_scan /// \brief Performs exclusive scan across threads in a logical warp. /// Invalid Warp Size template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto - exclusive_scan(T, T&, T, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto exclusive_scan(T, T&, T, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type { - (void) scan_op; - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size . Aborting warp sort."); + (void)scan_op; + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size " + ". Aborting warp sort."); return; } @@ -400,12 +407,13 @@ class warp_scan /// {10, 11, 12, 13, ..., 73}. The \p reduction will be 64. /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto exclusive_scan(T input, - T& output, - T init, - T& reduction, - storage_type& storage, - BinaryFunction scan_op = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto exclusive_scan(T input, + T& output, + T init, + T& reduction, + storage_type& storage, + BinaryFunction scan_op = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::exclusive_scan(input, output, init, reduction, storage, scan_op); @@ -414,12 +422,13 @@ class warp_scan /// \brief Performs exclusive scan and reduction across threads in a logical warp. /// Invalid Warp Size template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto - exclusive_scan(T, T&, T, T&, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto exclusive_scan(T, T&, T, T&, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type { - (void) scan_op; - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size . Aborting warp sort."); + (void)scan_op; + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size " + ". Aborting warp sort."); return; } @@ -432,10 +441,11 @@ class warp_scan /// \param [in] storage Reference to a temporary storage object of type storage_type. /// \param scan_op The function object used to combine elements used for the scan template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto exclusive_scan(T input, - T& output, - storage_type& storage, - BinaryFunction scan_op = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE + auto exclusive_scan(T input, + T& output, + storage_type& storage, + BinaryFunction scan_op = BinaryFunction()) #ifndef DOXYGEN_DOCUMENTATION_BUILD -> std::enable_if_t #else @@ -447,10 +457,11 @@ class warp_scan /// \cond template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto exclusive_scan(T /*input*/, - T& /*output*/, - storage_type& /*storage*/, - BinaryFunction /*scan_op*/ = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE + auto exclusive_scan(T /*input*/, + T& /*output*/, + storage_type& /*storage*/, + BinaryFunction /*scan_op*/ = BinaryFunction()) -> std::enable_if_t<(FunctionWarpSize > device_warp_size())> { ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size." @@ -469,11 +480,12 @@ class warp_scan /// \param [in] storage Reference to a temporary storage object of type storage_type. /// \param scan_op The function object used to combine elements used for the scan template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto exclusive_scan(T input, - T& output, - storage_type& storage, - T& reduction, - BinaryFunction scan_op = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE + auto exclusive_scan(T input, + T& output, + storage_type& storage, + T& reduction, + BinaryFunction scan_op = BinaryFunction()) #ifndef DOXYGEN_DOCUMENTATION_BUILD -> std::enable_if_t #else @@ -485,11 +497,12 @@ class warp_scan /// \cond template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto exclusive_scan(T /*input*/, - T& /*output*/, - storage_type& /*storage*/, - T& /*reduction*/, - BinaryFunction /*scan_op*/ = BinaryFunction()) + ROCPRIM_DEVICE ROCPRIM_INLINE + auto exclusive_scan(T /*input*/, + T& /*output*/, + storage_type& /*storage*/, + T& /*reduction*/, + BinaryFunction /*scan_op*/ = BinaryFunction()) -> std::enable_if_t<(FunctionWarpSize > device_warp_size())> { ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size." @@ -556,12 +569,13 @@ class warp_scan /// {100, 33, -34, -34, -36, ..., -62} etc. /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto scan(T input, - T& inclusive_output, - T& exclusive_output, - T init, - storage_type& storage, - BinaryFunction scan_op = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto scan(T input, + T& inclusive_output, + T& exclusive_output, + T init, + storage_type& storage, + BinaryFunction scan_op = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::scan(input, inclusive_output, exclusive_output, init, storage, scan_op); @@ -570,12 +584,13 @@ class warp_scan /// \brief Performs inclusive and exclusive scan operations across threads /// Invalid Warp Size template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto - scan(T, T&, T&, T, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto scan(T, T&, T&, T, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type { - (void) scan_op; - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size . Aborting warp sort."); + (void)scan_op; + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size " + ". Aborting warp sort."); return; } @@ -638,13 +653,14 @@ class warp_scan /// be {10, 11, 12, 13, ..., 73}. The \p reduction will be 64. /// \endparblock template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto scan(T input, - T& inclusive_output, - T& exclusive_output, - T init, - T& reduction, - storage_type& storage, - BinaryFunction scan_op = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto scan(T input, + T& inclusive_output, + T& exclusive_output, + T init, + T& reduction, + storage_type& storage, + BinaryFunction scan_op = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::scan( @@ -656,12 +672,13 @@ class warp_scan /// \brief Performs inclusive and exclusive scan operations across threads /// Invalid Warp Size template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto - scan(T, T&, T&, T, T&, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto scan(T, T&, T&, T, T&, storage_type&, BinaryFunction scan_op = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type { - (void) scan_op; - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size . Aborting warp sort."); + (void)scan_op; + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size " + ". Aborting warp sort."); return; } @@ -675,8 +692,8 @@ class warp_scan /// Synchronization barrier should be placed before \p storage is reused /// or repurposed: \p __syncthreads() or \p rocprim::syncthreads(). template - ROCPRIM_DEVICE ROCPRIM_INLINE auto - broadcast(T input, const unsigned int src_lane, storage_type& storage) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto broadcast(T input, const unsigned int src_lane, storage_type& storage) -> typename std::enable_if<(FunctionWarpSize <= device_warp_size()), T>::type { return base_type::broadcast(input, src_lane, storage); @@ -685,10 +702,12 @@ class warp_scan /// \brief Broadcasts value from one thread to all threads in logical warp. /// Invalid Warp Size template - ROCPRIM_DEVICE ROCPRIM_INLINE auto broadcast(T, const unsigned int, storage_type&) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto broadcast(T, const unsigned int, storage_type&) -> typename std::enable_if<(FunctionWarpSize > device_warp_size()), T>::type { - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp " + "size. Aborting warp sort."); return T(); } @@ -696,18 +715,20 @@ class warp_scan protected: // These undocumented functions are used by hipCUB prior to version 3.1 template - [[deprecated]] ROCPRIM_DEVICE ROCPRIM_INLINE auto - to_exclusive(T inclusive_input, T& exclusive_output, storage_type& storage) -> + [[deprecated]] ROCPRIM_DEVICE ROCPRIM_INLINE + auto to_exclusive(T inclusive_input, T& exclusive_output, storage_type& storage) -> typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { return base_type::to_exclusive(inclusive_input, exclusive_output, storage); } template - [[deprecated]] ROCPRIM_DEVICE ROCPRIM_INLINE auto to_exclusive(T, T&, storage_type&) -> + [[deprecated]] ROCPRIM_DEVICE ROCPRIM_INLINE + auto to_exclusive(T, T&, storage_type&) -> typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type { - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp " + "size. Aborting warp sort."); return; } #endif diff --git a/rocprim/include/rocprim/warp/warp_sort.hpp b/rocprim/include/rocprim/warp/warp_sort.hpp index 59ccdccb2..940fec91e 100644 --- a/rocprim/include/rocprim/warp/warp_sort.hpp +++ b/rocprim/include/rocprim/warp/warp_sort.hpp @@ -110,7 +110,8 @@ class warp_sort : detail::warp_sort_shuffle typedef typename detail::warp_sort_shuffle base_type; // Check if WarpSize is valid for the targets - static_assert(WarpSize <= ROCPRIM_MAX_WARP_SIZE, "WarpSize can't be greater than hardware warp size."); + static_assert(WarpSize <= ROCPRIM_MAX_WARP_SIZE, + "WarpSize can't be greater than hardware warp size."); public: /// \brief Struct used to allocate a temporary memory that is required for thread @@ -134,8 +135,8 @@ class warp_sort : detail::warp_sort_shuffle /// bool f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key& thread_key, - BinaryFunction compare_function = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key& thread_key, BinaryFunction compare_function = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::sort(thread_key, compare_function); @@ -144,12 +145,13 @@ class warp_sort : detail::warp_sort_shuffle /// \brief Warp sort for any data type. /// Invalid Warp Size template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key&, - BinaryFunction compare_function = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key&, BinaryFunction compare_function = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type { - (void) compare_function; // disables unused parameter warning - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + (void)compare_function; // disables unused parameter warning + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp " + "size. Aborting warp sort."); return; } @@ -166,8 +168,9 @@ class warp_sort : detail::warp_sort_shuffle template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], - BinaryFunction compare_function = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key (&thread_keys)[ItemsPerThread], + BinaryFunction compare_function = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::sort(thread_keys, compare_function); @@ -178,13 +181,15 @@ class warp_sort : detail::warp_sort_shuffle template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], - BinaryFunction compare_function = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key (&thread_keys)[ItemsPerThread], + BinaryFunction compare_function = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type { - (void) thread_keys; // disables unused parameter warning - (void) compare_function; // disables unused parameter warning - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + (void)thread_keys; // disables unused parameter warning + (void)compare_function; // disables unused parameter warning + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp " + "size. Aborting warp sort."); return; } @@ -217,9 +222,10 @@ class warp_sort : detail::warp_sort_shuffle /// } /// \endcode template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key& thread_key, - storage_type& storage, - BinaryFunction compare_function = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key& thread_key, + storage_type& storage, + BinaryFunction compare_function = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::sort( @@ -230,16 +236,16 @@ class warp_sort : detail::warp_sort_shuffle /// \brief Warp sort for any data type using temporary storage. /// Invalid Warp Size template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto - sort(Key&, storage_type&, BinaryFunction compare_function = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key&, storage_type&, BinaryFunction compare_function = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type { - (void) compare_function; // disables unused parameter warning - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + (void)compare_function; // disables unused parameter warning + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp " + "size. Aborting warp sort."); return; } - /// \brief Warp sort for any data type using temporary storage. /// /// \tparam BinaryFunction - type of binary function used for sort. Default type @@ -271,9 +277,10 @@ class warp_sort : detail::warp_sort_shuffle template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], - storage_type& storage, - BinaryFunction compare_function = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key (&thread_keys)[ItemsPerThread], + storage_type& storage, + BinaryFunction compare_function = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::sort( @@ -286,14 +293,16 @@ class warp_sort : detail::warp_sort_shuffle template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], - storage_type&, - BinaryFunction compare_function = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key (&thread_keys)[ItemsPerThread], + storage_type&, + BinaryFunction compare_function = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type { - (void) thread_keys; // disables unused parameter warning - (void) compare_function; // disables unused parameter warning - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + (void)thread_keys; // disables unused parameter warning + (void)compare_function; // disables unused parameter warning + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp " + "size. Aborting warp sort."); return; } @@ -309,9 +318,10 @@ class warp_sort : detail::warp_sort_shuffle /// bool f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key& thread_key, - Value& thread_value, - BinaryFunction compare_function = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key& thread_key, + Value& thread_value, + BinaryFunction compare_function = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::sort( @@ -322,12 +332,13 @@ class warp_sort : detail::warp_sort_shuffle /// \brief Warp sort by key for any data type. /// Invalid Warp Size template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto - sort(Key&, Value&, BinaryFunction compare_function = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key&, Value&, BinaryFunction compare_function = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type { - (void) compare_function; // disables unused parameter warning - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + (void)compare_function; // disables unused parameter warning + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp " + "size. Aborting warp sort."); return; } @@ -345,9 +356,10 @@ class warp_sort : detail::warp_sort_shuffle template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], - Value (&thread_values)[ItemsPerThread], - BinaryFunction compare_function = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key (&thread_keys)[ItemsPerThread], + Value (&thread_values)[ItemsPerThread], + BinaryFunction compare_function = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::sort( @@ -360,15 +372,17 @@ class warp_sort : detail::warp_sort_shuffle template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], - Value (&thread_values)[ItemsPerThread], - BinaryFunction compare_function = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key (&thread_keys)[ItemsPerThread], + Value (&thread_values)[ItemsPerThread], + BinaryFunction compare_function = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type { - (void) thread_keys; // disables unused parameter warning - (void) thread_values; // disables unused parameter warning - (void) compare_function; // disables unused parameter warning - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + (void)thread_keys; // disables unused parameter warning + (void)thread_values; // disables unused parameter warning + (void)compare_function; // disables unused parameter warning + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp " + "size. Aborting warp sort."); return; } @@ -402,10 +416,11 @@ class warp_sort : detail::warp_sort_shuffle /// } /// \endcode template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key& thread_key, - Value& thread_value, - storage_type& storage, - BinaryFunction compare_function = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key& thread_key, + Value& thread_value, + storage_type& storage, + BinaryFunction compare_function = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::sort( @@ -416,16 +431,16 @@ class warp_sort : detail::warp_sort_shuffle /// \brief Warp sort by key for any data type using temporary storage. /// Invalid Warp Size template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto - sort(Key&, Value&, storage_type&, BinaryFunction compare_function = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key&, Value&, storage_type&, BinaryFunction compare_function = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type { - (void) compare_function; // disables unused parameter warning - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + (void)compare_function; // disables unused parameter warning + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp " + "size. Aborting warp sort."); return; } - /// \brief Warp sort by key for any data type using temporary storage. /// /// \tparam BinaryFunction - type of binary function used for sort. Default type @@ -458,10 +473,11 @@ class warp_sort : detail::warp_sort_shuffle template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], - Value (&thread_values)[ItemsPerThread], - storage_type& storage, - BinaryFunction compare_function = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key (&thread_keys)[ItemsPerThread], + Value (&thread_values)[ItemsPerThread], + storage_type& storage, + BinaryFunction compare_function = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type { base_type::sort( @@ -474,16 +490,18 @@ class warp_sort : detail::warp_sort_shuffle template, unsigned int FunctionWarpSize = WarpSize> - ROCPRIM_DEVICE ROCPRIM_INLINE auto sort(Key (&thread_keys)[ItemsPerThread], - Value (&thread_values)[ItemsPerThread], - storage_type&, - BinaryFunction compare_function = BinaryFunction()) -> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto sort(Key (&thread_keys)[ItemsPerThread], + Value (&thread_values)[ItemsPerThread], + storage_type&, + BinaryFunction compare_function = BinaryFunction()) -> typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type { - (void) thread_keys; // disables unused parameter warning - (void) thread_values; // disables unused parameter warning - (void) compare_function; // disables unused parameter warning - ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size. Aborting warp sort."); + (void)thread_keys; // disables unused parameter warning + (void)thread_values; // disables unused parameter warning + (void)compare_function; // disables unused parameter warning + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp " + "size. Aborting warp sort."); return; } }; diff --git a/test/rocprim/test_warp_reduce.kernels.hpp b/test/rocprim/test_warp_reduce.kernels.hpp index 846d11a32..e1a8ee4b5 100644 --- a/test/rocprim/test_warp_reduce.kernels.hpp +++ b/test/rocprim/test_warp_reduce.kernels.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2019-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,11 +21,7 @@ #ifndef TEST_WARP_REDUCE_KERNELS_HPP_ #define TEST_WARP_REDUCE_KERNELS_HPP_ -template< - class T, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> +template __global__ __launch_bounds__(BlockSize) void warp_reduce_sum_kernel(T* device_input, T* device_output) @@ -46,11 +42,7 @@ void warp_reduce_sum_kernel(T* device_input, T* device_output) } } -template< - class T, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> +template __global__ __launch_bounds__(BlockSize) void warp_allreduce_sum_kernel(T* device_input, T* device_output) @@ -68,11 +60,7 @@ void warp_allreduce_sum_kernel(T* device_input, T* device_output) device_output[index] = value; } -template< - class T, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> +template __global__ __launch_bounds__(BlockSize) void warp_reduce_sum_kernel(T* device_input, T* device_output, size_t valid) @@ -93,11 +81,7 @@ void warp_reduce_sum_kernel(T* device_input, T* device_output, size_t valid) } } -template< - class T, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> +template __global__ __launch_bounds__(BlockSize) void warp_allreduce_sum_kernel(T* device_input, T* device_output, size_t valid) @@ -115,12 +99,7 @@ void warp_allreduce_sum_kernel(T* device_input, T* device_output, size_t valid) device_output[index] = value; } -template< - class T, - class Flag, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> +template __global__ __launch_bounds__(BlockSize) void head_segmented_warp_reduce_kernel(T* input, Flag* flags, T* output) @@ -139,12 +118,7 @@ void head_segmented_warp_reduce_kernel(T* input, Flag* flags, T* output) output[index] = value; } -template< - class T, - class Flag, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> +template __global__ __launch_bounds__(BlockSize) void tail_segmented_warp_reduce_kernel(T* input, Flag* flags, T* output) diff --git a/test/rocprim/test_warp_scan.kernels.hpp b/test/rocprim/test_warp_scan.kernels.hpp index 875389058..9047e437c 100644 --- a/test/rocprim/test_warp_scan.kernels.hpp +++ b/test/rocprim/test_warp_scan.kernels.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -23,11 +23,7 @@ #ifndef TEST_SCAN_REDUCE_KERNELS_HPP_ #define TEST_SCAN_REDUCE_KERNELS_HPP_ -template< - class T, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> +template __global__ __launch_bounds__(BlockSize) void warp_inclusive_scan_kernel(T* device_input, T* device_output) @@ -45,17 +41,12 @@ void warp_inclusive_scan_kernel(T* device_input, T* device_output) device_output[index] = value; } -template< - class T, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> +template __global__ __launch_bounds__(BlockSize) -void warp_inclusive_scan_reduce_kernel( - T* device_input, - T* device_output, - T* device_output_reductions) +void warp_inclusive_scan_reduce_kernel(T* device_input, + T* device_output, + T* device_output_reductions) { constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -75,11 +66,7 @@ void warp_inclusive_scan_reduce_kernel( } } -template< - class T, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> +template __global__ __launch_bounds__(BlockSize) void warp_exclusive_scan_kernel(T* device_input, T* device_output, T init) @@ -97,18 +84,13 @@ void warp_exclusive_scan_kernel(T* device_input, T* device_output, T init) device_output[index] = value; } -template< - class T, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> +template __global__ __launch_bounds__(BlockSize) -void warp_exclusive_scan_reduce_kernel( - T* device_input, - T* device_output, - T* device_output_reductions, - T init) +void warp_exclusive_scan_reduce_kernel(T* device_input, + T* device_output, + T* device_output_reductions, + T init) { constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -129,8 +111,8 @@ void warp_exclusive_scan_reduce_kernel( } template -__global__ __launch_bounds__(BlockSize) void warp_exclusive_scan_wo_init_kernel(T* device_input, - T* device_output) +__global__ __launch_bounds__(BlockSize) +void warp_exclusive_scan_wo_init_kernel(T* device_input, T* device_output) { static constexpr unsigned int block_warps_no = BlockSize / LogicalWarpSize; @@ -147,8 +129,10 @@ __global__ __launch_bounds__(BlockSize) void warp_exclusive_scan_wo_init_kernel( } template -__global__ __launch_bounds__(BlockSize) void warp_exclusive_scan_reduce_wo_init_kernel( - T* device_input, T* device_output, T* device_output_reductions) +__global__ __launch_bounds__(BlockSize) +void warp_exclusive_scan_reduce_wo_init_kernel(T* device_input, + T* device_output, + T* device_output_reductions) { static constexpr unsigned int block_warps_no = BlockSize / LogicalWarpSize; @@ -171,18 +155,13 @@ __global__ __launch_bounds__(BlockSize) void warp_exclusive_scan_reduce_wo_init_ } } -template< - class T, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> +template __global__ __launch_bounds__(BlockSize) -void warp_scan_kernel( - T* device_input, - T* device_inclusive_output, - T* device_exclusive_output, - T init) +void warp_scan_kernel(T* device_input, + T* device_inclusive_output, + T* device_exclusive_output, + T init) { constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); @@ -199,19 +178,14 @@ void warp_scan_kernel( device_exclusive_output[index] = exclusive_output; } -template< - class T, - unsigned int BlockSize, - unsigned int LogicalWarpSize -> +template __global__ __launch_bounds__(BlockSize) -void warp_scan_reduce_kernel( - T* device_input, - T* device_inclusive_output, - T* device_exclusive_output, - T* device_output_reductions, - T init) +void warp_scan_reduce_kernel(T* device_input, + T* device_inclusive_output, + T* device_exclusive_output, + T* device_output_reductions, + T init) { constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; const unsigned int warp_id = rocprim::detail::logical_warp_id(); diff --git a/test/rocprim/test_warp_sort.hpp b/test/rocprim/test_warp_sort.hpp index 85addaa4e..482453c43 100644 --- a/test/rocprim/test_warp_sort.hpp +++ b/test/rocprim/test_warp_sort.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -94,12 +94,12 @@ typed_test_def(RocprimWarpSortShuffleBasedTests, name_suffix, Sort) // Launching kernel hipLaunchKernelGGL( - HIP_KERNEL_NAME(test_hip_warp_sort< - items_per_thread, block_size, logical_warp_size, T - >), - dim3(grid_size), dim3(block_size), 0, 0, - d_output - ); + HIP_KERNEL_NAME(test_hip_warp_sort), + dim3(grid_size), + dim3(block_size), + 0, + 0, + d_output); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); diff --git a/test/rocprim/test_warp_sort.kernels.hpp b/test/rocprim/test_warp_sort.kernels.hpp index d00ea2380..2a69d4cb5 100644 --- a/test/rocprim/test_warp_sort.kernels.hpp +++ b/test/rocprim/test_warp_sort.kernels.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2021 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -23,18 +23,10 @@ #ifndef TEST_WARP_SORT_KERNELS_HPP_ #define TEST_WARP_SORT_KERNELS_HPP_ -template< - unsigned int ItemsPerThread, - unsigned int BlockSize, - unsigned int LogicalWarpSize, - class T -> +template __global__ __launch_bounds__(BlockSize) -auto test_hip_warp_sort(T* d_output) - -> typename std::enable_if< - ItemsPerThread == 1 , void - >::type +auto test_hip_warp_sort(T* d_output) -> typename std::enable_if::type { unsigned int i = threadIdx.x + (blockIdx.x * blockDim.x); T value = d_output[i]; @@ -43,41 +35,33 @@ auto test_hip_warp_sort(T* d_output) d_output[i] = value; } -template< - unsigned int ItemsPerThread, - unsigned int BlockSize, - unsigned int LogicalWarpSize, - class KeyType, - class ValueType -> +template __global__ __launch_bounds__(BlockSize) -auto test_hip_sort_key_value_kernel(KeyType* d_output_key, ValueType* d_output_value) - -> typename std::enable_if< - ItemsPerThread == 1 , void - >::type +auto test_hip_sort_key_value_kernel(KeyType* d_output_key, ValueType* d_output_value) -> + typename std::enable_if::type { - unsigned int i = threadIdx.x + (blockIdx.x * blockDim.x); - KeyType key = d_output_key[i]; - ValueType value = d_output_value[i]; + unsigned int i = threadIdx.x + (blockIdx.x * blockDim.x); + KeyType key = d_output_key[i]; + ValueType value = d_output_value[i]; rocprim::warp_sort wsort; wsort.sort(key, value); - d_output_key[i] = key; + d_output_key[i] = key; d_output_value[i] = value; } -template< - unsigned int ItemsPerThread, - unsigned int BlockSize, - unsigned int LogicalWarpSize, - class KeyType -> +template __global__ __launch_bounds__(BlockSize) -auto test_hip_warp_sort(KeyType * device_key_output) - -> typename std::enable_if< - (ItemsPerThread != 1 && LogicalWarpSize <= ::rocprim::device_warp_size()) , void - >::type +auto test_hip_warp_sort(KeyType* device_key_output) -> typename std:: + enable_if<(ItemsPerThread != 1 && LogicalWarpSize <= ::rocprim::device_warp_size()), void>::type { const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; @@ -91,66 +75,59 @@ auto test_hip_warp_sort(KeyType * device_key_output) ::rocprim::block_store_direct_blocked(lid, device_key_output + block_offset, keys); } -template< - unsigned int ItemsPerThread, - unsigned int BlockSize, - unsigned int LogicalWarpSize, - class KeyType -> +template __global__ __launch_bounds__(BlockSize) -auto test_hip_warp_sort(KeyType* /*device_key_output*/) - -> typename std::enable_if< - (ItemsPerThread != 1 && LogicalWarpSize > ::rocprim::device_warp_size()), void - >::type +auto test_hip_warp_sort(KeyType* /*device_key_output*/) -> typename std:: + enable_if<(ItemsPerThread != 1 && LogicalWarpSize > ::rocprim::device_warp_size()), void>::type { // This kernel will never be actually called, the tests are filtered at runtime if the warp size // of the current device is less than the tested LogicalWarpSize } -template< - unsigned int ItemsPerThread, - unsigned int BlockSize, - unsigned int LogicalWarpSize, - class KeyType, - class ValueType -> +template __global__ __launch_bounds__(BlockSize) -auto test_hip_sort_key_value_kernel(KeyType* device_key_output, ValueType* device_value_output) - -> typename std::enable_if< - (ItemsPerThread != 1 && LogicalWarpSize <= ::rocprim::device_warp_size()), void - >::type +auto test_hip_sort_key_value_kernel(KeyType* device_key_output, + ValueType* device_value_output) -> typename std:: + enable_if<(ItemsPerThread != 1 && LogicalWarpSize <= ::rocprim::device_warp_size()), void>::type { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; - KeyType keys[ItemsPerThread]; + KeyType keys[ItemsPerThread]; ValueType values[ItemsPerThread]; - ::rocprim::block_load_direct_warp_striped(lid, device_key_output + block_offset, keys); - ::rocprim::block_load_direct_warp_striped(lid, device_value_output + block_offset, values); + ::rocprim::block_load_direct_warp_striped(lid, + device_key_output + block_offset, + keys); + ::rocprim::block_load_direct_warp_striped(lid, + device_value_output + block_offset, + values); rocprim::warp_sort wsort; wsort.sort(keys, values); ::rocprim::block_store_direct_blocked(lid, device_key_output + block_offset, keys); ::rocprim::block_store_direct_blocked(lid, device_value_output + block_offset, values); - } -template< - unsigned int ItemsPerThread, - unsigned int BlockSize, - unsigned int LogicalWarpSize, - class KeyType, - class ValueType -> +template __global__ __launch_bounds__(BlockSize) -auto test_hip_sort_key_value_kernel(KeyType* /*device_key_output*/, ValueType* /*device_value_output*/) - -> typename std::enable_if< - (ItemsPerThread != 1 && LogicalWarpSize > ::rocprim::device_warp_size()), void - >::type +auto test_hip_sort_key_value_kernel(KeyType* /*device_key_output*/, + ValueType* /*device_value_output*/) -> typename std:: + enable_if<(ItemsPerThread != 1 && LogicalWarpSize > ::rocprim::device_warp_size()), void>::type { // This kernel will never be actually called, the tests are filtered at runtime if the warp size // of the current device is less than the tested LogicalWarpSize From 6fd035ee028b10f232dff995c7bbfe1408713ca8 Mon Sep 17 00:00:00 2001 From: Nick Breed Date: Thu, 21 Nov 2024 12:59:01 +0100 Subject: [PATCH 20/84] Temp fix: histogram dynamic memory failure on windows --- rocprim/include/rocprim/device/device_histogram.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rocprim/include/rocprim/device/device_histogram.hpp b/rocprim/include/rocprim/device/device_histogram.hpp index c18337b05..6c4846ec4 100644 --- a/rocprim/include/rocprim/device/device_histogram.hpp +++ b/rocprim/include/rocprim/device/device_histogram.hpp @@ -76,7 +76,12 @@ ROCPRIM_KERNEL __launch_bounds__( { static constexpr histogram_config_params params = device_params(); +// Temporary fix: issue with dynamic shared memory on windows. +#ifndef _WIN32 HIP_DYNAMIC_SHARED(unsigned int, block_histogram); +#else + __shared__ unsigned int block_histogram[params.shared_impl_max_bins]; +#endif histogram_shared Date: Thu, 21 Nov 2024 12:59:02 +0100 Subject: [PATCH 21/84] add ROCPRIM_ENABLE_ASSERTS to cmake to enable asserts in release build set ROCPRIM_ENABLE_ASSERTS for CI tests --- .gitlab-ci.yml | 2 +- CMakeLists.txt | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0f76d8c77..6c58d71e6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -170,7 +170,7 @@ build:cmake-minimum-apt: - | # Add hardened libc++ assertions for tests only if [[ $BUILD_TARGET == "TEST" ]]; then echo "Configuring with hardened libc++!" - EXTRA_CMAKE_CXX_FLAGS+=" -D_GLIBCXX_ASSERTIONS=ON" + EXTRA_CMAKE_CXX_FLAGS+=" -D_GLIBCXX_ASSERTIONS=ON -D ROCPRIM_ENABLE_ASSERTS=ON" fi - cmake -G Ninja diff --git a/CMakeLists.txt b/CMakeLists.txt index 47fda9ffa..773188d54 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,6 +74,7 @@ option(BUILD_NAIVE_BENCHMARK "Build naive benchmarks" OFF) cmake_dependent_option(BUILD_DOCS "Build documentation (requires sphinx)" OFF "NOT ONLY_INSTALL" OFF) option(BUILD_CODE_COVERAGE "Build with code coverage enabled" OFF) option(ROCPRIM_INSTALL "Enable installation of rocPRIM (projects embedding rocPRIM may want to turn this OFF)" ON) +option(ROCPRIM_ENABLE_ASSERTS "Enable asserts in release build)" OFF) check_language(HIP) cmake_dependent_option(USE_HIPCXX "Use CMake HIP language support" OFF CMAKE_HIP_COMPILER OFF) @@ -104,6 +105,21 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "" "Debug" "Release" "MinSizeRel" "RelWithDebInfo") endif() +if(ROCPRIM_ENABLE_ASSERTS) + if(NOT "${CMAKE_BUILD_TYPE}" STREQUAL "") + string(TOUPPER ${CMAKE_BUILD_TYPE} BUILD_TYPE) + set(BUILD_TYPE_CXX_FLAGS "CMAKE_CXX_FLAGS_${BUILD_TYPE}") + set(BUILD_TYPE_C_FLAGS "CMAKE_C_FLAGS_${BUILD_TYPE}") + endif() + + string(REGEX REPLACE "-DNDEBUG( |$)" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + string(REGEX REPLACE "-DNDEBUG( |$)" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") + if(NOT "${CMAKE_BUILD_TYPE}" STREQUAL "") + string(REGEX REPLACE "-DNDEBUG( |$)" "" ${BUILD_TYPE_CXX_FLAGS} "${${BUILD_TYPE_CXX_FLAGS}}") + string(REGEX REPLACE "-DNDEBUG( |$)" "" ${BUILD_TYPE_C_FLAGS} "${${BUILD_TYPE_C_FLAGS}}") + endif() +endif() + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE CACHE BOOL "Add paths to linker search and installed rpath") if(DEFINED BUILD_SHARED_LIBS) From 96d09c5c36aed98c9f5be710bb5538a42b960129 Mon Sep 17 00:00:00 2001 From: Sander Bos Date: Thu, 21 Nov 2024 12:59:02 +0100 Subject: [PATCH 22/84] Add missing docs for `block_run_length_decode` --- .../ops_classes/run_length_decode.rst | 12 ++++++++++++ docs/sphinx/_toc.yml.in | 1 + .../rocprim/block/block_run_length_decode.hpp | 18 +++++++++++------- 3 files changed, 24 insertions(+), 7 deletions(-) create mode 100644 docs/block_ops/ops_classes/run_length_decode.rst diff --git a/docs/block_ops/ops_classes/run_length_decode.rst b/docs/block_ops/ops_classes/run_length_decode.rst new file mode 100644 index 000000000..52d410318 --- /dev/null +++ b/docs/block_ops/ops_classes/run_length_decode.rst @@ -0,0 +1,12 @@ +.. meta:: + :description: rocPRIM documentation and API reference library + :keywords: rocPRIM, ROCm, API, documentation + +.. _blk-run_length_decode: + +******************************************************************** + Run-length decode +******************************************************************** + +.. doxygenclass:: rocprim::block_run_length_decode + :members: diff --git a/docs/sphinx/_toc.yml.in b/docs/sphinx/_toc.yml.in index 7c52b9a55..23ec331b4 100644 --- a/docs/sphinx/_toc.yml.in +++ b/docs/sphinx/_toc.yml.in @@ -53,6 +53,7 @@ subtrees: - file: block_ops/ops_classes/discontinuity.rst - file: block_ops/ops_classes/scan.rst - file: block_ops/ops_classes/reduce.rst + - file: block_ops/ops_classes/run_length_decode.rst - file: block_ops/ops_classes/shuffle.rst - file: block_ops/ops_classes/exchange.rst - file: block_ops/ops_classes/sort.rst diff --git a/rocprim/include/rocprim/block/block_run_length_decode.hpp b/rocprim/include/rocprim/block/block_run_length_decode.hpp index a302956a2..6cbe1c626 100644 --- a/rocprim/include/rocprim/block/block_run_length_decode.hpp +++ b/rocprim/include/rocprim/block/block_run_length_decode.hpp @@ -42,16 +42,18 @@ BEGIN_ROCPRIM_NAMESPACE /** * \brief The block_run_length_decode class supports decoding a run-length encoded array of items. That is, given - * the two arrays run_value[N] and run_lengths[N], run_value[i] is repeated run_lengths[i] many times in the output + * the two arrays `run_value[N]` and `run_lengths[N]`, `run_value[i]` is repeated `run_lengths[i]` many times in the output * array. - * Due to the nature of the run-length decoding algorithm ("decompression"), the output size of the run-length decoded - * array is runtime-dependent and potentially without any upper bound. To address this, block_run_length_decode allows - * retrieving a "window" from the run-length decoded array. The window's offset can be specified and BLOCK_THREADS * - * DECODED_ITEMS_PER_THREAD (i.e., referred to as window_size) decoded items from the specified window will be returned. - * - * \note: Trailing runs of length 0 are supported (i.e., they may only appear at the end of the run_lengths array). + * \note Trailing runs of length 0 are supported (i.e., they may only appear at the end of the `run_lengths` array). * A run of length zero may not be followed by a run length that is not zero. * + * \par Examples + * \parblock + * Due to the nature of the run-length decoding algorithm ("decompression"), the output size of the run-length decoded + * array is runtime-dependent and potentially without any upper bound. To address this, `block_run_length_decode` allows + * retrieving a "window" from the run-length decoded array. The window's offset can be specified and `BLOCK_THREADS * + * DECODED_ITEMS_PER_THREAD` (i.e., referred to as `window_size`) decoded items from the specified window will be returned. + * * \par * \code * __global__ void ExampleKernel(...) @@ -106,9 +108,11 @@ BEGIN_ROCPRIM_NAMESPACE * Suppose the set of input \p run_values across the block of threads is * { [0, 1], [2, 3], [4, 5], [6, 7], ..., [254, 255] } and * \p run_lengths is { [1, 2], [3, 4], [5, 1], [2, 3], ..., [5, 1] }. + * \par * The corresponding output \p decoded_items in those threads will be { [0, 1, 1, 2], [2, 2, 3, 3], [3, 3, 4, 4], * [4, 4, 4, 5], ..., [169, 169, 170, 171] } and \p relative_offsets will be { [0, 0, 1, 0], [1, 2, 0, 1], [2, * 3, 0, 1], [2, 3, 4, 0], ..., [3, 4, 0, 0] } during the first iteration of the while loop. + * \endparblock * * \tparam ItemT The data type of the items being run-length decoded * \tparam BLOCK_DIM_X The thread block length in threads along the X dimension From 85aaa4e826294a8e3a1eebfa4460d7e0da71a87a Mon Sep 17 00:00:00 2001 From: Bence Parajdi Date: Thu, 21 Nov 2024 12:59:09 +0100 Subject: [PATCH 23/84] Resolve "Add documentation and test for out-of-range behaviour on merge path search" add test for out of bounds diagonal test for rocprim::merge_path_search document behaviour for out of bounds diagonal test for rocprim::merge_path_search --- .../include/rocprim/thread/thread_search.hpp | 4 +- test/rocprim/test_thread_algos.cpp | 97 +++++++++++-------- 2 files changed, 58 insertions(+), 43 deletions(-) diff --git a/rocprim/include/rocprim/thread/thread_search.hpp b/rocprim/include/rocprim/thread/thread_search.hpp index d74a9c00d..0dd9d4e91 100644 --- a/rocprim/include/rocprim/thread/thread_search.hpp +++ b/rocprim/include/rocprim/thread/thread_search.hpp @@ -45,7 +45,9 @@ BEGIN_ROCPRIM_NAMESPACE /// \addtogroup thread_search /// @{ -/// \brief Computes the begin offsets into A and B for the specific diagonal +/// \brief Computes the begin offsets into A and B for the specific \p diagonal. If the \p diagonal +/// is larger than the sum of the two lengths the first component of the result will be clamped to +/// be no larger than \p a_len and the sum of the two components will be equal to \p diagonal. template #include @@ -407,6 +408,30 @@ __global__ void thread_search_kernel(Type* const device_input1, device_output_y[id] = coord.y; } +template +__global__ +void thread_search_out_of_bounds_kernel(Type* const device_input1, + Type* const device_input2, + OffsetT* device_output_x, + OffsetT* device_output_y, + const OffsetT input1_size, + const OffsetT input2_size, + BinaryFunction bin_op) +{ + const OffsetT partition_id = input1_size + input2_size + 1; + CoordinateT coord; + rocprim::merge_path_search(partition_id, + device_input1, + device_input2, + input1_size, + input2_size, + coord, + bin_op); + + *device_output_x = coord.x; + *device_output_y = coord.y; +} + template void merge_path_search_test() { @@ -431,53 +456,43 @@ void merge_path_search_test() std::sort(input1.begin(), input1.end(), bin_op); std::sort(input2.begin(), input2.end(), bin_op); - std::vector output_x(index_size); - std::vector output_y(index_size); + std::vector output_x; + std::vector output_y; + OffsetT output_oob_x, output_oob_y; // Preparing device - T* device_input1; - HIP_CHECK(test_common_utils::hipMallocHelper(reinterpret_cast(&device_input1), - input1.size() * sizeof(T))); - T* device_input2; - HIP_CHECK(test_common_utils::hipMallocHelper(reinterpret_cast(&device_input2), - input2.size() * sizeof(T))); - OffsetT* device_output_x; - HIP_CHECK(test_common_utils::hipMallocHelper(reinterpret_cast(&device_output_x), - output_x.size() * sizeof(OffsetT))); - OffsetT* device_output_y; - HIP_CHECK(test_common_utils::hipMallocHelper(reinterpret_cast(&device_output_y), - output_y.size() * sizeof(OffsetT))); - - HIP_CHECK(hipMemcpy(device_input1, - input1.data(), - input1.size() * sizeof(T), - hipMemcpyHostToDevice)); - - HIP_CHECK(hipMemcpy(device_input2, - input2.data(), - input2.size() * sizeof(T), - hipMemcpyHostToDevice)); + test_utils::device_ptr device_input1(input1); + test_utils::device_ptr device_input2(input2); + test_utils::device_ptr device_output_x(index_size); + test_utils::device_ptr device_output_y(index_size); + test_utils::device_ptr device_output_oob_x(1); + test_utils::device_ptr device_output_oob_y(1); thread_search_kernel - <<>>(device_input1, - device_input2, - device_output_x, - device_output_y, + <<>>(device_input1.get(), + device_input2.get(), + device_output_x.get(), + device_output_y.get(), input1.size(), input2.size(), bin_op); HIP_CHECK(hipGetLastError()); - // Reading results back - HIP_CHECK(hipMemcpy(output_x.data(), - device_output_x, - output_x.size() * sizeof(OffsetT), - hipMemcpyDeviceToHost)); + thread_search_out_of_bounds_kernel + <<>>(device_input1.get(), + device_input2.get(), + device_output_oob_x.get(), + device_output_oob_y.get(), + input1.size(), + input2.size(), + bin_op); + HIP_CHECK(hipGetLastError()); - HIP_CHECK(hipMemcpy(output_y.data(), - device_output_y, - output_y.size() * sizeof(OffsetT), - hipMemcpyDeviceToHost)); + // Reading results back + output_x = device_output_x.load(); + output_y = device_output_y.load(); + output_oob_x = device_output_oob_x.load()[0]; + output_oob_y = device_output_oob_y.load()[0]; std::vector combined_input(2 * size); std::merge(input1.begin(), @@ -487,6 +502,9 @@ void merge_path_search_test() combined_input.begin(), bin_op); + ASSERT_EQ(output_oob_x, input1.size()); + ASSERT_EQ(output_oob_y, input2.size()); + OffsetT slice_index = 0; for(OffsetT i = 0; i < index_size - 1; i++) { @@ -509,11 +527,6 @@ void merge_path_search_test() slice_index += length; } - - HIP_CHECK(hipFree(device_input1)); - HIP_CHECK(hipFree(device_input2)); - HIP_CHECK(hipFree(device_output_x)); - HIP_CHECK(hipFree(device_output_y)); } } From ccfb3a89ece24f515b404539bcdf3e44ecfa51a4 Mon Sep 17 00:00:00 2001 From: Cenxuan Tian Date: Thu, 21 Nov 2024 12:59:10 +0100 Subject: [PATCH 24/84] Resolve "Replace __launch_bounds__ with ROCPRIM_LAUNCH_BOUNDS in rocPRIM API" --- .../device/detail/device_adjacent_find.hpp | 25 ++--- .../device/detail/device_nth_element.hpp | 64 +++++------ .../device/detail/device_scan_common.hpp | 24 ++-- .../rocprim/device/detail/device_search.hpp | 28 ++--- .../rocprim/device/detail/device_search_n.hpp | 46 ++++---- .../device/device_adjacent_difference.hpp | 19 ++-- .../rocprim/device/device_find_first_of.hpp | 22 ++-- .../rocprim/device/device_histogram.hpp | 49 ++++----- .../include/rocprim/device/device_merge.hpp | 40 ++++--- .../rocprim/device/device_merge_sort.hpp | 77 ++++++------- .../rocprim/device/device_partition.hpp | 28 ++--- .../rocprim/device/device_radix_sort.hpp | 48 ++++---- .../include/rocprim/device/device_reduce.hpp | 13 +-- .../rocprim/device/device_reduce_by_key.hpp | 2 +- .../device/device_run_length_encode.hpp | 2 +- .../include/rocprim/device/device_scan.hpp | 37 +++---- .../rocprim/device/device_scan_by_key.hpp | 2 +- .../device/device_segmented_radix_sort.hpp | 103 +++++++++--------- .../device/device_segmented_reduce.hpp | 15 ++- .../rocprim/device/device_segmented_scan.hpp | 33 +++--- .../rocprim/device/device_transform.hpp | 4 +- .../device_radix_block_sort.hpp | 19 ++-- 22 files changed, 327 insertions(+), 373 deletions(-) diff --git a/rocprim/include/rocprim/device/detail/device_adjacent_find.hpp b/rocprim/include/rocprim/device/detail/device_adjacent_find.hpp index 6b292c76b..07b396bcf 100644 --- a/rocprim/include/rocprim/device/detail/device_adjacent_find.hpp +++ b/rocprim/include/rocprim/device/detail/device_adjacent_find.hpp @@ -41,11 +41,9 @@ template - static -ROCPRIM_KERNEL __launch_bounds__(1) - void init_adjacent_find(OutputT* reduce_output, - ordered_block_id ordered_tile_id, - const size_t size) + static ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(1) void init_adjacent_find(OutputT* reduce_output, + ordered_block_id ordered_tile_id, + const size_t size) { // Reset output value. *reduce_output = size; @@ -54,16 +52,13 @@ ROCPRIM_KERNEL __launch_bounds__(1) ordered_tile_id.reset(); } - static -ROCPRIM_KERNEL -#ifndef DOXYGEN_DOCUMENTATION_BUILD -__launch_bounds__(device_params().kernel_config.block_size) -#endif - void block_reduce_kernel(TransformedInputIterator transformed_input, - ReduceIndexIterator reduce_output, - const std::size_t size, - BinaryPred op, - OrderedTileIdType ordered_tile_id) + static ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params() + .kernel_config.block_size) void + block_reduce_kernel(TransformedInputIterator transformed_input, + ReduceIndexIterator reduce_output, + const std::size_t size, + BinaryPred op, + OrderedTileIdType ordered_tile_id) { static constexpr adjacent_find_config_params params = device_params(); static constexpr unsigned int block_size = params.kernel_config.block_size; diff --git a/rocprim/include/rocprim/device/detail/device_nth_element.hpp b/rocprim/include/rocprim/device/detail/device_nth_element.hpp index 2ed21127a..dbda6b86b 100644 --- a/rocprim/include/rocprim/device/detail/device_nth_element.hpp +++ b/rocprim/include/rocprim/device/detail/device_nth_element.hpp @@ -153,9 +153,8 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void } template -ROCPRIM_KERNEL - __launch_bounds__(device_params().stop_recursion_size) void kernel_block_sort( - KeysIterator keys, const size_t size, BinaryFunction compare_function) +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().stop_recursion_size) void + kernel_block_sort(KeysIterator keys, const size_t size, BinaryFunction compare_function) { kernel_block_sort_impl(keys, size, compare_function); } @@ -203,13 +202,14 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void } template -ROCPRIM_KERNEL __launch_bounds__( +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS( device_params().number_of_buckets - - 1) void kernel_find_splitters(KeysIterator keys, - typename std::iterator_traits::value_type* tree, - bool* equality_buckets, - const size_t size, - BinaryFunction compare_function) + - 1) void + kernel_find_splitters(KeysIterator keys, + typename std::iterator_traits::value_type* tree, + bool* equality_buckets, + const size_t size, + BinaryFunction compare_function) { kernel_find_splitters_impl(keys, tree, equality_buckets, size, compare_function); } @@ -319,16 +319,16 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void } template -ROCPRIM_KERNEL __launch_bounds__( +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS( device_params() .kernel_config - .block_size) void kernel_count_bucket_sizes(KeysIterator keys, - typename std::iterator_traits< - KeysIterator>::value_type* tree, - const size_t size, - size_t* buckets, - bool* equality_buckets, - BinaryFunction compare_function) + .block_size) void + kernel_count_bucket_sizes(KeysIterator keys, + typename std::iterator_traits::value_type* tree, + const size_t size, + size_t* buckets, + bool* equality_buckets, + BinaryFunction compare_function) { kernel_count_bucket_sizes_impl(keys, tree, @@ -385,13 +385,13 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void } template -ROCPRIM_KERNEL __launch_bounds__( +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS( device_params() - .number_of_buckets) void kernel_find_nth_element_bucket(size_t* buckets, - n_th_element_iteration_data* - nth_element_data, - bool* equality_buckets, - const size_t rank) + .number_of_buckets) void + kernel_find_nth_element_bucket(size_t* buckets, + n_th_element_iteration_data* nth_element_data, + bool* equality_buckets, + const size_t rank) { kernel_find_nth_element_bucket_impl(buckets, nth_element_data, equality_buckets, rank); @@ -588,15 +588,15 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void template ROCPRIM_KERNEL - __launch_bounds__(device_params().kernel_config.block_size) void kernel_copy_buckets( - KeysIterator keys, - typename std::iterator_traits::value_type* tree, - const size_t size, - nth_element_onesweep_lookback_state* lookback_states, - n_th_element_iteration_data* nth_element_data, - typename std::iterator_traits::value_type* keys_buffer, - bool* equality_buckets, - BinaryFunction compare_function) + ROCPRIM_LAUNCH_BOUNDS(device_params().kernel_config.block_size) void + kernel_copy_buckets(KeysIterator keys, + typename std::iterator_traits::value_type* tree, + const size_t size, + nth_element_onesweep_lookback_state* lookback_states, + n_th_element_iteration_data* nth_element_data, + typename std::iterator_traits::value_type* keys_buffer, + bool* equality_buckets, + BinaryFunction compare_function) { kernel_copy_buckets_impl(keys, tree, diff --git a/rocprim/include/rocprim/device/detail/device_scan_common.hpp b/rocprim/include/rocprim/device/detail/device_scan_common.hpp index 706f72aac..1a9008762 100644 --- a/rocprim/include/rocprim/device/detail/device_scan_common.hpp +++ b/rocprim/include/rocprim/device/detail/device_scan_common.hpp @@ -83,12 +83,13 @@ ROCPRIM_DEVICE ROCPRIM_INLINE void init_lookback_scan_state(LookBackScanState l template ROCPRIM_KERNEL - __launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) void init_lookback_scan_state_kernel( - LookBackScanState lookback_scan_state, - const unsigned int number_of_blocks, - ordered_block_id ordered_bid, - unsigned int save_index = 0, - typename LookBackScanState::value_type* const save_dest = nullptr) + ROCPRIM_LAUNCH_BOUNDS(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) void + init_lookback_scan_state_kernel(LookBackScanState lookback_scan_state, + const unsigned int number_of_blocks, + ordered_block_id ordered_bid, + unsigned int save_index = 0, + typename LookBackScanState::value_type* const save_dest + = nullptr) { const unsigned int block_id = ::rocprim::detail::block_id<0>(); const unsigned int block_size = ::rocprim::detail::block_size<0>(); @@ -111,11 +112,12 @@ ROCPRIM_KERNEL template ROCPRIM_KERNEL - __launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) void init_lookback_scan_state_kernel( - LookBackScanState lookback_scan_state, - const unsigned int number_of_blocks, - unsigned int save_index = 0, - typename LookBackScanState::value_type* const save_dest = nullptr) + ROCPRIM_LAUNCH_BOUNDS(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) void + init_lookback_scan_state_kernel(LookBackScanState lookback_scan_state, + const unsigned int number_of_blocks, + unsigned int save_index = 0, + typename LookBackScanState::value_type* const save_dest + = nullptr) { const unsigned int block_id = ::rocprim::detail::block_id<0>(); const unsigned int block_size = ::rocprim::detail::block_size<0>(); diff --git a/rocprim/include/rocprim/device/detail/device_search.hpp b/rocprim/include/rocprim/device/detail/device_search.hpp index 88532b66b..f6a9867d9 100644 --- a/rocprim/include/rocprim/device/detail/device_search.hpp +++ b/rocprim/include/rocprim/device/detail/device_search.hpp @@ -254,25 +254,25 @@ template().kernel_config.block_size) - void search_kernel_shared(InputIterator1 input, - InputIterator2 keys, - size_t* output, - size_t size, - size_t keys_size, - BinaryFunction compare_function) +ROCPRIM_LAUNCH_BOUNDS(device_params().kernel_config.block_size) void + search_kernel_shared(InputIterator1 input, + InputIterator2 keys, + size_t* output, + size_t size, + size_t keys_size, + BinaryFunction compare_function) { search_kernel_shared_impl(input, keys, output, size, keys_size, compare_function); } static ROCPRIM_KERNEL -__launch_bounds__(device_params().kernel_config.block_size) - void search_kernel(InputIterator1 input, - InputIterator2 keys, - size_t* output, - size_t size, - size_t keys_size, - BinaryFunction compare_function) +ROCPRIM_LAUNCH_BOUNDS(device_params().kernel_config.block_size) void + search_kernel(InputIterator1 input, + InputIterator2 keys, + size_t* output, + size_t size, + size_t keys_size, + BinaryFunction compare_function) { search_kernel_impl(input, keys, output, size, keys_size, compare_function); } diff --git a/rocprim/include/rocprim/device/detail/device_search_n.hpp b/rocprim/include/rocprim/device/detail/device_search_n.hpp index 32d7a2d5a..ec5386184 100644 --- a/rocprim/include/rocprim/device/detail/device_search_n.hpp +++ b/rocprim/include/rocprim/device/detail/device_search_n.hpp @@ -48,19 +48,17 @@ struct search_n_impl_kernels { template - static ROCPRIM_KERNEL __launch_bounds__(1) - void search_n_init_kernel(SizeType* __restrict__ output, const SizeType target) + static ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(1) void search_n_init_kernel(SizeType* __restrict__ output, + const SizeType target) { *output = target; } /// \brief Supports all forms of search_n operations, /// but the efficiency is insufficient when `items_per_block` is too large. - static ROCPRIM_KERNEL -#ifndef DOXYGEN_DOCUMENTATION_BUILD -__launch_bounds__(device_params().kernel_config.block_size) -#endif - void search_n_normal_kernel( + static ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params() + .kernel_config.block_size) void + search_n_normal_kernel( InputIterator input, size_t* __restrict__ output, const size_t size, @@ -109,11 +107,9 @@ __launch_bounds__(device_params().kernel_config.block_size) } } - static ROCPRIM_KERNEL -#ifndef DOXYGEN_DOCUMENTATION_BUILD -__launch_bounds__(device_params().kernel_config.block_size) -#endif - void search_n_find_heads_kernel( + static ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params() + .kernel_config.block_size) void + search_n_find_heads_kernel( InputIterator input, const size_t size, const typename std::iterator_traits::value_type* value, @@ -148,16 +144,14 @@ __launch_bounds__(device_params().kernel_config.block_size) } } - static ROCPRIM_KERNEL -#ifndef DOXYGEN_DOCUMENTATION_BUILD -__launch_bounds__(device_params().kernel_config.block_size) -#endif - void search_n_heads_filter_kernel(const size_t size, - const size_t count, - const size_t* __restrict__ heads, - const size_t heads_size, - size_t* __restrict__ filtered_heads, - size_t* __restrict__ filtered_heads_size) + static ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params() + .kernel_config.block_size) void + search_n_heads_filter_kernel(const size_t size, + const size_t count, + const size_t* __restrict__ heads, + const size_t heads_size, + size_t* __restrict__ filtered_heads, + size_t* __restrict__ filtered_heads_size) { constexpr auto params = device_params(); constexpr auto block_size = params.kernel_config.block_size; @@ -205,11 +199,9 @@ __launch_bounds__(device_params().kernel_config.block_size) } } - static ROCPRIM_KERNEL -#ifndef DOXYGEN_DOCUMENTATION_BUILD -__launch_bounds__(device_params().kernel_config.block_size) -#endif - void search_n_discard_heads_kernel( + static ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params() + .kernel_config.block_size) void + search_n_discard_heads_kernel( InputIterator input, const size_t size, const size_t count, diff --git a/rocprim/include/rocprim/device/device_adjacent_difference.hpp b/rocprim/include/rocprim/device/device_adjacent_difference.hpp index 39949c55d..dc3a0696b 100644 --- a/rocprim/include/rocprim/device/device_adjacent_difference.hpp +++ b/rocprim/include/rocprim/device/device_adjacent_difference.hpp @@ -62,15 +62,16 @@ template -void ROCPRIM_KERNEL - __launch_bounds__(device_params().adjacent_difference_kernel_config.block_size) - adjacent_difference_kernel( - const InputIt input, - const OutputIt output, - const std::size_t size, - const BinaryFunction op, - const typename std::iterator_traits::value_type* previous_values, - const std::size_t starting_block) +void ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS( + device_params() + .adjacent_difference_kernel_config + .block_size) adjacent_difference_kernel(const InputIt input, + const OutputIt output, + const std::size_t size, + const BinaryFunction op, + const typename std::iterator_traits::value_type* + previous_values, + const std::size_t starting_block) { adjacent_difference_kernel_impl(input, output, diff --git a/rocprim/include/rocprim/device/device_find_first_of.hpp b/rocprim/include/rocprim/device/device_find_first_of.hpp index cbffac4d0..c77142426 100644 --- a/rocprim/include/rocprim/device/device_find_first_of.hpp +++ b/rocprim/include/rocprim/device/device_find_first_of.hpp @@ -51,18 +51,16 @@ struct find_first_of_impl_kernels } static ROCPRIM_KERNEL -#ifndef DOXYGEN_DOCUMENTATION_BUILD - __launch_bounds__(device_params().kernel_config.block_size) -#endif -void find_first_of_kernel(InputIterator1 input, - InputIterator2 keys, - size_t* output, - size_t size, - size_t keys_size, - ordered_block_id ordered_bid, - BinaryFunction compare_function) -{ - constexpr find_first_of_config_params params = device_params(); + ROCPRIM_LAUNCH_BOUNDS(device_params().kernel_config.block_size) void + find_first_of_kernel(InputIterator1 input, + InputIterator2 keys, + size_t* output, + size_t size, + size_t keys_size, + ordered_block_id ordered_bid, + BinaryFunction compare_function) + { + constexpr find_first_of_config_params params = device_params(); constexpr unsigned int block_size = params.kernel_config.block_size; constexpr unsigned int items_per_thread = params.kernel_config.items_per_thread; diff --git a/rocprim/include/rocprim/device/device_histogram.hpp b/rocprim/include/rocprim/device/device_histogram.hpp index 6c4846ec4..3f5d36716 100644 --- a/rocprim/include/rocprim/device/device_histogram.hpp +++ b/rocprim/include/rocprim/device/device_histogram.hpp @@ -43,11 +43,9 @@ namespace detail { template -ROCPRIM_KERNEL __launch_bounds__( - device_params() - .histogram_config - .block_size) void init_histogram_kernel(fixed_array histogram, - fixed_array bins) +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().histogram_config.block_size) void + init_histogram_kernel(fixed_array histogram, + fixed_array bins) { static constexpr histogram_config_params params = device_params(); @@ -60,19 +58,16 @@ template -ROCPRIM_KERNEL __launch_bounds__( - device_params() - .histogram_config - .block_size) void histogram_shared_kernel(SampleIterator samples, - unsigned int columns, - unsigned int rows, - unsigned int row_stride, - unsigned int rows_per_block, - unsigned int shared_histograms, - fixed_array histogram, - fixed_array - sample_to_bin_op, - fixed_array bins) +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().histogram_config.block_size) void + histogram_shared_kernel(SampleIterator samples, + unsigned int columns, + unsigned int rows, + unsigned int row_stride, + unsigned int rows_per_block, + unsigned int shared_histograms, + fixed_array histogram, + fixed_array sample_to_bin_op, + fixed_array bins) { static constexpr histogram_config_params params = device_params(); @@ -104,17 +99,13 @@ template -ROCPRIM_KERNEL __launch_bounds__( - device_params() - .histogram_config - .block_size) void histogram_global_kernel(SampleIterator samples, - unsigned int columns, - unsigned int row_stride, - fixed_array histogram, - fixed_array - sample_to_bin_op, - fixed_array - bins_bits) +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().histogram_config.block_size) void + histogram_global_kernel(SampleIterator samples, + unsigned int columns, + unsigned int row_stride, + fixed_array histogram, + fixed_array sample_to_bin_op, + fixed_array bins_bits) { static constexpr histogram_config_params params = device_params(); diff --git a/rocprim/include/rocprim/device/device_merge.hpp b/rocprim/include/rocprim/device/device_merge.hpp index 5dcb00159..53f09a08c 100644 --- a/rocprim/include/rocprim/device/device_merge.hpp +++ b/rocprim/include/rocprim/device/device_merge.hpp @@ -46,15 +46,14 @@ template -ROCPRIM_KERNEL -__launch_bounds__(device_params().kernel_config.block_size) -void partition_kernel(IndexIterator index, - KeysInputIterator1 keys_input1, - KeysInputIterator2 keys_input2, - const size_t input1_size, - const size_t input2_size, - const unsigned int spacing, - BinaryFunction compare_function) +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().kernel_config.block_size) void + partition_kernel(IndexIterator index, + KeysInputIterator1 keys_input1, + KeysInputIterator2 keys_input2, + const size_t input1_size, + const size_t input2_size, + const unsigned int spacing, + BinaryFunction compare_function) { partition_kernel_impl( index, keys_input1, keys_input2, input1_size, input2_size, @@ -71,18 +70,17 @@ template -ROCPRIM_KERNEL -__launch_bounds__(device_params().kernel_config.block_size) -void merge_kernel(IndexIterator index, - KeysInputIterator1 keys_input1, - KeysInputIterator2 keys_input2, - KeysOutputIterator keys_output, - ValuesInputIterator1 values_input1, - ValuesInputIterator2 values_input2, - ValuesOutputIterator values_output, - const size_t input1_size, - const size_t input2_size, - BinaryFunction compare_function) +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().kernel_config.block_size) void + merge_kernel(IndexIterator index, + KeysInputIterator1 keys_input1, + KeysInputIterator2 keys_input2, + KeysOutputIterator keys_output, + ValuesInputIterator1 values_input1, + ValuesInputIterator2 values_input2, + ValuesOutputIterator values_output, + const size_t input1_size, + const size_t input2_size, + BinaryFunction compare_function) { static constexpr merge_config_params params = device_params(); merge_kernel_impl( diff --git a/rocprim/include/rocprim/device/device_merge_sort.hpp b/rocprim/include/rocprim/device/device_merge_sort.hpp index 872d4027a..4070d1634 100644 --- a/rocprim/include/rocprim/device/device_merge_sort.hpp +++ b/rocprim/include/rocprim/device/device_merge_sort.hpp @@ -52,15 +52,14 @@ template -ROCPRIM_KERNEL - __launch_bounds__(device_params().block_sort_config.block_size) -void block_sort_kernel(KeysInputIterator keys_input, - KeysOutputIterator keys_output, - ValuesInputIterator values_input, - ValuesOutputIterator values_output, - const OffsetT size, - const unsigned int num_blocks, - BinaryFunction compare_function) +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().block_sort_config.block_size) void + block_sort_kernel(KeysInputIterator keys_input, + KeysOutputIterator keys_output, + ValuesInputIterator values_input, + ValuesOutputIterator values_output, + const OffsetT size, + const unsigned int num_blocks, + BinaryFunction compare_function) { static constexpr merge_sort_block_sort_config_params params = device_params(); block_sort_kernel_impl -ROCPRIM_KERNEL __launch_bounds__( - device_params() - .merge_oddeven_config - .block_size) -void device_block_merge_oddeven_kernel(KeysInputIterator keys_input, - KeysOutputIterator keys_output, - ValuesInputIterator values_input, - ValuesOutputIterator values_output, - const OffsetT input_size, - const OffsetT sorted_block_size, - BinaryFunction compare_function) +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().merge_oddeven_config.block_size) void + device_block_merge_oddeven_kernel(KeysInputIterator keys_input, + KeysOutputIterator keys_output, + ValuesInputIterator values_input, + ValuesOutputIterator values_output, + const OffsetT input_size, + const OffsetT sorted_block_size, + BinaryFunction compare_function) { static constexpr merge_sort_block_merge_config_params params = device_params(); block_merge_oddeven_kernel -ROCPRIM_KERNEL __launch_bounds__( - device_params() - .merge_mergepath_config - .block_size) -void device_block_merge_mergepath_kernel(KeysInputIterator keys_input, - KeysOutputIterator keys_output, - ValuesInputIterator values_input, - ValuesOutputIterator values_output, - const OffsetT input_size, - const OffsetT sorted_block_size, - const unsigned int num_blocks, - BinaryFunction compare_function, - const OffsetT* merge_partitions) +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().merge_mergepath_config.block_size) void + device_block_merge_mergepath_kernel(KeysInputIterator keys_input, + KeysOutputIterator keys_output, + ValuesInputIterator values_input, + ValuesOutputIterator values_output, + const OffsetT input_size, + const OffsetT sorted_block_size, + const unsigned int num_blocks, + BinaryFunction compare_function, + const OffsetT* merge_partitions) { static constexpr merge_sort_block_merge_config_params params = device_params(); block_merge_mergepath_kernel -ROCPRIM_KERNEL __launch_bounds__( - device_params() - .merge_mergepath_partition_config - .block_size) -void device_block_merge_mergepath_partition_kernel(KeysInputIterator keys, - const OffsetT input_size, - const unsigned int num_partitions, - OffsetT* merge_partitions, - const CompareOpT compare_op, - const OffsetT sorted_block_size) +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params() + .merge_mergepath_partition_config.block_size) void + device_block_merge_mergepath_partition_kernel(KeysInputIterator keys, + const OffsetT input_size, + const unsigned int num_partitions, + OffsetT* merge_partitions, + const CompareOpT compare_op, + const OffsetT sorted_block_size) { static constexpr merge_sort_block_merge_config_params params = device_params(); static constexpr unsigned int items_per_tile diff --git a/rocprim/include/rocprim/device/device_partition.hpp b/rocprim/include/rocprim/device/device_partition.hpp index b2494c2cc..0ba83cff7 100644 --- a/rocprim/include/rocprim/device/device_partition.hpp +++ b/rocprim/include/rocprim/device/device_partition.hpp @@ -60,20 +60,20 @@ template ROCPRIM_KERNEL - __launch_bounds__(device_params().kernel_config.block_size) void partition_kernel( - KeyIterator keys_input, - ValueIterator values_input, - FlagIterator flags, - OutputKeyIterator keys_output, - OutputValueIterator values_output, - size_t* selected_count, - size_t* prev_selected_count, - size_t prev_processed, - const size_t total_size, - InequalityOp inequality_op, - OffsetLookbackScanState offset_scan_state, - const unsigned int number_of_blocks, - UnaryPredicates... predicates) + ROCPRIM_LAUNCH_BOUNDS(device_params().kernel_config.block_size) void + partition_kernel(KeyIterator keys_input, + ValueIterator values_input, + FlagIterator flags, + OutputKeyIterator keys_output, + OutputValueIterator values_output, + size_t* selected_count, + size_t* prev_selected_count, + size_t prev_processed, + const size_t total_size, + InequalityOp inequality_op, + OffsetLookbackScanState offset_scan_state, + const unsigned int number_of_blocks, + UnaryPredicates... predicates) { partition_kernel_impl(keys_input, values_input, diff --git a/rocprim/include/rocprim/device/device_radix_sort.hpp b/rocprim/include/rocprim/device/device_radix_sort.hpp index d7fbc92d2..03af23902 100644 --- a/rocprim/include/rocprim/device/device_radix_sort.hpp +++ b/rocprim/include/rocprim/device/device_radix_sort.hpp @@ -83,14 +83,14 @@ using offset_type_t = std::conditional_t< template ROCPRIM_KERNEL - __launch_bounds__(device_params().histogram.block_size) void onesweep_histograms_kernel( - KeysInputIterator keys_input, - Offset* global_digit_counts, - const Offset size, - const Offset full_blocks, - Decomposer decomposer, - const unsigned int begin_bit, - const unsigned int end_bit) + ROCPRIM_LAUNCH_BOUNDS(device_params().histogram.block_size) void + onesweep_histograms_kernel(KeysInputIterator keys_input, + Offset* global_digit_counts, + const Offset size, + const Offset full_blocks, + Decomposer decomposer, + const unsigned int begin_bit, + const unsigned int end_bit) { static constexpr radix_sort_onesweep_config_params params = device_params(); onesweep_histograms -ROCPRIM_KERNEL __launch_bounds__( - device_params() - .histogram.block_size) void onesweep_scan_histograms_kernel(Offset* global_digit_offsets) +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().histogram.block_size) void + onesweep_scan_histograms_kernel(Offset* global_digit_offsets) { static constexpr radix_sort_onesweep_config_params params = device_params(); onesweep_scan_histograms( @@ -208,20 +207,19 @@ template -ROCPRIM_KERNEL - __launch_bounds__(device_params().sort.block_size) void onesweep_iteration_kernel( - KeysInputIterator keys_input, - KeysOutputIterator keys_output, - ValuesInputIterator values_input, - ValuesOutputIterator values_output, - const unsigned int size, - Offset* global_digit_offsets_in, - Offset* global_digit_offsets_out, - onesweep_lookback_state* lookback_states, - Decomposer decomposer, - const unsigned int bit, - const unsigned int current_radix_bits, - const unsigned int full_blocks) +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().sort.block_size) void + onesweep_iteration_kernel(KeysInputIterator keys_input, + KeysOutputIterator keys_output, + ValuesInputIterator values_input, + ValuesOutputIterator values_output, + const unsigned int size, + Offset* global_digit_offsets_in, + Offset* global_digit_offsets_out, + onesweep_lookback_state* lookback_states, + Decomposer decomposer, + const unsigned int bit, + const unsigned int current_radix_bits, + const unsigned int full_blocks) { static constexpr radix_sort_onesweep_config_params params = device_params(); onesweep_iteration -ROCPRIM_KERNEL - __launch_bounds__(device_params().reduce_config.block_size) -void block_reduce_kernel(InputIterator input, - const size_t size, - OutputIterator output, - InitValueType initial_value, - BinaryFunction reduce_op) +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().reduce_config.block_size) void + block_reduce_kernel(InputIterator input, + const size_t size, + OutputIterator output, + InitValueType initial_value, + BinaryFunction reduce_op) { block_reduce_kernel_impl( input, diff --git a/rocprim/include/rocprim/device/device_reduce_by_key.hpp b/rocprim/include/rocprim/device/device_reduce_by_key.hpp index 190d19f28..c8607afec 100644 --- a/rocprim/include/rocprim/device/device_reduce_by_key.hpp +++ b/rocprim/include/rocprim/device/device_reduce_by_key.hpp @@ -103,7 +103,7 @@ template -ROCPRIM_KERNEL __launch_bounds__(device_params().kernel_config.block_size) void +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().kernel_config.block_size) void reduce_by_key_kernel(const KeyIterator keys_input, const ValueIterator values_input, const UniqueIterator unique_keys, diff --git a/rocprim/include/rocprim/device/device_run_length_encode.hpp b/rocprim/include/rocprim/device/device_run_length_encode.hpp index e96b57a98..4d55cf8a8 100644 --- a/rocprim/include/rocprim/device/device_run_length_encode.hpp +++ b/rocprim/include/rocprim/device/device_run_length_encode.hpp @@ -100,7 +100,7 @@ template -ROCPRIM_KERNEL __launch_bounds__(device_params().kernel_config.block_size) void +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().kernel_config.block_size) void non_trivial_kernel(const InputIterator input, const OffsetsOutputIterator offsets_output, const CountsOutputIterator counts_output, diff --git a/rocprim/include/rocprim/device/device_scan.hpp b/rocprim/include/rocprim/device/device_scan.hpp index 0633222e5..79977eb3b 100644 --- a/rocprim/include/rocprim/device/device_scan.hpp +++ b/rocprim/include/rocprim/device/device_scan.hpp @@ -100,13 +100,12 @@ template -ROCPRIM_KERNEL - __launch_bounds__(device_params().kernel_config.block_size) void single_scan_kernel( - InputIterator input, - const size_t size, - const InitValueType initial_value, - OutputIterator output, - BinaryFunction scan_op) +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().kernel_config.block_size) void + single_scan_kernel(InputIterator input, + const size_t size, + const InitValueType initial_value, + OutputIterator output, + BinaryFunction scan_op) { single_scan_kernel_impl(input, size, @@ -127,18 +126,18 @@ template ROCPRIM_KERNEL - __launch_bounds__(device_params().kernel_config.block_size) void lookback_scan_kernel( - InputIterator input, - OutputIterator output, - const size_t size, - const InitValueType initial_value, - BinaryFunction scan_op, - LookBackScanState lookback_scan_state, - const unsigned int number_of_blocks, - AccType* previous_last_element = nullptr, - AccType* new_last_element = nullptr, - bool override_first_value = false, - bool save_last_value = false) + ROCPRIM_LAUNCH_BOUNDS(device_params().kernel_config.block_size) void + lookback_scan_kernel(InputIterator input, + OutputIterator output, + const size_t size, + const InitValueType initial_value, + BinaryFunction scan_op, + LookBackScanState lookback_scan_state, + const unsigned int number_of_blocks, + AccType* previous_last_element = nullptr, + AccType* new_last_element = nullptr, + bool override_first_value = false, + bool save_last_value = false) { lookback_scan_kernel_impl( input, diff --git a/rocprim/include/rocprim/device/device_scan_by_key.hpp b/rocprim/include/rocprim/device/device_scan_by_key.hpp index 29d93c4db..a0f762675 100644 --- a/rocprim/include/rocprim/device/device_scan_by_key.hpp +++ b/rocprim/include/rocprim/device/device_scan_by_key.hpp @@ -57,7 +57,7 @@ template -void __global__ __launch_bounds__(device_params().kernel_config.block_size) +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().kernel_config.block_size) void device_scan_by_key_kernel(const KeyInputIterator keys, const InputIterator values, const OutputIterator output, diff --git a/rocprim/include/rocprim/device/device_segmented_radix_sort.hpp b/rocprim/include/rocprim/device/device_segmented_radix_sort.hpp index e0c21cb7a..fd63149e4 100644 --- a/rocprim/include/rocprim/device/device_segmented_radix_sort.hpp +++ b/rocprim/include/rocprim/device/device_segmented_radix_sort.hpp @@ -60,8 +60,7 @@ template ROCPRIM_KERNEL - __launch_bounds__(device_params().kernel_config.block_size) -void segmented_sort_kernel( + ROCPRIM_LAUNCH_BOUNDS(device_params().kernel_config.block_size) void segmented_sort_kernel( KeysInputIterator keys_input, typename std::iterator_traits::value_type* keys_tmp, KeysOutputIterator keys_output, @@ -97,24 +96,24 @@ template -ROCPRIM_KERNEL __launch_bounds__( +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS( device_params() .kernel_config - .block_size) -void segmented_sort_large_kernel( - KeysInputIterator keys_input, - typename std::iterator_traits::value_type* keys_tmp, - KeysOutputIterator keys_output, - ValuesInputIterator values_input, - typename std::iterator_traits::value_type* values_tmp, - ValuesOutputIterator values_output, - bool to_output, - SegmentIndexIterator segment_indices, - OffsetIterator begin_offsets, - OffsetIterator end_offsets, - unsigned int iterations, - unsigned int begin_bit, - unsigned int end_bit) + .block_size) void + segmented_sort_large_kernel( + KeysInputIterator keys_input, + typename std::iterator_traits::value_type* keys_tmp, + KeysOutputIterator keys_output, + ValuesInputIterator values_input, + typename std::iterator_traits::value_type* values_tmp, + ValuesOutputIterator values_output, + bool to_output, + SegmentIndexIterator segment_indices, + OffsetIterator begin_offsets, + OffsetIterator end_offsets, + unsigned int iterations, + unsigned int begin_bit, + unsigned int end_bit) { segmented_sort_large(keys_input, keys_tmp, @@ -139,26 +138,24 @@ template -ROCPRIM_KERNEL __launch_bounds__( +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS( device_params() .warp_sort_config - .block_size_small) void segmented_sort_small_kernel(KeysInputIterator keys_input, - typename std::iterator_traits< - KeysInputIterator>::value_type* - keys_tmp, - KeysOutputIterator keys_output, - ValuesInputIterator values_input, - typename std::iterator_traits< - ValuesInputIterator>::value_type* - values_tmp, - ValuesOutputIterator values_output, - bool to_output, - unsigned int num_segments, - SegmentIndexIterator segment_indices, - OffsetIterator begin_offsets, - OffsetIterator end_offsets, - unsigned int begin_bit, - unsigned int end_bit) + .block_size_small) void + segmented_sort_small_kernel( + KeysInputIterator keys_input, + typename std::iterator_traits::value_type* keys_tmp, + KeysOutputIterator keys_output, + ValuesInputIterator values_input, + typename std::iterator_traits::value_type* values_tmp, + ValuesOutputIterator values_output, + bool to_output, + unsigned int num_segments, + SegmentIndexIterator segment_indices, + OffsetIterator begin_offsets, + OffsetIterator end_offsets, + unsigned int begin_bit, + unsigned int end_bit) { segmented_sort_small( keys_input, keys_tmp, keys_output, values_input, values_tmp, values_output, @@ -176,26 +173,24 @@ template -ROCPRIM_KERNEL __launch_bounds__( +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS( device_params() .warp_sort_config - .block_size_medium) void segmented_sort_medium_kernel(KeysInputIterator keys_input, - typename std::iterator_traits< - KeysInputIterator>::value_type* - keys_tmp, - KeysOutputIterator keys_output, - ValuesInputIterator values_input, - typename std::iterator_traits< - ValuesInputIterator>::value_type* - values_tmp, - ValuesOutputIterator values_output, - bool to_output, - unsigned int num_segments, - SegmentIndexIterator segment_indices, - OffsetIterator begin_offsets, - OffsetIterator end_offsets, - unsigned int begin_bit, - unsigned int end_bit) + .block_size_medium) void + segmented_sort_medium_kernel( + KeysInputIterator keys_input, + typename std::iterator_traits::value_type* keys_tmp, + KeysOutputIterator keys_output, + ValuesInputIterator values_input, + typename std::iterator_traits::value_type* values_tmp, + ValuesOutputIterator values_output, + bool to_output, + unsigned int num_segments, + SegmentIndexIterator segment_indices, + OffsetIterator begin_offsets, + OffsetIterator end_offsets, + unsigned int begin_bit, + unsigned int end_bit) { segmented_sort_medium(keys_input, keys_tmp, diff --git a/rocprim/include/rocprim/device/device_segmented_reduce.hpp b/rocprim/include/rocprim/device/device_segmented_reduce.hpp index b0c5826d0..6b396d8f6 100644 --- a/rocprim/include/rocprim/device/device_segmented_reduce.hpp +++ b/rocprim/include/rocprim/device/device_segmented_reduce.hpp @@ -48,14 +48,13 @@ template -ROCPRIM_KERNEL __launch_bounds__( - device_params() - .reduce_config.block_size) void segmented_reduce_kernel(InputIterator input, - OutputIterator output, - OffsetIterator begin_offsets, - OffsetIterator end_offsets, - BinaryFunction reduce_op, - ResultType initial_value) +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().reduce_config.block_size) void + segmented_reduce_kernel(InputIterator input, + OutputIterator output, + OffsetIterator begin_offsets, + OffsetIterator end_offsets, + BinaryFunction reduce_op, + ResultType initial_value) { segmented_reduce( input, output, diff --git a/rocprim/include/rocprim/device/device_segmented_scan.hpp b/rocprim/include/rocprim/device/device_segmented_scan.hpp index 833650390..28d43d5a5 100644 --- a/rocprim/include/rocprim/device/device_segmented_scan.hpp +++ b/rocprim/include/rocprim/device/device_segmented_scan.hpp @@ -72,24 +72,21 @@ struct transform_op_t } }; -template< - bool Exclusive, - class Config, - class ResultType, - class InputIterator, - class OutputIterator, - class OffsetIterator, - class InitValueType, - class BinaryFunction -> -ROCPRIM_KERNEL -__launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) -void segmented_scan_kernel(InputIterator input, - OutputIterator output, - OffsetIterator begin_offsets, - OffsetIterator end_offsets, - InitValueType initial_value, - BinaryFunction scan_op) +template +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) void + segmented_scan_kernel(InputIterator input, + OutputIterator output, + OffsetIterator begin_offsets, + OffsetIterator end_offsets, + InitValueType initial_value, + BinaryFunction scan_op) { segmented_scan( input, output, begin_offsets, end_offsets, diff --git a/rocprim/include/rocprim/device/device_transform.hpp b/rocprim/include/rocprim/device/device_transform.hpp index 8beee4f5a..d0e1e47f0 100644 --- a/rocprim/include/rocprim/device/device_transform.hpp +++ b/rocprim/include/rocprim/device/device_transform.hpp @@ -49,8 +49,8 @@ template ROCPRIM_KERNEL - __launch_bounds__(device_params().kernel_config.block_size) void transform_kernel( - InputIterator input, const size_t size, OutputIterator output, UnaryFunction transform_op) + ROCPRIM_LAUNCH_BOUNDS(device_params().kernel_config.block_size) void transform_kernel( + InputIterator input, const size_t size, OutputIterator output, UnaryFunction transform_op) { transform_kernel_impl().kernel_config.block_size, device_params().kernel_config.items_per_thread, diff --git a/rocprim/include/rocprim/device/specialization/device_radix_block_sort.hpp b/rocprim/include/rocprim/device/specialization/device_radix_block_sort.hpp index b05c5cc29..7a206f200 100644 --- a/rocprim/include/rocprim/device/specialization/device_radix_block_sort.hpp +++ b/rocprim/include/rocprim/device/specialization/device_radix_block_sort.hpp @@ -37,16 +37,15 @@ template -ROCPRIM_KERNEL - __launch_bounds__(device_params().block_size) void radix_sort_block_sort_kernel( - KeysInputIterator keys_input, - KeysOutputIterator keys_output, - ValuesInputIterator values_input, - ValuesOutputIterator values_output, - unsigned int size, - Decomposer decomposer, - unsigned int bit, - unsigned int current_radix_bits) +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().block_size) void + radix_sort_block_sort_kernel(KeysInputIterator keys_input, + KeysOutputIterator keys_output, + ValuesInputIterator values_input, + ValuesOutputIterator values_output, + unsigned int size, + Decomposer decomposer, + unsigned int bit, + unsigned int current_radix_bits) { static constexpr kernel_config_params params = device_params(); sort_single(keys_input, From 33e447e63f2343d2f3c18c0d01dcc7545a690910 Mon Sep 17 00:00:00 2001 From: Beatriz Navidad Vilches Date: Thu, 21 Nov 2024 13:08:33 +0100 Subject: [PATCH 25/84] Update rmake.py copyright --- rmake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rmake.py b/rmake.py index 986ad6087..75397b0c2 100644 --- a/rmake.py +++ b/rmake.py @@ -1,5 +1,5 @@ #!/usr/bin/python3 -""" Copyright (c) 2021-2023 Advanced Micro Devices, Inc. All rights reserved. +""" Copyright (c) 2021-2024 Advanced Micro Devices, Inc. All rights reserved. Manage build and installation""" import re From f64016c06ab11b1a12dfba5c0cd779a27c977f95 Mon Sep 17 00:00:00 2001 From: Nara Prasetya Date: Tue, 26 Nov 2024 09:17:41 +0000 Subject: [PATCH 26/84] fix double buffers in device radix sort --- CHANGELOG.md | 4 ++ ...rk_device_radix_sort_onesweep.parallel.hpp | 6 +++ .../rocprim/device/device_merge_sort.hpp | 16 ++++--- .../rocprim/device/device_partial_sort.hpp | 5 ++- .../rocprim/device/device_radix_sort.hpp | 43 +++++++++++++++---- .../device_radix_merge_sort.hpp | 28 ++++++++---- 6 files changed, 78 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfff549e9..31d678556 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ Full documentation for rocPRIM is available at [https://rocm.docs.amd.com/projec * Added gfx950 support. +### Resolved issues + +* Fixed device radix sort not returning the correct required temporary storage when a double buffer contains `nullptr`. + ## rocPRIM 3.4.0 for ROCm 6.4.0 ### Added diff --git a/benchmark/benchmark_device_radix_sort_onesweep.parallel.hpp b/benchmark/benchmark_device_radix_sort_onesweep.parallel.hpp index 40495dd63..c43696700 100644 --- a/benchmark/benchmark_device_radix_sort_onesweep.parallel.hpp +++ b/benchmark/benchmark_device_radix_sort_onesweep.parallel.hpp @@ -136,6 +136,7 @@ struct device_radix_sort_onesweep_benchmark : public config_autotune_interface 0, sizeof(key_type) * 8, stream, + false, false))); HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); @@ -159,6 +160,7 @@ struct device_radix_sort_onesweep_benchmark : public config_autotune_interface 0, sizeof(key_type) * 8, stream, + false, false))); } HIP_CHECK(hipDeviceSynchronize()); @@ -190,6 +192,7 @@ struct device_radix_sort_onesweep_benchmark : public config_autotune_interface 0, sizeof(key_type) * 8, stream, + false, false))); } @@ -276,6 +279,7 @@ struct device_radix_sort_onesweep_benchmark : public config_autotune_interface 0, sizeof(key_type) * 8, stream, + false, false))); HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); @@ -299,6 +303,7 @@ struct device_radix_sort_onesweep_benchmark : public config_autotune_interface 0, sizeof(key_type) * 8, stream, + false, false))); } HIP_CHECK(hipDeviceSynchronize()); @@ -330,6 +335,7 @@ struct device_radix_sort_onesweep_benchmark : public config_autotune_interface 0, sizeof(key_type) * 8, stream, + false, false))); } diff --git a/rocprim/include/rocprim/device/device_merge_sort.hpp b/rocprim/include/rocprim/device/device_merge_sort.hpp index 4070d1634..035d93adc 100644 --- a/rocprim/include/rocprim/device/device_merge_sort.hpp +++ b/rocprim/include/rocprim/device/device_merge_sort.hpp @@ -197,8 +197,9 @@ inline hipError_t merge_sort_block_merge( BinaryFunction compare_function, const hipStream_t stream, bool debug_synchronous, - typename std::iterator_traits::value_type* keys_double_buffer = nullptr, - typename std::iterator_traits::value_type* values_double_buffer = nullptr) + typename std::iterator_traits::value_type* keys_double_buffer = nullptr, + typename std::iterator_traits::value_type* values_double_buffer = nullptr, + const bool no_allocate_tmp_buffer = false) { using key_type = typename std::iterator_traits::value_type; using value_type = typename std::iterator_traits::value_type; @@ -244,7 +245,7 @@ inline hipError_t merge_sort_block_merge( value_type* values_buffer = nullptr; hipError_t partition_result; - if(keys_double_buffer == nullptr) + if(!no_allocate_tmp_buffer) { partition_result = detail::temp_storage::partition( temporary_storage, @@ -539,7 +540,8 @@ inline hipError_t merge_sort_impl( const hipStream_t stream, bool debug_synchronous, typename std::iterator_traits::value_type* keys_buffer = nullptr, - typename std::iterator_traits::value_type* values_buffer = nullptr) + typename std::iterator_traits::value_type* values_buffer = nullptr, + bool no_allocate_tmp_buffer = false) { using key_type = typename std::iterator_traits::value_type; using value_type = typename std::iterator_traits::value_type; @@ -573,7 +575,8 @@ inline hipError_t merge_sort_impl( stream, debug_synchronous, keys_buffer, - values_buffer); + values_buffer, + no_allocate_tmp_buffer); } if(size == size_t(0)) @@ -608,7 +611,8 @@ inline hipError_t merge_sort_impl( stream, debug_synchronous, keys_buffer, - values_buffer); + values_buffer, + no_allocate_tmp_buffer); } return hipSuccess; } diff --git a/rocprim/include/rocprim/device/device_partial_sort.hpp b/rocprim/include/rocprim/device/device_partial_sort.hpp index 857a8c4a6..1ce720ab7 100644 --- a/rocprim/include/rocprim/device/device_partial_sort.hpp +++ b/rocprim/include/rocprim/device/device_partial_sort.hpp @@ -99,7 +99,9 @@ struct SortImpl compare_function, stream, debug_synchronous, - keys_buffer); + keys_buffer, + static_cast(nullptr), + true); } }; @@ -148,6 +150,7 @@ struct SortImpl::value_type) * 8, stream, + false, debug_synchronous); } }; diff --git a/rocprim/include/rocprim/device/device_radix_sort.hpp b/rocprim/include/rocprim/device/device_radix_sort.hpp index 03af23902..04746e8fa 100644 --- a/rocprim/include/rocprim/device/device_radix_sort.hpp +++ b/rocprim/include/rocprim/device/device_radix_sort.hpp @@ -441,6 +441,7 @@ hipError_t radix_sort_onesweep_impl( const unsigned int begin_bit, const unsigned int end_bit, const hipStream_t stream, + const bool no_allocate_tmp_buffer, const bool debug_synchronous) { using key_type = typename std::iterator_traits::value_type; @@ -470,8 +471,7 @@ hipError_t radix_sort_onesweep_impl( const unsigned int num_lookback_states = radix_size_per_place * ceiling_div(items_per_batch, sort_items_per_block); - constexpr bool with_values = !std::is_same::value; - const bool with_double_buffer = keys_tmp != nullptr; + constexpr bool with_values = !std::is_same::value; offset_type* global_digit_offsets; offset_type* global_digit_offsets_tmp; @@ -488,10 +488,10 @@ hipError_t radix_sort_onesweep_impl( radix_size_per_place), detail::temp_storage::ptr_aligned_array(&lookback_states, num_lookback_states), detail::temp_storage::ptr_aligned_array(&keys_tmp_storage, - !with_double_buffer ? size : 0), + !no_allocate_tmp_buffer ? size : 0), detail::temp_storage::ptr_aligned_array(&values_tmp_storage, - !with_double_buffer && with_values ? size - : 0))); + !no_allocate_tmp_buffer && with_values ? size + : 0))); if(partition_result != hipSuccess || temporary_storage == nullptr) { @@ -529,16 +529,16 @@ hipError_t radix_sort_onesweep_impl( return error; } - if(!with_double_buffer) + if(!no_allocate_tmp_buffer) { keys_tmp = keys_tmp_storage; values_tmp = values_tmp_storage; } // Copy input keys and values if necessary (in-place sorting: input and output iterators are equal). - bool to_output = with_double_buffer || (places - 1) % 2 == 0; + bool to_output = no_allocate_tmp_buffer || (places - 1) % 2 == 0; bool from_input = true; - if(!with_double_buffer && to_output) + if(!no_allocate_tmp_buffer && to_output) { const bool keys_alias = ::rocprim::detail::can_iterators_alias(keys_input, keys_output, size); @@ -628,6 +628,7 @@ hipError_t unsigned int begin_bit, unsigned int end_bit, hipStream_t stream, + bool no_allocate_tmp_buffer, bool debug_synchronous) { using key_type = typename std::iterator_traits::value_type; @@ -721,6 +722,7 @@ hipError_t begin_bit, end_bit, stream, + no_allocate_tmp_buffer, debug_synchronous); } else @@ -741,6 +743,7 @@ hipError_t begin_bit, end_bit, stream, + no_allocate_tmp_buffer, debug_synchronous); } } @@ -863,6 +866,7 @@ hipError_t radix_sort_keys(void* temporary_storage, begin_bit, end_bit, stream, + false, debug_synchronous); } @@ -979,6 +983,7 @@ hipError_t radix_sort_keys(void* temporary_storage, begin_bit, end_bit, stream, + true, debug_synchronous); if(temporary_storage != nullptr && error == hipSuccess && is_result_in_output) { @@ -1128,6 +1133,7 @@ auto radix_sort_keys(void* temporary_storage, begin_bit, end_bit, stream, + false, debug_synchronous); } @@ -1260,6 +1266,7 @@ auto radix_sort_keys(void* temporary_storage, 0, detail::decomposer_max_bits::value, stream, + false, debug_synchronous); } @@ -1401,6 +1408,7 @@ auto radix_sort_keys(void* temporary_storage, begin_bit, end_bit, stream, + true, debug_synchronous); if(temporary_storage != nullptr && error == hipSuccess && is_result_in_output) { @@ -1535,6 +1543,7 @@ auto radix_sort_keys(void* temporary_storage, 0, detail::decomposer_max_bits::value, stream, + true, debug_synchronous); if(temporary_storage != nullptr && error == hipSuccess && is_result_in_output) { @@ -1658,6 +1667,7 @@ hipError_t radix_sort_keys_desc(void* temporary_storage, begin_bit, end_bit, stream, + false, debug_synchronous); } @@ -1774,6 +1784,7 @@ hipError_t radix_sort_keys_desc(void* temporary_storage, begin_bit, end_bit, stream, + true, debug_synchronous); if(temporary_storage != nullptr && error == hipSuccess && is_result_in_output) { @@ -1923,6 +1934,7 @@ auto radix_sort_keys_desc(void* temporary_storage, begin_bit, end_bit, stream, + false, debug_synchronous); } @@ -2055,6 +2067,7 @@ auto radix_sort_keys_desc(void* temporary_storage, 0, detail::decomposer_max_bits::value, stream, + false, debug_synchronous); } @@ -2196,6 +2209,7 @@ auto radix_sort_keys_desc(void* temporary_storage, begin_bit, end_bit, stream, + true, debug_synchronous); if(temporary_storage != nullptr && error == hipSuccess && is_result_in_output) { @@ -2330,6 +2344,7 @@ auto radix_sort_keys_desc(void* temporary_storage, 0, detail::decomposer_max_bits::value, stream, + true, debug_synchronous); if(temporary_storage != nullptr && error == hipSuccess && is_result_in_output) { @@ -2471,6 +2486,7 @@ hipError_t radix_sort_pairs(void* temporary_storage, begin_bit, end_bit, stream, + false, debug_synchronous); } @@ -2600,6 +2616,7 @@ hipError_t radix_sort_pairs(void* temporary_storage, begin_bit, end_bit, stream, + true, debug_synchronous); if(temporary_storage != nullptr && error == hipSuccess && is_result_in_output) { @@ -2768,6 +2785,7 @@ auto radix_sort_pairs(void* temporary_storage, begin_bit, end_bit, stream, + false, debug_synchronous); } @@ -2914,6 +2932,7 @@ auto radix_sort_pairs(void* temporary_storage, 0, detail::decomposer_max_bits::value, stream, + false, debug_synchronous); } @@ -3066,6 +3085,7 @@ auto radix_sort_pairs(void* temporary_storage, begin_bit, end_bit, stream, + true, debug_synchronous); if(temporary_storage != nullptr && error == hipSuccess && is_result_in_output) { @@ -3208,6 +3228,7 @@ auto radix_sort_pairs(void* temporary_storage, 0, detail::decomposer_max_bits::value, stream, + true, debug_synchronous); if(temporary_storage != nullptr && error == hipSuccess && is_result_in_output) { @@ -3346,6 +3367,7 @@ hipError_t radix_sort_pairs_desc(void* temporary_storage, begin_bit, end_bit, stream, + false, debug_synchronous); } @@ -3469,6 +3491,7 @@ hipError_t radix_sort_pairs_desc(void* temporary_storage, begin_bit, end_bit, stream, + true, debug_synchronous); if(temporary_storage != nullptr && error == hipSuccess && is_result_in_output) { @@ -3637,6 +3660,7 @@ auto radix_sort_pairs_desc(void* temporary_storage, begin_bit, end_bit, stream, + false, debug_synchronous); } @@ -3783,6 +3807,7 @@ auto radix_sort_pairs_desc(void* temporary_storage, 0, detail::decomposer_max_bits::value, stream, + false, debug_synchronous); } @@ -3935,6 +3960,7 @@ auto radix_sort_pairs_desc(void* temporary_storage, begin_bit, end_bit, stream, + true, debug_synchronous); if(temporary_storage != nullptr && error == hipSuccess && is_result_in_output) { @@ -4077,6 +4103,7 @@ auto radix_sort_pairs_desc(void* temporary_storage, 0, detail::decomposer_max_bits::value, stream, + true, debug_synchronous); if(temporary_storage != nullptr && error == hipSuccess && is_result_in_output) { diff --git a/rocprim/include/rocprim/device/specialization/device_radix_merge_sort.hpp b/rocprim/include/rocprim/device/specialization/device_radix_merge_sort.hpp index 0d1d94602..a5b9e39f7 100644 --- a/rocprim/include/rocprim/device/specialization/device_radix_merge_sort.hpp +++ b/rocprim/include/rocprim/device/specialization/device_radix_merge_sort.hpp @@ -47,7 +47,8 @@ auto invoke_merge_sort_block_merge( const hipStream_t stream, bool debug_synchronous, typename std::iterator_traits::value_type* keys_buffer, - typename std::iterator_traits::value_type* values_buffer) + typename std::iterator_traits::value_type* values_buffer, + const bool no_allocate_tmp_buffer) -> std::enable_if_t::value_type>::value, hipError_t> { @@ -65,7 +66,8 @@ auto invoke_merge_sort_block_merge( stream, debug_synchronous, keys_buffer, - values_buffer); + values_buffer, + no_allocate_tmp_buffer); } else { @@ -80,7 +82,8 @@ auto invoke_merge_sort_block_merge( stream, debug_synchronous, keys_buffer, - values_buffer); + values_buffer, + no_allocate_tmp_buffer); } } @@ -98,7 +101,8 @@ auto invoke_merge_sort_block_merge( const hipStream_t stream, bool debug_synchronous, typename std::iterator_traits::value_type* keys_buffer, - typename std::iterator_traits::value_type* values_buffer) + typename std::iterator_traits::value_type* values_buffer, + const bool no_allocate_tmp_buffer) -> std::enable_if_t< !is_integral::value_type>::value, hipError_t> @@ -117,7 +121,8 @@ auto invoke_merge_sort_block_merge( stream, debug_synchronous, keys_buffer, - values_buffer); + values_buffer, + no_allocate_tmp_buffer); } template::value_type* keys_buffer, - typename std::iterator_traits::value_type* values_buffer) + typename std::iterator_traits::value_type* values_buffer, + const bool no_allocate_tmp_buffer) -> std::enable_if_t::value, hipError_t> { using key_type = typename std::iterator_traits::value_type; @@ -156,7 +162,8 @@ auto invoke_merge_sort_block_merge( stream, debug_synchronous, keys_buffer, - values_buffer); + values_buffer, + no_allocate_tmp_buffer); } /// In device_radix_sort, we use this device_radix_sort_merge_sort specialization only @@ -182,6 +189,7 @@ hipError_t radix_sort_merge_impl( unsigned int bit, unsigned int end_bit, hipStream_t stream, + const bool no_allocate_tmp_buffer, bool debug_synchronous) { using key_type = typename std::iterator_traits::value_type; @@ -244,7 +252,8 @@ hipError_t radix_sort_merge_impl( stream, debug_synchronous, keys_buffer, - values_buffer); + values_buffer, + no_allocate_tmp_buffer); } if(size == size_t(0)) @@ -285,7 +294,8 @@ hipError_t radix_sort_merge_impl( stream, debug_synchronous, keys_buffer, - values_buffer); + values_buffer, + no_allocate_tmp_buffer); } return hipSuccess; } From 326716b0ff04de17e59f7229b4d3d416c0bd7b4c Mon Sep 17 00:00:00 2001 From: Cenxuan Tian Date: Wed, 27 Nov 2024 16:27:55 +0000 Subject: [PATCH 27/84] Resolve "Replace hipMalloc, hipMallocHelper, and hipFree in tests with device_ptr" --- test/hipgraph/test_hipgraph_algs.cpp | 73 +- test/rocprim/detail/get_rocprim_version.cpp | 27 +- test/rocprim/test_arg_index_iterator.cpp | 70 +- test/rocprim/test_block_histogram.kernels.hpp | 46 +- test/rocprim/test_block_radix_rank.cpp.in | 3 +- test/rocprim/test_block_radix_rank.hpp | 23 +- test/rocprim/test_block_radix_sort.cpp.in | 1 + .../rocprim/test_block_radix_sort.kernels.hpp | 51 +- test/rocprim/test_block_reduce.cpp | 3 +- test/rocprim/test_block_reduce.kernels.hpp | 45 +- test/rocprim/test_block_run_length_decode.cpp | 56 +- test/rocprim/test_config_dispatch.cpp | 34 +- test/rocprim/test_constant_iterator.cpp | 28 +- test/rocprim/test_counting_iterator.cpp | 28 +- test/rocprim/test_device_adjacent_find.cpp | 33 +- test/rocprim/test_device_partition.cpp | 410 +++----- test/rocprim/test_device_search_n.cpp | 100 +- test/rocprim/test_device_segmented_scan.cpp | 451 +++----- test/rocprim/test_device_select.cpp | 975 ++++++------------ test/rocprim/test_device_transform.cpp | 95 +- test/rocprim/test_discard_iterator.cpp | 136 +-- .../rocprim/test_lookback_reproducibility.cpp | 248 ++--- test/rocprim/test_predicate_iterator.cpp | 51 +- test/rocprim/test_reverse_iterator.cpp | 31 +- test/rocprim/test_texture_cache_iterator.cpp | 34 +- test/rocprim/test_thread.cpp | 57 +- test/rocprim/test_thread_algos.cpp | 126 +-- test/rocprim/test_transform_iterator.cpp | 77 +- test/rocprim/test_utils_device_ptr.hpp | 14 +- test/rocprim/test_warp_exchange.cpp | 57 +- test/rocprim/test_warp_load.cpp | 34 +- test/rocprim/test_warp_reduce.cpp | 3 +- test/rocprim/test_warp_reduce.hpp | 353 +++---- test/rocprim/test_warp_scan.cpp | 3 +- test/rocprim/test_warp_scan.hpp | 508 +++------ test/rocprim/test_warp_sort.cpp | 3 +- test/rocprim/test_warp_sort.hpp | 90 +- test/rocprim/test_warp_store.cpp | 33 +- test/rocprim/test_zip_iterator.cpp | 199 +--- 39 files changed, 1527 insertions(+), 3082 deletions(-) diff --git a/test/hipgraph/test_hipgraph_algs.cpp b/test/hipgraph/test_hipgraph_algs.cpp index 125ab6c44..cf6453eda 100644 --- a/test/hipgraph/test_hipgraph_algs.cpp +++ b/test/hipgraph/test_hipgraph_algs.cpp @@ -26,6 +26,7 @@ // required rocprim headers #include "../rocprim/test_seed.hpp" #include "../rocprim/test_utils.hpp" +#include "../rocprim/test_utils_device_ptr.hpp" #include #include @@ -96,14 +97,10 @@ TEST(TestHipGraphAlgs, SortAndSearch) SCOPED_TRACE(testing::Message() << "with search_needle_size = " << search_needle_size); // Allocate device buffers and copy data into them - key_type* d_sort_input = nullptr; - key_type* d_sort_output = nullptr; // also used as search_input - key_type* d_search_output = nullptr; - key_type* d_search_needles = nullptr; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_sort_input, sort_data_size * sizeof(key_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_sort_output, sort_data_size * sizeof(key_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_search_output, search_needle_size * sizeof(key_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_search_needles, search_needle_size * sizeof(key_type))); + test_utils::device_ptr d_sort_input(sort_data_size); + test_utils::device_ptr d_sort_output(sort_data_size); // also used as search_input + test_utils::device_ptr d_search_output(search_needle_size); + test_utils::device_ptr d_search_needles(search_needle_size); // Default stream does not support hipGraph stream capture, so create a non-blocking one hipStream_t stream = 0; @@ -116,27 +113,25 @@ TEST(TestHipGraphAlgs, SortAndSearch) size_t sort_temp_storage_bytes = 0; HIP_CHECK(rocprim::merge_sort(nullptr, sort_temp_storage_bytes, - d_sort_input, - d_sort_output, + d_sort_input.get(), + d_sort_output.get(), sort_data_size, compare_op, stream, - debug_synchronous - )); + debug_synchronous)); // Get size of temporary storage required for binary_search size_t search_temp_storage_bytes = 0; HIP_CHECK(rocprim::binary_search(nullptr, search_temp_storage_bytes, - d_sort_output, - d_search_needles, - d_search_output, + d_sort_output.get(), + d_search_needles.get(), + d_search_output.get(), sort_data_size, search_needle_size, compare_op, stream, - debug_synchronous - )); + debug_synchronous)); // Allocate the temp storage // Note: a single store will be used for both the sort and search algorithms @@ -144,41 +139,33 @@ TEST(TestHipGraphAlgs, SortAndSearch) // temp_storage_size_bytes must be > 0 ASSERT_GT(temp_storage_bytes, 0); - void* d_temp_storage = nullptr; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_bytes); // Begin graph capture test_utils::GraphHelper gHelper; gHelper.startStreamCapture(stream); // Launch merge_sort - HIP_CHECK( - rocprim::merge_sort(d_temp_storage, + HIP_CHECK(rocprim::merge_sort(d_temp_storage.get(), sort_temp_storage_bytes, - d_sort_input, - d_sort_output, + d_sort_input.get(), + d_sort_output.get(), sort_data_size, compare_op, stream, - false - ) - ); + false)); // Launch binary_search - HIP_CHECK( - rocprim::binary_search(d_temp_storage, + HIP_CHECK(rocprim::binary_search(d_temp_storage.get(), search_temp_storage_bytes, - d_sort_output, - d_search_needles, - d_search_output, + d_sort_output.get(), + d_search_needles.get(), + d_search_output.get(), sort_data_size, search_needle_size, compare_op, stream, - false - ); - ); + false);); // End graph capture, but do not execute the graph yet. gHelper.endStreamCapture(stream); @@ -200,25 +187,21 @@ TEST(TestHipGraphAlgs, SortAndSearch) computeExpectedSortAndSearchResult(sort_input, search_needles, expected_search_output, compare_op); // Copy input data to the device - HIP_CHECK(hipMemcpy(d_sort_input, sort_input.data(), sort_data_size * sizeof(key_type), hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_search_needles, search_needles.data(), search_needle_size * sizeof(key_type), hipMemcpyHostToDevice)); - + d_sort_input.store(sort_input); + d_search_needles.store(search_needles); + // Launch the graph gHelper.launchGraph(stream, true); // Copy output back to host - HIP_CHECK(hipMemcpy(device_output.data(), d_search_output, search_needle_size * sizeof(key_type), hipMemcpyDeviceToHost)); - + device_output = d_search_output.load(); + // Validate the results ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(device_output, expected_search_output)); } // Clean up - HIP_CHECK(hipFree(d_sort_input)); - HIP_CHECK(hipFree(d_sort_output)); - HIP_CHECK(hipFree(d_search_output)); - HIP_CHECK(hipFree(d_search_needles)); - HIP_CHECK(hipFree(d_temp_storage)); + gHelper.cleanupGraphHelper(); HIP_CHECK(hipStreamDestroy(stream)); } diff --git a/test/rocprim/detail/get_rocprim_version.cpp b/test/rocprim/detail/get_rocprim_version.cpp index 0ffb93286..fec53dd71 100644 --- a/test/rocprim/detail/get_rocprim_version.cpp +++ b/test/rocprim/detail/get_rocprim_version.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2022 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,6 +21,7 @@ // SOFTWARE. #include "../../common_test_header.hpp" +#include "../../rocprim/test_utils_device_ptr.hpp" #include "get_rocprim_version.hpp" @@ -33,29 +34,11 @@ void get_version_kernel(unsigned int * version) unsigned int get_rocprim_version_on_device() { - unsigned int version = 0; + test_utils::device_ptr d_version(1); - unsigned int * d_version; - HIP_CHECK(hipMalloc(reinterpret_cast(&d_version), sizeof(unsigned int))); - HIP_CHECK(hipDeviceSynchronize()); - - hipLaunchKernelGGL( - get_version_kernel, - dim3(1), dim3(1), 0, 0, - d_version - ); + hipLaunchKernelGGL(get_version_kernel, dim3(1), dim3(1), 0, 0, d_version.get()); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); - HIP_CHECK( - hipMemcpy( - &version, d_version, - sizeof(unsigned int), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); - HIP_CHECK(hipFree(d_version)); - - return version; + return d_version.load()[0]; } diff --git a/test/rocprim/test_arg_index_iterator.cpp b/test/rocprim/test_arg_index_iterator.cpp index 32022d0d2..d9f134b07 100644 --- a/test/rocprim/test_arg_index_iterator.cpp +++ b/test/rocprim/test_arg_index_iterator.cpp @@ -29,7 +29,7 @@ // required test headers #include "test_seed.hpp" #include "test_utils.hpp" - +#include "test_utils_device_ptr.hpp" // Params for tests template @@ -129,20 +129,10 @@ TYPED_TEST(RocprimArgIndexIteratorTests, ReduceArgMinimum) std::vector input = test_utils::get_random_data(size, 1, 200, seed_value); std::vector output(1); - T * d_input; - key_value * d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(key_value))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(output.size()); - Iterator d_iter(d_input); + Iterator d_iter(d_input.get()); arg_min reduce_op; const key_value max(test_utils::numeric_limits::max(), @@ -154,50 +144,40 @@ TYPED_TEST(RocprimArgIndexIteratorTests, ReduceArgMinimum) // temp storage size_t temp_storage_size_bytes; - void * d_temp_storage = nullptr; + // Get size of d_temp_storage - HIP_CHECK( - rocprim::reduce( - d_temp_storage, temp_storage_size_bytes, - d_iter, d_output, max, input.size(), - reduce_op, stream, debug_synchronous - ) - ); + HIP_CHECK(rocprim::reduce(nullptr, + temp_storage_size_bytes, + d_iter, + d_output.get(), + max, + input.size(), + reduce_op, + stream, + debug_synchronous)); // temp_storage_size_bytes must be >0 ASSERT_GT(temp_storage_size_bytes, 0); - - // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); // Run - HIP_CHECK( - rocprim::reduce( - d_temp_storage, temp_storage_size_bytes, - d_iter, d_output, max, input.size(), - reduce_op, stream, debug_synchronous - ) - ); + HIP_CHECK(rocprim::reduce(d_temp_storage.get(), + temp_storage_size_bytes, + d_iter, + d_output.get(), + max, + input.size(), + reduce_op, + stream, + debug_synchronous)); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - output.size() * sizeof(key_value), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + output = d_output.load(); // Check if output values are as expected test_utils::assert_eq(output[0].key, expected.key); test_utils::assert_eq(output[0].value, expected.value); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_temp_storage)); } } diff --git a/test/rocprim/test_block_histogram.kernels.hpp b/test/rocprim/test_block_histogram.kernels.hpp index e6e285ccd..62fb8d97d 100644 --- a/test/rocprim/test_block_histogram.kernels.hpp +++ b/test/rocprim/test_block_histogram.kernels.hpp @@ -30,6 +30,7 @@ // required test headers #include "../common_test_header.hpp" +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" #include @@ -153,48 +154,25 @@ void test_block_histogram_input_arrays() } // Preparing device - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(T))); - BinType* device_output_bin; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output_bin, output_bin.size() * sizeof(BinType))); - - HIP_CHECK( - hipMemcpy( - device_output, output.data(), - output.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - - HIP_CHECK( - hipMemcpy( - device_output_bin, output_bin.data(), - output_bin.size() * sizeof(BinType), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_output(output); + test_utils::device_ptr device_output_bin(output_bin); // Running kernel hipLaunchKernelGGL( - HIP_KERNEL_NAME(histogram_kernel), - dim3(grid_size), dim3(block_size), 0, 0, - device_output, device_output_bin - ); + HIP_KERNEL_NAME( + histogram_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_output.get(), + device_output_bin.get()); HIP_CHECK(hipGetLastError()); // Reading results back - HIP_CHECK( - hipMemcpy( - output_bin.data(), device_output_bin, - output_bin.size() * sizeof(BinType), - hipMemcpyDeviceToHost - ) - ); + output_bin = device_output_bin.load(); test_utils::assert_eq(output_bin, expected_bin); - - HIP_CHECK(hipFree(device_output)); - HIP_CHECK(hipFree(device_output_bin)); } } diff --git a/test/rocprim/test_block_radix_rank.cpp.in b/test/rocprim/test_block_radix_rank.cpp.in index dc4170ade..4e3f7f9be 100644 --- a/test/rocprim/test_block_radix_rank.cpp.in +++ b/test/rocprim/test_block_radix_rank.cpp.in @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -29,6 +29,7 @@ #include // required test headers +#include "test_utils_device_ptr.hpp" #include "test_utils_sort_comparator.hpp" #include "test_utils_types.hpp" diff --git a/test/rocprim/test_block_radix_rank.hpp b/test/rocprim/test_block_radix_rank.hpp index c05059d1e..4f35e6474 100644 --- a/test/rocprim/test_block_radix_rank.hpp +++ b/test/rocprim/test_block_radix_rank.hpp @@ -168,13 +168,8 @@ void test_block_radix_rank() } } - T* d_keys_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_input, size * sizeof(T))); - unsigned int* d_ranks_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_ranks_output, size * sizeof(unsigned int))); - - HIP_CHECK( - hipMemcpy(d_keys_input, keys_input.data(), size * sizeof(T), hipMemcpyHostToDevice)); + test_utils::device_ptr d_keys_input(keys_input); + test_utils::device_ptr d_ranks_output(size); // Running kernel hipLaunchKernelGGL( @@ -184,25 +179,17 @@ void test_block_radix_rank() dim3(block_size), 0, 0, - d_keys_input, - d_ranks_output, + d_keys_input.get(), + d_ranks_output.get(), descending, start_bit, radix_bits); HIP_CHECK(hipGetLastError()); // Getting results to host - std::vector ranks_output(size); - HIP_CHECK(hipMemcpy(ranks_output.data(), - d_ranks_output, - size * sizeof(unsigned int), - hipMemcpyDeviceToHost)); - HIP_CHECK(hipDeviceSynchronize()); + auto ranks_output = d_ranks_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(ranks_output, expected)); - - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_ranks_output)); } } diff --git a/test/rocprim/test_block_radix_sort.cpp.in b/test/rocprim/test_block_radix_sort.cpp.in index 1fdf44f33..9320a14c5 100644 --- a/test/rocprim/test_block_radix_sort.cpp.in +++ b/test/rocprim/test_block_radix_sort.cpp.in @@ -28,6 +28,7 @@ #include // required test headers +#include "test_utils_device_ptr.hpp" #include "test_utils_sort_comparator.hpp" #include "test_utils_types.hpp" diff --git a/test/rocprim/test_block_radix_sort.kernels.hpp b/test/rocprim/test_block_radix_sort.kernels.hpp index 83dc8a0c2..ab808e22d 100644 --- a/test/rocprim/test_block_radix_sort.kernels.hpp +++ b/test/rocprim/test_block_radix_sort.kernels.hpp @@ -216,16 +216,10 @@ auto test_block_radix_sort() -> typename std::enable_if::type } // Preparing device - key_type* device_keys_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_keys_output, size * sizeof(key_type))); - - HIP_CHECK(hipMemcpy(device_keys_output, - keys_output.get(), - size * sizeof(keys_output[0]), - hipMemcpyHostToDevice)); + test_utils::device_ptr device_keys_output(keys_output.get(), size); sort_key_kernel - <<>>(device_keys_output, + <<>>(device_keys_output.get(), to_striped, descending, start_bit, @@ -234,7 +228,7 @@ auto test_block_radix_sort() -> typename std::enable_if::type // Getting results to host HIP_CHECK(hipMemcpy(keys_output.get(), - device_keys_output, + device_keys_output.get(), size * sizeof(keys_output[0]), hipMemcpyDeviceToHost)); @@ -243,8 +237,6 @@ auto test_block_radix_sort() -> typename std::enable_if::type keys_output.get() + size, expected.begin(), expected.end())); - - HIP_CHECK(hipFree(device_keys_output)); } } @@ -332,23 +324,8 @@ auto test_block_radix_sort() -> typename std::enable_if::type values_expected[i] = expected[i].second; } - key_type* device_keys_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_keys_output, size * sizeof(key_type))); - value_type* device_values_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_values_output, values_output.size() * sizeof(value_type))); - - HIP_CHECK(hipMemcpy(device_keys_output, - keys_output.get(), - size * sizeof(keys_output[0]), - hipMemcpyHostToDevice)); - - HIP_CHECK( - hipMemcpy( - device_values_output, values_output.data(), - values_output.size() * sizeof(typename decltype(values_output)::value_type), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_keys_output(keys_output.get(), size); + test_utils::device_ptr device_values_output(values_output); // Running kernel sort_key_value_kernel typename std::enable_if::type radix_bits_per_pass, key_type, value_type> - <<>>(device_keys_output, - device_values_output, + <<>>(device_keys_output.get(), + device_values_output.get(), to_striped, descending, start_bit, @@ -366,26 +343,16 @@ auto test_block_radix_sort() -> typename std::enable_if::type // Getting results to host HIP_CHECK(hipMemcpy(keys_output.get(), - device_keys_output, + device_keys_output.get(), size * sizeof(keys_output[0]), hipMemcpyDeviceToHost)); - - HIP_CHECK( - hipMemcpy( - values_output.data(), device_values_output, - values_output.size() * sizeof(typename decltype(values_output)::value_type), - hipMemcpyDeviceToHost - ) - ); + values_output = device_values_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(keys_output.get(), keys_output.get() + size, keys_expected.begin(), keys_expected.end())); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(values_output, values_expected)); - - HIP_CHECK(hipFree(device_keys_output)); - HIP_CHECK(hipFree(device_values_output)); } } diff --git a/test/rocprim/test_block_reduce.cpp b/test/rocprim/test_block_reduce.cpp index aac92dc10..d344d8ec9 100644 --- a/test/rocprim/test_block_reduce.cpp +++ b/test/rocprim/test_block_reduce.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2022 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -28,6 +28,7 @@ #include // required test headers +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" // kernel definitions diff --git a/test/rocprim/test_block_reduce.kernels.hpp b/test/rocprim/test_block_reduce.kernels.hpp index 971a273ee..874776fc0 100644 --- a/test/rocprim/test_block_reduce.kernels.hpp +++ b/test/rocprim/test_block_reduce.kernels.hpp @@ -267,49 +267,26 @@ void test_block_reduce_input_arrays() } // Preparing device - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(T))); - T* device_output_reductions; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output_reductions, output_reductions.size() * sizeof(T))); - - HIP_CHECK( - hipMemcpy( - device_output, output.data(), - output.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - - HIP_CHECK( - hipMemcpy( - device_output_reductions, output_reductions.data(), - output_reductions.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_output(output); + test_utils::device_ptr device_output_reductions(output_reductions); // Running kernel hipLaunchKernelGGL( - HIP_KERNEL_NAME(reduce_array_kernel), - dim3(grid_size), dim3(block_size), 0, 0, - device_output, device_output_reductions - ); + HIP_KERNEL_NAME( + reduce_array_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_output.get(), + device_output_reductions.get()); HIP_CHECK(hipGetLastError()); // Reading results back - HIP_CHECK( - hipMemcpy( - output_reductions.data(), device_output_reductions, - output_reductions.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + output_reductions = device_output_reductions.load(); // Verifying results test_utils::assert_eq(output_reductions, expected_reductions); - - HIP_CHECK(hipFree(device_output)); - HIP_CHECK(hipFree(device_output_reductions)); } } diff --git a/test/rocprim/test_block_run_length_decode.cpp b/test/rocprim/test_block_run_length_decode.cpp index 8afd944bf..5f60754b2 100644 --- a/test/rocprim/test_block_run_length_decode.cpp +++ b/test/rocprim/test_block_run_length_decode.cpp @@ -32,6 +32,7 @@ #include "rocprim/block/block_load_func.hpp" #include "rocprim/block/block_store_func.hpp" #include "rocprim/functional.hpp" +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" template d_run_items(run_items); + test_utils::device_ptr d_run_lengths(run_lengths); + test_utils::device_ptr d_decoded_runs(expected.size()); + test_utils::device_ptr d_decoded_offsets(expected.size()); + block_run_length_decode_kernel - <<>>(d_run_items, - d_run_lengths, - d_decoded_runs, - d_decoded_offsets); + <<>>(d_run_items.get(), + d_run_lengths.get(), + d_decoded_runs.get(), + d_decoded_offsets.get()); HIP_CHECK(hipPeekAtLastError()); HIP_CHECK(hipDeviceSynchronize()); - std::vector output(expected.size()); - HIP_CHECK(hipMemcpy(output.data(), - d_decoded_runs, - output.size() * sizeof(ItemT), - hipMemcpyDeviceToHost)); - HIP_CHECK(hipGetLastError()) - - std::vector offsets(expected.size()); - HIP_CHECK(hipMemcpy(offsets.data(), - d_decoded_offsets, - offsets.size() * sizeof(LengthT), - hipMemcpyDeviceToHost)); - - HIP_CHECK(hipFree(d_run_items)); - HIP_CHECK(hipFree(d_run_lengths)); - HIP_CHECK(hipFree(d_decoded_runs)); - HIP_CHECK(hipFree(d_decoded_offsets)); + std::vector output = d_decoded_runs.load(); + std::vector offsets = d_decoded_offsets.load(); unsigned int expected_offset = -1; ItemT previous_value = ItemT{}; diff --git a/test/rocprim/test_config_dispatch.cpp b/test/rocprim/test_config_dispatch.cpp index 62ebda06b..7f5f47a3e 100644 --- a/test/rocprim/test_config_dispatch.cpp +++ b/test/rocprim/test_config_dispatch.cpp @@ -1,9 +1,33 @@ +// MIT License +// +// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + #include "../common_test_header.hpp" #include #include +#include "test_utils_device_ptr.hpp" + using rocprim::detail::target_arch; __global__ void write_target_arch(target_arch* dest_arch) @@ -49,19 +73,15 @@ TEST(RocprimConfigDispatchTests, HostMatchesDevice) target_arch host_arch; HIP_CHECK(rocprim::detail::host_target_arch(stream, host_arch)); - target_arch* device_arch_ptr; - HIP_CHECK(hipMalloc(&device_arch_ptr, sizeof(*device_arch_ptr))); + test_utils::device_ptr device_arch_ptr(1); - hipLaunchKernelGGL(write_target_arch, dim3(1), dim3(1), 0, stream, device_arch_ptr); + hipLaunchKernelGGL(write_target_arch, dim3(1), dim3(1), 0, stream, device_arch_ptr.get()); HIP_CHECK(hipGetLastError()); - target_arch device_arch; - HIP_CHECK(hipMemcpy(&device_arch, device_arch_ptr, sizeof(device_arch), hipMemcpyDeviceToHost)); + target_arch device_arch = device_arch_ptr.load()[0]; ASSERT_NE(host_arch, target_arch::invalid); ASSERT_EQ(host_arch, device_arch); - - HIP_CHECK(hipFree(device_arch_ptr)) } TEST(RocprimConfigDispatchTests, ParseCommonArches) diff --git a/test/rocprim/test_constant_iterator.cpp b/test/rocprim/test_constant_iterator.cpp index dfdf132e1..2224ce360 100644 --- a/test/rocprim/test_constant_iterator.cpp +++ b/test/rocprim/test_constant_iterator.cpp @@ -27,6 +27,7 @@ #include // required test headers +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" // Params for tests @@ -87,37 +88,26 @@ TYPED_TEST(RocprimConstantIteratorTests, Transform) Iterator input_begin(value); std::vector output(size); - T * d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(T))); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_output(output.size()); // Calculate expected results on host std::vector expected(size, transform()(value)); // Run - HIP_CHECK( - rocprim::transform( - input_begin, d_output, size, - transform(), stream, debug_synchronous - ) - ); + HIP_CHECK(rocprim::transform(input_begin, + d_output.get(), + size, + transform(), + stream, + debug_synchronous)); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + output = d_output.load(); // Validating results test_utils::assert_near(output, expected, test_utils::precision); - - HIP_CHECK(hipFree(d_output)); } } diff --git a/test/rocprim/test_counting_iterator.cpp b/test/rocprim/test_counting_iterator.cpp index 7273eed72..555acd114 100644 --- a/test/rocprim/test_counting_iterator.cpp +++ b/test/rocprim/test_counting_iterator.cpp @@ -27,6 +27,7 @@ #include // required test headers +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" // Params for tests @@ -87,9 +88,7 @@ TYPED_TEST(RocprimCountingIteratorTests, Transform) Iterator input_begin(test_utils::get_random_value(0, 200, seed_value)); std::vector output(size); - T * d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(T))); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_output(output.size()); // Calculate expected results on host std::vector expected(size); @@ -101,32 +100,23 @@ TYPED_TEST(RocprimCountingIteratorTests, Transform) ); // Run - HIP_CHECK( - rocprim::transform( - input_begin, d_output, size, - transform(), stream, debug_synchronous - ) - ); + HIP_CHECK(rocprim::transform(input_begin, + d_output.get(), + size, + transform(), + stream, + debug_synchronous)); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + output = d_output.load(); // Validating results for(size_t i = 0; i < output.size(); i++) { ASSERT_EQ(output[i], expected[i]) << "where index = " << i; } - - HIP_CHECK(hipFree(d_output)); } } diff --git a/test/rocprim/test_device_adjacent_find.cpp b/test/rocprim/test_device_adjacent_find.cpp index 50ee889a9..614dba654 100644 --- a/test/rocprim/test_device_adjacent_find.cpp +++ b/test/rocprim/test_device_adjacent_find.cpp @@ -24,6 +24,7 @@ #include "indirect_iterator.hpp" #include "test_utils_custom_test_types.hpp" +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" #include @@ -159,32 +160,24 @@ TYPED_TEST(RocprimDeviceAdjacentFindTests, AdjacentFind) std::iota(input.begin(), input.begin() + first_adj_index, 0); std::fill(input.begin(), input.end(), first_adj_index); - T* d_input; - output_type* d_output; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(*d_input))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, sizeof(*d_output))); - HIP_CHECK(hipMemcpy(d_input, - input.data(), - input.size() * sizeof(*d_input), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(1); const auto output_it - = test_utils::wrap_in_identity_iterator(d_output); + = test_utils::wrap_in_identity_iterator(d_output.get()); // Allocate temporary storage std::size_t tmp_storage_size; - void* d_tmp_storage = nullptr; - HIP_CHECK(::rocprim::adjacent_find(d_tmp_storage, + HIP_CHECK(::rocprim::adjacent_find(nullptr, tmp_storage_size, - d_input, + d_input.get(), output_it, size, op, stream, debug_synchronous)); ASSERT_GT(tmp_storage_size, 0); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_tmp_storage, tmp_storage_size)); + test_utils::device_ptr d_tmp_storage(tmp_storage_size); test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) @@ -193,9 +186,9 @@ TYPED_TEST(RocprimDeviceAdjacentFindTests, AdjacentFind) } // Run - HIP_CHECK(::rocprim::adjacent_find(d_tmp_storage, + HIP_CHECK(::rocprim::adjacent_find(d_tmp_storage.get(), tmp_storage_size, - d_input, + d_input.get(), output_it, size, op, @@ -211,8 +204,7 @@ TYPED_TEST(RocprimDeviceAdjacentFindTests, AdjacentFind) HIP_CHECK(hipDeviceSynchronize()); // Allocate memory for output and copy to host side - output_type output; - HIP_CHECK(hipMemcpy(&output, d_output, sizeof(*d_output), hipMemcpyDeviceToHost)); + auto output = d_output.load()[0]; // Calculate expected results on host const auto expected @@ -222,11 +214,6 @@ TYPED_TEST(RocprimDeviceAdjacentFindTests, AdjacentFind) // Check if output values are as expected ASSERT_EQ(output, expected); - // Cleanup - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_tmp_storage)); - if(TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); diff --git a/test/rocprim/test_device_partition.cpp b/test/rocprim/test_device_partition.cpp index a3dedc186..951cb769d 100644 --- a/test/rocprim/test_device_partition.cpp +++ b/test/rocprim/test_device_partition.cpp @@ -29,6 +29,7 @@ #include // required test headers +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" // Params for tests @@ -117,18 +118,10 @@ TYPED_TEST(RocprimDevicePartitionTests, Flagged) std::vector input = test_utils::get_random_data(size, 1, 100, seed_value); std::vector flags = test_utils::get_random_data01(size, 0.25, seed_value); - T * d_input; - F * d_flags; - U * d_output; - unsigned int * d_selected_count_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_flags, flags.size() * sizeof(F))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, input.size() * sizeof(U))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_selected_count_output, sizeof(unsigned int))); - HIP_CHECK( - hipMemcpy(d_input, input.data(), input.size() * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK( - hipMemcpy(d_flags, flags.data(), flags.size() * sizeof(F), hipMemcpyHostToDevice)); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_flags(flags); + test_utils::device_ptr d_output(input.size()); + test_utils::device_ptr d_selected_count_output(1); // Calculate expected_selected and expected_rejected results on host std::vector expected_selected; @@ -155,10 +148,10 @@ TYPED_TEST(RocprimDevicePartitionTests, Flagged) HIP_CHECK(rocprim::partition( nullptr, temp_storage_size_bytes, - d_input, - d_flags, - test_utils::wrap_in_identity_iterator(d_output), - d_selected_count_output, + d_input.get(), + d_flags.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), + d_selected_count_output.get(), input.size(), stream, debug_synchronous)); @@ -167,8 +160,7 @@ TYPED_TEST(RocprimDevicePartitionTests, Flagged) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - void* d_temp_storage = nullptr; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::use_graphs) @@ -178,12 +170,12 @@ TYPED_TEST(RocprimDevicePartitionTests, Flagged) // Run HIP_CHECK(rocprim::partition( - d_temp_storage, + d_temp_storage.get(), temp_storage_size_bytes, - d_input, - d_flags, - test_utils::wrap_in_identity_iterator(d_output), - d_selected_count_output, + d_input.get(), + d_flags.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), + d_selected_count_output.get(), input.size(), stream, debug_synchronous)); @@ -194,19 +186,11 @@ TYPED_TEST(RocprimDevicePartitionTests, Flagged) } // Check if number of selected value is as expected_selected - unsigned int selected_count_output = 0; - HIP_CHECK(hipMemcpy(&selected_count_output, - d_selected_count_output, - sizeof(unsigned int), - hipMemcpyDeviceToHost)); + unsigned int selected_count_output = d_selected_count_output.load()[0]; ASSERT_EQ(selected_count_output, expected_selected.size()); // Check if output values are as expected_selected - std::vector output(input.size()); - HIP_CHECK(hipMemcpy(output.data(), - d_output, - output.size() * sizeof(U), - hipMemcpyDeviceToHost)); + auto output = d_output.load(); std::vector output_rejected; for(size_t i = 0; i < expected_rejected.size(); i++) @@ -217,12 +201,6 @@ TYPED_TEST(RocprimDevicePartitionTests, Flagged) ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected_selected, expected_selected.size())); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output_rejected, expected_rejected, expected_rejected.size())); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_flags)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_selected_count_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if(TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -270,18 +248,12 @@ TYPED_TEST(RocprimDevicePartitionTests, PredicateEmptyInput) auto select_op = select_op_t{}; - U * d_output; - unsigned int * d_selected_count_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, sizeof(U))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_selected_count_output, sizeof(unsigned int))); unsigned int selected_count_output = 123; - HIP_CHECK(hipMemcpy(d_selected_count_output, - &selected_count_output, - sizeof(unsigned int), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_output(1); + test_utils::device_ptr d_selected_count_output(selected_count_output); test_utils::out_of_bounds_flag out_of_bounds; - test_utils::bounds_checking_iterator d_checking_output(d_output, + test_utils::bounds_checking_iterator d_checking_output(d_output.get(), out_of_bounds.device_pointer(), 0); @@ -292,15 +264,14 @@ TYPED_TEST(RocprimDevicePartitionTests, PredicateEmptyInput) temp_storage_size_bytes, rocprim::make_constant_iterator(T(345)), d_checking_output, - d_selected_count_output, + d_selected_count_output.get(), 0, select_op, stream, debug_synchronous)); // allocate temporary storage - void* d_temp_storage = nullptr; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::use_graphs) @@ -309,17 +280,16 @@ TYPED_TEST(RocprimDevicePartitionTests, PredicateEmptyInput) } // Run - HIP_CHECK(rocprim::partition(d_temp_storage, + HIP_CHECK(rocprim::partition(d_temp_storage.get(), temp_storage_size_bytes, rocprim::make_constant_iterator(T(345)), d_checking_output, - d_selected_count_output, + d_selected_count_output.get(), 0, select_op, stream, debug_synchronous)); - if(TestFixture::use_graphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -329,19 +299,9 @@ TYPED_TEST(RocprimDevicePartitionTests, PredicateEmptyInput) ASSERT_FALSE(out_of_bounds.get()); // Check if number of selected value is 0 - HIP_CHECK( - hipMemcpy( - &selected_count_output, d_selected_count_output, - sizeof(unsigned int), - hipMemcpyDeviceToHost - ) - ); + selected_count_output = d_selected_count_output.load()[0]; ASSERT_EQ(selected_count_output, 0); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_selected_count_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if (TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -382,14 +342,9 @@ TYPED_TEST(RocprimDevicePartitionTests, Predicate) // Generate data std::vector input = test_utils::get_random_data(size, 1, 100, seed_value); - T * d_input; - U * d_output; - unsigned int * d_selected_count_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, input.size() * sizeof(U))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_selected_count_output, sizeof(unsigned int))); - HIP_CHECK( - hipMemcpy(d_input, input.data(), input.size() * sizeof(T), hipMemcpyHostToDevice)); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(input.size()); + test_utils::device_ptr d_selected_count_output(1); // Calculate expected_selected and expected_rejected results on host std::vector expected_selected; @@ -415,9 +370,9 @@ TYPED_TEST(RocprimDevicePartitionTests, Predicate) HIP_CHECK(rocprim::partition( nullptr, temp_storage_size_bytes, - d_input, - test_utils::wrap_in_identity_iterator(d_output), - d_selected_count_output, + d_input.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), + d_selected_count_output.get(), input.size(), select_op, stream, @@ -427,8 +382,7 @@ TYPED_TEST(RocprimDevicePartitionTests, Predicate) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - void* d_temp_storage = nullptr; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::use_graphs) @@ -438,17 +392,16 @@ TYPED_TEST(RocprimDevicePartitionTests, Predicate) // Run HIP_CHECK(rocprim::partition( - d_temp_storage, + d_temp_storage.get(), temp_storage_size_bytes, - d_input, - test_utils::wrap_in_identity_iterator(d_output), - d_selected_count_output, + d_input.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), + d_selected_count_output.get(), input.size(), select_op, stream, debug_synchronous)); - if(TestFixture::use_graphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -457,19 +410,11 @@ TYPED_TEST(RocprimDevicePartitionTests, Predicate) HIP_CHECK(hipDeviceSynchronize()); // Check if number of selected value is as expected_selected - unsigned int selected_count_output = 0; - HIP_CHECK(hipMemcpy(&selected_count_output, - d_selected_count_output, - sizeof(unsigned int), - hipMemcpyDeviceToHost)); + unsigned int selected_count_output = d_selected_count_output.load()[0]; ASSERT_EQ(selected_count_output, expected_selected.size()); // Check if output values are as expected_selected - std::vector output(input.size()); - HIP_CHECK(hipMemcpy(output.data(), - d_output, - output.size() * sizeof(U), - hipMemcpyDeviceToHost)); + const auto output = d_output.load(); std::vector output_rejected; for(size_t i = 0; i < expected_rejected.size(); i++) @@ -480,11 +425,6 @@ TYPED_TEST(RocprimDevicePartitionTests, Predicate) ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected_selected, expected_selected.size())); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output_rejected, expected_rejected, expected_rejected.size())); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_selected_count_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if(TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -532,19 +472,10 @@ TYPED_TEST(RocprimDevicePartitionTests, PredicateTwoWay) // Generate data std::vector input = test_utils::get_random_data(size, 1, 100, seed_value); - T* d_input; - U* d_selected; - U* d_rejected; - - unsigned int* d_selected_count_output; - - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_selected, input.size() * sizeof(U))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_rejected, input.size() * sizeof(U))); - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_selected_count_output, sizeof(unsigned int))); - HIP_CHECK( - hipMemcpy(d_input, input.data(), input.size() * sizeof(T), hipMemcpyHostToDevice)); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_selected(input.size()); + test_utils::device_ptr d_rejected(input.size()); + test_utils::device_ptr d_selected_count_output(1); // Calculate expected_selected and expected_rejected results on host std::vector expected_selected; @@ -569,10 +500,10 @@ TYPED_TEST(RocprimDevicePartitionTests, PredicateTwoWay) HIP_CHECK(rocprim::partition_two_way( nullptr, temp_storage_size_bytes, - d_input, - test_utils::wrap_in_identity_iterator(d_selected), - test_utils::wrap_in_identity_iterator(d_rejected), - d_selected_count_output, + d_input.get(), + test_utils::wrap_in_identity_iterator(d_selected.get()), + test_utils::wrap_in_identity_iterator(d_rejected.get()), + d_selected_count_output.get(), input.size(), select_op, stream, @@ -582,8 +513,7 @@ TYPED_TEST(RocprimDevicePartitionTests, PredicateTwoWay) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - void* d_temp_storage = nullptr; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::use_graphs) @@ -593,18 +523,17 @@ TYPED_TEST(RocprimDevicePartitionTests, PredicateTwoWay) // Run HIP_CHECK(rocprim::partition_two_way( - d_temp_storage, + d_temp_storage.get(), temp_storage_size_bytes, - d_input, - test_utils::wrap_in_identity_iterator(d_selected), - test_utils::wrap_in_identity_iterator(d_rejected), - d_selected_count_output, + d_input.get(), + test_utils::wrap_in_identity_iterator(d_selected.get()), + test_utils::wrap_in_identity_iterator(d_rejected.get()), + d_selected_count_output.get(), input.size(), select_op, stream, debug_synchronous)); - if(TestFixture::use_graphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -613,36 +542,18 @@ TYPED_TEST(RocprimDevicePartitionTests, PredicateTwoWay) HIP_CHECK(hipDeviceSynchronize()); // Check if number of selected value is as expected - unsigned int selected_count_output = 0; - HIP_CHECK(hipMemcpy(&selected_count_output, - d_selected_count_output, - sizeof(unsigned int), - hipMemcpyDeviceToHost)); + unsigned int selected_count_output = d_selected_count_output.load()[0]; ASSERT_EQ(selected_count_output, expected_selected.size()); // Check if output values are as expected - std::vector selected(input.size()); - std::vector rejected(input.size()); - HIP_CHECK(hipMemcpy(selected.data(), - d_selected, - selected.size() * sizeof(U), - hipMemcpyDeviceToHost)); - HIP_CHECK(hipMemcpy(rejected.data(), - d_rejected, - rejected.size() * sizeof(U), - hipMemcpyDeviceToHost)); + const auto selected = d_selected.load(); + const auto rejected = d_rejected.load(); ASSERT_NO_FATAL_FAILURE( test_utils::assert_eq(selected, expected_selected, expected_selected.size())); ASSERT_NO_FATAL_FAILURE( test_utils::assert_eq(rejected, expected_rejected, expected_rejected.size())); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_selected)); - HIP_CHECK(hipFree(d_rejected)); - HIP_CHECK(hipFree(d_selected_count_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if(TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -713,25 +624,11 @@ TYPED_TEST(RocprimDevicePartitionTests, PredicateThreeWay) // Output auto selected_counts = std::array{}; - T* d_input = nullptr; - U* d_first_output = nullptr; - U* d_second_output = nullptr; - U* d_unselected_output = nullptr; - unsigned int* d_selected_counts = nullptr; - - HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(T))); - HIP_CHECK(hipMalloc(&d_first_output, input.size() * sizeof(U))); - HIP_CHECK(hipMalloc(&d_second_output, input.size() * sizeof(U))); - HIP_CHECK(hipMalloc(&d_unselected_output, input.size() * sizeof(U))); - HIP_CHECK(hipMalloc(&d_selected_counts, sizeof(selected_counts))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_first_output(input.size()); + test_utils::device_ptr d_second_output(input.size()); + test_utils::device_ptr d_unselected_output(input.size()); + test_utils::device_ptr d_selected_counts(2); const auto first_op = LessOp{std::get<0>(limits)}; const auto second_op = LessOp{std::get<1>(limits)}; @@ -760,12 +657,14 @@ TYPED_TEST(RocprimDevicePartitionTests, PredicateThreeWay) HIP_CHECK(rocprim::partition_three_way( nullptr, temp_storage_size_bytes, - d_input, - test_utils::wrap_in_identity_iterator(d_first_output), - test_utils::wrap_in_identity_iterator(d_second_output), + d_input.get(), test_utils::wrap_in_identity_iterator( - d_unselected_output), - d_selected_counts, + d_first_output.get()), + test_utils::wrap_in_identity_iterator( + d_second_output.get()), + test_utils::wrap_in_identity_iterator( + d_unselected_output.get()), + d_selected_counts.get(), input.size(), first_op, second_op, @@ -776,8 +675,7 @@ TYPED_TEST(RocprimDevicePartitionTests, PredicateThreeWay) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - void* d_temp_storage = nullptr; - HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_size_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::use_graphs) @@ -787,21 +685,22 @@ TYPED_TEST(RocprimDevicePartitionTests, PredicateThreeWay) // Run HIP_CHECK(rocprim::partition_three_way( - d_temp_storage, + d_temp_storage.get(), temp_storage_size_bytes, - d_input, - test_utils::wrap_in_identity_iterator(d_first_output), - test_utils::wrap_in_identity_iterator(d_second_output), + d_input.get(), + test_utils::wrap_in_identity_iterator( + d_first_output.get()), test_utils::wrap_in_identity_iterator( - d_unselected_output), - d_selected_counts, + d_second_output.get()), + test_utils::wrap_in_identity_iterator( + d_unselected_output.get()), + d_selected_counts.get(), input.size(), first_op, second_op, stream, debug_synchronous)); - if(TestFixture::use_graphs) { gHelper.createAndLaunchGraph(stream); @@ -810,52 +709,33 @@ TYPED_TEST(RocprimDevicePartitionTests, PredicateThreeWay) HIP_CHECK(hipDeviceSynchronize()); // Check if number of selected value is as expected_selected - HIP_CHECK( - hipMemcpy( - selected_counts.data(), d_selected_counts, - sizeof(selected_counts), - hipMemcpyDeviceToHost - ) - ); + HIP_CHECK(hipMemcpy(selected_counts.data(), + d_selected_counts.get(), + sizeof(selected_counts), + hipMemcpyDeviceToHost)); ASSERT_EQ(selected_counts, expected_counts); // Check if output values are as expected_selected const auto output = [&]{ auto result = std::vector(input.size()); - HIP_CHECK( - hipMemcpy( - result.data(), d_first_output, - expected_counts[0] * sizeof(result[0]), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK( - hipMemcpy( - result.data() + expected_counts[0], d_second_output, - expected_counts[1] * sizeof(result[0]), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK( - hipMemcpy( - result.data() + expected_counts[0] + expected_counts[1], - d_unselected_output, - (input.size() - expected_counts[0] - expected_counts[1]) * sizeof(result[0]), - hipMemcpyDeviceToHost - ) - ); + HIP_CHECK(hipMemcpy(result.data(), + d_first_output.get(), + expected_counts[0] * sizeof(result[0]), + hipMemcpyDeviceToHost)); + HIP_CHECK(hipMemcpy(result.data() + expected_counts[0], + d_second_output.get(), + expected_counts[1] * sizeof(result[0]), + hipMemcpyDeviceToHost)); + HIP_CHECK(hipMemcpy(result.data() + expected_counts[0] + expected_counts[1], + d_unselected_output.get(), + (input.size() - expected_counts[0] - expected_counts[1]) + * sizeof(result[0]), + hipMemcpyDeviceToHost)); return result; }(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected, expected.size())); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_first_output)); - HIP_CHECK(hipFree(d_second_output)); - HIP_CHECK(hipFree(d_unselected_output)); - HIP_CHECK(hipFree(d_selected_counts)); - HIP_CHECK(hipFree(d_temp_storage)); - if(TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -1152,9 +1032,8 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartition) size_t* d_count_output{}; HIP_CHECK(test_common_utils::hipMallocHelper(&d_count_output, sizeof(*d_count_output))); - void* d_temporary_storage{}; size_t temporary_storage_size{}; - HIP_CHECK(rocprim::partition(d_temporary_storage, + HIP_CHECK(rocprim::partition(nullptr, temporary_storage_size, input_iterator, output_checker_it, @@ -1163,9 +1042,8 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartition) predicate, stream, debug_synchronous)); - ASSERT_NE(0, temporary_storage_size); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_size)); + test_utils::device_ptr d_temporary_storage(temporary_storage_size); test_utils::GraphHelper gHelper;; if(use_graphs) @@ -1173,7 +1051,7 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartition) gHelper.startStreamCapture(stream); } - HIP_CHECK(rocprim::partition(d_temporary_storage, + HIP_CHECK(rocprim::partition(d_temporary_storage.get(), temporary_storage_size, input_iterator, output_checker_it, @@ -1183,7 +1061,6 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartition) stream, debug_synchronous)); - if(use_graphs) { gHelper.createAndLaunchGraph(stream); @@ -1208,7 +1085,6 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartition) ASSERT_EQ(incorrect_flag, 0); - HIP_CHECK(hipFree(d_temporary_storage)); HIP_CHECK(hipFree(d_count_output)); HIP_CHECK(hipFree(d_incorrect_flag)); @@ -1272,9 +1148,8 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartitionTwoWay) size_t* d_count_output{}; HIP_CHECK(test_common_utils::hipMallocHelper(&d_count_output, sizeof(*d_count_output))); - void* d_temporary_storage{}; size_t temporary_storage_size{}; - HIP_CHECK(rocprim::partition_two_way(d_temporary_storage, + HIP_CHECK(rocprim::partition_two_way(nullptr, temporary_storage_size, input_iterator, select_checker_it, @@ -1286,7 +1161,7 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartitionTwoWay) debug_synchronous)); ASSERT_NE(0, temporary_storage_size); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_size)); + test_utils::device_ptr d_temporary_storage(temporary_storage_size); test_utils::GraphHelper gHelper;; if(use_graphs) @@ -1294,7 +1169,7 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartitionTwoWay) gHelper.startStreamCapture(stream); } - HIP_CHECK(rocprim::partition_two_way(d_temporary_storage, + HIP_CHECK(rocprim::partition_two_way(d_temporary_storage.get(), temporary_storage_size, input_iterator, select_checker_it, @@ -1305,7 +1180,6 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartitionTwoWay) stream, debug_synchronous)); - if(use_graphs) { gHelper.createAndLaunchGraph(stream); @@ -1337,7 +1211,6 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartitionTwoWay) ASSERT_EQ(incorrect_select_flag, 0); ASSERT_EQ(incorrect_reject_flag, 0); - HIP_CHECK(hipFree(d_temporary_storage)); HIP_CHECK(hipFree(d_count_output)); HIP_CHECK(hipFree(d_incorrect_select_flag)); HIP_CHECK(hipFree(d_incorrect_reject_flag)); @@ -1399,9 +1272,8 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartitionThreeWay) size_t* d_count_output{}; HIP_CHECK(test_common_utils::hipMallocHelper(&d_count_output, 2 * sizeof(*d_count_output))); - void* d_temporary_storage{}; size_t temporary_storage_size{}; - HIP_CHECK(rocprim::partition_three_way(d_temporary_storage, + HIP_CHECK(rocprim::partition_three_way(nullptr, temporary_storage_size, input_iterator, output_checker_a, @@ -1413,17 +1285,16 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartitionThreeWay) predicate_b, stream, debug_synchronous)); - ASSERT_NE(0, temporary_storage_size); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_size)); + test_utils::device_ptr d_temporary_storage(temporary_storage_size); - test_utils::GraphHelper gHelper;; + test_utils::GraphHelper gHelper; if(use_graphs) { gHelper.startStreamCapture(stream); } - HIP_CHECK(rocprim::partition_three_way(d_temporary_storage, + HIP_CHECK(rocprim::partition_three_way(d_temporary_storage.get(), temporary_storage_size, input_iterator, output_checker_a, @@ -1436,7 +1307,6 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartitionThreeWay) stream, debug_synchronous)); - if(use_graphs) { gHelper.createAndLaunchGraph(stream); @@ -1465,7 +1335,6 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartitionThreeWay) ASSERT_EQ(incorrect_flag, 0); - HIP_CHECK(hipFree(d_temporary_storage)); HIP_CHECK(hipFree(d_count_output)); HIP_CHECK(hipFree(d_incorrect_flag)); @@ -1553,14 +1422,9 @@ TEST(RocprimDevicePartitionBlockSizeTests, BlockSize) for (size_t i = 0; i < size; i++) memcpy(input[i].data, input_data.data() + i * test_obj_size, test_obj_size); - T * d_input; - U * d_output; - unsigned int * d_selected_count_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, input.size() * sizeof(U))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_selected_count_output, sizeof(unsigned int))); - HIP_CHECK( - hipMemcpy(d_input, input.data(), input.size() * sizeof(T), hipMemcpyHostToDevice)); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(input.size()); + test_utils::device_ptr d_selected_count_output(1); // Calculate expected_selected and expected_rejected results on host std::vector expected_selected; @@ -1583,52 +1447,41 @@ TEST(RocprimDevicePartitionBlockSizeTests, BlockSize) // temp storage size_t temp_storage_size_bytes; // Get size of d_temp_storage - HIP_CHECK(rocprim::partition( - nullptr, - temp_storage_size_bytes, - d_input, - d_output, - d_selected_count_output, - input.size(), - select_op, - stream, - debug_synchronous)); + HIP_CHECK(rocprim::partition(nullptr, + temp_storage_size_bytes, + d_input.get(), + d_output.get(), + d_selected_count_output.get(), + input.size(), + select_op, + stream, + debug_synchronous)); // temp_storage_size_bytes must be >0 ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - void* d_temp_storage = nullptr; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); // Run - HIP_CHECK(rocprim::partition( - d_temp_storage, - temp_storage_size_bytes, - d_input, - d_output, - d_selected_count_output, - input.size(), - select_op, - stream, - debug_synchronous)); + HIP_CHECK(rocprim::partition(d_temp_storage.get(), + temp_storage_size_bytes, + d_input.get(), + d_output.get(), + d_selected_count_output.get(), + input.size(), + select_op, + stream, + debug_synchronous)); HIP_CHECK(hipDeviceSynchronize()); // Check if number of selected value is as expected_selected - unsigned int selected_count_output = 0; - HIP_CHECK(hipMemcpy(&selected_count_output, - d_selected_count_output, - sizeof(unsigned int), - hipMemcpyDeviceToHost)); + unsigned int selected_count_output = d_selected_count_output.load()[0]; ASSERT_EQ(selected_count_output, expected_selected.size()); // Check if output values are as expected_selected - std::vector output(input.size()); - HIP_CHECK(hipMemcpy(output.data(), - d_output, - output.size() * sizeof(U), - hipMemcpyDeviceToHost)); + const auto output = d_output.load(); std::vector output_rejected; for(size_t i = 0; i < expected_rejected.size(); i++) @@ -1637,12 +1490,9 @@ TEST(RocprimDevicePartitionBlockSizeTests, BlockSize) output_rejected.push_back(output[j]); } ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected_selected, expected_selected.size())); - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output_rejected, expected_rejected, expected_rejected.size())); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_selected_count_output)); - HIP_CHECK(hipFree(d_temp_storage)); + ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output_rejected, + expected_rejected, + expected_rejected.size())); } } } \ No newline at end of file diff --git a/test/rocprim/test_device_search_n.cpp b/test/rocprim/test_device_search_n.cpp index af4c2fd09..32ae29507 100644 --- a/test/rocprim/test_device_search_n.cpp +++ b/test/rocprim/test_device_search_n.cpp @@ -117,10 +117,10 @@ TYPED_TEST(RocprimDeviceSearchNTests, RandomTest) for(const auto size : test_utils::get_sizes(seed_value)) { - hipStream_t stream = 0; // default - size_t count = test_utils::get_random_value(0, size, ++seed_value); - size_t temp_storage_size; - input_type h_value + hipStream_t stream = 0; // default + size_t count = test_utils::get_random_value(0, size, ++seed_value); + size_t temp_storage_size; + input_type h_value = test_utils::get_random_value(0, limit_type::max(), ++seed_value); @@ -140,7 +140,6 @@ TYPED_TEST(RocprimDeviceSearchNTests, RandomTest) test_utils::device_ptr d_input(h_input); test_utils::device_ptr d_value(&h_value, 1); test_utils::device_ptr d_output(1); - test_utils::device_ptr d_temp_storage; SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); SCOPED_TRACE(testing::Message() << "with count = " << count); @@ -159,8 +158,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, RandomTest) h_input.size(), count, nullptr)); - - d_temp_storage.resize(temp_storage_size); + test_utils::device_ptr d_temp_storage(temp_storage_size); test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) @@ -223,10 +221,10 @@ TYPED_TEST(RocprimDeviceSearchNTests, MaxCount) for(const auto size : test_utils::get_sizes(seed_value)) { - hipStream_t stream = 0; // default - size_t count = size; - size_t temp_storage_size; - input_type h_value + hipStream_t stream = 0; // default + size_t count = size; + size_t temp_storage_size; + input_type h_value = test_utils::get_random_value(0, limit_type::max(), ++seed_value); @@ -235,7 +233,6 @@ TYPED_TEST(RocprimDeviceSearchNTests, MaxCount) test_utils::device_ptr d_input(h_input); test_utils::device_ptr d_value(&h_value, 1); test_utils::device_ptr d_output(1); - test_utils::device_ptr d_temp_storage; SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); SCOPED_TRACE(testing::Message() << "with count = " << count); @@ -254,8 +251,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, MaxCount) h_input.size(), count, nullptr)); - - d_temp_storage.resize(temp_storage_size); + test_utils::device_ptr d_temp_storage(temp_storage_size); test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) @@ -320,10 +316,10 @@ TYPED_TEST(RocprimDeviceSearchNTests, MinCount) for(const auto size : test_utils::get_sizes(seed_value)) { - hipStream_t stream = 0; // default - size_t count = 0; - size_t temp_storage_size; - input_type h_value + hipStream_t stream = 0; // default + size_t count = 0; + size_t temp_storage_size; + input_type h_value = test_utils::get_random_value(0, limit_type::max(), ++seed_value); @@ -332,7 +328,6 @@ TYPED_TEST(RocprimDeviceSearchNTests, MinCount) test_utils::device_ptr d_input(h_input); test_utils::device_ptr d_value(&h_value, 1); test_utils::device_ptr d_output(1); - test_utils::device_ptr d_temp_storage; SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); SCOPED_TRACE(testing::Message() << "with count = " << count); @@ -352,9 +347,8 @@ TYPED_TEST(RocprimDeviceSearchNTests, MinCount) count, nullptr)); - d_temp_storage.resize(temp_storage_size); - - test_utils::GraphHelper gHelper; + test_utils::device_ptr d_temp_storage(temp_storage_size); + test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) { gHelper.startStreamCapture(stream); @@ -427,7 +421,6 @@ TYPED_TEST(RocprimDeviceSearchNTests, StartFromBegin) test_utils::device_ptr d_input(h_input); test_utils::device_ptr d_value(&h_value, 1); test_utils::device_ptr d_output(1); - test_utils::device_ptr d_temp_storage; SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); SCOPED_TRACE(testing::Message() << "with count = " << count); @@ -447,9 +440,8 @@ TYPED_TEST(RocprimDeviceSearchNTests, StartFromBegin) count, nullptr)); - d_temp_storage.resize(temp_storage_size); - - test_utils::GraphHelper gHelper; + test_utils::device_ptr d_temp_storage(temp_storage_size); + test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) { gHelper.startStreamCapture(stream); @@ -523,7 +515,6 @@ TYPED_TEST(RocprimDeviceSearchNTests, StartFromMiddle) test_utils::device_ptr d_input(h_input); test_utils::device_ptr d_value(&h_value, 1); test_utils::device_ptr d_output(1); - test_utils::device_ptr d_temp_storage; SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); SCOPED_TRACE(testing::Message() << "with count = " << count); @@ -543,9 +534,8 @@ TYPED_TEST(RocprimDeviceSearchNTests, StartFromMiddle) count, nullptr)); - d_temp_storage.resize(temp_storage_size); - - test_utils::GraphHelper gHelper; + test_utils::device_ptr d_temp_storage(temp_storage_size); + test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) { gHelper.startStreamCapture(stream); @@ -608,10 +598,10 @@ TYPED_TEST(RocprimDeviceSearchNTests, StartFromEnd) for(const auto size : test_utils::get_sizes(seed_value)) { - hipStream_t stream = 0; // default - size_t count = test_utils::get_random_value(0, size, ++seed_value); - size_t temp_storage_size; - input_type h_value{1}; + hipStream_t stream = 0; // default + size_t count = test_utils::get_random_value(0, size, ++seed_value); + size_t temp_storage_size; + input_type h_value{1}; std::vector h_input(size); std::fill(h_input.begin(), h_input.begin() + (size - count), 0); std::fill(h_input.begin() + (size - count), h_input.end(), h_value); @@ -619,7 +609,6 @@ TYPED_TEST(RocprimDeviceSearchNTests, StartFromEnd) test_utils::device_ptr d_input(h_input); test_utils::device_ptr d_value(&h_value, 1); test_utils::device_ptr d_output(1); - test_utils::device_ptr d_temp_storage; SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); SCOPED_TRACE(testing::Message() << "with count = " << count); @@ -639,9 +628,8 @@ TYPED_TEST(RocprimDeviceSearchNTests, StartFromEnd) count, nullptr)); - d_temp_storage.resize(temp_storage_size); - - test_utils::GraphHelper gHelper; + test_utils::device_ptr d_temp_storage(temp_storage_size); + test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) { gHelper.startStreamCapture(stream); @@ -704,10 +692,10 @@ TYPED_TEST(RocprimDeviceSearchNTests, StartFromEndButFail) for(const auto size : test_utils::get_sizes(seed_value)) { - hipStream_t stream = 0; // default - size_t count = test_utils::get_random_value(0, size, ++seed_value); - size_t temp_storage_size; - input_type h_value{1}; + hipStream_t stream = 0; // default + size_t count = test_utils::get_random_value(0, size, ++seed_value); + size_t temp_storage_size; + input_type h_value{1}; std::vector h_input(size); std::fill(h_input.begin(), h_input.begin() + (size - count), 0); std::fill(h_input.begin() + (size - count), h_input.end(), h_value); @@ -719,7 +707,6 @@ TYPED_TEST(RocprimDeviceSearchNTests, StartFromEndButFail) test_utils::device_ptr d_input(h_input); test_utils::device_ptr d_value(&h_value, 1); test_utils::device_ptr d_output(1); - test_utils::device_ptr d_temp_storage; SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); SCOPED_TRACE(testing::Message() << "with count = " << count); @@ -739,9 +726,8 @@ TYPED_TEST(RocprimDeviceSearchNTests, StartFromEndButFail) count, nullptr)); - d_temp_storage.resize(temp_storage_size); - - test_utils::GraphHelper gHelper; + test_utils::device_ptr d_temp_storage(temp_storage_size); + test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) { gHelper.startStreamCapture(stream); @@ -841,7 +827,6 @@ TYPED_TEST(RocprimDeviceSearchNTests, NoiseTest_1block) test_utils::device_ptr d_input(h_input); test_utils::device_ptr d_value(&h_value, 1); test_utils::device_ptr d_output(1); - test_utils::device_ptr d_temp_storage; SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); SCOPED_TRACE(testing::Message() << "with count = " << count); @@ -861,9 +846,8 @@ TYPED_TEST(RocprimDeviceSearchNTests, NoiseTest_1block) count, nullptr)); - d_temp_storage.resize(temp_storage_size); - - test_utils::GraphHelper gHelper; + test_utils::device_ptr d_temp_storage(temp_storage_size); + test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) { gHelper.startStreamCapture(stream); @@ -963,7 +947,6 @@ TYPED_TEST(RocprimDeviceSearchNTests, NoiseTest_2block) test_utils::device_ptr d_input(h_input); test_utils::device_ptr d_value(&h_value, 1); test_utils::device_ptr d_output(1); - test_utils::device_ptr d_temp_storage; SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); SCOPED_TRACE(testing::Message() << "with count = " << count); @@ -983,8 +966,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, NoiseTest_2block) count, nullptr)); - d_temp_storage.resize(temp_storage_size); - + test_utils::device_ptr d_temp_storage(temp_storage_size); test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) { @@ -1085,7 +1067,6 @@ TYPED_TEST(RocprimDeviceSearchNTests, NoiseTest_3block) test_utils::device_ptr d_input(h_input); test_utils::device_ptr d_value(&h_value, 1); test_utils::device_ptr d_output(1); - test_utils::device_ptr d_temp_storage; SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); SCOPED_TRACE(testing::Message() << "with count = " << count); @@ -1105,8 +1086,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, NoiseTest_3block) count, nullptr)); - d_temp_storage.resize(temp_storage_size); - + test_utils::device_ptr d_temp_storage(temp_storage_size); test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) { @@ -1209,7 +1189,6 @@ TYPED_TEST(RocprimDeviceSearchNTests, MultiResult1) test_utils::device_ptr d_input(h_input); test_utils::device_ptr d_value(&h_value, 1); test_utils::device_ptr d_output(1); - test_utils::device_ptr d_temp_storage; SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); SCOPED_TRACE(testing::Message() << "with count = " << count); @@ -1229,8 +1208,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, MultiResult1) count, nullptr)); - d_temp_storage.resize(temp_storage_size); - + test_utils::device_ptr d_temp_storage(temp_storage_size); test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) { @@ -1327,7 +1305,6 @@ TYPED_TEST(RocprimDeviceSearchNTests, MultiResult2) test_utils::device_ptr d_input(h_input); test_utils::device_ptr d_value(&h_value, 1); test_utils::device_ptr d_output(1); - test_utils::device_ptr d_temp_storage; SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); SCOPED_TRACE(testing::Message() << "with count = " << count); @@ -1347,8 +1324,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, MultiResult2) count, nullptr)); - d_temp_storage.resize(temp_storage_size); - + test_utils::device_ptr d_temp_storage(temp_storage_size); test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) { diff --git a/test/rocprim/test_device_segmented_scan.cpp b/test/rocprim/test_device_segmented_scan.cpp index f19236fd7..e4492b98a 100644 --- a/test/rocprim/test_device_segmented_scan.cpp +++ b/test/rocprim/test_device_segmented_scan.cpp @@ -26,37 +26,38 @@ #include // required test headers +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" #include -template< - class Input, - class Output, - class ScanOp = ::rocprim::plus, - int Init = 0, // as only integral types supported, int is used here even for floating point inputs - unsigned int MinSegmentLength = 0, - unsigned int MaxSegmentLength = 1000, - // Tests output iterator with void value_type (OutputIterator concept) - // Segmented scan primitives which use head flags do not support this kind - // of output iterators. - bool UseIdentityIterator = false, - bool UseGraphs = false -> +template, + int Init + = 0, // as only integral types supported, int is used here even for floating point inputs + unsigned int MinSegmentLength = 0, + unsigned int MaxSegmentLength = 1000, + // Tests output iterator with void value_type (OutputIterator concept) + // Segmented scan primitives which use head flags do not support this kind + // of output iterators. + bool UseIdentityIterator = false, + bool UseGraphs = false> struct params { - using input_type = Input; - using output_type = Output; - using scan_op_type = ScanOp; - static constexpr int init = Init; - static constexpr unsigned int min_segment_length = MinSegmentLength; - static constexpr unsigned int max_segment_length = MaxSegmentLength; - static constexpr bool use_identity_iterator = UseIdentityIterator; - static constexpr bool use_graphs = UseGraphs; + using input_type = Input; + using output_type = Output; + using scan_op_type = ScanOp; + static constexpr int init = Init; + static constexpr unsigned int min_segment_length = MinSegmentLength; + static constexpr unsigned int max_segment_length = MaxSegmentLength; + static constexpr bool use_identity_iterator = UseIdentityIterator; + static constexpr bool use_graphs = UseGraphs; }; template -class RocprimDeviceSegmentedScan : public ::testing::Test { +class RocprimDeviceSegmentedScan : public ::testing::Test +{ public: using params = Params; }; @@ -98,28 +99,28 @@ TYPED_TEST(RocprimDeviceSegmentedScan, InclusiveScan) using offset_type = unsigned int; constexpr bool use_identity_iterator = TestFixture::params::use_identity_iterator; - const bool debug_synchronous = false; + const bool debug_synchronous = false; scan_op_type scan_op; - std::random_device rd; + std::random_device rd; std::default_random_engine gen(rd()); std::uniform_int_distribution segment_length_dis( TestFixture::params::min_segment_length, - TestFixture::params::max_segment_length - ); + TestFixture::params::max_segment_length); hipStream_t stream = 0; // default stream - if (TestFixture::params::use_graphs) + if(TestFixture::params::use_graphs) { // Default stream does not support hipGraph stream capture, so create one HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { - unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); for(size_t size : test_utils::get_sizes(seed_value)) @@ -128,18 +129,19 @@ TYPED_TEST(RocprimDeviceSegmentedScan, InclusiveScan) // Generate data and calculate expected results std::vector values_expected(size); - std::vector values_input = test_utils::get_random_data(size, 0, 100, seed_value); + std::vector values_input + = test_utils::get_random_data(size, 0, 100, seed_value); std::vector offsets; - unsigned int segments_count = 0; - size_t offset = 0; + unsigned int segments_count = 0; + size_t offset = 0; size_t max_segment_length = 0; while(offset < size) { const size_t segment_length = segment_length_dis(gen); offsets.push_back(offset); - const size_t end = std::min(size, offset + segment_length); + const size_t end = std::min(size, offset + segment_length); max_segment_length = std::max(max_segment_length, end - offset); input_type aggregate = values_input[offset]; @@ -172,44 +174,25 @@ TYPED_TEST(RocprimDeviceSegmentedScan, InclusiveScan) continue; } - input_type * d_values_input; - offset_type * d_offsets; - output_type * d_values_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_values_input, size * sizeof(input_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_offsets, (segments_count + 1) * sizeof(offset_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_values_output, size * sizeof(output_type))); - HIP_CHECK( - hipMemcpy( - d_values_input, values_input.data(), - size * sizeof(input_type), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_offsets, offsets.data(), - (segments_count + 1) * sizeof(offset_type), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_values_input(values_input); + test_utils::device_ptr d_offsets(offsets); + test_utils::device_ptr d_values_output(size); size_t temporary_storage_bytes; HIP_CHECK(rocprim::segmented_inclusive_scan( nullptr, temporary_storage_bytes, - d_values_input, - test_utils::wrap_in_identity_iterator(d_values_output), + d_values_input.get(), + test_utils::wrap_in_identity_iterator(d_values_output.get()), segments_count, - d_offsets, - d_offsets + 1, + d_offsets.get(), + d_offsets.get() + 1, scan_op, stream, debug_synchronous)); ASSERT_GT(temporary_storage_bytes, 0); - void * d_temporary_storage; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); test_utils::GraphHelper gHelper; if(TestFixture::params::use_graphs) @@ -217,19 +200,18 @@ TYPED_TEST(RocprimDeviceSegmentedScan, InclusiveScan) gHelper.startStreamCapture(stream); } - HIP_CHECK( - rocprim::segmented_inclusive_scan( - d_temporary_storage, temporary_storage_bytes, - d_values_input, - test_utils::wrap_in_identity_iterator(d_values_output), - segments_count, - d_offsets, d_offsets + 1, - scan_op, - stream, debug_synchronous - ) - ); - - + HIP_CHECK(rocprim::segmented_inclusive_scan( + d_temporary_storage.get(), + temporary_storage_bytes, + d_values_input.get(), + test_utils::wrap_in_identity_iterator(d_values_output.get()), + segments_count, + d_offsets.get(), + d_offsets.get() + 1, + scan_op, + stream, + debug_synchronous)); + if(TestFixture::params::use_graphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -237,27 +219,14 @@ TYPED_TEST(RocprimDeviceSegmentedScan, InclusiveScan) HIP_CHECK(hipDeviceSynchronize()); - std::vector values_output(size); - HIP_CHECK( - hipMemcpy( - values_output.data(), d_values_output, - values_output.size() * sizeof(output_type), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + auto values_output = d_values_output.load(); ASSERT_NO_FATAL_FAILURE( test_utils::assert_near(values_output, values_expected, precision)); - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_values_input)); - HIP_CHECK(hipFree(d_offsets)); - HIP_CHECK(hipFree(d_values_output)); - if(TestFixture::params::use_graphs) { - gHelper.cleanupGraphHelper(); + gHelper.cleanupGraphHelper(); } } } @@ -287,24 +256,24 @@ TYPED_TEST(RocprimDeviceSegmentedScan, ExclusiveScan) scan_op_type scan_op; - std::random_device rd; + std::random_device rd; std::default_random_engine gen(rd()); std::uniform_int_distribution segment_length_dis( TestFixture::params::min_segment_length, - TestFixture::params::max_segment_length - ); + TestFixture::params::max_segment_length); hipStream_t stream = 0; // default stream - if (TestFixture::params::use_graphs) + if(TestFixture::params::use_graphs) { // Default stream does not support hipGraph stream capture, so create one HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { - unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); for(size_t size : test_utils::get_sizes(seed_value)) @@ -313,18 +282,19 @@ TYPED_TEST(RocprimDeviceSegmentedScan, ExclusiveScan) // Generate data and calculate expected results std::vector values_expected(size); - std::vector values_input = test_utils::get_random_data(size, 0, 100, seed_value); + std::vector values_input + = test_utils::get_random_data(size, 0, 100, seed_value); std::vector offsets; - unsigned int segments_count = 0; - size_t offset = 0; + unsigned int segments_count = 0; + size_t offset = 0; size_t max_segment_length = 0; while(offset < size) { const size_t segment_length = segment_length_dis(gen); offsets.push_back(offset); - const size_t end = std::min(size, offset + segment_length); + const size_t end = std::min(size, offset + segment_length); max_segment_length = std::max(max_segment_length, end - offset); input_type aggregate = init; @@ -357,66 +327,47 @@ TYPED_TEST(RocprimDeviceSegmentedScan, ExclusiveScan) continue; } - input_type * d_values_input; - offset_type * d_offsets; - output_type * d_values_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_values_input, size * sizeof(input_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_offsets, (segments_count + 1) * sizeof(offset_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_values_output, size * sizeof(output_type))); - HIP_CHECK( - hipMemcpy( - d_values_input, values_input.data(), - size * sizeof(input_type), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_offsets, offsets.data(), - (segments_count + 1) * sizeof(offset_type), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_values_input(values_input); + test_utils::device_ptr d_offsets(offsets); + test_utils::device_ptr d_values_output(size); size_t temporary_storage_bytes; - HIP_CHECK( - rocprim::segmented_exclusive_scan( - nullptr, temporary_storage_bytes, - d_values_input, - test_utils::wrap_in_identity_iterator(d_values_output), - segments_count, - d_offsets, d_offsets + 1, - init, scan_op, - stream, debug_synchronous - ) - ); + HIP_CHECK(rocprim::segmented_exclusive_scan( + nullptr, + temporary_storage_bytes, + d_values_input.get(), + test_utils::wrap_in_identity_iterator(d_values_output.get()), + segments_count, + d_offsets.get(), + d_offsets.get() + 1, + init, + scan_op, + stream, + debug_synchronous)); HIP_CHECK(hipDeviceSynchronize()); ASSERT_GT(temporary_storage_bytes, 0); - void * d_temporary_storage; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); - - test_utils::GraphHelper gHelper; + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); + test_utils::GraphHelper gHelper; if(TestFixture::params::use_graphs) { gHelper.startStreamCapture(stream); } - HIP_CHECK( - rocprim::segmented_exclusive_scan( - d_temporary_storage, temporary_storage_bytes, - d_values_input, - test_utils::wrap_in_identity_iterator(d_values_output), - segments_count, - d_offsets, d_offsets + 1, - init, scan_op, - stream, debug_synchronous - ) - ); - - + HIP_CHECK(rocprim::segmented_exclusive_scan( + d_temporary_storage.get(), + temporary_storage_bytes, + d_values_input.get(), + test_utils::wrap_in_identity_iterator(d_values_output.get()), + segments_count, + d_offsets.get(), + d_offsets.get() + 1, + init, + scan_op, + stream, + debug_synchronous)); + if(TestFixture::params::use_graphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -424,27 +375,14 @@ TYPED_TEST(RocprimDeviceSegmentedScan, ExclusiveScan) HIP_CHECK(hipDeviceSynchronize()); - std::vector values_output(size); - HIP_CHECK( - hipMemcpy( - values_output.data(), d_values_output, - values_output.size() * sizeof(output_type), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + auto values_output = d_values_output.load(); ASSERT_NO_FATAL_FAILURE( test_utils::assert_near(values_output, values_expected, precision)); - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_values_input)); - HIP_CHECK(hipFree(d_offsets)); - HIP_CHECK(hipFree(d_values_output)); - if(TestFixture::params::use_graphs) { - gHelper.cleanupGraphHelper(); + gHelper.cleanupGraphHelper(); } } } @@ -474,15 +412,16 @@ TYPED_TEST(RocprimDeviceSegmentedScan, InclusiveScanUsingHeadFlags) scan_op_type scan_op; hipStream_t stream = 0; // default stream - if (TestFixture::params::use_graphs) + if(TestFixture::params::use_graphs) { // Default stream does not support hipGraph stream capture, so create one HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { - unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); for(auto size : test_utils::get_sizes(seed_value)) @@ -490,8 +429,10 @@ TYPED_TEST(RocprimDeviceSegmentedScan, InclusiveScanUsingHeadFlags) SCOPED_TRACE(testing::Message() << "with size = " << size); // Generate data - std::vector input = test_utils::get_random_data(size, 1, 10, seed_value); - std::vector flags = test_utils::get_random_data(size, 0, 10, seed_value); + std::vector input + = test_utils::get_random_data(size, 1, 10, seed_value); + std::vector flags + = test_utils::get_random_data(size, 0, 10, seed_value); if(size != 0) flags[0] = 1U; @@ -534,27 +475,9 @@ TYPED_TEST(RocprimDeviceSegmentedScan, InclusiveScanUsingHeadFlags) continue; } - input_type * d_input; - flag_type * d_flags; - output_type * d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(input_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_flags, flags.size() * sizeof(flag_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, input.size() * sizeof(output_type))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - input.size() * sizeof(input_type), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_flags, flags.data(), - flags.size() * sizeof(flag_type), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_flags(flags); + test_utils::device_ptr d_output(input.size()); // Calculate expected results on host std::vector expected(input.size()); @@ -568,14 +491,15 @@ TYPED_TEST(RocprimDeviceSegmentedScan, InclusiveScanUsingHeadFlags) // temp storage size_t temp_storage_size_bytes; // Get size of d_temp_storage - HIP_CHECK( - rocprim::segmented_inclusive_scan( - nullptr, temp_storage_size_bytes, - d_input, d_output, d_flags, - input.size(), scan_op, stream, - debug_synchronous - ) - ); + HIP_CHECK(rocprim::segmented_inclusive_scan(nullptr, + temp_storage_size_bytes, + d_input.get(), + d_output.get(), + d_flags.get(), + input.size(), + scan_op, + stream, + debug_synchronous)); HIP_CHECK(hipDeviceSynchronize()); @@ -583,9 +507,7 @@ TYPED_TEST(RocprimDeviceSegmentedScan, InclusiveScanUsingHeadFlags) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - void * d_temp_storage = nullptr; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper; if(TestFixture::params::use_graphs) @@ -594,16 +516,16 @@ TYPED_TEST(RocprimDeviceSegmentedScan, InclusiveScanUsingHeadFlags) } // Run - HIP_CHECK( - rocprim::segmented_inclusive_scan( - d_temp_storage, temp_storage_size_bytes, - d_input, d_output, d_flags, - input.size(), scan_op, stream, - debug_synchronous - ) - ); - - + HIP_CHECK(rocprim::segmented_inclusive_scan(d_temp_storage.get(), + temp_storage_size_bytes, + d_input.get(), + d_output.get(), + d_flags.get(), + input.size(), + scan_op, + stream, + debug_synchronous)); + if(TestFixture::params::use_graphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -612,26 +534,13 @@ TYPED_TEST(RocprimDeviceSegmentedScan, InclusiveScanUsingHeadFlags) HIP_CHECK(hipDeviceSynchronize()); // Check if output values are as expected - std::vector output(input.size()); - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - output.size() * sizeof(output_type), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + auto output = d_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_near(output, expected, precision)); - HIP_CHECK(hipFree(d_temp_storage)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_flags)); - HIP_CHECK(hipFree(d_output)); - if(TestFixture::params::use_graphs) { - gHelper.cleanupGraphHelper(); + gHelper.cleanupGraphHelper(); } } } @@ -663,15 +572,16 @@ TYPED_TEST(RocprimDeviceSegmentedScan, ExclusiveScanUsingHeadFlags) scan_op_type scan_op; hipStream_t stream = 0; // default stream - if (TestFixture::params::use_graphs) + if(TestFixture::params::use_graphs) { // Default stream does not support hipGraph stream capture, so create one HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { - unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); for(auto size : test_utils::get_sizes(seed_value)) @@ -679,8 +589,10 @@ TYPED_TEST(RocprimDeviceSegmentedScan, ExclusiveScanUsingHeadFlags) SCOPED_TRACE(testing::Message() << "with size = " << size); // Generate data - std::vector input = test_utils::get_random_data(size, 1, 10, seed_value); - std::vector flags = test_utils::get_random_data(size, 0, 10, seed_value); + std::vector input + = test_utils::get_random_data(size, 1, 10, seed_value); + std::vector flags + = test_utils::get_random_data(size, 0, 10, seed_value); if(size != 0) flags[0] = 1U; @@ -723,27 +635,9 @@ TYPED_TEST(RocprimDeviceSegmentedScan, ExclusiveScanUsingHeadFlags) continue; } - input_type * d_input; - flag_type * d_flags; - output_type * d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(input_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_flags, flags.size() * sizeof(flag_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, input.size() * sizeof(output_type))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - input.size() * sizeof(input_type), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_flags, flags.data(), - flags.size() * sizeof(flag_type), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_flags(flags); + test_utils::device_ptr d_output(input.size()); // Calculate expected results on host std::vector expected(input.size()); @@ -758,13 +652,16 @@ TYPED_TEST(RocprimDeviceSegmentedScan, ExclusiveScanUsingHeadFlags) // temp storage size_t temp_storage_size_bytes; // Get size of d_temp_storage - HIP_CHECK( - rocprim::segmented_exclusive_scan( - nullptr, temp_storage_size_bytes, - d_input, d_output, d_flags, init, - input.size(), scan_op, stream, debug_synchronous - ) - ); + HIP_CHECK(rocprim::segmented_exclusive_scan(nullptr, + temp_storage_size_bytes, + d_input.get(), + d_output.get(), + d_flags.get(), + init, + input.size(), + scan_op, + stream, + debug_synchronous)); HIP_CHECK(hipDeviceSynchronize()); @@ -772,9 +669,7 @@ TYPED_TEST(RocprimDeviceSegmentedScan, ExclusiveScanUsingHeadFlags) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - void * d_temp_storage = nullptr; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper; if(TestFixture::params::use_graphs) @@ -783,15 +678,17 @@ TYPED_TEST(RocprimDeviceSegmentedScan, ExclusiveScanUsingHeadFlags) } // Run - HIP_CHECK( - rocprim::segmented_exclusive_scan( - d_temp_storage, temp_storage_size_bytes, - d_input, d_output, d_flags, init, - input.size(), scan_op, stream, debug_synchronous - ) - ); - - + HIP_CHECK(rocprim::segmented_exclusive_scan(d_temp_storage.get(), + temp_storage_size_bytes, + d_input.get(), + d_output.get(), + d_flags.get(), + init, + input.size(), + scan_op, + stream, + debug_synchronous)); + if(TestFixture::params::use_graphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -800,28 +697,16 @@ TYPED_TEST(RocprimDeviceSegmentedScan, ExclusiveScanUsingHeadFlags) HIP_CHECK(hipDeviceSynchronize()); // Check if output values are as expected - std::vector output(input.size()); - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - output.size() * sizeof(output_type), - hipMemcpyDeviceToHost - ) - ); + auto output = d_output.load(); if(TestFixture::params::use_graphs) { - gHelper.cleanupGraphHelper(); + gHelper.cleanupGraphHelper(); } HIP_CHECK(hipDeviceSynchronize()); ASSERT_NO_FATAL_FAILURE(test_utils::assert_near(output, expected, precision)); - - HIP_CHECK(hipFree(d_temp_storage)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_flags)); - HIP_CHECK(hipFree(d_output)); } } diff --git a/test/rocprim/test_device_select.cpp b/test/rocprim/test_device_select.cpp index ffe841f0a..4f30d675a 100644 --- a/test/rocprim/test_device_select.cpp +++ b/test/rocprim/test_device_select.cpp @@ -25,40 +25,39 @@ // required rocprim headers #include #include -#include #include +#include // required test headers +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" #include // Params for tests -template< - class InputType, - class OutputType = InputType, - class FlagType = unsigned int, - bool UseIdentityIterator = false, - bool UseGraphs = false -> +template struct DeviceSelectParams { - using input_type = InputType; - using output_type = OutputType; - using flag_type = FlagType; + using input_type = InputType; + using output_type = OutputType; + using flag_type = FlagType; static constexpr bool use_identity_iterator = UseIdentityIterator; - static constexpr bool use_graphs = UseGraphs; + static constexpr bool use_graphs = UseGraphs; }; template class RocprimDeviceSelectTests : public ::testing::Test { public: - using input_type = typename Params::input_type; - using output_type = typename Params::output_type; - using flag_type = typename Params::flag_type; - const bool debug_synchronous = false; + using input_type = typename Params::input_type; + using output_type = typename Params::output_type; + using flag_type = typename Params::flag_type; + const bool debug_synchronous = false; static constexpr bool use_identity_iterator = Params::use_identity_iterator; - static constexpr bool use_graphs = Params::use_graphs; + static constexpr bool use_graphs = Params::use_graphs; }; typedef ::testing::Types, @@ -84,21 +83,22 @@ TYPED_TEST(RocprimDeviceSelectTests, Flagged) SCOPED_TRACE(testing::Message() << "with device_id = " << device_id); HIP_CHECK(hipSetDevice(device_id)); - using T = typename TestFixture::input_type; - using U = typename TestFixture::output_type; - using F = typename TestFixture::flag_type; + using T = typename TestFixture::input_type; + using U = typename TestFixture::output_type; + using F = typename TestFixture::flag_type; static constexpr bool use_identity_iterator = TestFixture::use_identity_iterator; hipStream_t stream = 0; // default stream - if (TestFixture::use_graphs) + if(TestFixture::use_graphs) { // Default stream does not support hipGraph stream capture, so create one HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { - unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); for(auto size : test_utils::get_sizes(seed_value)) @@ -109,29 +109,10 @@ TYPED_TEST(RocprimDeviceSelectTests, Flagged) std::vector input = test_utils::get_random_data(size, 1, 100, seed_value); std::vector flags = test_utils::get_random_data(size, 0, 1, seed_value); - T * d_input; - F * d_flags; - U * d_output; - unsigned int * d_selected_count_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_flags, flags.size() * sizeof(F))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, input.size() * sizeof(U))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_selected_count_output, sizeof(unsigned int))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_flags, flags.data(), - flags.size() * sizeof(F), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_flags(flags); + test_utils::device_ptr d_output(input.size()); + test_utils::device_ptr d_selected_count_output(1); // Calculate expected results on host std::vector expected; @@ -150,10 +131,10 @@ TYPED_TEST(RocprimDeviceSelectTests, Flagged) HIP_CHECK(rocprim::select( nullptr, temp_storage_size_bytes, - d_input, - d_flags, - test_utils::wrap_in_identity_iterator(d_output), - d_selected_count_output, + d_input.get(), + d_flags.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), + d_selected_count_output.get(), input.size(), stream, TestFixture::debug_synchronous)); @@ -164,9 +145,7 @@ TYPED_TEST(RocprimDeviceSelectTests, Flagged) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - void * d_temp_storage = nullptr; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::use_graphs) @@ -175,21 +154,17 @@ TYPED_TEST(RocprimDeviceSelectTests, Flagged) } // Run - HIP_CHECK( - rocprim::select( - d_temp_storage, - temp_storage_size_bytes, - d_input, - d_flags, - test_utils::wrap_in_identity_iterator(d_output), - d_selected_count_output, - input.size(), - stream, - TestFixture::debug_synchronous - ) - ); + HIP_CHECK(rocprim::select( + d_temp_storage.get(), + temp_storage_size_bytes, + d_input.get(), + d_flags.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), + d_selected_count_output.get(), + input.size(), + stream, + TestFixture::debug_synchronous)); - if(TestFixture::use_graphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -198,35 +173,13 @@ TYPED_TEST(RocprimDeviceSelectTests, Flagged) HIP_CHECK(hipDeviceSynchronize()); // Check if number of selected value is as expected - unsigned int selected_count_output = 0; - HIP_CHECK( - hipMemcpy( - &selected_count_output, d_selected_count_output, - sizeof(unsigned int), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + unsigned int selected_count_output = d_selected_count_output.load()[0]; ASSERT_EQ(selected_count_output, expected.size()); // Check if output values are as expected - std::vector output(input.size()); - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - output.size() * sizeof(U), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + std::vector output = d_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected, expected.size())); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_flags)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_selected_count_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if(TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -240,12 +193,12 @@ TYPED_TEST(RocprimDeviceSelectTests, Flagged) } } - template struct select_op { - __device__ __host__ inline - bool operator()(const T& value) const + __device__ __host__ + inline bool + operator()(const T& value) const { return rocprim::less()(value, T(50)); } @@ -257,21 +210,22 @@ TYPED_TEST(RocprimDeviceSelectTests, SelectOp) SCOPED_TRACE(testing::Message() << "with device_id = " << device_id); HIP_CHECK(hipSetDevice(device_id)); - using T = typename TestFixture::input_type; - using U = typename TestFixture::output_type; + using T = typename TestFixture::input_type; + using U = typename TestFixture::output_type; static constexpr bool use_identity_iterator = TestFixture::use_identity_iterator; - const bool debug_synchronous = TestFixture::debug_synchronous; + const bool debug_synchronous = TestFixture::debug_synchronous; hipStream_t stream = 0; // default stream - if (TestFixture::use_graphs) + if(TestFixture::use_graphs) { // Default stream does not support hipGraph stream capture, so create one HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { - unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); for(auto size : test_utils::get_sizes(seed_value)) @@ -281,20 +235,9 @@ TYPED_TEST(RocprimDeviceSelectTests, SelectOp) // Generate data std::vector input = test_utils::get_random_data(size, 0, 100, seed_value); - T * d_input; - U * d_output; - unsigned int * d_selected_count_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, input.size() * sizeof(U))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_selected_count_output, sizeof(unsigned int))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(input.size()); + test_utils::device_ptr d_selected_count_output(1); // Calculate expected results on host std::vector expected; @@ -313,9 +256,9 @@ TYPED_TEST(RocprimDeviceSelectTests, SelectOp) HIP_CHECK(rocprim::select( nullptr, temp_storage_size_bytes, - d_input, - test_utils::wrap_in_identity_iterator(d_output), - d_selected_count_output, + d_input.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), + d_selected_count_output.get(), input.size(), select_op(), stream, @@ -327,9 +270,7 @@ TYPED_TEST(RocprimDeviceSelectTests, SelectOp) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - void * d_temp_storage = nullptr; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::use_graphs) @@ -338,21 +279,17 @@ TYPED_TEST(RocprimDeviceSelectTests, SelectOp) } // Run - HIP_CHECK( - rocprim::select( - d_temp_storage, - temp_storage_size_bytes, - d_input, - test_utils::wrap_in_identity_iterator(d_output), - d_selected_count_output, - input.size(), - select_op(), - stream, - debug_synchronous - ) - ); + HIP_CHECK(rocprim::select( + d_temp_storage.get(), + temp_storage_size_bytes, + d_input.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), + d_selected_count_output.get(), + input.size(), + select_op(), + stream, + debug_synchronous)); - if(TestFixture::use_graphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -361,34 +298,13 @@ TYPED_TEST(RocprimDeviceSelectTests, SelectOp) HIP_CHECK(hipDeviceSynchronize()); // Check if number of selected value is as expected - unsigned int selected_count_output = 0; - HIP_CHECK( - hipMemcpy( - &selected_count_output, d_selected_count_output, - sizeof(unsigned int), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + unsigned int selected_count_output = d_selected_count_output.load()[0]; ASSERT_EQ(selected_count_output, expected.size()); // Check if output values are as expected - std::vector output(input.size()); - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - output.size() * sizeof(U), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + std::vector output = d_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected, expected.size())); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_selected_count_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if(TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -546,9 +462,7 @@ TYPED_TEST(RocprimDeviceSelectTests, SelectFlagged) std::vector get_discontinuity_probabilities() { - std::vector probabilities = { - 0.05, 0.25, 0.5, 0.75, 0.95, 1 - }; + std::vector probabilities = {0.05, 0.25, 0.5, 0.75, 0.95, 1}; return probabilities; } @@ -565,18 +479,19 @@ TYPED_TEST(RocprimDeviceSelectTests, Unique) using scan_op_type = rocprim::plus; static constexpr bool use_identity_iterator = TestFixture::use_identity_iterator; - const bool debug_synchronous = TestFixture::debug_synchronous; + const bool debug_synchronous = TestFixture::debug_synchronous; hipStream_t stream = 0; // default stream - if (TestFixture::use_graphs) + if(TestFixture::use_graphs) { // Default stream does not support hipGraph stream capture, so create one HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { - unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); const auto probabilities = get_discontinuity_probabilities(); @@ -591,26 +506,13 @@ TYPED_TEST(RocprimDeviceSelectTests, Unique) std::vector input(size); { std::vector input01 = test_utils::get_random_data01(size, p, seed_value); - std::partial_sum( - input01.begin(), input01.end(), input.begin(), scan_op_type() - ); + std::partial_sum(input01.begin(), input01.end(), input.begin(), scan_op_type()); } // Allocate and copy to device - T * d_input; - U * d_output; - unsigned int * d_selected_count_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, input.size() * sizeof(U))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_selected_count_output, sizeof(unsigned int))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(input.size()); + test_utils::device_ptr d_selected_count_output(1); // Calculate expected results on host std::vector expected; @@ -633,9 +535,9 @@ TYPED_TEST(RocprimDeviceSelectTests, Unique) HIP_CHECK(rocprim::unique( nullptr, temp_storage_size_bytes, - d_input, - test_utils::wrap_in_identity_iterator(d_output), - d_selected_count_output, + d_input.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), + d_selected_count_output.get(), input.size(), op_type(), stream, @@ -647,9 +549,7 @@ TYPED_TEST(RocprimDeviceSelectTests, Unique) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - void * d_temp_storage = nullptr; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::use_graphs) @@ -658,21 +558,17 @@ TYPED_TEST(RocprimDeviceSelectTests, Unique) } // Run - HIP_CHECK( - rocprim::unique( - d_temp_storage, - temp_storage_size_bytes, - d_input, - test_utils::wrap_in_identity_iterator(d_output), - d_selected_count_output, - input.size(), - op_type(), - stream, - debug_synchronous - ) - ); - - + HIP_CHECK(rocprim::unique( + d_temp_storage.get(), + temp_storage_size_bytes, + d_input.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), + d_selected_count_output.get(), + input.size(), + op_type(), + stream, + debug_synchronous)); + if(TestFixture::use_graphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -681,34 +577,13 @@ TYPED_TEST(RocprimDeviceSelectTests, Unique) HIP_CHECK(hipDeviceSynchronize()); // Check if number of selected value is as expected - unsigned int selected_count_output = 0; - HIP_CHECK( - hipMemcpy( - &selected_count_output, d_selected_count_output, - sizeof(unsigned int), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + unsigned int selected_count_output = d_selected_count_output.load()[0]; ASSERT_EQ(selected_count_output, expected.size()); // Check if output values are as expected - std::vector output(input.size()); - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - output.size() * sizeof(U), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + std::vector output = d_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected, expected.size())); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_selected_count_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if(TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -727,18 +602,19 @@ TYPED_TEST(RocprimDeviceSelectTests, Unique) template struct element_equal_operator { - F *data; + F* data; element_equal_operator(F* _data) { - this->data = _data; + this->data = _data; } __host__ __device__ - bool operator()(const T& index_a, const T& index_b) const + bool operator()(const T& index_a, const T& index_b) const { F lhs = data[index_a]; F rhs = data[index_b]; - if (lhs != rhs) { + if(lhs != rhs) + { return false; } return true; @@ -752,23 +628,24 @@ void testUniqueGuardedOperator() SCOPED_TRACE(testing::Message() << "with device_id = " << device_id); HIP_CHECK(hipSetDevice(device_id)); - using T = int64_t; - using F = int64_t; - using U = int64_t; - using scan_op_type = rocprim::plus; + using T = int64_t; + using F = int64_t; + using U = int64_t; + using scan_op_type = rocprim::plus; static constexpr bool use_identity_iterator = false; - const bool debug_synchronous = false; + const bool debug_synchronous = false; hipStream_t stream = 0; // default stream - if (UseGraphs) + if(UseGraphs) { // Default stream does not support hipGraph stream capture, so create one HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { - unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); const auto probabilities = get_discontinuity_probabilities(); @@ -780,42 +657,26 @@ void testUniqueGuardedOperator() SCOPED_TRACE(testing::Message() << "with p = " << p); // Generate data - std::vector input = test_utils::get_random_data(size, 0, size - 1, seed_value); + std::vector input + = test_utils::get_random_data(size, 0, size - 1, seed_value); std::vector input_flag(size); { - std::vector input01 = test_utils::get_random_data01(size, p, seed_value + 1); - std::partial_sum( - input01.begin(), input01.end(), input_flag.begin(), scan_op_type() - ); + std::vector input01 + = test_utils::get_random_data01(size, p, seed_value + 1); + std::partial_sum(input01.begin(), + input01.end(), + input_flag.begin(), + scan_op_type()); } // Allocate and copy to device - T * d_input; - F * d_flag; - U * d_output; - unsigned int * d_selected_count_output; - HIP_CHECK(hipMalloc(reinterpret_cast(&d_input), input.size() * sizeof(T))); - HIP_CHECK(hipMalloc(reinterpret_cast(&d_flag), input_flag.size() * sizeof(F))); - HIP_CHECK(hipMalloc(reinterpret_cast(&d_output), input.size() * sizeof(U))); - HIP_CHECK(hipMalloc(reinterpret_cast(&d_selected_count_output), sizeof(unsigned int))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_flag, input_flag.data(), - input_flag.size() * sizeof(F), - hipMemcpyHostToDevice - ) - ); - element_equal_operator device_equal_op(d_flag); - element_equal_operator host_equal_op(input_flag.data()); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_flag(input_flag); + test_utils::device_ptr d_output(input.size()); + test_utils::device_ptr d_selected_count_output(1); + element_equal_operator device_equal_op(d_flag.get()); + element_equal_operator host_equal_op(input_flag.data()); // Calculate expected results on host std::vector expected; @@ -835,19 +696,16 @@ void testUniqueGuardedOperator() // temp storage size_t temp_storage_size_bytes; // Get size of d_temp_storage - HIP_CHECK( - rocprim::unique( - nullptr, - temp_storage_size_bytes, - d_input, - test_utils::wrap_in_identity_iterator(d_output), - d_selected_count_output, - input.size(), - device_equal_op, - stream, - debug_synchronous - ) - ); + HIP_CHECK(rocprim::unique( + nullptr, + temp_storage_size_bytes, + d_input.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), + d_selected_count_output.get(), + input.size(), + device_equal_op, + stream, + debug_synchronous)); HIP_CHECK(hipDeviceSynchronize()); @@ -855,9 +713,7 @@ void testUniqueGuardedOperator() ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - void * d_temp_storage = nullptr; - HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(UseGraphs) @@ -866,21 +722,17 @@ void testUniqueGuardedOperator() } // Run - HIP_CHECK( - rocprim::unique( - d_temp_storage, - temp_storage_size_bytes, - d_input, - test_utils::wrap_in_identity_iterator(d_output), - d_selected_count_output, - input.size(), - device_equal_op, - stream, - debug_synchronous - ) - ); - - + HIP_CHECK(rocprim::unique( + d_temp_storage.get(), + temp_storage_size_bytes, + d_input.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), + d_selected_count_output.get(), + input.size(), + device_equal_op, + stream, + debug_synchronous)); + if(UseGraphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -889,35 +741,13 @@ void testUniqueGuardedOperator() HIP_CHECK(hipDeviceSynchronize()); // Check if number of selected value is as expected - unsigned int selected_count_output = 0; - HIP_CHECK( - hipMemcpy( - &selected_count_output, d_selected_count_output, - sizeof(unsigned int), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + unsigned int selected_count_output = d_selected_count_output.load()[0]; ASSERT_EQ(selected_count_output, expected.size()); // Check if output values are as expected - std::vector output(input.size()); - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - output.size() * sizeof(U), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + auto output = d_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected, expected.size())); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_flag)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_selected_count_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if(UseGraphs) { gHelper.cleanupGraphHelper(); @@ -943,48 +773,46 @@ TEST(RocprimDeviceSelectTests, UniqueGuardedOperatorWithGraphs) } // Params for tests -template< - typename KeyType, - typename ValueType, - typename OutputKeyType = KeyType, - typename OutputValueType = ValueType, - bool UseIdentityIterator = false, - bool UseGraphs = false -> +template struct DeviceUniqueByKeyParams { - using key_type = KeyType; - using value_type = ValueType; - using output_key_type = OutputKeyType; - using output_value_type = OutputValueType; + using key_type = KeyType; + using value_type = ValueType; + using output_key_type = OutputKeyType; + using output_value_type = OutputValueType; static constexpr bool use_identity_iterator = UseIdentityIterator; - static constexpr bool use_graphs = UseGraphs; + static constexpr bool use_graphs = UseGraphs; }; template class RocprimDeviceUniqueByKeyTests : public ::testing::Test { public: - using key_type = typename Params::key_type; - using value_type = typename Params::value_type; - using output_key_type = typename Params::output_key_type; - using output_value_type = typename Params::output_value_type; - const bool debug_synchronous = false; + using key_type = typename Params::key_type; + using value_type = typename Params::value_type; + using output_key_type = typename Params::output_key_type; + using output_value_type = typename Params::output_value_type; + const bool debug_synchronous = false; static constexpr bool use_identity_iterator = Params::use_identity_iterator; - const bool use_graphs = Params::use_graphs; + const bool use_graphs = Params::use_graphs; }; -typedef ::testing::Types< - DeviceUniqueByKeyParams, - DeviceUniqueByKeyParams, - DeviceUniqueByKeyParams, - DeviceUniqueByKeyParams, - DeviceUniqueByKeyParams, - DeviceUniqueByKeyParams, - DeviceUniqueByKeyParams, - DeviceUniqueByKeyParams, test_utils::custom_test_type>, - DeviceUniqueByKeyParams -> RocprimDeviceUniqueByKeyTestParams; +typedef ::testing::Types, + DeviceUniqueByKeyParams, + DeviceUniqueByKeyParams, + DeviceUniqueByKeyParams, + DeviceUniqueByKeyParams, + DeviceUniqueByKeyParams, + DeviceUniqueByKeyParams, + DeviceUniqueByKeyParams, + test_utils::custom_test_type>, + DeviceUniqueByKeyParams> + RocprimDeviceUniqueByKeyTestParams; TYPED_TEST_SUITE(RocprimDeviceUniqueByKeyTests, RocprimDeviceUniqueByKeyTestParams); @@ -994,27 +822,28 @@ TYPED_TEST(RocprimDeviceUniqueByKeyTests, UniqueByKey) SCOPED_TRACE(testing::Message() << "with device_id = " << device_id); HIP_CHECK(hipSetDevice(device_id)); - using key_type = typename TestFixture::key_type; - using value_type = typename TestFixture::value_type; - using output_key_type = typename TestFixture::output_key_type; + using key_type = typename TestFixture::key_type; + using value_type = typename TestFixture::value_type; + using output_key_type = typename TestFixture::output_key_type; using output_value_type = typename TestFixture::output_value_type; using op_type = rocprim::equal_to; - using scan_op_type = rocprim::plus; + using scan_op_type = rocprim::plus; static constexpr bool use_identity_iterator = TestFixture::use_identity_iterator; - const bool debug_synchronous = TestFixture::debug_synchronous; + const bool debug_synchronous = TestFixture::debug_synchronous; hipStream_t stream = 0; // default stream - if (TestFixture::use_graphs) + if(TestFixture::use_graphs) { // Default stream does not support hipGraph stream capture, so create one HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { - unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); const auto probabilities = get_discontinuity_probabilities(); @@ -1028,43 +857,25 @@ TYPED_TEST(RocprimDeviceUniqueByKeyTests, UniqueByKey) // Generate data std::vector input_keys(size); { - std::vector input01 = test_utils::get_random_data01(size, p, seed_value); - std::partial_sum( - input01.begin(), input01.end(), input_keys.begin(), scan_op_type() - ); + std::vector input01 + = test_utils::get_random_data01(size, p, seed_value); + std::partial_sum(input01.begin(), + input01.end(), + input_keys.begin(), + scan_op_type()); } const auto input_values = test_utils::get_random_data(size, -1000, 1000, seed_value); // Allocate and copy to device - key_type* d_keys_input; - value_type* d_values_input; - output_key_type* d_keys_output; - output_value_type* d_values_output; - unsigned int * d_selected_count_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_input, input_keys.size() * sizeof(input_keys[0]))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_values_input, input_values.size() * sizeof(input_values[0]))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_output, input_keys.size() * sizeof(output_key_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_values_output, input_values.size() * sizeof(output_value_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_selected_count_output, sizeof(unsigned int))); - HIP_CHECK( - hipMemcpy( - d_keys_input, input_keys.data(), - input_keys.size() * sizeof(input_keys[0]), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_values_input, input_values.data(), - input_values.size() * sizeof(input_values[0]), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_keys_input(input_keys); + test_utils::device_ptr d_values_input(input_values); + test_utils::device_ptr d_keys_output(input_keys.size()); + test_utils::device_ptr d_values_output(input_values.size()); + test_utils::device_ptr d_selected_count_output(1); // Calculate expected results on host - std::vector expected_keys; + std::vector expected_keys; std::vector expected_values; expected_keys.reserve(input_keys.size()); expected_values.reserve(input_values.size()); @@ -1085,21 +896,20 @@ TYPED_TEST(RocprimDeviceUniqueByKeyTests, UniqueByKey) // temp storage size_t temp_storage_size_bytes; // Get size of d_temp_storage - HIP_CHECK( - rocprim::unique_by_key( - nullptr, - temp_storage_size_bytes, - d_keys_input, - d_values_input, - test_utils::wrap_in_identity_iterator(d_keys_output), - test_utils::wrap_in_identity_iterator(d_values_output), - d_selected_count_output, - input_keys.size(), - op_type(), - stream, - debug_synchronous - ) - ); + HIP_CHECK(rocprim::unique_by_key( + nullptr, + temp_storage_size_bytes, + d_keys_input.get(), + d_values_input.get(), + test_utils::wrap_in_identity_iterator( + d_keys_output.get()), + test_utils::wrap_in_identity_iterator( + d_values_output.get()), + d_selected_count_output.get(), + input_keys.size(), + op_type(), + stream, + debug_synchronous)); HIP_CHECK(hipDeviceSynchronize()); @@ -1107,9 +917,7 @@ TYPED_TEST(RocprimDeviceUniqueByKeyTests, UniqueByKey) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - void * d_temp_storage = nullptr; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::use_graphs) @@ -1118,23 +926,21 @@ TYPED_TEST(RocprimDeviceUniqueByKeyTests, UniqueByKey) } // Run - HIP_CHECK( - rocprim::unique_by_key( - d_temp_storage, - temp_storage_size_bytes, - d_keys_input, - d_values_input, - test_utils::wrap_in_identity_iterator(d_keys_output), - test_utils::wrap_in_identity_iterator(d_values_output), - d_selected_count_output, - input_keys.size(), - op_type(), - stream, - debug_synchronous - ) - ); - - + HIP_CHECK(rocprim::unique_by_key( + d_temp_storage.get(), + temp_storage_size_bytes, + d_keys_input.get(), + d_values_input.get(), + test_utils::wrap_in_identity_iterator( + d_keys_output.get()), + test_utils::wrap_in_identity_iterator( + d_values_output.get()), + d_selected_count_output.get(), + input_keys.size(), + op_type(), + stream, + debug_synchronous)); + if(TestFixture::use_graphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -1143,44 +949,16 @@ TYPED_TEST(RocprimDeviceUniqueByKeyTests, UniqueByKey) HIP_CHECK(hipDeviceSynchronize()); // Check if number of selected value is as expected - unsigned int selected_count_output = 0; - HIP_CHECK( - hipMemcpy( - &selected_count_output, d_selected_count_output, - sizeof(unsigned int), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + auto selected_count_output = d_selected_count_output.load()[0]; ASSERT_EQ(selected_count_output, expected_keys.size()); // Check if outputs are as expected - std::vector output_keys(input_keys.size()); - HIP_CHECK( - hipMemcpy( - output_keys.data(), d_keys_output, - output_keys.size() * sizeof(output_keys[0]), - hipMemcpyDeviceToHost - ) - ); - std::vector output_values(input_values.size()); - HIP_CHECK( - hipMemcpy( - output_values.data(), d_values_output, - output_values.size() * sizeof(output_values[0]), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output_keys, expected_keys, expected_keys.size())); - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output_values, expected_values, expected_values.size())); - - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_values_input)); - HIP_CHECK(hipFree(d_keys_output)); - HIP_CHECK(hipFree(d_values_output)); - HIP_CHECK(hipFree(d_selected_count_output)); - HIP_CHECK(hipFree(d_temp_storage)); + auto output_keys = d_keys_output.load(); + auto output_values = d_values_output.load(); + ASSERT_NO_FATAL_FAILURE( + test_utils::assert_eq(output_keys, expected_keys, expected_keys.size())); + ASSERT_NO_FATAL_FAILURE( + test_utils::assert_eq(output_values, expected_values, expected_values.size())); if(TestFixture::use_graphs) { @@ -1250,26 +1028,9 @@ TYPED_TEST(RocprimDeviceUniqueByKeyTests, UniqueByKeyAlias) = test_utils::get_random_data(size, -1000, 1000, seed_value); // Allocate and copy to device - key_type* d_keys_input; - value_type* d_values_input; - unsigned int* d_selected_count_output; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_keys_input, - input_keys.size() * sizeof(input_keys[0]))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_values_input, - input_values.size() - * sizeof(input_values[0]))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_selected_count_output, - sizeof(unsigned int))); - HIP_CHECK(hipMemcpy(d_keys_input, - input_keys.data(), - input_keys.size() * sizeof(input_keys[0]), - hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_values_input, - input_values.data(), - input_values.size() * sizeof(input_values[0]), - hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_keys_input(input_keys); + test_utils::device_ptr d_values_input(input_values); + test_utils::device_ptr d_selected_count_output(1); // Calculate expected results on host std::vector expected_keys; @@ -1296,11 +1057,13 @@ TYPED_TEST(RocprimDeviceUniqueByKeyTests, UniqueByKeyAlias) HIP_CHECK(rocprim::unique_by_key( nullptr, temp_storage_size_bytes, - d_keys_input, - d_values_input, - test_utils::wrap_in_identity_iterator(d_keys_input), - test_utils::wrap_in_identity_iterator(d_values_input), - d_selected_count_output, + d_keys_input.get(), + d_values_input.get(), + test_utils::wrap_in_identity_iterator( + d_keys_input.get()), + test_utils::wrap_in_identity_iterator( + d_values_input.get()), + d_selected_count_output.get(), input_keys.size(), op_type(), stream, @@ -1312,10 +1075,7 @@ TYPED_TEST(RocprimDeviceUniqueByKeyTests, UniqueByKeyAlias) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - void* d_temp_storage = nullptr; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::use_graphs) @@ -1325,19 +1085,20 @@ TYPED_TEST(RocprimDeviceUniqueByKeyTests, UniqueByKeyAlias) // Run HIP_CHECK(rocprim::unique_by_key( - d_temp_storage, + d_temp_storage.get(), temp_storage_size_bytes, - d_keys_input, - d_values_input, - test_utils::wrap_in_identity_iterator(d_keys_input), - test_utils::wrap_in_identity_iterator(d_values_input), - d_selected_count_output, + d_keys_input.get(), + d_values_input.get(), + test_utils::wrap_in_identity_iterator( + d_keys_input.get()), + test_utils::wrap_in_identity_iterator( + d_values_input.get()), + d_selected_count_output.get(), input_keys.size(), op_type(), stream, debug_synchronous)); - if(TestFixture::use_graphs) { gHelper.createAndLaunchGraph(stream); @@ -1346,36 +1107,17 @@ TYPED_TEST(RocprimDeviceUniqueByKeyTests, UniqueByKeyAlias) HIP_CHECK(hipDeviceSynchronize()); // Check if number of selected value is as expected - unsigned int selected_count_output = 0; - HIP_CHECK(hipMemcpy(&selected_count_output, - d_selected_count_output, - sizeof(unsigned int), - hipMemcpyDeviceToHost)); - HIP_CHECK(hipDeviceSynchronize()); + auto selected_count_output = d_selected_count_output.load()[0]; ASSERT_EQ(selected_count_output, expected_keys.size()); // Check if outputs are as expected - std::vector output_keys(input_keys.size()); - HIP_CHECK(hipMemcpy(output_keys.data(), - d_keys_input, - output_keys.size() * sizeof(output_keys[0]), - hipMemcpyDeviceToHost)); - std::vector output_values(input_values.size()); - HIP_CHECK(hipMemcpy(output_values.data(), - d_values_input, - output_values.size() * sizeof(output_values[0]), - hipMemcpyDeviceToHost)); - HIP_CHECK(hipDeviceSynchronize()); + auto output_keys = d_keys_input.load(); + auto output_values = d_values_input.load(); ASSERT_NO_FATAL_FAILURE( test_utils::assert_eq(output_keys, expected_keys, expected_keys.size())); ASSERT_NO_FATAL_FAILURE( test_utils::assert_eq(output_values, expected_values, expected_values.size())); - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_values_input)); - HIP_CHECK(hipFree(d_selected_count_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if(TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -1390,18 +1132,22 @@ TYPED_TEST(RocprimDeviceUniqueByKeyTests, UniqueByKeyAlias) } } -class RocprimDeviceSelectLargeInputTests : public ::testing::TestWithParam> { - public: - const bool debug_synchronous = false; +class RocprimDeviceSelectLargeInputTests + : public ::testing::TestWithParam> +{ +public: + const bool debug_synchronous = false; }; -INSTANTIATE_TEST_SUITE_P(RocprimDeviceSelectLargeInputFlaggedTest, RocprimDeviceSelectLargeInputTests, - ::testing::Values(std::make_pair(2048, false), // params: flag_selector/segment_length, use_graphs - std::make_pair(9643, false), - std::make_pair(32768, false), - std::make_pair(38713, false), - std::make_pair(38713, true) -)); +INSTANTIATE_TEST_SUITE_P( + RocprimDeviceSelectLargeInputFlaggedTest, + RocprimDeviceSelectLargeInputTests, + ::testing::Values(std::make_pair(2048, + false), // params: flag_selector/segment_length, use_graphs + std::make_pair(9643, false), + std::make_pair(32768, false), + std::make_pair(38713, false), + std::make_pair(38713, true))); TEST_P(RocprimDeviceSelectLargeInputTests, LargeInputFlagged) { @@ -1409,16 +1155,16 @@ TEST_P(RocprimDeviceSelectLargeInputTests, LargeInputFlagged) SCOPED_TRACE(testing::Message() << "with device_id = " << device_id); HIP_CHECK(hipSetDevice(device_id)); - auto param = GetParam(); + auto param = GetParam(); unsigned int flag_selector = std::get<0>(param); - const bool use_graphs = std::get<1>(param); + const bool use_graphs = std::get<1>(param); using InputIterator = typename rocprim::counting_iterator; const bool debug_synchronous = RocprimDeviceSelectLargeInputTests::debug_synchronous; hipStream_t stream = 0; // default stream - if (use_graphs) + if(use_graphs) { // Default stream does not support hipGraph stream capture, so create one HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); @@ -1432,61 +1178,51 @@ TEST_P(RocprimDeviceSelectLargeInputTests, LargeInputFlagged) SCOPED_TRACE(testing::Message() << "with size = " << size); // Generate data - size_t initial_value = 0; + size_t initial_value = 0; InputIterator input_begin(initial_value); - auto flags_it = rocprim::make_transform_iterator(rocprim::make_counting_iterator(size_t(0)), [flag_selector](size_t i){ - if (i % flag_selector == 0) return 1; - else return 0; - }); + auto flags_it = rocprim::make_transform_iterator(rocprim::make_counting_iterator(size_t(0)), + [flag_selector](size_t i) + { + if(i % flag_selector == 0) + return 1; + else + return 0; + }); - size_t selected_count_output = 0; - size_t *d_selected_count_output; + size_t selected_count_output = 0; + test_utils::device_ptr d_selected_count_output(1); size_t expected_output_size = rocprim::detail::ceiling_div(size, flag_selector); - - size_t *d_output; + std::vector output(expected_output_size); + test_utils::device_ptr d_output(expected_output_size); // Calculate expected results on host std::vector expected_output(expected_output_size); - for (size_t i = 0; i < expected_output_size; i++) + for(size_t i = 0; i < expected_output_size; i++) { - expected_output[i] = input_begin[i*flag_selector]; + expected_output[i] = input_begin[i * flag_selector]; } - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, sizeof(d_output[0]) * expected_output_size)); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_selected_count_output, sizeof(d_selected_count_output[0]))); - HIP_CHECK(hipDeviceSynchronize()); - // temp storage size_t temp_storage_size_bytes; - void *d_temp_storage = nullptr; - // Get size of d_temp_storage - HIP_CHECK( - rocprim::select( - d_temp_storage, - temp_storage_size_bytes, - input_begin, - flags_it, - d_output, - d_selected_count_output, - size, - stream, - debug_synchronous - ) - ); + HIP_CHECK(rocprim::select(nullptr, + temp_storage_size_bytes, + input_begin, + flags_it, + d_output.get(), + d_selected_count_output.get(), + size, + stream, + debug_synchronous)); HIP_CHECK(hipDeviceSynchronize()); // temp_storage_size_bytes must be >0 ASSERT_GT(temp_storage_size_bytes, 0); - - // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(use_graphs) { @@ -1494,21 +1230,16 @@ TEST_P(RocprimDeviceSelectLargeInputTests, LargeInputFlagged) } // Run - HIP_CHECK( - rocprim::select( - d_temp_storage, - temp_storage_size_bytes, - input_begin, - flags_it, - d_output, - d_selected_count_output, - size, - stream, - debug_synchronous - ) - ); + HIP_CHECK(rocprim::select(d_temp_storage.get(), + temp_storage_size_bytes, + input_begin, + flags_it, + d_output.get(), + d_selected_count_output.get(), + size, + stream, + debug_synchronous)); - if(use_graphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -1517,31 +1248,14 @@ TEST_P(RocprimDeviceSelectLargeInputTests, LargeInputFlagged) HIP_CHECK(hipDeviceSynchronize()); // Check if number of selected value is as expected - HIP_CHECK( - hipMemcpy( - &selected_count_output, d_selected_count_output, - sizeof(size_t), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + selected_count_output = d_selected_count_output.load()[0]; ASSERT_EQ(selected_count_output, expected_output_size); // Check if output values are as expected - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - sizeof(output[0]) * expected_output_size, - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); - - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected_output, expected_output.size())); + output = d_output.load(); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_selected_count_output)); - HIP_CHECK(hipFree(d_temp_storage)); + ASSERT_NO_FATAL_FAILURE( + test_utils::assert_eq(output, expected_output, expected_output.size())); if(use_graphs) { @@ -1549,7 +1263,7 @@ TEST_P(RocprimDeviceSelectLargeInputTests, LargeInputFlagged) } } - if (use_graphs) + if(use_graphs) HIP_CHECK(hipStreamDestroy(stream)); } @@ -1824,14 +1538,14 @@ TEST_P(RocprimDeviceSelectLargeInputTests, LargeInputSelectFlagged) TEST_P(RocprimDeviceSelectLargeInputTests, LargeInputUnique) { - static constexpr bool debug_synchronous = false; + static constexpr bool debug_synchronous = false; - auto param = GetParam(); + auto param = GetParam(); const unsigned int segment_length = std::get<0>(param); - const bool use_graphs = std::get<1>(param); + const bool use_graphs = std::get<1>(param); hipStream_t stream = 0; - if (use_graphs) + if(use_graphs) { // Default stream does not support hipGraph stream capture, so create one HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); @@ -1856,76 +1570,55 @@ TEST_P(RocprimDeviceSelectLargeInputTests, LargeInputUnique) std::vector expected_output(expected_output_size); std::iota(expected_output.begin(), expected_output.end(), 0); - size_t* d_output{}; - size_t* d_unique_count_output{}; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, - sizeof(*d_output) * expected_output_size)); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_unique_count_output, - sizeof(*d_unique_count_output))); + test_utils::device_ptr d_output(expected_output_size); + test_utils::device_ptr d_unique_count_output(1); size_t temp_storage_size_bytes{}; - void* d_temp_storage{}; - HIP_CHECK(rocprim::unique(d_temp_storage, + + HIP_CHECK(rocprim::unique(nullptr, temp_storage_size_bytes, input_it, - d_output, - d_unique_count_output, + d_output.get(), + d_unique_count_output.get(), size, rocprim::equal_to{}, stream, debug_synchronous)); ASSERT_GT(temp_storage_size_bytes, 0); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(use_graphs) { gHelper.startStreamCapture(stream); } - HIP_CHECK(rocprim::unique(d_temp_storage, + HIP_CHECK(rocprim::unique(d_temp_storage.get(), temp_storage_size_bytes, input_it, - d_output, - d_unique_count_output, + d_output.get(), + d_unique_count_output.get(), size, rocprim::equal_to{}, stream, debug_synchronous)); - if(use_graphs) { gHelper.createAndLaunchGraph(stream); } - size_t unique_count_output{}; - HIP_CHECK(hipMemcpyWithStream(&unique_count_output, - d_unique_count_output, - sizeof(unique_count_output), - hipMemcpyDeviceToHost, - stream)); + auto unique_count_output = d_unique_count_output.load()[0]; ASSERT_EQ(unique_count_output, expected_output_size); - std::vector output(expected_output_size); - HIP_CHECK(hipMemcpyWithStream(output.data(), - d_output, - sizeof(output[0]) * expected_output_size, - hipMemcpyDeviceToHost, - stream)); - + auto output = d_output.load(); ASSERT_NO_FATAL_FAILURE( test_utils::assert_eq(output, expected_output, expected_output.size())); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_unique_count_output)); - HIP_CHECK(hipFree(d_temp_storage)); - - if (use_graphs) + if(use_graphs) gHelper.cleanupGraphHelper(); } - if (use_graphs) + if(use_graphs) HIP_CHECK(hipStreamDestroy(stream)); } diff --git a/test/rocprim/test_device_transform.cpp b/test/rocprim/test_device_transform.cpp index 6efa48c6e..5e8106818 100644 --- a/test/rocprim/test_device_transform.cpp +++ b/test/rocprim/test_device_transform.cpp @@ -28,6 +28,7 @@ #include // required test headers +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" // Params for tests @@ -141,18 +142,8 @@ TYPED_TEST(RocprimDeviceTransformTests, Transform) std::vector input = test_utils::get_random_data(size, 1, 100, seed_value); std::vector output(input.size(), (U)0); - T * d_input; - U * d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(U))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(output.size()); // Calculate expected results on host std::vector expected(input.size()); @@ -165,15 +156,14 @@ TYPED_TEST(RocprimDeviceTransformTests, Transform) } // Run - HIP_CHECK( - rocprim::transform( - d_input, - test_utils::wrap_in_identity_iterator(d_output), - input.size(), transform(), stream, TestFixture::debug_synchronous - ) - ); + HIP_CHECK(rocprim::transform( + d_input.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), + input.size(), + transform(), + stream, + TestFixture::debug_synchronous)); - if(TestFixture::use_graphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -183,22 +173,12 @@ TYPED_TEST(RocprimDeviceTransformTests, Transform) HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - output.size() * sizeof(U), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + output = d_output.load(); // Check if output values are as expected ASSERT_NO_FATAL_FAILURE( test_utils::assert_near(output, expected, test_utils::precision)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - if (TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -252,27 +232,9 @@ TYPED_TEST(RocprimDeviceTransformTests, BinaryTransform) std::vector input2 = test_utils::get_random_data(size, 1, 100, seed_value); std::vector output(input1.size(), (U)0); - T1 * d_input1; - T2 * d_input2; - U * d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input1, input1.size() * sizeof(T1))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input2, input2.size() * sizeof(T2))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(U))); - HIP_CHECK( - hipMemcpy( - d_input1, input1.data(), - input1.size() * sizeof(T1), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_input2, input2.data(), - input2.size() * sizeof(T2), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_input1(input1); + test_utils::device_ptr d_input2(input2); + test_utils::device_ptr d_output(output.size()); // Calculate expected results on host std::vector expected(input1.size()); @@ -288,15 +250,15 @@ TYPED_TEST(RocprimDeviceTransformTests, BinaryTransform) } // Run - HIP_CHECK( - rocprim::transform( - d_input1, d_input2, - test_utils::wrap_in_identity_iterator(d_output), - input1.size(), binary_transform(), stream, debug_synchronous - ) - ); + HIP_CHECK(rocprim::transform( + d_input1.get(), + d_input2.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), + input1.size(), + binary_transform(), + stream, + debug_synchronous)); - if(TestFixture::use_graphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -306,23 +268,12 @@ TYPED_TEST(RocprimDeviceTransformTests, BinaryTransform) HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - output.size() * sizeof(U), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + output = d_output.load(); // Check if output values are as expected ASSERT_NO_FATAL_FAILURE( test_utils::assert_near(output, expected, test_utils::precision)); - HIP_CHECK(hipFree(d_input1)); - HIP_CHECK(hipFree(d_input2)); - HIP_CHECK(hipFree(d_output)); - if (TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); diff --git a/test/rocprim/test_discard_iterator.cpp b/test/rocprim/test_discard_iterator.cpp index 4e895055b..7b949e642 100644 --- a/test/rocprim/test_discard_iterator.cpp +++ b/test/rocprim/test_discard_iterator.cpp @@ -23,12 +23,13 @@ #include "../common_test_header.hpp" // required rocprim headers +#include #include #include -#include // required test headers #include "test_seed.hpp" +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" TEST(RocprimDiscardIteratorTests, Equal) @@ -39,24 +40,25 @@ TEST(RocprimDiscardIteratorTests, Equal) using Iterator = typename rocprim::discard_iterator; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { - unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); Iterator x(test_utils::get_random_value(0, 200, seed_value)); - Iterator y = x; - ASSERT_TRUE(x == y); - - x += 100; - for(size_t i = 0; i < 100; i++) - { - y++; - } - ASSERT_TRUE(x == y); - - y--; - ASSERT_TRUE(x != y); + Iterator y = x; + ASSERT_TRUE(x == y); + + x += 100; + for(size_t i = 0; i < 100; i++) + { + y++; + } + ASSERT_TRUE(x == y); + + y--; + ASSERT_TRUE(x != y); } } @@ -68,9 +70,10 @@ TEST(RocprimDiscardIteratorTests, Less) using Iterator = typename rocprim::discard_iterator; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { - unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); Iterator x(test_utils::get_random_value(0, 200, seed_value)); @@ -91,94 +94,61 @@ TEST(RocprimDiscardIteratorTests, ReduceByKey) int device_id = test_common_utils::obtain_device_from_ctest(); SCOPED_TRACE(testing::Message() << "with device_id = " << device_id); HIP_CHECK(hipSetDevice(device_id)); - + const bool debug_synchronous = false; hipStream_t stream = 0; // default // host input - std::vector keys_input = { - 0, 0, 0, 1, 1, 2, 2, 0, 0, 0, 0 - }; + std::vector keys_input = {0, 0, 0, 1, 1, 2, 2, 0, 0, 0, 0}; std::vector values_input(keys_input.size(), 1); // expected output - std::vector aggregates_expected = { 3, 2, 2, 4 }; + std::vector aggregates_expected = {3, 2, 2, 4}; // device input/output - int * d_keys_input; - int * d_values_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_input, keys_input.size() * sizeof(int))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_values_input, values_input.size() * sizeof(int))); - HIP_CHECK( - hipMemcpy( - d_keys_input, keys_input.data(), - keys_input.size() * sizeof(int), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_values_input, values_input.data(), - values_input.size() * sizeof(int), - hipMemcpyHostToDevice - ) - ); - int * d_aggregates_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_aggregates_output, aggregates_expected.size() * sizeof(int))); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_keys_input(keys_input); + test_utils::device_ptr d_values_input(values_input); + test_utils::device_ptr d_aggregates_output(aggregates_expected.size()); // Get temporary storage size size_t temporary_storage_bytes; - HIP_CHECK( - rocprim::reduce_by_key( - nullptr, temporary_storage_bytes, - d_keys_input, - d_values_input, values_input.size(), - rocprim::make_discard_iterator(), - d_aggregates_output, - rocprim::make_discard_iterator(), - rocprim::plus(), rocprim::equal_to(), - stream, debug_synchronous - ) - ); + HIP_CHECK(rocprim::reduce_by_key(nullptr, + temporary_storage_bytes, + d_keys_input.get(), + d_values_input.get(), + values_input.size(), + rocprim::make_discard_iterator(), + d_aggregates_output.get(), + rocprim::make_discard_iterator(), + rocprim::plus(), + rocprim::equal_to(), + stream, + debug_synchronous)); HIP_CHECK(hipDeviceSynchronize()); ASSERT_GT(temporary_storage_bytes, 0); - void * d_temporary_storage; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); - - HIP_CHECK( - rocprim::reduce_by_key( - d_temporary_storage, temporary_storage_bytes, - d_keys_input, - d_values_input, values_input.size(), - rocprim::make_discard_iterator(), - d_aggregates_output, - rocprim::make_discard_iterator(), - rocprim::plus(), rocprim::equal_to(), - stream, debug_synchronous - ) - ); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); + + HIP_CHECK(rocprim::reduce_by_key(d_temporary_storage.get(), + temporary_storage_bytes, + d_keys_input.get(), + d_values_input.get(), + values_input.size(), + rocprim::make_discard_iterator(), + d_aggregates_output.get(), + rocprim::make_discard_iterator(), + rocprim::plus(), + rocprim::equal_to(), + stream, + debug_synchronous)); HIP_CHECK(hipDeviceSynchronize()); // Check if output values are as expected - std::vector aggregates_output(aggregates_expected.size()); - HIP_CHECK( - hipMemcpy( - aggregates_output.data(), d_aggregates_output, - aggregates_expected.size() * sizeof(int), - hipMemcpyDeviceToHost - ) - ); + std::vector aggregates_output = d_aggregates_output.load(); for(size_t i = 0; i < aggregates_output.size(); i++) { ASSERT_EQ(aggregates_output[i], aggregates_expected[i]); } - - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_values_input)); - HIP_CHECK(hipFree(d_aggregates_output)); - HIP_CHECK(hipFree(d_temporary_storage)); } diff --git a/test/rocprim/test_lookback_reproducibility.cpp b/test/rocprim/test_lookback_reproducibility.cpp index 2c50818f5..60c29eb94 100644 --- a/test/rocprim/test_lookback_reproducibility.cpp +++ b/test/rocprim/test_lookback_reproducibility.cpp @@ -27,8 +27,11 @@ #include #include +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" +#include + template std::vector get_sizes(T seed_value) { @@ -78,8 +81,8 @@ TYPED_TEST_SUITE(RocprimLookbackReproducibilityTests, Suite); template void test_reproducibility(S scan_op, F run_test) { - bool* d_enable_sleep; - HIP_CHECK(hipMalloc(&d_enable_sleep, sizeof(*d_enable_sleep))); + test_utils::device_ptr d_enable_sleep_ptr(1); + int* d_enable_sleep = d_enable_sleep_ptr.get(); // Delay the operator by a semi-random amount to increase the likelyhood // of changing the number of lookback steps between the runs. @@ -95,19 +98,16 @@ void test_reproducibility(S scan_op, F run_test) return scan_op(a, b); }; - bool enable_sleep = false; - HIP_CHECK( - hipMemcpy(d_enable_sleep, &enable_sleep, sizeof(enable_sleep), hipMemcpyHostToDevice)); + std::vector enable_sleep = {0}; + d_enable_sleep_ptr.store(enable_sleep); auto first = run_test(eepy_scan_op); - enable_sleep = true; - HIP_CHECK( - hipMemcpy(d_enable_sleep, &enable_sleep, sizeof(enable_sleep), hipMemcpyHostToDevice)); + enable_sleep[0] = 1; + d_enable_sleep_ptr.store(enable_sleep); auto second = run_test(eepy_scan_op); + // We want the result to be bitwise equal, even if the inputs/outputs are floats. ASSERT_NO_FATAL_FAILURE(test_utils::assert_bit_eq(first, second)); - - HIP_CHECK(hipFree(d_enable_sleep)); } template @@ -159,13 +159,8 @@ TYPED_TEST(RocprimLookbackReproducibilityTests, Scan) // Generate data std::vector input = test_utils::get_random_data(size, -1000, 1000, seed_value); - T* d_input; - T* d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, input.size() * sizeof(T))); - HIP_CHECK( - hipMemcpy(d_input, input.data(), input.size() * sizeof(T), hipMemcpyHostToDevice)); - + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(input.size()); scan_op_type scan_op; test_reproducibility( @@ -173,40 +168,31 @@ TYPED_TEST(RocprimLookbackReproducibilityTests, Scan) [&](auto test_scan_op) { size_t temp_storage_size_bytes; - void* d_temp_storage = nullptr; - HIP_CHECK(rocprim::deterministic_inclusive_scan(d_temp_storage, + test_utils::device_ptr d_temp_storage; + HIP_CHECK(rocprim::deterministic_inclusive_scan(nullptr, temp_storage_size_bytes, - d_input, - d_output, + d_input.get(), + d_output.get(), input.size(), scan_op, stream, debug_synchronous)); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, - temp_storage_size_bytes)); + d_temp_storage.resize(temp_storage_size_bytes); - HIP_CHECK(rocprim::deterministic_inclusive_scan(d_temp_storage, + HIP_CHECK(rocprim::deterministic_inclusive_scan(d_temp_storage.get(), temp_storage_size_bytes, - d_input, - d_output, + d_input.get(), + d_output.get(), input.size(), test_scan_op, stream, debug_synchronous)); HIP_CHECK(hipGetLastError()); - std::vector output(input.size()); - HIP_CHECK(hipMemcpy(output.data(), - d_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost)); - HIP_CHECK(hipFree(d_temp_storage)); + std::vector output = d_output.load(); return output; }); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); } } } @@ -244,66 +230,48 @@ TYPED_TEST(RocprimLookbackReproducibilityTests, ScanByKey) std::vector keys = generate_segments(seed_value, size, min_segment_length, max_segment_length); - K* d_keys; - V* d_input; - V* d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys, keys.size() * sizeof(K))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(V))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, input.size() * sizeof(V))); - HIP_CHECK( - hipMemcpy(d_keys, keys.data(), keys.size() * sizeof(K), hipMemcpyHostToDevice)); - HIP_CHECK( - hipMemcpy(d_input, input.data(), input.size() * sizeof(V), hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_keys(keys); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(input.size()); scan_op_type scan_op; compare_op_type compare_op; - test_reproducibility( - scan_op, - [&](auto test_scan_op) - { - size_t temp_storage_size_bytes; - void* d_temp_storage = nullptr; - HIP_CHECK(rocprim::deterministic_inclusive_scan_by_key( - d_temp_storage, - temp_storage_size_bytes, - d_keys, - d_input, - d_output, - input.size(), - scan_op, - compare_op, - stream, - debug_synchronous)); - - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, - temp_storage_size_bytes)); - - HIP_CHECK(rocprim::deterministic_inclusive_scan_by_key( - d_temp_storage, - temp_storage_size_bytes, - d_keys, - d_input, - d_output, - input.size(), - test_scan_op, - compare_op, - stream, - debug_synchronous)); - HIP_CHECK(hipGetLastError()); - - std::vector output(input.size()); - HIP_CHECK(hipMemcpy(output.data(), - d_output, - output.size() * sizeof(V), - hipMemcpyDeviceToHost)); - HIP_CHECK(hipFree(d_temp_storage)); - return output; - }); - - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_input)); + test_reproducibility(scan_op, + [&](auto test_scan_op) + { + size_t temp_storage_size_bytes; + test_utils::device_ptr d_temp_storage; + HIP_CHECK(rocprim::deterministic_inclusive_scan_by_key( + nullptr, + temp_storage_size_bytes, + d_keys.get(), + d_input.get(), + d_output.get(), + input.size(), + scan_op, + compare_op, + stream, + debug_synchronous)); + + d_temp_storage.resize(temp_storage_size_bytes); + + HIP_CHECK(rocprim::deterministic_inclusive_scan_by_key( + d_temp_storage.get(), + temp_storage_size_bytes, + d_keys.get(), + d_input.get(), + d_output.get(), + input.size(), + test_scan_op, + compare_op, + stream, + debug_synchronous)); + HIP_CHECK(hipGetLastError()); + + std::vector output = d_output.load(); + return output; + }); } } } @@ -342,20 +310,10 @@ TYPED_TEST(RocprimLookbackReproducibilityTests, ReduceByKey) std::vector keys = generate_segments(seed_value, size, min_segment_length, max_segment_length); - K* d_keys; - V* d_input; - V* d_output; - size_t* d_unique_count_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys, keys.size() * sizeof(K))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(V))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, input.size() * sizeof(V))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_unique_count_output, - sizeof(*d_unique_count_output))); - HIP_CHECK( - hipMemcpy(d_keys, keys.data(), keys.size() * sizeof(K), hipMemcpyHostToDevice)); - HIP_CHECK( - hipMemcpy(d_input, input.data(), input.size() * sizeof(V), hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_keys(keys); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(input.size()); + test_utils::device_ptr d_unique_count_output(1); scan_op_type scan_op; compare_op_type compare_op; @@ -368,57 +326,43 @@ TYPED_TEST(RocprimLookbackReproducibilityTests, ReduceByKey) scan_op, [&](auto test_scan_op) { - size_t temp_storage_size_bytes; - void* d_temp_storage = nullptr; - HIP_CHECK(rocprim::deterministic_reduce_by_key(d_temp_storage, - temp_storage_size_bytes, - d_keys, - d_input, - input.size(), - d_discard_unique_output, - d_output, - d_unique_count_output, - test_scan_op, - compare_op, - stream, - debug_synchronous)); - - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, - temp_storage_size_bytes)); - - HIP_CHECK(rocprim::deterministic_reduce_by_key(d_temp_storage, - temp_storage_size_bytes, - d_keys, - d_input, - input.size(), - d_discard_unique_output, - d_output, - d_unique_count_output, - test_scan_op, - compare_op, - stream, - debug_synchronous)); + size_t temp_storage_size_bytes; + test_utils::device_ptr d_temp_storage; + HIP_CHECK( + rocprim::deterministic_reduce_by_key(nullptr, + temp_storage_size_bytes, + d_keys.get(), + d_input.get(), + input.size(), + d_discard_unique_output, + d_output.get(), + d_unique_count_output.get(), + test_scan_op, + compare_op, + stream, + debug_synchronous)); + d_temp_storage.resize(temp_storage_size_bytes); + + HIP_CHECK( + rocprim::deterministic_reduce_by_key(d_temp_storage.get(), + temp_storage_size_bytes, + d_keys.get(), + d_input.get(), + input.size(), + d_discard_unique_output, + d_output.get(), + d_unique_count_output.get(), + test_scan_op, + compare_op, + stream, + debug_synchronous)); HIP_CHECK(hipGetLastError()); - size_t unique_count_output; - HIP_CHECK(hipMemcpy(&unique_count_output, - d_unique_count_output, - sizeof(unique_count_output), - hipMemcpyDeviceToHost)); - - std::vector output(unique_count_output); - HIP_CHECK(hipMemcpy(output.data(), - d_output, - output.size() * sizeof(V), - hipMemcpyDeviceToHost)); - HIP_CHECK(hipFree(d_temp_storage)); + [[maybe_unused]] size_t unique_count_output = d_unique_count_output.load()[0]; + + std::vector output = d_output.load(); return output; }); - - HIP_CHECK(hipFree(d_keys)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_unique_count_output)); } } } diff --git a/test/rocprim/test_predicate_iterator.cpp b/test/rocprim/test_predicate_iterator.cpp index ff0b70128..c8acc06e5 100644 --- a/test/rocprim/test_predicate_iterator.cpp +++ b/test/rocprim/test_predicate_iterator.cpp @@ -22,6 +22,7 @@ #include "rocprim/iterator/transform_iterator.hpp" #include "test_utils_data_generation.hpp" +#include "test_utils_device_ptr.hpp" #include @@ -43,7 +44,8 @@ struct is_odd // While this can be "constexpr T(const T&) const", we want to verify that // it compiles without the constness. template - __device__ __host__ bool operator()(T& a) + __device__ __host__ + bool operator()(T& a) { return a % 2; } @@ -53,7 +55,9 @@ template struct set_to { template - __device__ __host__ constexpr T operator()(const T&) const + __device__ __host__ + constexpr T + operator()(const T&) const { return V; } @@ -63,7 +67,9 @@ template struct increment_by { template - __device__ __host__ T constexpr operator()(const T& a) const + __device__ __host__ + T constexpr + operator()(const T& a) const { return a + V; } @@ -72,7 +78,9 @@ struct increment_by struct identity { template - __device__ __host__ constexpr T operator()(const T& a) const + __device__ __host__ + constexpr T + operator()(const T& a) const { return a; } @@ -215,22 +223,17 @@ TEST(RocprimPredicateIteratorTests, DeviceInplace) using predicate = is_odd; using transform = increment_by<5>; - constexpr size_t size = 100; - constexpr size_t data_size = sizeof(T) * size; + constexpr size_t size = 100; std::vector h_data(size); std::iota(h_data.begin(), h_data.end(), 0); - T* d_data; - HIP_CHECK(hipMalloc(&d_data, data_size)); - HIP_CHECK(hipMemcpy(d_data, h_data.data(), data_size, hipMemcpyHostToDevice)); + test_utils::device_ptr d_data(h_data); - auto w_it = rocprim::make_predicate_iterator(d_data, predicate{}); + auto w_it = rocprim::make_predicate_iterator(d_data.get(), predicate{}); - HIP_CHECK(rocprim::transform(d_data, w_it, size, transform{})); - - HIP_CHECK(hipMemcpy(h_data.data(), d_data, data_size, hipMemcpyDeviceToHost)); - HIP_CHECK(hipFree(d_data)); + HIP_CHECK(rocprim::transform(d_data.get(), w_it, size, transform{})); + h_data = d_data.load(); for(T i = 0; i < T{size}; ++i) { @@ -252,26 +255,20 @@ TEST(RocprimPredicateIteratorTests, DeviceRead) using predicate = is_odd; using transform = increment_by<5>; - constexpr size_t size = 100; - constexpr size_t data_size = sizeof(T) * size; + constexpr size_t size = 100; std::vector h_data(size); std::iota(h_data.begin(), h_data.end(), 0); - T* d_input; - T* d_output; - HIP_CHECK(hipMalloc(&d_input, data_size)); - HIP_CHECK(hipMalloc(&d_output, data_size)); - HIP_CHECK(hipMemcpy(d_input, h_data.data(), data_size, hipMemcpyHostToDevice)); + test_utils::device_ptr d_input(h_data); + test_utils::device_ptr d_output(size); - auto t_it = rocprim::make_transform_iterator(d_input, transform{}); - auto r_it = rocprim::make_predicate_iterator(t_it, d_input, predicate{}); + auto t_it = rocprim::make_transform_iterator(d_input.get(), transform{}); + auto r_it = rocprim::make_predicate_iterator(t_it, d_input.get(), predicate{}); - HIP_CHECK(rocprim::transform(r_it, d_output, size, identity{})); + HIP_CHECK(rocprim::transform(r_it, d_output.get(), size, identity{})); - HIP_CHECK(hipMemcpy(h_data.data(), d_output, data_size, hipMemcpyDeviceToHost)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); + h_data = d_output.load(); for(T i = 0; i < T{size}; ++i) { diff --git a/test/rocprim/test_reverse_iterator.cpp b/test/rocprim/test_reverse_iterator.cpp index 8966723cf..ec2c60fa8 100644 --- a/test/rocprim/test_reverse_iterator.cpp +++ b/test/rocprim/test_reverse_iterator.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -22,6 +22,7 @@ #include "../common_test_header.hpp" #include "test_utils.hpp" +#include "test_utils_device_ptr.hpp" #include #include @@ -59,30 +60,26 @@ TEST(RocprimReverseIteratorTests, DeviceVector) std::sort(expected.rbegin(), expected.rend()); std::vector output(size); - T* d_input{}; - T* d_output{}; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, size * sizeof(T))); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, size * sizeof(T))); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(size); - auto d_output_it = rocprim::make_reverse_iterator(d_output + size); + auto d_output_it = rocprim::make_reverse_iterator(d_output.get() + size); - void* d_temp_storage{}; size_t temp_storage_bytes; HIP_CHECK( - rocprim::radix_sort_keys(d_temp_storage, temp_storage_bytes, d_input, d_output_it, size)); + rocprim::radix_sort_keys(nullptr, temp_storage_bytes, d_input.get(), d_output_it, size)); ASSERT_NE(temp_storage_bytes, static_cast(0)); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_bytes)); - HIP_CHECK( - rocprim::radix_sort_keys(d_temp_storage, temp_storage_bytes, d_input, d_output_it, size)); - HIP_CHECK(hipMemcpy(output.data(), d_output, size * sizeof(T), hipMemcpyDeviceToHost)); + test_utils::device_ptr d_temp_storage(temp_storage_bytes); - ASSERT_EQ(expected, output); + HIP_CHECK(rocprim::radix_sort_keys(d_temp_storage.get(), + temp_storage_bytes, + d_input.get(), + d_output_it, + size)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_temp_storage)); + output = d_output.load(); + ASSERT_EQ(expected, output); } TEST(RocprimReverseIteratorTests, TestIncrement) diff --git a/test/rocprim/test_texture_cache_iterator.cpp b/test/rocprim/test_texture_cache_iterator.cpp index 4e60180d3..0bf60e0a6 100644 --- a/test/rocprim/test_texture_cache_iterator.cpp +++ b/test/rocprim/test_texture_cache_iterator.cpp @@ -27,6 +27,7 @@ #include // required test headers +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" // Params for tests @@ -102,21 +103,11 @@ TYPED_TEST(RocprimTextureCacheIteratorTests, Transform) } std::vector output(size); - T * d_input; - T * d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(T))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(output.size()); Iterator x; - HIP_CHECK(x.bind_texture(d_input, sizeof(T) * input.size())); + HIP_CHECK(x.bind_texture(d_input.get(), sizeof(T) * input.size())); // Calculate expected results on host std::vector expected(size); @@ -129,23 +120,12 @@ TYPED_TEST(RocprimTextureCacheIteratorTests, Transform) // Run HIP_CHECK( - rocprim::transform( - x, d_output, size, - transform(), stream, debug_synchronous - ) - ); + rocprim::transform(x, d_output.get(), size, transform(), stream, debug_synchronous)); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + output = d_output.load(); // Validating results for(size_t i = 0; i < output.size(); i++) @@ -154,7 +134,5 @@ TYPED_TEST(RocprimTextureCacheIteratorTests, Transform) } HIP_CHECK(x.unbind_texture()); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); } } diff --git a/test/rocprim/test_thread.cpp b/test/rocprim/test_thread.cpp index ed5b54ada..705f86cc8 100644 --- a/test/rocprim/test_thread.cpp +++ b/test/rocprim/test_thread.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,6 +21,7 @@ // SOFTWARE. #include "../common_test_header.hpp" +#include "test_utils_device_ptr.hpp" // required rocprim headers #include @@ -112,33 +113,22 @@ TYPED_TEST(RocprimThreadTests, FlatBlockThreadID) } // Preparing device - Type* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, block_size * sizeof(typename decltype(output)::value_type))); + test_utils::device_ptr device_output(block_size); // Running kernel hipLaunchKernelGGL( - HIP_KERNEL_NAME( - flat_id_kernel< - block_size_x, block_size_y, block_size_z - > - ), - dim3(1), dim3(block_size_x, block_size_y, block_size_z), 0, 0, - device_output - ); + HIP_KERNEL_NAME(flat_id_kernel), + dim3(1), + dim3(block_size_x, block_size_y, block_size_z), + 0, + 0, + device_output.get()); HIP_CHECK(hipGetLastError()); // Reading results from device - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(typename decltype(output)::value_type), - hipMemcpyDeviceToHost - ) - ); - + output = device_output.load(); // Validating results ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); - HIP_CHECK(hipFree(device_output)); } } @@ -190,32 +180,21 @@ TYPED_TEST(RocprimThreadTests, FlatBlockID) } // Preparing device - Type* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - + test_utils::device_ptr device_output(block_size); // Running kernel hipLaunchKernelGGL( - HIP_KERNEL_NAME( - block_id_kernel< - block_size_x, block_size_y, block_size_z - > - ), - dim3(block_size_x, block_size_y, block_size_z), dim3(block_size_x, block_size_y, block_size_z), 0, 0, - device_output - ); + HIP_KERNEL_NAME(block_id_kernel), + dim3(block_size_x, block_size_y, block_size_z), + dim3(block_size_x, block_size_y, block_size_z), + 0, + 0, + device_output.get()); HIP_CHECK(hipGetLastError()); // Reading results from device - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(typename decltype(output)::value_type), - hipMemcpyDeviceToHost - ) - ); + output = device_output.load(); // Validating results ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); - HIP_CHECK(hipFree(device_output)); } } diff --git a/test/rocprim/test_thread_algos.cpp b/test/rocprim/test_thread_algos.cpp index 7311a5190..de8bb381e 100644 --- a/test/rocprim/test_thread_algos.cpp +++ b/test/rocprim/test_thread_algos.cpp @@ -98,38 +98,17 @@ TYPED_TEST(RocprimThreadOperationTests, Load) std::vector expected = input; // Preparing device - T* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(reinterpret_cast(&device_input), - input.size() * sizeof(T))); - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(reinterpret_cast(&device_output), - output.size() * sizeof(T))); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - - thread_load_kernel<<>>(device_input, device_output); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(input.size()); + + thread_load_kernel<<>>(device_input.get(), device_output.get()); HIP_CHECK(hipGetLastError()); // Reading results back - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + output = device_output.load(); // Verifying results ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); } } @@ -163,38 +142,17 @@ TYPED_TEST(RocprimThreadOperationTests, Store) std::vector expected = input; // Preparing device - T* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(reinterpret_cast(&device_input), - input.size() * sizeof(T))); - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(reinterpret_cast(&device_output), - output.size() * sizeof(T))); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - - thread_store_kernel<<>>(device_input, device_output); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(input.size()); + + thread_store_kernel<<>>(device_input.get(), device_output.get()); HIP_CHECK(hipGetLastError()); // Reading results back - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + output = device_output.load(); // Verifying results ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); } } @@ -253,41 +211,21 @@ TYPED_TEST(RocprimThreadOperationTests, Reduction) //std::vector expected = input; // Preparing device - T* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(reinterpret_cast(&device_input), - input.size() * sizeof(T))); - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(reinterpret_cast(&device_output), - output.size() * sizeof(T))); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - - thread_reduce_kernel<<>>(device_input, device_output); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(input.size()); + + thread_reduce_kernel + <<>>(device_input.get(), device_output.get()); HIP_CHECK(hipGetLastError()); // Reading results back - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + output = device_output.load(); // Verifying results for(size_t i = 0; i < output.size(); i+=length) { ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output[i], expected[i])); } - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); } } @@ -339,38 +277,18 @@ TYPED_TEST(RocprimThreadOperationTests, Scan) } // Preparing device - T* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(reinterpret_cast(&device_input), - input.size() * sizeof(T))); - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(reinterpret_cast(&device_output), - output.size() * sizeof(T))); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - - thread_scan_kernel<<>>(device_input, device_output); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(input.size()); + + thread_scan_kernel + <<>>(device_input.get(), device_output.get()); HIP_CHECK(hipGetLastError()); // Reading results back - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + output = device_output.load(); // Verifying results ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); } } diff --git a/test/rocprim/test_transform_iterator.cpp b/test/rocprim/test_transform_iterator.cpp index b8b51a457..b4549856a 100644 --- a/test/rocprim/test_transform_iterator.cpp +++ b/test/rocprim/test_transform_iterator.cpp @@ -21,6 +21,7 @@ // SOFTWARE. #include "../common_test_header.hpp" +#include "test_utils_device_ptr.hpp" // required rocprim headers #include @@ -108,18 +109,8 @@ TYPED_TEST(RocprimTransformIteratorTests, TransformReduce) std::vector input = test_utils::get_random_data(size, 1, 200, seed_value); std::vector output(1); - input_type * d_input; - value_type * d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(input_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(value_type))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - input.size() * sizeof(input_type), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(output.size()); auto reduce_op = rocprim::plus(); unary_function transform; @@ -128,63 +119,41 @@ TYPED_TEST(RocprimTransformIteratorTests, TransformReduce) iterator_type x(input.data(), transform); value_type expected = std::accumulate(x, x + size, value_type(0), reduce_op); - auto d_iter = iterator_type(d_input, transform); + auto d_iter = iterator_type(d_input.get(), transform); // temp storage size_t temp_storage_size_bytes; // Get size of d_temp_storage - HIP_CHECK( - rocprim::reduce( - nullptr, - temp_storage_size_bytes, - d_iter, - d_output, - value_type(0), - input.size(), - reduce_op, - stream - ) - ); + HIP_CHECK(rocprim::reduce(nullptr, + temp_storage_size_bytes, + d_iter, + d_output.get(), + value_type(0), + input.size(), + reduce_op, + stream)); // temp_storage_size_bytes must be >0 ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - void * d_temp_storage = nullptr; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); // Run - HIP_CHECK( - rocprim::reduce( - d_temp_storage, - temp_storage_size_bytes, - d_iter, - d_output, - value_type(0), - input.size(), - reduce_op, - stream, - TestFixture::debug_synchronous - ) - ); + HIP_CHECK(rocprim::reduce(d_temp_storage.get(), + temp_storage_size_bytes, + d_iter, + d_output.get(), + value_type(0), + input.size(), + reduce_op, + stream, + TestFixture::debug_synchronous)); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - output.size() * sizeof(value_type), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + output = d_output.load(); // Check if output values are as expected test_utils::assert_near(output[0], expected, test_utils::precision * size); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_temp_storage)); } } diff --git a/test/rocprim/test_utils_device_ptr.hpp b/test/rocprim/test_utils_device_ptr.hpp index badad886e..f9a570978 100644 --- a/test/rocprim/test_utils_device_ptr.hpp +++ b/test/rocprim/test_utils_device_ptr.hpp @@ -203,9 +203,9 @@ class device_ptr hipMemcpyHostToDevice)); } - /// \brief Copy from host to device + /// \brief Copy from device to device template - void store(device_ptr const& device_ptr, size_type offset = 0) + void replace(device_ptr const& device_ptr, size_type offset = 0) { static_assert(sizeof(InPtrValueType) == sizeof(value_type), "sizeof(InPtrValueType) must equal to sizeof(value_type)"); @@ -231,6 +231,16 @@ class device_ptr hipMemcpyDeviceToHost)); return ret; } + /// \brief Copy from device to host + std::vector load(size_type target_size, size_type offset = 0) const + { + std::vector ret(target_size); + HIP_CHECK(hipMemcpy(ret.data(), + device_raw_ptr_ + offset, + target_size * sizeof(value_type), + hipMemcpyDeviceToHost)); + return ret; + } private: value_type* device_raw_ptr_; diff --git a/test/rocprim/test_warp_exchange.cpp b/test/rocprim/test_warp_exchange.cpp index 636a84788..04bdf3e77 100644 --- a/test/rocprim/test_warp_exchange.cpp +++ b/test/rocprim/test_warp_exchange.cpp @@ -23,6 +23,7 @@ #include "../common_test_header.hpp" #include "rocprim/types.hpp" #include "test_utils.hpp" +#include "test_utils_device_ptr.hpp" #include @@ -324,23 +325,16 @@ TYPED_TEST(WarpExchangeTest, WarpExchange) input = stripe_vector(input, warp_size, items_per_thread); } - T* d_input{}; - HIP_CHECK(hipMalloc(&d_input, items_count * sizeof(T))); - HIP_CHECK(hipMemcpy(d_input, input.data(), items_count * sizeof(T), hipMemcpyHostToDevice)); - T* d_output{}; - HIP_CHECK(hipMalloc(&d_output, items_count * sizeof(T))); - HIP_CHECK(hipMemset(d_output, 0, items_count * sizeof(T))); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(items_count); + HIP_CHECK(hipMemset(d_output.get(), 0, items_count * sizeof(T))); warp_exchange_kernel - <<>>(d_input, d_output); + <<>>(d_input.get(), d_output.get()); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); - std::vector output(items_count); - HIP_CHECK(hipMemcpy(output.data(), d_output, items_count * sizeof(T), hipMemcpyDeviceToHost)); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); + auto output = d_output.load(); if(std::is_same::value || std::is_same::value) @@ -377,23 +371,16 @@ TYPED_TEST(WarpExchangeTest, WarpExchangeNotInplace) input = stripe_vector(input, warp_size, items_per_thread); } - T* d_input{}; - HIP_CHECK(hipMalloc(&d_input, items_count * sizeof(T))); - HIP_CHECK(hipMemcpy(d_input, input.data(), items_count * sizeof(T), hipMemcpyHostToDevice)); - T* d_output{}; - HIP_CHECK(hipMalloc(&d_output, items_count * sizeof(T))); - HIP_CHECK(hipMemset(d_output, 0, items_count * sizeof(T))); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(items_count); + HIP_CHECK(hipMemset(d_output.get(), 0, items_count * sizeof(T))); warp_exchange_kernel - <<>>(d_input, d_output, false); + <<>>(d_input.get(), d_output.get(), false); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); - std::vector output(items_count); - HIP_CHECK(hipMemcpy(output.data(), d_output, items_count * sizeof(T), hipMemcpyDeviceToHost)); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); + auto output = d_output.load(); if(std::is_same::value || std::is_same::value) @@ -489,27 +476,17 @@ TYPED_TEST(WarpExchangeScatterTest, WarpExchangeScatter) [](const T input_val) { return static_cast(input_val) % (warp_size * items_per_thread); }); - T* d_input{}; - HIP_CHECK(hipMalloc(&d_input, items_count * sizeof(T))); - HIP_CHECK(hipMemcpy(d_input, input.data(), items_count * sizeof(T), hipMemcpyHostToDevice)); - T* d_output{}; - HIP_CHECK(hipMalloc(&d_output, items_count * sizeof(T))); - HIP_CHECK(hipMemset(d_output, 0, items_count * sizeof(T))); - OffsetT* d_ranks{}; - HIP_CHECK(hipMalloc(&d_ranks, items_count * sizeof(OffsetT))); - HIP_CHECK(hipMemcpy(d_ranks, ranks.data(), items_count * sizeof(OffsetT), hipMemcpyHostToDevice)); + test_utils::device_ptr d_ranks(ranks); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(items_count); + HIP_CHECK(hipMemset(d_output.get(), 0, items_count * sizeof(T))); warp_exchange_scatter_kernel - <<>>(d_input, d_output, d_ranks); + <<>>(d_input.get(), d_output.get(), d_ranks.get()); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); - std::vector output(items_count); - HIP_CHECK(hipMemcpy(output.data(), d_output, items_count * sizeof(T), hipMemcpyDeviceToHost)); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_ranks)); + auto output = d_output.load(); ASSERT_EQ(expected, output); } diff --git a/test/rocprim/test_warp_load.cpp b/test/rocprim/test_warp_load.cpp index b4fac714a..0cfd9f254 100644 --- a/test/rocprim/test_warp_load.cpp +++ b/test/rocprim/test_warp_load.cpp @@ -23,6 +23,7 @@ #include "../common_test_header.hpp" #include "rocprim/types.hpp" #include "test_utils.hpp" +#include "test_utils_device_ptr.hpp" #include #include @@ -244,22 +245,15 @@ TYPED_TEST(WarpLoadTest, WarpLoad) std::vector input(items_count); std::iota(input.begin(), input.end(), static_cast(0)); - T* d_input{}; - HIP_CHECK(hipMalloc(&d_input, items_count * sizeof(T))); - HIP_CHECK(hipMemcpy(d_input, input.data(), items_count * sizeof(T), hipMemcpyHostToDevice)); - T* d_output{}; - HIP_CHECK(hipMalloc(&d_output, items_count * sizeof(T))); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(items_count); warp_load_kernel - <<>>(d_input, d_output); + <<>>(d_input.get(), d_output.get()); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); - std::vector output(items_count); - HIP_CHECK(hipMemcpy(output.data(), d_output, items_count * sizeof(T), hipMemcpyDeviceToHost)); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); + std::vector output = d_output.load(); auto expected = input; if(method == ::rocprim::warp_load_method::warp_load_striped) @@ -288,22 +282,18 @@ TYPED_TEST(WarpLoadTest, WarpLoadGuarded) std::vector input(items_count); std::iota(input.begin(), input.end(), static_cast(0)); - T* d_input{}; - HIP_CHECK(hipMalloc(&d_input, items_count * sizeof(T))); - HIP_CHECK(hipMemcpy(d_input, input.data(), items_count * sizeof(T), hipMemcpyHostToDevice)); - T* d_output{}; - HIP_CHECK(hipMalloc(&d_output, items_count * sizeof(T))); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(items_count); warp_load_guarded_kernel - <<>>(d_input, d_output, valid_items, oob_default); + <<>>(d_input.get(), + d_output.get(), + valid_items, + oob_default); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); - std::vector output(items_count); - HIP_CHECK(hipMemcpy(output.data(), d_output, items_count * sizeof(T), hipMemcpyDeviceToHost)); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); + std::vector output = d_output.load(); auto expected = input; for(size_t warp_idx = 0; warp_idx < block_size / warp_size; ++warp_idx) diff --git a/test/rocprim/test_warp_reduce.cpp b/test/rocprim/test_warp_reduce.cpp index 88b6bfaf9..aaf75d7a9 100644 --- a/test/rocprim/test_warp_reduce.cpp +++ b/test/rocprim/test_warp_reduce.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2021 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -26,6 +26,7 @@ #include // required test headers +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" // kernel definitions diff --git a/test/rocprim/test_warp_reduce.hpp b/test/rocprim/test_warp_reduce.hpp index 996808472..8fad511c1 100644 --- a/test/rocprim/test_warp_reduce.hpp +++ b/test/rocprim/test_warp_reduce.hpp @@ -93,53 +93,40 @@ typed_test_def(RocprimWarpReduceTests, name_suffix, ReduceSum) expected[i] = static_cast(value); } - T* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_input, input.size() * sizeof(typename decltype(input)::value_type))); - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(output.size()); // Launching kernel if (current_device_warp_size == ws32) { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_reduce_sum_kernel), - dim3(size/block_size_ws32), dim3(block_size_ws32), 0, 0, - device_input, device_output - ); + dim3(size / block_size_ws32), + dim3(block_size_ws32), + 0, + 0, + device_input.get(), + device_output.get()); } else if (current_device_warp_size == ws64) { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_reduce_sum_kernel), - dim3(size/block_size_ws64), dim3(block_size_ws64), 0, 0, - device_input, device_output - ); + dim3(size / block_size_ws64), + dim3(block_size_ws64), + 0, + 0, + device_input.get(), + device_output.get()); } HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + output = device_output.load(); test_utils::assert_near(output, expected, test_utils::precision * logical_warp_size); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); } } @@ -216,53 +203,40 @@ typed_test_def(RocprimWarpReduceTests, name_suffix, AllReduceSum) } } - T* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_input, input.size() * sizeof(typename decltype(input)::value_type))); - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(output.size()); // Launching kernel if (current_device_warp_size == ws32) { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_allreduce_sum_kernel), - dim3(size/block_size_ws32), dim3(block_size_ws32), 0, 0, - device_input, device_output - ); + dim3(size / block_size_ws32), + dim3(block_size_ws32), + 0, + 0, + device_input.get(), + device_output.get()); } else if (current_device_warp_size == ws64) { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_allreduce_sum_kernel), - dim3(size/block_size_ws64), dim3(block_size_ws64), 0, 0, - device_input, device_output - ); + dim3(size / block_size_ws64), + dim3(block_size_ws64), + 0, + 0, + device_input.get(), + device_output.get()); } HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + output = device_output.load(); test_utils::assert_near(output, expected, test_utils::precision * logical_warp_size); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); } } @@ -336,53 +310,42 @@ typed_test_def(RocprimWarpReduceTests, name_suffix, ReduceSumValid) expected[i] = static_cast(value); } - T* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_input, input.size() * sizeof(typename decltype(input)::value_type))); - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(output.size()); // Launching kernel if (current_device_warp_size == ws32) { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_reduce_sum_kernel), - dim3(size/block_size_ws32), dim3(block_size_ws32), 0, 0, - device_input, device_output, valid - ); + dim3(size / block_size_ws32), + dim3(block_size_ws32), + 0, + 0, + device_input.get(), + device_output.get(), + valid); } else if (current_device_warp_size == ws64) { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_reduce_sum_kernel), - dim3(size/block_size_ws64), dim3(block_size_ws64), 0, 0, - device_input, device_output, valid - ); + dim3(size / block_size_ws64), + dim3(block_size_ws64), + 0, + 0, + device_input.get(), + device_output.get(), + valid); } HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + output = device_output.load(); test_utils::assert_near(output, expected, test_utils::precision * logical_warp_size); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); } } @@ -461,53 +424,42 @@ typed_test_def(RocprimWarpReduceTests, name_suffix, AllReduceSumValid) } } - T* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_input, input.size() * sizeof(typename decltype(input)::value_type))); - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(output.size()); // Launching kernel if (current_device_warp_size == ws32) { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_allreduce_sum_kernel), - dim3(size/block_size_ws32), dim3(block_size_ws32), 0, 0, - device_input, device_output, valid - ); + dim3(size / block_size_ws32), + dim3(block_size_ws32), + 0, + 0, + device_input.get(), + device_output.get(), + valid); } else if (current_device_warp_size == ws64) { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_allreduce_sum_kernel), - dim3(size/block_size_ws64), dim3(block_size_ws64), 0, 0, - device_input, device_output, valid - ); + dim3(size / block_size_ws64), + dim3(block_size_ws64), + 0, + 0, + device_input.get(), + device_output.get(), + valid); } HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + output = device_output.load(); test_utils::assert_near(output, expected, test_utils::precision * valid); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); } } @@ -588,55 +540,42 @@ typed_test_def(RocprimWarpReduceTests, name_suffix, ReduceCustomStruct) expected[i] = static_cast(value); } - T* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_input, input.size() * sizeof(typename decltype(input)::value_type))); - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(output.size()); // Launching kernel if (current_device_warp_size == ws32) { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_reduce_sum_kernel), - dim3(size/block_size_ws32), dim3(block_size_ws32), 0, 0, - device_input, device_output - ); + dim3(size / block_size_ws32), + dim3(block_size_ws32), + 0, + 0, + device_input.get(), + device_output.get()); } else if (current_device_warp_size == ws64) { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_reduce_sum_kernel), - dim3(size/block_size_ws64), dim3(block_size_ws64), 0, 0, - device_input, device_output - ); + dim3(size / block_size_ws64), + dim3(block_size_ws64), + 0, + 0, + device_input.get(), + device_output.get()); } HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + output = device_output.load(); test_utils::assert_near(output, expected, test_utils::precision * logical_warp_size); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); } } @@ -703,27 +642,9 @@ typed_test_def(RocprimWarpReduceTests, name_suffix, HeadSegmentedReduceSum) } std::vector output(input.size()); - T* device_input; - flag_type* device_flags; - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_input, input.size() * sizeof(typename decltype(input)::value_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&device_flags, flags.size() * sizeof(typename decltype(flags)::value_type))); - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - device_flags, flags.data(), - flags.size() * sizeof(flag_type), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_flags(flags); + test_utils::device_ptr device_output(output.size()); // Calculate expected results on host std::vector expected(output.size()); @@ -748,32 +669,39 @@ typed_test_def(RocprimWarpReduceTests, name_suffix, HeadSegmentedReduceSum) if (current_device_warp_size == ws32) { hipLaunchKernelGGL( - HIP_KERNEL_NAME(head_segmented_warp_reduce_kernel), - dim3(size/block_size_ws32), dim3(block_size_ws32), 0, 0, - device_input, device_flags, device_output - ); + HIP_KERNEL_NAME(head_segmented_warp_reduce_kernel), + dim3(size / block_size_ws32), + dim3(block_size_ws32), + 0, + 0, + device_input.get(), + device_flags.get(), + device_output.get()); } else if (current_device_warp_size == ws64) { hipLaunchKernelGGL( - HIP_KERNEL_NAME(head_segmented_warp_reduce_kernel), - dim3(size/block_size_ws64), dim3(block_size_ws64), 0, 0, - device_input, device_flags, device_output - ); + HIP_KERNEL_NAME(head_segmented_warp_reduce_kernel), + dim3(size / block_size_ws64), + dim3(block_size_ws64), + 0, + 0, + device_input.get(), + device_flags.get(), + device_output.get()); } HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + output = device_output.load(); std::vector output_segment(output.size(), T(0)); std::vector expected_segment(output.size(), T(0)); @@ -788,10 +716,6 @@ typed_test_def(RocprimWarpReduceTests, name_suffix, HeadSegmentedReduceSum) test_utils::assert_near(output_segment, expected_segment, test_utils::precision * logical_warp_size); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_flags)); - HIP_CHECK(hipFree(device_output)); } } @@ -858,27 +782,9 @@ typed_test_def(RocprimWarpReduceTests, name_suffix, TailSegmentedReduceSum) } std::vector output(input.size()); - T* device_input; - flag_type* device_flags; - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_input, input.size() * sizeof(typename decltype(input)::value_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&device_flags, flags.size() * sizeof(typename decltype(flags)::value_type))); - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - device_flags, flags.data(), - flags.size() * sizeof(flag_type), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_flags(flags); + test_utils::device_ptr device_output(output.size()); // Calculate expected results on host std::vector expected(output.size()); @@ -914,32 +820,39 @@ typed_test_def(RocprimWarpReduceTests, name_suffix, TailSegmentedReduceSum) if (current_device_warp_size == ws32) { hipLaunchKernelGGL( - HIP_KERNEL_NAME(tail_segmented_warp_reduce_kernel), - dim3(size/block_size_ws32), dim3(block_size_ws32), 0, 0, - device_input, device_flags, device_output - ); + HIP_KERNEL_NAME(tail_segmented_warp_reduce_kernel), + dim3(size / block_size_ws32), + dim3(block_size_ws32), + 0, + 0, + device_input.get(), + device_flags.get(), + device_output.get()); } else if (current_device_warp_size == ws64) { hipLaunchKernelGGL( - HIP_KERNEL_NAME(tail_segmented_warp_reduce_kernel), - dim3(size/block_size_ws64), dim3(block_size_ws64), 0, 0, - device_input, device_flags, device_output - ); + HIP_KERNEL_NAME(tail_segmented_warp_reduce_kernel), + dim3(size / block_size_ws64), + dim3(block_size_ws64), + 0, + 0, + device_input.get(), + device_flags.get(), + device_output.get()); } HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + output = device_output.load(); std::vector output_segment(segment_indexes.size()); std::vector expected_segment(segment_indexes.size()); @@ -952,10 +865,6 @@ typed_test_def(RocprimWarpReduceTests, name_suffix, TailSegmentedReduceSum) test_utils::assert_near(output_segment, expected_segment, test_utils::precision * logical_warp_size); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_flags)); - HIP_CHECK(hipFree(device_output)); } } diff --git a/test/rocprim/test_warp_scan.cpp b/test/rocprim/test_warp_scan.cpp index cb4e5f362..bb3d1369e 100644 --- a/test/rocprim/test_warp_scan.cpp +++ b/test/rocprim/test_warp_scan.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2021 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -26,6 +26,7 @@ #include // required test headers +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" // kernel definitions diff --git a/test/rocprim/test_warp_scan.hpp b/test/rocprim/test_warp_scan.hpp index c1f9c1e57..8c0e15101 100644 --- a/test/rocprim/test_warp_scan.hpp +++ b/test/rocprim/test_warp_scan.hpp @@ -94,54 +94,41 @@ typed_test_def(RocprimWarpScanTests, name_suffix, InclusiveScan) } // Writing to device memory - T* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_input, input.size() * sizeof(typename decltype(input)::value_type))); - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(output.size()); // Launching kernel if (current_device_warp_size == ws32) { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_inclusive_scan_kernel), - dim3(grid_size), dim3(block_size), 0, 0, - device_input, device_output - ); + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_input.get(), + device_output.get()); } else if (current_device_warp_size == ws64) { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_inclusive_scan_kernel), - dim3(grid_size), dim3(block_size), 0, 0, - device_input, device_output - ); + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_input.get(), + device_output.get()); } HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + output = device_output.load(); // Validating results test_utils::assert_near(output, expected, test_utils::precision * logical_warp_size); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); } } @@ -219,73 +206,50 @@ typed_test_def(RocprimWarpScanTests, name_suffix, InclusiveScanReduce) } // Writing to device memory - T* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_input, input.size() * sizeof(typename decltype(input)::value_type))); - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - T* device_output_reductions; - HIP_CHECK( - test_common_utils::hipMallocHelper( - &device_output_reductions, - output_reductions.size() * sizeof(typename decltype(output_reductions)::value_type) - ) - ); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(output.size()); + test_utils::device_ptr device_output_reductions(output_reductions.size()); // Launching kernel if (current_device_warp_size == ws32) { hipLaunchKernelGGL( - HIP_KERNEL_NAME(warp_inclusive_scan_reduce_kernel), - dim3(grid_size), dim3(block_size_ws32), 0, 0, - device_input, device_output, device_output_reductions - ); + HIP_KERNEL_NAME( + warp_inclusive_scan_reduce_kernel), + dim3(grid_size), + dim3(block_size_ws32), + 0, + 0, + device_input.get(), + device_output.get(), + device_output_reductions.get()); } else if(current_device_warp_size == ws64) { hipLaunchKernelGGL( - HIP_KERNEL_NAME(warp_inclusive_scan_reduce_kernel), - dim3(grid_size), dim3(block_size_ws64), 0, 0, - device_input, device_output, device_output_reductions - ); + HIP_KERNEL_NAME( + warp_inclusive_scan_reduce_kernel), + dim3(grid_size), + dim3(block_size_ws64), + 0, + 0, + device_input.get(), + device_output.get(), + device_output_reductions.get()); } HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); - - HIP_CHECK( - hipMemcpy( - output_reductions.data(), device_output_reductions, - output_reductions.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + output = device_output.load(); + output_reductions = device_output_reductions.load(); // Validating results test_utils::assert_near(output, expected, test_utils::precision * logical_warp_size); test_utils::assert_near(output_reductions, expected_reductions, test_utils::precision * logical_warp_size); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); - HIP_CHECK(hipFree(device_output_reductions)); } } @@ -362,54 +326,43 @@ typed_test_def(RocprimWarpScanTests, name_suffix, ExclusiveScan) } // Writing to device memory - T* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_input, input.size() * sizeof(typename decltype(input)::value_type))); - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(output.size()); // Launching kernel if (current_device_warp_size == ws32) { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_exclusive_scan_kernel), - dim3(grid_size), dim3(block_size_ws32), 0, 0, - device_input, device_output, init - ); + dim3(grid_size), + dim3(block_size_ws32), + 0, + 0, + device_input.get(), + device_output.get(), + init); } else if (current_device_warp_size == ws64) { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_exclusive_scan_kernel), - dim3(grid_size), dim3(block_size_ws64), 0, 0, - device_input, device_output, init - ); + dim3(grid_size), + dim3(block_size_ws64), + 0, + 0, + device_input.get(), + device_output.get(), + init); } HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + output = device_output.load(); // Validating results test_utils::assert_near(output, expected, test_utils::precision * logical_warp_size); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); } } @@ -494,17 +447,8 @@ typed_test_def(RocprimWarpScanTests, name_suffix, ExclusiveScanWoInit) } // Writing to device memory - T* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper( - &device_input, - input.size() * sizeof(typename decltype(input)::value_type))); - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper( - &device_output, - output.size() * sizeof(typename decltype(output)::value_type))); - - HIP_CHECK( - hipMemcpy(device_input, input.data(), input.size() * sizeof(T), hipMemcpyHostToDevice)); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(output.size()); // Launching kernel if(current_device_warp_size == ws32) @@ -516,8 +460,8 @@ typed_test_def(RocprimWarpScanTests, name_suffix, ExclusiveScanWoInit) dim3(block_size_ws32), 0, 0, - device_input, - device_output); + device_input.get(), + device_output.get()); } else if(current_device_warp_size == ws64) { @@ -528,18 +472,15 @@ typed_test_def(RocprimWarpScanTests, name_suffix, ExclusiveScanWoInit) dim3(block_size_ws64), 0, 0, - device_input, - device_output); + device_input.get(), + device_output.get()); } HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK(hipMemcpy(output.data(), - device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost)); + output = device_output.load(); // The first value of each logical warp has an unspecified result, expect whatever we got // for those values to not fail the test. @@ -550,9 +491,6 @@ typed_test_def(RocprimWarpScanTests, name_suffix, ExclusiveScanWoInit) // Validating results test_utils::assert_near(output, expected, test_utils::precision * logical_warp_size); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); } } @@ -639,72 +577,51 @@ typed_test_def(RocprimWarpScanTests, name_suffix, ExclusiveReduceScan) } // Writing to device memory - T* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_input, input.size() * sizeof(typename decltype(input)::value_type))); - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - T* device_output_reductions; - HIP_CHECK( - test_common_utils::hipMallocHelper( - &device_output_reductions, - output_reductions.size() * sizeof(typename decltype(output_reductions)::value_type) - ) - ); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(output.size()); + test_utils::device_ptr device_output_reductions(output_reductions.size()); // Launching kernel if (current_device_warp_size == ws32) { hipLaunchKernelGGL( - HIP_KERNEL_NAME(warp_exclusive_scan_reduce_kernel), - dim3(grid_size), dim3(block_size_ws32), 0, 0, - device_input, device_output, device_output_reductions, init - ); + HIP_KERNEL_NAME( + warp_exclusive_scan_reduce_kernel), + dim3(grid_size), + dim3(block_size_ws32), + 0, + 0, + device_input.get(), + device_output.get(), + device_output_reductions.get(), + init); } else if (current_device_warp_size == ws64) { hipLaunchKernelGGL( - HIP_KERNEL_NAME(warp_exclusive_scan_reduce_kernel), - dim3(grid_size), dim3(block_size_ws64), 0, 0, - device_input, device_output, device_output_reductions, init - ); + HIP_KERNEL_NAME( + warp_exclusive_scan_reduce_kernel), + dim3(grid_size), + dim3(block_size_ws64), + 0, + 0, + device_input.get(), + device_output.get(), + device_output_reductions.get(), + init); } HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); - - HIP_CHECK( - hipMemcpy( - output_reductions.data(), device_output_reductions, - output_reductions.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + output = device_output.load(); + output_reductions = device_output_reductions.load(); // Validating results test_utils::assert_near(output, expected, test_utils::precision * logical_warp_size); test_utils::assert_near(output_reductions, expected_reductions, test_utils::precision * logical_warp_size); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); - HIP_CHECK(hipFree(device_output_reductions)); } } @@ -799,21 +716,9 @@ typed_test_def(RocprimWarpScanTests, name_suffix, ExclusiveReduceScanWoInit) } // Writing to device memory - T* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper( - &device_input, - input.size() * sizeof(typename decltype(input)::value_type))); - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper( - &device_output, - output.size() * sizeof(typename decltype(output)::value_type))); - T* device_output_reductions; - HIP_CHECK(test_common_utils::hipMallocHelper( - &device_output_reductions, - output_reductions.size() * sizeof(typename decltype(output_reductions)::value_type))); - - HIP_CHECK( - hipMemcpy(device_input, input.data(), input.size() * sizeof(T), hipMemcpyHostToDevice)); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(output.size()); + test_utils::device_ptr device_output_reductions(output_reductions.size()); // Launching kernel if(current_device_warp_size == ws32) @@ -826,9 +731,9 @@ typed_test_def(RocprimWarpScanTests, name_suffix, ExclusiveReduceScanWoInit) dim3(block_size_ws32), 0, 0, - device_input, - device_output, - device_output_reductions); + device_input.get(), + device_output.get(), + device_output_reductions.get()); } else if(current_device_warp_size == ws64) { @@ -840,23 +745,16 @@ typed_test_def(RocprimWarpScanTests, name_suffix, ExclusiveReduceScanWoInit) dim3(block_size_ws64), 0, 0, - device_input, - device_output, - device_output_reductions); + device_input.get(), + device_output.get(), + device_output_reductions.get()); } HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK(hipMemcpy(output.data(), - device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost)); - - HIP_CHECK(hipMemcpy(output_reductions.data(), - device_output_reductions, - output_reductions.size() * sizeof(T), - hipMemcpyDeviceToHost)); + output = device_output.load(); + output_reductions = device_output_reductions.load(); // The first value of each logical warp has an unspecified result, expect whatever we got // for those values to not fail the test. @@ -870,10 +768,6 @@ typed_test_def(RocprimWarpScanTests, name_suffix, ExclusiveReduceScanWoInit) test_utils::assert_near(output_reductions, expected_reductions, test_utils::precision * logical_warp_size); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); - HIP_CHECK(hipFree(device_output_reductions)); } } @@ -957,68 +851,44 @@ typed_test_def(RocprimWarpScanTests, name_suffix, Scan) } // Writing to device memory - T* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_input, input.size() * sizeof(typename decltype(input)::value_type))); - T* device_inclusive_output; - HIP_CHECK( - test_common_utils::hipMallocHelper( - &device_inclusive_output, - output_inclusive.size() * sizeof(typename decltype(output_inclusive)::value_type) - ) - ); - T* device_exclusive_output; - HIP_CHECK( - test_common_utils::hipMallocHelper( - &device_exclusive_output, - output_exclusive.size() * sizeof(typename decltype(output_exclusive)::value_type) - ) - ); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_inclusive_output(output_inclusive.size()); + test_utils::device_ptr device_exclusive_output(output_exclusive.size()); // Launching kernel if (current_device_warp_size == ws32) { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_scan_kernel), - dim3(grid_size), dim3(block_size_ws32), 0, 0, - device_input, device_inclusive_output, device_exclusive_output, init - ); + dim3(grid_size), + dim3(block_size_ws32), + 0, + 0, + device_input.get(), + device_inclusive_output.get(), + device_exclusive_output.get(), + init); } else if (current_device_warp_size == ws64) { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_scan_kernel), - dim3(grid_size), dim3(block_size_ws64), 0, 0, - device_input, device_inclusive_output, device_exclusive_output, init - ); + dim3(grid_size), + dim3(block_size_ws64), + 0, + 0, + device_input.get(), + device_inclusive_output.get(), + device_exclusive_output.get(), + init); } HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK( - hipMemcpy( - output_inclusive.data(), device_inclusive_output, - output_inclusive.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); - - HIP_CHECK( - hipMemcpy( - output_exclusive.data(), device_exclusive_output, - output_exclusive.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + output_inclusive = device_inclusive_output.load(); + output_exclusive = device_exclusive_output.load(); // Validating results test_utils::assert_near(output_inclusive, @@ -1027,12 +897,7 @@ typed_test_def(RocprimWarpScanTests, name_suffix, Scan) test_utils::assert_near(output_exclusive, expected_exclusive, test_utils::precision * logical_warp_size); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_inclusive_output)); - HIP_CHECK(hipFree(device_exclusive_output)); } - } typed_test_def(RocprimWarpScanTests, name_suffix, ScanReduce) @@ -1118,85 +983,48 @@ typed_test_def(RocprimWarpScanTests, name_suffix, ScanReduce) } // Writing to device memory - T* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_input, input.size() * sizeof(typename decltype(input)::value_type))); - T* device_inclusive_output; - HIP_CHECK( - test_common_utils::hipMallocHelper( - &device_inclusive_output, - output_inclusive.size() * sizeof(typename decltype(output_inclusive)::value_type) - ) - ); - T* device_exclusive_output; - HIP_CHECK( - test_common_utils::hipMallocHelper( - &device_exclusive_output, - output_exclusive.size() * sizeof(typename decltype(output_exclusive)::value_type) - ) - ); - T* device_output_reductions; - HIP_CHECK( - test_common_utils::hipMallocHelper( - &device_output_reductions, - output_reductions.size() * sizeof(typename decltype(output_reductions)::value_type) - ) - ); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_inclusive_output(output_inclusive.size()); + test_utils::device_ptr device_exclusive_output(output_exclusive.size()); + test_utils::device_ptr device_output_reductions(output_reductions.size()); // Launching kernel if (current_device_warp_size == ws32) { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_scan_reduce_kernel), - dim3(grid_size), dim3(block_size_ws32), 0, 0, - device_input, - device_inclusive_output, device_exclusive_output, device_output_reductions, init - ); + dim3(grid_size), + dim3(block_size_ws32), + 0, + 0, + device_input.get(), + device_inclusive_output.get(), + device_exclusive_output.get(), + device_output_reductions.get(), + init); } else if (current_device_warp_size == ws64) { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_scan_reduce_kernel), - dim3(grid_size), dim3(block_size_ws64), 0, 0, - device_input, - device_inclusive_output, device_exclusive_output, device_output_reductions, init - ); + dim3(grid_size), + dim3(block_size_ws64), + 0, + 0, + device_input.get(), + device_inclusive_output.get(), + device_exclusive_output.get(), + device_output_reductions.get(), + init); } HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK( - hipMemcpy( - output_inclusive.data(), device_inclusive_output, - output_inclusive.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); - - HIP_CHECK( - hipMemcpy( - output_exclusive.data(), device_exclusive_output, - output_exclusive.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); - - HIP_CHECK( - hipMemcpy( - output_reductions.data(), device_output_reductions, - output_reductions.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + output_inclusive = device_inclusive_output.load(); + output_exclusive = device_exclusive_output.load(); + output_reductions = device_output_reductions.load(); // Validating results test_utils::assert_near(output_inclusive, @@ -1208,11 +1036,6 @@ typed_test_def(RocprimWarpScanTests, name_suffix, ScanReduce) test_utils::assert_near(output_reductions, expected_reductions, test_utils::precision * logical_warp_size); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_inclusive_output)); - HIP_CHECK(hipFree(device_exclusive_output)); - HIP_CHECK(hipFree(device_output_reductions)); } } @@ -1295,55 +1118,42 @@ typed_test_def(RocprimWarpScanTests, name_suffix, InclusiveScanCustomType) } // Writing to device memory - T* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_input, input.size() * sizeof(typename decltype(input)::value_type))); - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(output.size()); // Launching kernel if (current_device_warp_size == ws32) { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_inclusive_scan_kernel), - dim3(grid_size), dim3(block_size_ws32), 0, 0, - device_input, device_output - ); + dim3(grid_size), + dim3(block_size_ws32), + 0, + 0, + device_input.get(), + device_output.get()); } else if (current_device_warp_size == ws64) { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_inclusive_scan_kernel), - dim3(grid_size), dim3(block_size_ws64), 0, 0, - device_input, device_output - ); + dim3(grid_size), + dim3(block_size_ws64), + 0, + 0, + device_input.get(), + device_output.get()); } HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + output = device_output.load(); // Validating results test_utils::assert_near(output, expected, test_utils::precision * logical_warp_size); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); } } diff --git a/test/rocprim/test_warp_sort.cpp b/test/rocprim/test_warp_sort.cpp index 21ae49fd1..8978d419e 100644 --- a/test/rocprim/test_warp_sort.cpp +++ b/test/rocprim/test_warp_sort.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2021 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -26,6 +26,7 @@ #include // required test headers +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" // kernel definitions diff --git a/test/rocprim/test_warp_sort.hpp b/test/rocprim/test_warp_sort.hpp index 482453c43..19e8b88f7 100644 --- a/test/rocprim/test_warp_sort.hpp +++ b/test/rocprim/test_warp_sort.hpp @@ -79,18 +79,7 @@ typed_test_def(RocprimWarpSortShuffleBasedTests, name_suffix, Sort) } // Writing to device memory - T* d_output; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(typename decltype(output)::value_type)) - ); - - HIP_CHECK( - hipMemcpy( - d_output, output.data(), - output.size() * sizeof(typename decltype(output)::value_type), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr d_output(output); // Launching kernel hipLaunchKernelGGL( @@ -99,23 +88,15 @@ typed_test_def(RocprimWarpSortShuffleBasedTests, name_suffix, Sort) dim3(block_size), 0, 0, - d_output); + d_output.get()); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - output.size() * sizeof(typename decltype(output)::value_type), - hipMemcpyDeviceToHost - ) - ); + output = d_output.load(); test_utils::assert_eq(output, expected); - - HIP_CHECK(hipFree(d_output)); } } @@ -185,59 +166,28 @@ typed_test_def(RocprimWarpSortShuffleBasedTests, name_suffix, SortKeyInt) } // Writing to device memory - T* d_output_key; - T* d_output_value; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_output_key, output_key.size() * sizeof(typename decltype(output_key)::value_type)) - ); - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_output_value, output_value.size() * sizeof(typename decltype(output_value)::value_type)) - ); - - HIP_CHECK( - hipMemcpy( - d_output_key, output_key.data(), - output_key.size() * sizeof(typename decltype(output_key)::value_type), - hipMemcpyHostToDevice - ) - ); - - HIP_CHECK( - hipMemcpy( - d_output_value, output_value.data(), - output_value.size() * sizeof(typename decltype(output_value)::value_type), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr d_output_key(output_key); + test_utils::device_ptr d_output_value(output_value); // Launching kernel - hipLaunchKernelGGL( - HIP_KERNEL_NAME(test_hip_sort_key_value_kernel< - items_per_thread, block_size, logical_warp_size, T, T - >), - dim3(grid_size), dim3(block_size), 0, 0, - d_output_key, d_output_value - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(test_hip_sort_key_value_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + d_output_key.get(), + d_output_value.get()); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK( - hipMemcpy( - output_key.data(), d_output_key, - output_key.size() * sizeof(typename decltype(output_key)::value_type), - hipMemcpyDeviceToHost - ) - ); - - HIP_CHECK( - hipMemcpy( - output_value.data(), d_output_value, - output_value.size() * sizeof(typename decltype(output_value)::value_type), - hipMemcpyDeviceToHost - ) - ); + output_key = d_output_key.load(); + output_value = d_output_value.load(); std::vector expected_key(expected.size()); std::vector expected_value(expected.size()); @@ -262,9 +212,5 @@ typed_test_def(RocprimWarpSortShuffleBasedTests, name_suffix, SortKeyInt) test_utils::assert_eq(output_key, expected_key); test_utils::assert_eq(output_value, expected_value); - - HIP_CHECK(hipFree(d_output_key)); - HIP_CHECK(hipFree(d_output_value)); } - } diff --git a/test/rocprim/test_warp_store.cpp b/test/rocprim/test_warp_store.cpp index 987177f04..d0fc7c6c1 100644 --- a/test/rocprim/test_warp_store.cpp +++ b/test/rocprim/test_warp_store.cpp @@ -23,6 +23,7 @@ #include "../common_test_header.hpp" #include "rocprim/types.hpp" #include "test_utils.hpp" +#include "test_utils_device_ptr.hpp" #include #include @@ -232,22 +233,15 @@ TYPED_TEST(WarpStoreTest, WarpLoad) std::vector input(items_count); std::iota(input.begin(), input.end(), static_cast(0)); - T* d_input{}; - HIP_CHECK(hipMalloc(&d_input, items_count * sizeof(T))); - HIP_CHECK(hipMemcpy(d_input, input.data(), items_count * sizeof(T), hipMemcpyHostToDevice)); - T* d_output{}; - HIP_CHECK(hipMalloc(&d_output, items_count * sizeof(T))); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(items_count); warp_store_kernel - <<>>(d_input, d_output); + <<>>(d_input.get(), d_output.get()); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); - std::vector output(items_count); - HIP_CHECK(hipMemcpy(output.data(), d_output, items_count * sizeof(T), hipMemcpyDeviceToHost)); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); + std::vector output = d_output.load(); auto expected = input; if(method == ::rocprim::warp_store_method::warp_store_striped) @@ -274,23 +268,16 @@ TYPED_TEST(WarpStoreTest, WarpStoreGuarded) std::vector input(items_count); std::iota(input.begin(), input.end(), static_cast(0)); - T* d_input{}; - HIP_CHECK(hipMalloc(&d_input, items_count * sizeof(T))); - HIP_CHECK(hipMemcpy(d_input, input.data(), items_count * sizeof(T), hipMemcpyHostToDevice)); - T* d_output{}; - HIP_CHECK(hipMalloc(&d_output, items_count * sizeof(T))); - HIP_CHECK(hipMemset(d_output, 0, items_count * sizeof(T))); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(items_count); + HIP_CHECK(hipMemset(d_output.get(), 0, items_count * sizeof(T))); warp_store_guarded_kernel - <<>>(d_input, d_output, valid_items); + <<>>(d_input.get(), d_output.get(), valid_items); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); - std::vector output(items_count); - HIP_CHECK(hipMemcpy(output.data(), d_output, items_count * sizeof(T), hipMemcpyDeviceToHost)); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); + std::vector output = d_output.load(); auto expected = input; if(method == ::rocprim::warp_store_method::warp_store_striped) diff --git a/test/rocprim/test_zip_iterator.cpp b/test/rocprim/test_zip_iterator.cpp index 5e105184e..9ab54d429 100644 --- a/test/rocprim/test_zip_iterator.cpp +++ b/test/rocprim/test_zip_iterator.cpp @@ -21,7 +21,7 @@ // SOFTWARE. #include "../common_test_header.hpp" - +#include "test_utils_device_ptr.hpp" // required rocprim headers #include #include @@ -174,35 +174,11 @@ TEST(RocprimZipIteratorTests, Transform) std::vector input3 = test_utils::get_random_data(size, 1, 100, seed_value); std::vector output(input1.size()); - T1 * d_input1; - T2 * d_input2; - T3 * d_input3; - U * d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input1, input1.size() * sizeof(T1))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input2, input2.size() * sizeof(T2))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input3, input3.size() * sizeof(T3))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(U))); - HIP_CHECK( - hipMemcpy( - d_input1, input1.data(), - input1.size() * sizeof(T1), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_input2, input2.data(), - input2.size() * sizeof(T2), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_input3, input3.data(), - input3.size() * sizeof(T3), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr d_input1(input1); + test_utils::device_ptr d_input2(input2); + test_utils::device_ptr d_input3(input3); + test_utils::device_ptr d_output(output.size()); + HIP_CHECK(hipDeviceSynchronize()); // Calculate expected results on host @@ -219,31 +195,18 @@ TEST(RocprimZipIteratorTests, Transform) ); // Run - HIP_CHECK( - rocprim::transform( - rocprim::make_zip_iterator( - rocprim::make_tuple( - d_input1, d_input2, d_input3 - ) - ), - d_output, - input1.size(), - tuple3_transform_op(), - stream, - debug_synchronous - ) - ); + HIP_CHECK(rocprim::transform( + rocprim::make_zip_iterator( + rocprim::make_tuple(d_input1.get(), d_input2.get(), d_input3.get())), + d_output.get(), + input1.size(), + tuple3_transform_op(), + stream, + debug_synchronous)); HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - output.size() * sizeof(U), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + output = d_output.load(output.size()); // Check if output values are as expected // precision of tuple3_transform_op is precision * 2 @@ -251,11 +214,6 @@ TEST(RocprimZipIteratorTests, Transform) test_utils::assert_near(output, expected, std::max(test_utils::precision, test_utils::precision * 2)); - - HIP_CHECK(hipFree(d_input1)); - HIP_CHECK(hipFree(d_input2)); - HIP_CHECK(hipFree(d_input3)); - HIP_CHECK(hipFree(d_output)); } } @@ -315,40 +273,11 @@ TEST(RocprimZipIteratorTests, TransformReduce) std::vector output1(1, 0); std::vector output2(1, 0); - T1* d_input1; - T2* d_input2; - T3* d_input3; - U1* d_output1; - U2* d_output2; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input1, input1.size() * sizeof(T1))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input2, input2.size() * sizeof(T2))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input3, input3.size() * sizeof(T3))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output1, output1.size() * sizeof(U1))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output2, output2.size() * sizeof(U2))); - - // Copy input data to device - HIP_CHECK( - hipMemcpy( - d_input1, input1.data(), - input1.size() * sizeof(T1), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_input2, input2.data(), - input2.size() * sizeof(T2), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_input3, input3.data(), - input3.size() * sizeof(T3), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_input1(input1); + test_utils::device_ptr d_input2(input2); + test_utils::device_ptr d_input3(input3); + test_utils::device_ptr d_output1(output1.size()); + test_utils::device_ptr d_output2(output2.size()); // Calculate expected results on host U1 expected1 = std::accumulate(input1.begin(), input1.end(), T1(0)); @@ -358,74 +287,45 @@ TEST(RocprimZipIteratorTests, TransformReduce) // temp storage size_t temp_storage_size_bytes; // Get size of d_temp_storage - HIP_CHECK( - rocprim::reduce( - nullptr, - temp_storage_size_bytes, - rocprim::make_transform_iterator( - rocprim::make_zip_iterator( - rocprim::make_tuple(d_input1, d_input2, d_input3) - ), - tuple3to2_transform_op() - ), + HIP_CHECK(rocprim::reduce( + nullptr, + temp_storage_size_bytes, + rocprim::make_transform_iterator( rocprim::make_zip_iterator( - rocprim::make_tuple(d_output1, d_output2) - ), - input1.size(), - tuple2_reduce_op(), - stream, - debug_synchronous - ) - ); + rocprim::make_tuple(d_input1.get(), d_input2.get(), d_input3.get())), + tuple3to2_transform_op()), + rocprim::make_zip_iterator(rocprim::make_tuple(d_output1.get(), d_output2.get())), + input1.size(), + tuple2_reduce_op(), + stream, + debug_synchronous)); HIP_CHECK(hipDeviceSynchronize()); // temp_storage_size_bytes must be >0 ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - void * d_temp_storage = nullptr; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - ASSERT_NE(d_temp_storage, nullptr); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); + ASSERT_NE(d_temp_storage.get(), nullptr); // Run - HIP_CHECK( - rocprim::reduce( - d_temp_storage, - temp_storage_size_bytes, - rocprim::make_transform_iterator( - rocprim::make_zip_iterator( - rocprim::make_tuple(d_input1, d_input2, d_input3) - ), - tuple3to2_transform_op() - ), + HIP_CHECK(rocprim::reduce( + d_temp_storage.get(), + temp_storage_size_bytes, + rocprim::make_transform_iterator( rocprim::make_zip_iterator( - rocprim::make_tuple(d_output1, d_output2) - ), - input1.size(), - tuple2_reduce_op(), - stream, - debug_synchronous - ) - ); + rocprim::make_tuple(d_input1.get(), d_input2.get(), d_input3.get())), + tuple3to2_transform_op()), + rocprim::make_zip_iterator(rocprim::make_tuple(d_output1.get(), d_output2.get())), + input1.size(), + tuple2_reduce_op(), + stream, + debug_synchronous)); HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - HIP_CHECK( - hipMemcpy( - output1.data(), d_output1, - output1.size() * sizeof(U1), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK( - hipMemcpy( - output2.data(), d_output2, - output2.size() * sizeof(U2), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + output1 = d_output1.load(); + output2 = d_output2.load(); // Check if output values are as expected // precision of tuple3to2_transform_op is (0, precision) @@ -440,13 +340,6 @@ TEST(RocprimZipIteratorTests, TransformReduce) expected2, (std::max(test_utils::precision, test_utils::precision) + test_utils::precision)*size); - - HIP_CHECK(hipFree(d_input1)); - HIP_CHECK(hipFree(d_input2)); - HIP_CHECK(hipFree(d_input3)); - HIP_CHECK(hipFree(d_output1)); - HIP_CHECK(hipFree(d_output2)); - HIP_CHECK(hipFree(d_temp_storage)); } } From 4aa1ca54db32d2d28148cedf41f374c2b8bf47b6 Mon Sep 17 00:00:00 2001 From: Anton Gorenko Date: Fri, 29 Nov 2024 09:32:35 +0000 Subject: [PATCH 28/84] fix device adjacent difference tuning benchmark --- ...rk_device_adjacent_difference.parallel.hpp | 45 +++++++++---------- ...device_merge_sort_block_merge.parallel.hpp | 6 +-- 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/benchmark/benchmark_device_adjacent_difference.parallel.hpp b/benchmark/benchmark_device_adjacent_difference.parallel.hpp index 02133b913..66b67ca06 100644 --- a/benchmark/benchmark_device_adjacent_difference.parallel.hpp +++ b/benchmark/benchmark_device_adjacent_difference.parallel.hpp @@ -35,11 +35,11 @@ #include #include +#include +#include #include #include -#include - template std::string config_name() { @@ -239,40 +239,37 @@ struct device_adjacent_difference_benchmark : public config_autotune_interface template struct device_adjacent_difference_benchmark_generator { - static constexpr unsigned int min_items_per_thread = 0; + // Device Adjacent difference uses block_load/store_transpose to coalesce memory transaction to global memory + // However it accesses shared memory with a stride of items per thread, which leads to reduced performance if power + // of two is used for small types. Experiments shown that primes are the best choice for performance. + static constexpr std::array primes{1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31}; + static constexpr unsigned int max_items_per_thread_arg = TUNING_SHARED_MEMORY_MAX / (BlockSize * sizeof(T) * 2 + sizeof(T)); template struct create_ipt { - // Device Adjacent difference uses block_load/store_transpose to coalesc memory transaction to global memory - // However it accesses shared memory with a stride of items per thread, which leads to reduced performance if power - // of two is used for small types. Experiments shown that primes are the best choice for performance. - static constexpr int primes[] = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, - 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97}; - static constexpr uint ipt_num = primes[IptValueIndex]; - using generated_config = rocprim::adjacent_difference_config; - - void operator()(std::vector>& storage) + template + auto operator()(std::vector>& storage) + -> std::enable_if_t<(ipt_num < max_items_per_thread_arg)> { - if(ipt_num < max_items_per_thread_arg) - { - storage.emplace_back( - std::make_unique>()); - } + using generated_config = rocprim::adjacent_difference_config; + + storage.emplace_back( + std::make_unique< + device_adjacent_difference_benchmark>()); } + + template + auto operator()(std::vector>&) + -> std::enable_if_t + {} }; static void create(std::vector>& storage) { - static constexpr unsigned int max_items_per_thread - = rocprim::Log2::VALUE; - static_for_each, - create_ipt>(storage); + static_for_each, create_ipt>(storage); } }; diff --git a/benchmark/benchmark_device_merge_sort_block_merge.parallel.hpp b/benchmark/benchmark_device_merge_sort_block_merge.parallel.hpp index ff04119ff..960652e5d 100644 --- a/benchmark/benchmark_device_merge_sort_block_merge.parallel.hpp +++ b/benchmark/benchmark_device_merge_sort_block_merge.parallel.hpp @@ -87,7 +87,7 @@ struct device_merge_sort_block_merge_benchmark : public config_autotune_interfac { using key_type = Key; - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(key_type); // Generate data @@ -223,7 +223,7 @@ struct device_merge_sort_block_merge_benchmark : public config_autotune_interfac using key_type = Key; using value_type = Value; - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(key_type); // Generate data @@ -332,7 +332,7 @@ struct device_merge_sort_block_merge_benchmark : public config_autotune_interfac stream)); HIP_CHECK(hipMemcpyAsync(d_values, d_values_input, - size * sizeof(key_type), + size * sizeof(value_type), hipMemcpyDeviceToDevice, stream)); HIP_CHECK(hipEventRecord(start, stream)); From 9cf8ca94b3be6dc9bb14dafa74c302cf2a9eb8ce Mon Sep 17 00:00:00 2001 From: Beatriz Navidad Vilches Date: Fri, 29 Nov 2024 12:29:33 +0000 Subject: [PATCH 29/84] Fix warp dpp reduction --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31d678556..13a8d6ad8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ Full documentation for rocPRIM is available at [https://rocm.docs.amd.com/projec * Fixed device radix sort not returning the correct required temporary storage when a double buffer contains `nullptr`. +### Resolved issues + +* Fixed the warp-level reduction `rocprim::warp_reduce.reduce` DPP implementation to avoid undefined intermediate values during the reduction. + ## rocPRIM 3.4.0 for ROCm 6.4.0 ### Added From 7e81f1e8391ce8640788d194175329181b49534d Mon Sep 17 00:00:00 2001 From: Anton Gorenko Date: Fri, 29 Nov 2024 14:54:38 +0000 Subject: [PATCH 30/84] Fix static assertion for int128 in ONNX Runtime with GNU extensions --- .../include/rocprim/type_traits_interface.hpp | 43 ++++++++++++++----- test/rocprim/CMakeLists.txt | 21 ++++++--- 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/rocprim/include/rocprim/type_traits_interface.hpp b/rocprim/include/rocprim/type_traits_interface.hpp index 2efcc5372..25eccc628 100644 --- a/rocprim/include/rocprim/type_traits_interface.hpp +++ b/rocprim/include/rocprim/type_traits_interface.hpp @@ -31,9 +31,6 @@ #ifndef ROCPRIM_DO_NOT_COMPILE_IF #define ROCPRIM_DO_NOT_COMPILE_IF(condition, msg) static_assert(!(condition), msg) #endif -#ifndef ROCPRIM_COMPILE_ERROR - #define ROCPRIM_COMPILE_ERROR(msg) static_assert(false, msg) -#endif /// \brief Wrapper macro for std::enable_if aims to increase code readability #ifndef ROCPRIM_REQUIRES #define ROCPRIM_REQUIRES(...) typename std::enable_if<(__VA_ARGS__)>::type* = nullptr @@ -687,10 +684,16 @@ struct traits::define using float_bit_mask = traits::float_bit_mask::values; }; -/// \brief This is the definition of traits of `rocprim::int128_t` -/// rocPRIM arithmetic type -template<> -struct traits::define +// Type traits like std::is_integral and std::is_arithmetic may be defined for 128-bit integral +// types (__int128_t and __uint128_t) in several cases: +// * with libstdc++ when GNU extensions are enabled (-std=gnu++17, which is the default C++ +// standard in clang); +// * always with libc++ (it is used on HIP SDK for Windows). + +namespace detail +{ + +struct define_int128_t { /// \brief Trait `is_arithmetic` for this type using is_arithmetic = traits::is_arithmetic::values; @@ -700,10 +703,7 @@ struct traits::define using integral_sign = traits::integral_sign::values; }; -/// \brief This is the definition of traits of `rocprim::uint128_t` -/// rocPRIM arithmetic type -template<> -struct traits::define +struct define_uint128_t { /// \brief Trait `is_arithmetic` for this type using is_arithmetic = traits::is_arithmetic::values; @@ -712,6 +712,27 @@ struct traits::define /// \brief Trait `integral_sign` for this type using integral_sign = traits::integral_sign::values; }; + +} // namespace detail + +/// \brief This is the definition of traits of `rocprim::int128_t` +/// rocPRIM arithmetic type +template<> +struct traits::define + : std::conditional_t::value, + traits::define, + detail::define_int128_t> +{}; + +/// \brief This is the definition of traits of `rocprim::uint128_t` +/// rocPRIM arithmetic type +template<> +struct traits::define + : std::conditional_t::value, + traits::define, + detail::define_uint128_t> +{}; + /// @} /// \defgroup rocprim_type_traits_wrapper Handy wrappers for obtaining type traits diff --git a/test/rocprim/CMakeLists.txt b/test/rocprim/CMakeLists.txt index c6d5cf258..bfc447cc4 100644 --- a/test/rocprim/CMakeLists.txt +++ b/test/rocprim/CMakeLists.txt @@ -214,20 +214,26 @@ ${TEST_TYPE_SLICE_COUNT} test type slice(s) for test target ${TEST_TARGET}") set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${SOURCES}) endfunction() -function(add_rocprim_cpp17_test TEST_NAME TEST_SOURCES) - add_rocprim_test(${TEST_NAME} ${TEST_SOURCES}) +function(add_rocprim_cpp_standard_test STANDARD EXTENSIONS TARGET_SUFFIX TEST_NAME TEST_SOURCES) get_rocprim_test_target(${TEST_SOURCES} TEST_TARGET) + set(TEST_TARGET ${TEST_TARGET}${TARGET_SUFFIX}) + add_rocprim_test_internal(${TEST_NAME}${TARGET_SUFFIX} "${TEST_SOURCES}" ${TEST_TARGET}) # Request C++ standard 17, but decay to a previous version if not available: set_target_properties(${TEST_TARGET} PROPERTIES - CXX_STANDARD 17) + CXX_STANDARD ${STANDARD} + CXX_EXTENSIONS ${EXTENSIONS}) if(USE_HIPCXX) set_target_properties(${TEST_TARGET} PROPERTIES - HIP_STANDARD 17) + HIP_STANDARD ${STANDARD} + HIP_EXTENSIONS ${EXTENSIONS}) endif() endfunction() +function(add_rocprim_cpp17_test TEST_NAME TEST_SOURCES) + add_rocprim_cpp_standard_test(17 OFF "" ${TEST_NAME} ${TEST_SOURCES}) +endfunction() # **************************************************************************** # Tests @@ -294,7 +300,12 @@ add_rocprim_test("rocprim.texture_cache_iterator" test_texture_cache_iterator.cp add_rocprim_test("rocprim.thread" test_thread.cpp) add_rocprim_test("rocprim.thread_algos" test_thread_algos.cpp) add_rocprim_test("rocprim.transform_iterator" test_transform_iterator.cpp) -add_rocprim_test("rocprim.type_traits_interface" test_type_traits_interface.cpp) +add_rocprim_cpp_standard_test(14 OFF "_cpp14" "rocprim.type_traits_interface" test_type_traits_interface.cpp) +add_rocprim_cpp_standard_test(14 ON "_gnupp14" "rocprim.type_traits_interface" test_type_traits_interface.cpp) +add_rocprim_cpp_standard_test(17 OFF "_cpp17" "rocprim.type_traits_interface" test_type_traits_interface.cpp) +add_rocprim_cpp_standard_test(17 ON "_gnupp17" "rocprim.type_traits_interface" test_type_traits_interface.cpp) +add_rocprim_cpp_standard_test(20 OFF "_cpp20" "rocprim.type_traits_interface" test_type_traits_interface.cpp) +add_rocprim_cpp_standard_test(20 ON "_gnupp20" "rocprim.type_traits_interface" test_type_traits_interface.cpp) add_rocprim_test("rocprim.no_half_operators" test_no_half_operators.cpp) add_rocprim_test("rocprim.intrinsics" test_intrinsics.cpp) add_rocprim_test("rocprim.intrinsics_atomic" test_intrinsics_atomic.cpp) From 0e572d2b5438d9e4dd2763f86b741f3d734ee2fd Mon Sep 17 00:00:00 2001 From: Robin Voetter Date: Mon, 2 Dec 2024 15:56:19 +0000 Subject: [PATCH 31/84] fix GenerateResourceSpec for gfx90a --- cmake/GenerateResourceSpec.cmake | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cmake/GenerateResourceSpec.cmake b/cmake/GenerateResourceSpec.cmake index 213f451b9..ebb6139ed 100755 --- a/cmake/GenerateResourceSpec.cmake +++ b/cmake/GenerateResourceSpec.cmake @@ -37,7 +37,7 @@ string(REGEX MATCHALL [[--(gfx[0-9a-f]+)]] # std::string ip; # int id; # }; -# +# # std::vector GFXIP_AND_ID{ {"gfx900",0},{"gfx803",1},{"gfx900",2} }; # std::sort(GFXIP_AND_ID.begin(), GFXIP_AND_ID.end(), # [](const device& lhs, const device& rhs) @@ -85,14 +85,14 @@ list(SORT GFXIP_AND_ID) set(JSON_PAYLOAD) set(IT1 0) list(GET GFXIP_AND_ID ${IT1} I1) -string(REGEX REPLACE ":[0-9]+" "" IP1 ${I1}) +string(REGEX REPLACE ":[0-9a-f]+" "" IP1 ${I1}) list(LENGTH GFXIP_AND_ID COUNT) while(IT1 LESS COUNT) string(APPEND JSON_PAYLOAD "\n \"${IP1}\": [") set(IT2 ${IT1}) list(GET GFXIP_AND_ID ${IT2} I2) - string(REGEX REPLACE [[:[0-9]+$]] "" IP2 ${I2}) - string(REGEX REPLACE [[^gfx[0-9]+:]] "" ID2 ${I2}) + string(REGEX REPLACE [[:[0-9a-f]+$]] "" IP2 ${I2}) + string(REGEX REPLACE [[^gfx[0-9a-f]+:]] "" ID2 ${I2}) while(${IP2} STREQUAL ${IP1} AND IT2 LESS COUNT) string(APPEND JSON_PAYLOAD "\n {\n" @@ -102,8 +102,8 @@ while(IT1 LESS COUNT) math(EXPR IT2 "${IT2} + 1") if(IT2 LESS COUNT) list(GET GFXIP_AND_ID ${IT2} I2) - string(REGEX REPLACE [[:[0-9]+$]] "" IP2 ${I2}) - string(REGEX REPLACE [[^gfx[0-9]+:]] "" ID2 ${I2}) + string(REGEX REPLACE [[:[0-9a-f]+$]] "" IP2 ${I2}) + string(REGEX REPLACE [[^gfx[0-9a-f]+:]] "" ID2 ${I2}) endif() endwhile() string(REGEX REPLACE [[,$]] "" JSON_PAYLOAD ${JSON_PAYLOAD}) @@ -132,4 +132,4 @@ file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/resources.json ${JSON_HEAD} ${JSON_PAYLOAD} ${JSON_TAIL} -) \ No newline at end of file +) From 0654d1a9ff335160082f8edbcbe3259587ed28ec Mon Sep 17 00:00:00 2001 From: Bence Parajdi Date: Mon, 2 Dec 2024 15:57:09 +0000 Subject: [PATCH 32/84] Resolve "ROCPRIM_TARGET_ARCH deprecation" --- README.md | 5 ++--- docs/device_ops/config.rst | 16 ++-------------- rocprim/include/rocprim/config.hpp | 11 ----------- rocprim/include/rocprim/device/config_types.hpp | 7 +++---- 4 files changed, 7 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 329b17042..4c7088284 100644 --- a/README.md +++ b/README.md @@ -314,9 +314,8 @@ different types and operations, by passing compile-time configuration structures parameter. The main "knobs" are usually the size of the block and the number of items processed by a single thread. -rocPRIM has built-in default configurations for each of its primitives. In order to use the included -configurations, you need to define the macro `ROCPRIM_TARGET_ARCH` as `803` if you want the -algorithms optimized for gfx803 GCN version, or to `900` for gfx900. +rocPRIM has built-in default configurations for each of its primitives, these will be used automatically +based on the input types and the target architecture from the stream used. ## hipCUB diff --git a/docs/device_ops/config.rst b/docs/device_ops/config.rst index badfae3c5..875c4979f 100644 --- a/docs/device_ops/config.rst +++ b/docs/device_ops/config.rst @@ -20,17 +20,5 @@ the GPU model .. doxygenstruct:: rocprim::default_config -.. warning:: - - To provide information about the GPU you're targeting, you have to - set ``ROCPRIM_TARGET_ARCH``. - - If the target is not supported by ``rocPRIM``, the templates will - use the configuration for the model ``900``. - - If ``ROCPRIM_TARGET_TARGET`` is not defined, it defaults to ``0``, - which is not supported by ``rocPRIM`` and thus the configurations - will be for the model ``900``. - - - +The default configuration. When used the dynamic dispatch will find an optimal configuration +based on the type of the input data and the target architecture of the stream. diff --git a/rocprim/include/rocprim/config.hpp b/rocprim/include/rocprim/config.hpp index 7c96b54cc..393297317 100644 --- a/rocprim/include/rocprim/config.hpp +++ b/rocprim/include/rocprim/config.hpp @@ -131,17 +131,6 @@ #define ROCPRIM_THREAD_STORE_USE_CACHE_MODIFIERS 1 #endif - -// Defines targeted AMD architecture. Supported values: -// * 803 (gfx803) -// * 900 (gfx900) -// * 906 (gfx906) -// * 908 (gfx908) -// * 910 (gfx90a) -#ifndef ROCPRIM_TARGET_ARCH - #define ROCPRIM_TARGET_ARCH 0 -#endif - #ifndef ROCPRIM_NAVI #if defined(__HIP_DEVICE_COMPILE__) \ && (defined(__GFX10__) || defined(__GFX11__) || defined(__GFX12__)) diff --git a/rocprim/include/rocprim/device/config_types.hpp b/rocprim/include/rocprim/device/config_types.hpp index 7dfe5e740..2569aa3c4 100644 --- a/rocprim/include/rocprim/device/config_types.hpp +++ b/rocprim/include/rocprim/device/config_types.hpp @@ -37,10 +37,9 @@ BEGIN_ROCPRIM_NAMESPACE /// \brief Special type used to show that the given device-level operation -/// will be executed with optimal configuration dependent on types of the function's parameters -/// and the target device architecture specified by ROCPRIM_TARGET_ARCH. -/// Algorithms supporting dynamic dispatch will ignore ROCPRIM_TARGET_ARCH and -/// launch using optimal configuration based on the target architecture derived from the stream. +/// will be executed with optimal configuration dependent on types of the function's parameters. +/// With dynamic dispatch algorithms will launch using optimal configuration based on the target +/// architecture derived from the stream. struct default_config { #ifndef DOXYGEN_SHOULD_SKIP_THIS From be04730bff593c0592ce5d72adc39afffe3f5e09 Mon Sep 17 00:00:00 2001 From: Sander Bos Date: Mon, 2 Dec 2024 16:19:03 +0000 Subject: [PATCH 33/84] Fix warnings when generating sphinx docs --- docs/device_ops/adjacent_find.rst | 3 +- docs/device_ops/find_end.rst | 2 +- docs/device_ops/merge.rst | 2 +- docs/device_ops/partial_sort.rst | 4 +- docs/device_ops/search_n.rst | 4 +- docs/doxygen/Doxyfile | 695 ++++++++++++------ .../device/detail/device_config_helper.hpp | 2 +- .../rocprim/device/device_adjacent_find.hpp | 6 +- .../rocprim/device/device_search_n.hpp | 3 + 9 files changed, 487 insertions(+), 234 deletions(-) diff --git a/docs/device_ops/adjacent_find.rst b/docs/device_ops/adjacent_find.rst index 30e6d99d8..a0fee45ec 100644 --- a/docs/device_ops/adjacent_find.rst +++ b/docs/device_ops/adjacent_find.rst @@ -16,5 +16,4 @@ Configuring the kernel adjacent_find ======================== -.. doxygenfunction:: rocprim::adjacent_find(void* const temporary_storage, std::size_t& storage_size, InputIteratorType input, OutputIteratorType output, const std::size_t size, const BinaryPred op=BinaryPred{}, const hipStream_t stream=0, const bool debug_synchronous=false) - +.. doxygenfunction:: rocprim::adjacent_find(void* const temporary_storage, std::size_t& storage_size, InputIterator input, OutputIterator output, const std::size_t size, BinaryPred op=BinaryPred{}, const hipStream_t stream=0, const bool debug_synchronous=false) diff --git a/docs/device_ops/find_end.rst b/docs/device_ops/find_end.rst index 70fa80811..07a1c1649 100644 --- a/docs/device_ops/find_end.rst +++ b/docs/device_ops/find_end.rst @@ -11,7 +11,7 @@ Find end Configuring the kernel ~~~~~~~~~~~~~~~~~~~~~~ -.. doxygenstruct:: rocprim::search_config +See :ref:`Configuring the kernel of Search `. find_end ~~~~~~~~ diff --git a/docs/device_ops/merge.rst b/docs/device_ops/merge.rst index 604e25675..9c088af39 100644 --- a/docs/device_ops/merge.rst +++ b/docs/device_ops/merge.rst @@ -11,7 +11,7 @@ Configuring the kernel ======================= -.. doxygentypedef:: rocprim::merge_config +.. doxygenstruct:: rocprim::merge_config merge ========== diff --git a/docs/device_ops/partial_sort.rst b/docs/device_ops/partial_sort.rst index 222ed25f1..856766c2b 100644 --- a/docs/device_ops/partial_sort.rst +++ b/docs/device_ops/partial_sort.rst @@ -16,5 +16,5 @@ Configuring the kernel partial_sort ~~~~~~~~~~~~ -.. doxygenfunction:: rocprim::partial_sort(void* temporary_storage, size_t& storage_size, KeysIterator keys, size_t middle, size_t size, BinaryFunction compare_function = BinaryFunction(), hipStream_t stream = 0, bool debug_synchronous = false) -.. doxygenfunction:: rocprim::partial_sort_copy(void* temporary_storage, size_t& storage_size, KeysInputIterator keys_input, KeysOutputIterator keys_output, size_t middle, size_t size, BinaryFunction compare_function = BinaryFunction(), hipStream_t stream = 0, bool debug_synchronous = false) +.. doxygenfunction:: rocprim::partial_sort(void* temporary_storage, size_t& storage_size, KeysIterator keys, size_t middle, size_t size, BinaryFunction compare_function = BinaryFunction(), hipStream_t stream = 0, bool debug_synchronous = false, Decomposer decomposer = {}) +.. doxygenfunction:: rocprim::partial_sort_copy(void* temporary_storage, size_t& storage_size, KeysInputIterator keys_input, KeysOutputIterator keys_output, size_t middle, size_t size, BinaryFunction compare_function = BinaryFunction(), hipStream_t stream = 0, bool debug_synchronous = false, Decomposer decomposer = {}) diff --git a/docs/device_ops/search_n.rst b/docs/device_ops/search_n.rst index fba3386ad..a214042f4 100644 --- a/docs/device_ops/search_n.rst +++ b/docs/device_ops/search_n.rst @@ -11,9 +11,9 @@ Configuring the kernel ======================== -.. doxygenstruct:: rocprim::search_n +.. doxygenstruct:: rocprim::search_n_config search_n ======================== -.. doxygenfunction:: rocprim::search_n(void* const temporary_storage, size_t& storage_size, InputIterator input, OutputIterator output, const size_t size, const size_t count, const typename std::iterator_traits::value_type* value, const BinaryPredicate binary_predicate = BinaryPredicate(), const hipStream_t stream = static_cast(0), const bool debug_synchronous = false) +.. doxygenfunction:: rocprim::search_n(void* temporary_storage, size_t& storage_size, InputIterator input, OutputIterator output, const size_t size, const size_t count, const typename std::iterator_traits::value_type* value, const BinaryPredicate binary_predicate = BinaryPredicate(), const hipStream_t stream = static_cast(0), const bool debug_synchronous = false) diff --git a/docs/doxygen/Doxyfile b/docs/doxygen/Doxyfile index 5dc7aa7ca..2795eec11 100644 --- a/docs/doxygen/Doxyfile +++ b/docs/doxygen/Doxyfile @@ -1,4 +1,4 @@ -# Doxyfile 1.8.11 +# Doxyfile 1.9.4 # This file describes the settings to be used by the documentation system # doxygen (www.doxygen.org) for a project. @@ -12,16 +12,25 @@ # For lists, items can also be appended using: # TAG += value [value, ...] # Values that contain spaces should be placed between quotes (\" \"). +# +# Note: +# +# Use doxygen to compare the used configuration file with the template +# configuration file: +# doxygen -x [configFile] +# Use doxygen to compare the used configuration file with the template +# configuration file without replacing the environment variables: +# doxygen -x_noenv [configFile] #--------------------------------------------------------------------------- # Project related configuration options #--------------------------------------------------------------------------- -# This tag specifies the encoding used for all characters in the config file -# that follow. The default is UTF-8 which is also the encoding used for all text -# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv -# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv -# for the list of possible encodings. +# This tag specifies the encoding used for all characters in the configuration +# file that follow. The default is UTF-8 which is also the encoding used for all +# text before the first occurrence of this tag. Doxygen uses libiconv (or the +# iconv built into libc) for the transcoding. See +# https://www.gnu.org/software/libiconv/ for the list of possible encodings. # The default value is: UTF-8. DOXYFILE_ENCODING = UTF-8 @@ -60,16 +69,28 @@ PROJECT_LOGO = OUTPUT_DIRECTORY = . -# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- -# directories (in 2 levels) under the output directory of each output format and -# will distribute the generated files over these directories. Enabling this +# If the CREATE_SUBDIRS tag is set to YES then doxygen will create up to 4096 +# sub-directories (in 2 levels) under the output directory of each output format +# and will distribute the generated files over these directories. Enabling this # option can be useful when feeding doxygen a huge amount of source files, where # putting all generated files in the same directory would otherwise causes -# performance problems for the file system. +# performance problems for the file system. Adapt CREATE_SUBDIRS_LEVEL to +# control the number of sub-directories. # The default value is: NO. CREATE_SUBDIRS = NO +# Controls the number of sub-directories that will be created when +# CREATE_SUBDIRS tag is set to YES. Level 0 represents 16 directories, and every +# level increment doubles the number of directories, resulting in 4096 +# directories at level 8 which is the default and also the maximum value. The +# sub-directories are organized in 2 levels, the first level always has a fixed +# numer of 16 directories. +# Minimum value: 0, maximum value: 8, default value: 8. +# This tag requires that the tag CREATE_SUBDIRS is set to YES. + +CREATE_SUBDIRS_LEVEL = 8 + # If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII # characters to appear in the names of generated files. If set to NO, non-ASCII # characters will be escaped, for example _xE3_x81_x84 will be used for Unicode @@ -81,14 +102,14 @@ ALLOW_UNICODE_NAMES = NO # The OUTPUT_LANGUAGE tag is used to specify the language in which all # documentation generated by doxygen is written. Doxygen will use this # information to generate all constant output in the proper language. -# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, -# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), -# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, -# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), -# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, -# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, -# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, -# Ukrainian and Vietnamese. +# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Bulgarian, +# Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, Dutch, English +# (United States), Esperanto, Farsi (Persian), Finnish, French, German, Greek, +# Hindi, Hungarian, Indonesian, Italian, Japanese, Japanese-en (Japanese with +# English messages), Korean, Korean-en (Korean with English messages), Latvian, +# Lithuanian, Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, +# Romanian, Russian, Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, +# Swedish, Turkish, Ukrainian and Vietnamese. # The default value is: English. OUTPUT_LANGUAGE = English @@ -179,6 +200,16 @@ SHORT_NAMES = NO JAVADOC_AUTOBRIEF = NO +# If the JAVADOC_BANNER tag is set to YES then doxygen will interpret a line +# such as +# /*************** +# as being the beginning of a Javadoc-style comment "banner". If set to NO, the +# Javadoc-style will behave just like regular comments and it will not be +# interpreted by doxygen. +# The default value is: NO. + +JAVADOC_BANNER = NO + # If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first # line (until the first dot) of a Qt-style comment as the brief description. If # set to NO, the Qt-style will behave just like regular Qt-style comments (thus @@ -199,6 +230,14 @@ QT_AUTOBRIEF = NO MULTILINE_CPP_IS_BRIEF = NO +# By default Python docstrings are displayed as preformatted text and doxygen's +# special commands cannot be used. By setting PYTHON_DOCSTRING to NO the +# doxygen's special commands can be used and the contents of the docstring +# documentation blocks is shown as doxygen documentation. +# The default value is: YES. + +PYTHON_DOCSTRING = YES + # If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the # documentation from any documented member that it re-implements. # The default value is: YES. @@ -222,20 +261,19 @@ TAB_SIZE = 8 # the documentation. An alias has the form: # name=value # For example adding -# "sideeffect=@par Side Effects:\n" +# "sideeffect=@par Side Effects:^^" # will allow you to put the command \sideeffect (or @sideeffect) in the # documentation, which will result in a user-defined paragraph with heading -# "Side Effects:". You can put \n's in the value part of an alias to insert -# newlines. - -ALIASES = skip_doxy_start="\{" -ALIASES += skip_doxy_end="\}" - -# This tag can be used to specify a number of word-keyword mappings (TCL only). -# A mapping has the form "name=value". For example adding "class=itcl::class" -# will allow you to use the command class in the itcl::class meaning. +# "Side Effects:". Note that you cannot put \n's in the value part of an alias +# to insert newlines (in the resulting output). You can put ^^ in the value part +# of an alias to insert a newline as if a physical newline was in the original +# file. When you need a literal { or } or , in the value part of an alias you +# have to escape them by means of a backslash (\), this can lead to conflicts +# with the commands \{ and \} for these it is advised to use the version @{ and +# @} or use a double escape (\\{ and \\}) -TCL_SUBST = +ALIASES = skip_doxy_start=\{ \ + skip_doxy_end=\} # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources # only. Doxygen will then generate output that is more tailored for C. For @@ -265,28 +303,40 @@ OPTIMIZE_FOR_FORTRAN = NO OPTIMIZE_OUTPUT_VHDL = NO +# Set the OPTIMIZE_OUTPUT_SLICE tag to YES if your project consists of Slice +# sources only. Doxygen will then generate output that is more tailored for that +# language. For instance, namespaces will be presented as modules, types will be +# separated into more groups, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_SLICE = NO + # Doxygen selects the parser to use depending on the extension of the files it # parses. With this tag you can assign which parser to use for a given # extension. Doxygen has a built-in mapping, but you can override or extend it # using this tag. The format is ext=language, where ext is a file extension, and -# language is one of the parsers supported by doxygen: IDL, Java, Javascript, -# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran: -# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran: -# Fortran. In the later case the parser tries to guess whether the code is fixed -# or free formatted code, this is the default for Fortran type files), VHDL. For -# instance to make doxygen treat .inc files as Fortran files (default is PHP), -# and .f files as C (default is Fortran), use: inc=Fortran f=C. +# language is one of the parsers supported by doxygen: IDL, Java, JavaScript, +# Csharp (C#), C, C++, Lex, D, PHP, md (Markdown), Objective-C, Python, Slice, +# VHDL, Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: +# FortranFree, unknown formatted Fortran: Fortran. In the later case the parser +# tries to guess whether the code is fixed or free formatted code, this is the +# default for Fortran type files). For instance to make doxygen treat .inc files +# as Fortran files (default is PHP), and .f files as C (default is Fortran), +# use: inc=Fortran f=C. # # Note: For files without extension you can use no_extension as a placeholder. # # Note that for custom extensions you also need to set FILE_PATTERNS otherwise -# the files are not read by doxygen. +# the files are not read by doxygen. When specifying no_extension you should add +# * to the FILE_PATTERNS. +# +# Note see also the list of default file extension mappings. EXTENSION_MAPPING = # If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments # according to the Markdown format, which allows for more readable -# documentation. See http://daringfireball.net/projects/markdown/ for details. +# documentation. See https://daringfireball.net/projects/markdown/ for details. # The output of markdown processing is further processed by doxygen, so you can # mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in # case of backward compatibilities issues. @@ -294,6 +344,15 @@ EXTENSION_MAPPING = MARKDOWN_SUPPORT = YES +# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up +# to that level are automatically included in the table of contents, even if +# they do not have an id attribute. +# Note: This feature currently applies only to Markdown headings. +# Minimum value: 0, maximum value: 99, default value: 5. +# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. + +TOC_INCLUDE_HEADINGS = 5 + # When enabled doxygen tries to link words that correspond to documented # classes, or namespaces to their corresponding documentation. Such a link can # be prevented in individual cases by putting a % sign in front of the word or @@ -319,7 +378,7 @@ BUILTIN_STL_SUPPORT = NO CPP_CLI_SUPPORT = NO # Set the SIP_SUPPORT tag to YES if your project consists of sip (see: -# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen +# https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen # will parse them like normal C++ but will assume all classes use public instead # of private inheritance when no explicit protection keyword is present. # The default value is: NO. @@ -405,6 +464,19 @@ TYPEDEF_HIDES_STRUCT = NO LOOKUP_CACHE_SIZE = 0 +# The NUM_PROC_THREADS specifies the number of threads doxygen is allowed to use +# during processing. When set to 0 doxygen will based this on the number of +# cores available in the system. You can set it explicitly to a value larger +# than 0 to get more control over the balance between CPU load and processing +# speed. At this moment only the input processing can be done using multiple +# threads. Since this is still an experimental feature the default is set to 1, +# which effectively disables parallel processing. Please report any issues you +# encounter. Generating dot graphs in parallel is controlled by the +# DOT_NUM_THREADS setting. +# Minimum value: 0, maximum value: 32, default value: 1. + +NUM_PROC_THREADS = 1 + #--------------------------------------------------------------------------- # Build related configuration options #--------------------------------------------------------------------------- @@ -425,6 +497,12 @@ EXTRACT_ALL = NO EXTRACT_PRIVATE = NO +# If the EXTRACT_PRIV_VIRTUAL tag is set to YES, documented private virtual +# methods of a class will be included in the documentation. +# The default value is: NO. + +EXTRACT_PRIV_VIRTUAL = NO + # If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal # scope will be included in the documentation. # The default value is: NO. @@ -462,6 +540,13 @@ EXTRACT_LOCAL_METHODS = NO EXTRACT_ANON_NSPACES = NO +# If this flag is set to YES, the name of an unnamed parameter in a declaration +# will be determined by the corresponding definition. By default unnamed +# parameters remain unnamed in the output. +# The default value is: YES. + +RESOLVE_UNNAMED_PARAMS = YES + # If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all # undocumented members inside documented classes or files. If set to NO these # members will be included in the various overviews, but no documentation @@ -479,8 +564,8 @@ HIDE_UNDOC_MEMBERS = NO HIDE_UNDOC_CLASSES = NO # If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend -# (class|struct|union) declarations. If set to NO, these declarations will be -# included in the documentation. +# declarations. If set to NO, these declarations will be included in the +# documentation. # The default value is: NO. HIDE_FRIEND_COMPOUNDS = NO @@ -499,11 +584,18 @@ HIDE_IN_BODY_DOCS = NO INTERNAL_DOCS = NO -# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file -# names in lower-case letters. If set to YES, upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# and Mac users are advised to set this option to NO. +# With the correct setting of option CASE_SENSE_NAMES doxygen will better be +# able to match the capabilities of the underlying filesystem. In case the +# filesystem is case sensitive (i.e. it supports files in the same directory +# whose names only differ in casing), the option must be set to YES to properly +# deal with such files in case they appear in the input. For filesystems that +# are not case sensitive the option should be set to NO to properly deal with +# output files written for symbols that only differ in casing, such as for two +# classes, one named CLASS and the other named Class, and to also support +# references to files without having to specify the exact matching casing. On +# Windows (including Cygwin) and MacOS, users should typically set this option +# to NO, whereas on Linux or other Unix flavors it should typically be set to +# YES. # The default value is: system dependent. CASE_SENSE_NAMES = YES @@ -522,6 +614,12 @@ HIDE_SCOPE_NAMES = NO HIDE_COMPOUND_REFERENCE= NO +# If the SHOW_HEADERFILE tag is set to YES then the documentation for a class +# will show which file needs to be included to use the class. +# The default value is: YES. + +SHOW_HEADERFILE = YES + # If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of # the files that are included by a file in the documentation of that file. # The default value is: YES. @@ -679,7 +777,8 @@ FILE_VERSION_FILTER = # output files in an output format independent way. To create the layout file # that represents doxygen's defaults, run doxygen with the -l option. You can # optionally specify a file name after the option, if omitted DoxygenLayout.xml -# will be used as the name of the layout file. +# will be used as the name of the layout file. See also section "Changing the +# layout of pages" for information. # # Note that if you run doxygen from a directory containing a file called # DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE @@ -690,7 +789,7 @@ LAYOUT_FILE = # The CITE_BIB_FILES tag can be used to specify one or more bib files containing # the reference definitions. This must be a list of .bib files. The .bib # extension is automatically appended if omitted. This requires the bibtex tool -# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info. +# to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info. # For LaTeX the style of the bibliography can be controlled using # LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the # search path. See also \cite for info how to create references. @@ -725,23 +824,35 @@ WARNINGS = YES WARN_IF_UNDOCUMENTED = YES # If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some parameters -# in a documented function, or documenting parameters that don't exist or using -# markup commands wrongly. +# potential errors in the documentation, such as documenting some parameters in +# a documented function twice, or documenting parameters that don't exist or +# using markup commands wrongly. # The default value is: YES. WARN_IF_DOC_ERROR = YES +# If WARN_IF_INCOMPLETE_DOC is set to YES, doxygen will warn about incomplete +# function parameter documentation. If set to NO, doxygen will accept that some +# parameters have no documentation without warning. +# The default value is: YES. + +WARN_IF_INCOMPLETE_DOC = YES + # This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that # are documented, but have no documentation for their parameters or return -# value. If set to NO, doxygen will only warn about wrong or incomplete -# parameter documentation, but not about the absence of documentation. +# value. If set to NO, doxygen will only warn about wrong parameter +# documentation, but not about the absence of documentation. If EXTRACT_ALL is +# set to YES then this flag will automatically be disabled. See also +# WARN_IF_INCOMPLETE_DOC # The default value is: NO. WARN_NO_PARAMDOC = NO # If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when -# a warning is encountered. +# a warning is encountered. If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS +# then doxygen will continue running as if WARN_AS_ERROR tag is set to NO, but +# at the end of the doxygen process doxygen will return with a non-zero status. +# Possible values are: NO, YES and FAIL_ON_WARNINGS. # The default value is: NO. WARN_AS_ERROR = YES @@ -752,13 +863,27 @@ WARN_AS_ERROR = YES # and the warning text. Optionally the format may contain $version, which will # be replaced by the version of the file (if it could be obtained via # FILE_VERSION_FILTER) +# See also: WARN_LINE_FORMAT # The default value is: $file:$line: $text. WARN_FORMAT = "$file:$line: $text" +# In the $text part of the WARN_FORMAT command it is possible that a reference +# to a more specific place is given. To make it easier to jump to this place +# (outside of doxygen) the user can define a custom "cut" / "paste" string. +# Example: +# WARN_LINE_FORMAT = "'vi $file +$line'" +# See also: WARN_FORMAT +# The default value is: at line $line of file $file. + +WARN_LINE_FORMAT = "at line $line of file $file" + # The WARN_LOGFILE tag can be used to specify a file to which warning and error # messages should be written. If left blank the output is written to standard -# error (stderr). +# error (stderr). In case the file specified cannot be opened for writing the +# warning and error messages are written to standard error. When as file - is +# specified the warning and error messages are written to standard output +# (stdout). WARN_LOGFILE = @@ -787,8 +912,8 @@ INPUT = mainpage.dox \ # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses # libiconv (or the iconv built into libc) for the transcoding. See the libiconv -# documentation (see: http://www.gnu.org/software/libiconv) for the list of -# possible encodings. +# documentation (see: +# https://www.gnu.org/software/libiconv/) for the list of possible encodings. # The default value is: UTF-8. INPUT_ENCODING = UTF-8 @@ -801,11 +926,15 @@ INPUT_ENCODING = UTF-8 # need to set EXTENSION_MAPPING for the extension otherwise the files are not # read by doxygen. # +# Note the list of default checked file patterns might differ from the list of +# default file extension mappings. +# # If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, # *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, -# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, -# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f, *.for, *.tcl, -# *.vhd, *.vhdl, *.ucf, *.qsf, *.as and *.js. +# *.hh, *.hxx, *.hpp, *.h++, *.l, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, +# *.inc, *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C +# comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, +# *.vhdl, *.ucf, *.qsf and *.ice. FILE_PATTERNS = @@ -844,7 +973,7 @@ EXCLUDE_PATTERNS = # (namespaces, classes, functions, etc.) that should be excluded from the # output. The symbol name can be a fully qualified name, a word, or if the # wildcard * is used, a substring. Examples: ANamespace, AClass, -# AClass::ANamespace, ANamespace::*Test +# ANamespace::AClass, ANamespace::*Test # # Note that the wildcards are matched against the file with absolute path, so to # exclude all test directories use the pattern */test/* @@ -960,7 +1089,7 @@ INLINE_SOURCES = NO STRIP_CODE_COMMENTS = YES # If the REFERENCED_BY_RELATION tag is set to YES then for each documented -# function all documented functions referencing it will be listed. +# entity all documented functions referencing it will be listed. # The default value is: NO. REFERENCED_BY_RELATION = NO @@ -992,12 +1121,12 @@ SOURCE_TOOLTIPS = YES # If the USE_HTAGS tag is set to YES then the references to source code will # point to the HTML generated by the htags(1) tool instead of doxygen built-in # source browser. The htags tool is part of GNU's global source tagging system -# (see http://www.gnu.org/software/global/global.html). You will need version +# (see https://www.gnu.org/software/global/global.html). You will need version # 4.8.6 or higher. # # To use it do the following: # - Install the latest version of global -# - Enable SOURCE_BROWSER and USE_HTAGS in the config file +# - Enable SOURCE_BROWSER and USE_HTAGS in the configuration file # - Make sure the INPUT points to the root of the source tree # - Run doxygen as normal # @@ -1020,16 +1149,24 @@ USE_HTAGS = NO VERBATIM_HEADERS = YES # If the CLANG_ASSISTED_PARSING tag is set to YES then doxygen will use the -# clang parser (see: http://clang.llvm.org/) for more accurate parsing at the -# cost of reduced performance. This can be particularly helpful with template -# rich C++ code for which doxygen's built-in parser lacks the necessary type -# information. +# clang parser (see: +# http://clang.llvm.org/) for more accurate parsing at the cost of reduced +# performance. This can be particularly helpful with template rich C++ code for +# which doxygen's built-in parser lacks the necessary type information. # Note: The availability of this option depends on whether or not doxygen was -# generated with the -Duse-libclang=ON option for CMake. +# generated with the -Duse_libclang=ON option for CMake. # The default value is: NO. CLANG_ASSISTED_PARSING = NO +# If the CLANG_ASSISTED_PARSING tag is set to YES and the CLANG_ADD_INC_PATHS +# tag is set to YES then doxygen will add the directory of each input to the +# include path. +# The default value is: YES. +# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES. + +CLANG_ADD_INC_PATHS = YES + # If clang assisted parsing is enabled you can provide the compiler with command # line options that you would normally use when invoking the compiler. Note that # the include paths will already be set by doxygen for the files and directories @@ -1038,6 +1175,19 @@ CLANG_ASSISTED_PARSING = NO CLANG_OPTIONS = +# If clang assisted parsing is enabled you can provide the clang parser with the +# path to the directory containing a file called compile_commands.json. This +# file is the compilation database (see: +# http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html) containing the +# options used when the source files were built. This is equivalent to +# specifying the -p option to a clang tool, such as clang-check. These options +# will then be passed to the parser. Any options specified with CLANG_OPTIONS +# will be added as well. +# Note: The availability of this option depends on whether or not doxygen was +# generated with the -Duse_libclang=ON option for CMake. + +CLANG_DATABASE_PATH = + #--------------------------------------------------------------------------- # Configuration options related to the alphabetical class index #--------------------------------------------------------------------------- @@ -1049,13 +1199,6 @@ CLANG_OPTIONS = ALPHABETICAL_INDEX = NO -# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in -# which the alphabetical index list will be split. -# Minimum value: 1, maximum value: 20, default value: 5. -# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. - -COLS_IN_ALPHA_INDEX = 5 - # In case all classes in a project start with a common prefix, all classes will # be put under the same header in the alphabetical index. The IGNORE_PREFIX tag # can be used to specify a prefix (or a list of prefixes) that should be ignored @@ -1155,8 +1298,8 @@ HTML_EXTRA_FILES = # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen # will adjust the colors in the style sheet and background images according to -# this color. Hue is specified as an angle on a colorwheel, see -# http://en.wikipedia.org/wiki/Hue for more information. For instance the value +# this color. Hue is specified as an angle on a color-wheel, see +# https://en.wikipedia.org/wiki/Hue for more information. For instance the value # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 # purple, and 360 is red again. # Minimum value: 0, maximum value: 359, default value: 220. @@ -1165,7 +1308,7 @@ HTML_EXTRA_FILES = HTML_COLORSTYLE_HUE = 220 # The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors -# in the HTML output. For a value of 0 the output will use grayscales only. A +# in the HTML output. For a value of 0 the output will use gray-scales only. A # value of 255 will produce the most vivid colors. # Minimum value: 0, maximum value: 255, default value: 100. # This tag requires that the tag GENERATE_HTML is set to YES. @@ -1192,6 +1335,17 @@ HTML_COLORSTYLE_GAMMA = 80 HTML_TIMESTAMP = NO +# If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML +# documentation will contain a main index with vertical navigation menus that +# are dynamically created via JavaScript. If disabled, the navigation index will +# consists of multiple levels of tabs that are statically embedded in every HTML +# page. Disable this option to support browsers that do not have JavaScript, +# like the Qt help browser. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_DYNAMIC_MENUS = YES + # If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML # documentation will contain sections that can be hidden and shown after the # page has loaded. @@ -1215,13 +1369,14 @@ HTML_INDEX_NUM_ENTRIES = 100 # If the GENERATE_DOCSET tag is set to YES, additional index files will be # generated that can be used as input for Apple's Xcode 3 integrated development -# environment (see: http://developer.apple.com/tools/xcode/), introduced with -# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a -# Makefile in the HTML output directory. Running make will produce the docset in -# that directory and running make install will install the docset in +# environment (see: +# https://developer.apple.com/xcode/), introduced with OSX 10.5 (Leopard). To +# create a documentation set, doxygen will generate a Makefile in the HTML +# output directory. Running make will produce the docset in that directory and +# running make install will install the docset in # ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at -# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html -# for more information. +# startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy +# genXcode/_index.html for more information. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. @@ -1235,6 +1390,13 @@ GENERATE_DOCSET = NO DOCSET_FEEDNAME = "Doxygen generated docs" +# This tag determines the URL of the docset feed. A documentation feed provides +# an umbrella under which multiple documentation sets from a single provider +# (such as a company or product suite) can be grouped. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_FEEDURL = + # This tag specifies a string that should uniquely identify the documentation # set bundle. This should be a reverse domain-name style string, e.g. # com.mycompany.MyDocSet. Doxygen will append .docset to the name. @@ -1260,8 +1422,12 @@ DOCSET_PUBLISHER_NAME = Publisher # If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three # additional HTML index files: index.hhp, index.hhc, and index.hhk. The # index.hhp is a project file that can be read by Microsoft's HTML Help Workshop -# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on -# Windows. +# on Windows. In the beginning of 2021 Microsoft took the original page, with +# a.o. the download links, offline the HTML help workshop was already many years +# in maintenance mode). You can download the HTML help workshop from the web +# archives at Installation executable (see: +# http://web.archive.org/web/20160201063255/http://download.microsoft.com/downlo +# ad/0/A/9/0A939EF6-E31C-430F-A3DF-DFAE7960D564/htmlhelp.exe). # # The HTML Help Workshop contains a compiler that can convert all HTML output # generated by doxygen into a single compiled HTML file (.chm). Compiled HTML @@ -1291,7 +1457,7 @@ CHM_FILE = HHC_LOCATION = # The GENERATE_CHI flag controls if a separate .chi index file is generated -# (YES) or that it should be included in the master .chm file (NO). +# (YES) or that it should be included in the main .chm file (NO). # The default value is: NO. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. @@ -1336,7 +1502,8 @@ QCH_FILE = # The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help # Project output. For more information please see Qt Help Project / Namespace -# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace). +# (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_QHP is set to YES. @@ -1344,8 +1511,8 @@ QHP_NAMESPACE = # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt # Help Project output. For more information please see Qt Help Project / Virtual -# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual- -# folders). +# Folders (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual-folders). # The default value is: doc. # This tag requires that the tag GENERATE_QHP is set to YES. @@ -1353,30 +1520,30 @@ QHP_VIRTUAL_FOLDER = doc # If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom # filter to add. For more information please see Qt Help Project / Custom -# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- -# filters). +# Filters (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_NAME = # The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the # custom filter to add. For more information please see Qt Help Project / Custom -# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- -# filters). +# Filters (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_ATTRS = # The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this # project's filter section matches. Qt Help Project / Filter Attributes (see: -# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes). +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_SECT_FILTER_ATTRS = -# The QHG_LOCATION tag can be used to specify the location of Qt's -# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the -# generated .qhp file. +# The QHG_LOCATION tag can be used to specify the location (absolute path +# including file name) of Qt's qhelpgenerator. If non-empty doxygen will try to +# run qhelpgenerator on the generated .qhp file. # This tag requires that the tag GENERATE_QHP is set to YES. QHG_LOCATION = @@ -1419,16 +1586,28 @@ DISABLE_INDEX = NO # to work a browser that supports JavaScript, DHTML, CSS and frames is required # (i.e. any modern browser). Windows users are probably better off using the # HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can -# further fine-tune the look of the index. As an example, the default style -# sheet generated by doxygen has an example that shows how to put an image at -# the root of the tree instead of the PROJECT_NAME. Since the tree basically has -# the same information as the tab index, you could consider setting -# DISABLE_INDEX to YES when enabling this option. +# further fine tune the look of the index (see "Fine-tuning the output"). As an +# example, the default style sheet generated by doxygen has an example that +# shows how to put an image at the root of the tree instead of the PROJECT_NAME. +# Since the tree basically has the same information as the tab index, you could +# consider setting DISABLE_INDEX to YES when enabling this option. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_TREEVIEW = NONE +# When both GENERATE_TREEVIEW and DISABLE_INDEX are set to YES, then the +# FULL_SIDEBAR option determines if the side bar is limited to only the treeview +# area (value NO) or if it should extend to the full height of the window (value +# YES). Setting this to YES gives a layout similar to +# https://docs.readthedocs.io with more room for contents, but less room for the +# project logo, title, and description. If either GENERATE_TREEVIEW or +# DISABLE_INDEX is set to NO, this option has no effect. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FULL_SIDEBAR = NO + # The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that # doxygen will group on one line in the generated HTML documentation. # @@ -1453,6 +1632,24 @@ TREEVIEW_WIDTH = 250 EXT_LINKS_IN_WINDOW = NO +# If the OBFUSCATE_EMAILS tag is set to YES, doxygen will obfuscate email +# addresses. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +OBFUSCATE_EMAILS = YES + +# If the HTML_FORMULA_FORMAT option is set to svg, doxygen will use the pdf2svg +# tool (see https://github.com/dawbarton/pdf2svg) or inkscape (see +# https://inkscape.org) to generate formulas as SVG images instead of PNGs for +# the HTML output. These images will generally look nicer at scaled resolutions. +# Possible values are: png (the default) and svg (looks nicer but requires the +# pdf2svg or inkscape tool). +# The default value is: png. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FORMULA_FORMAT = png + # Use this tag to change the font size of LaTeX formulas included as images in # the HTML documentation. When you change the font size after a successful # doxygen run you need to manually remove any form_*.png images from the HTML @@ -1462,7 +1659,7 @@ EXT_LINKS_IN_WINDOW = NO FORMULA_FONTSIZE = 10 -# Use the FORMULA_TRANPARENT tag to determine whether or not the images +# Use the FORMULA_TRANSPARENT tag to determine whether or not the images # generated for formulas are transparent PNGs. Transparent PNGs are not # supported properly for IE 6.0, but are supported on all modern browsers. # @@ -1473,8 +1670,14 @@ FORMULA_FONTSIZE = 10 FORMULA_TRANSPARENT = YES +# The FORMULA_MACROFILE can contain LaTeX \newcommand and \renewcommand commands +# to create new LaTeX commands to be used in formulas as building blocks. See +# the section "Including formulas" for details. + +FORMULA_MACROFILE = + # Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see -# http://www.mathjax.org) which uses client side Javascript for the rendering +# https://www.mathjax.org) which uses client side JavaScript for the rendering # instead of using pre-rendered bitmaps. Use this if you do not have LaTeX # installed or if you want to formulas look prettier in the HTML output. When # enabled you may also need to install MathJax separately and configure the path @@ -1484,11 +1687,29 @@ FORMULA_TRANSPARENT = YES USE_MATHJAX = NO +# With MATHJAX_VERSION it is possible to specify the MathJax version to be used. +# Note that the different versions of MathJax have different requirements with +# regards to the different settings, so it is possible that also other MathJax +# settings have to be changed when switching between the different MathJax +# versions. +# Possible values are: MathJax_2 and MathJax_3. +# The default value is: MathJax_2. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_VERSION = MathJax_2 + # When MathJax is enabled you can set the default output format to be used for -# the MathJax output. See the MathJax site (see: -# http://docs.mathjax.org/en/latest/output.html) for more details. +# the MathJax output. For more details about the output format see MathJax +# version 2 (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) and MathJax version 3 +# (see: +# http://docs.mathjax.org/en/latest/web/components/output.html). # Possible values are: HTML-CSS (which is slower, but has the best -# compatibility), NativeMML (i.e. MathML) and SVG. +# compatibility. This is the name for Mathjax version 2, for MathJax version 3 +# this will be translated into chtml), NativeMML (i.e. MathML. Only supported +# for NathJax 2. For MathJax version 3 chtml will be used instead.), chtml (This +# is the name for Mathjax version 3, for MathJax version 2 this will be +# translated into HTML-CSS) and SVG. # The default value is: HTML-CSS. # This tag requires that the tag USE_MATHJAX is set to YES. @@ -1501,22 +1722,29 @@ MATHJAX_FORMAT = HTML-CSS # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax # Content Delivery Network so you can quickly see the result without installing # MathJax. However, it is strongly recommended to install a local copy of -# MathJax from http://www.mathjax.org before deployment. -# The default value is: http://cdn.mathjax.org/mathjax/latest. +# MathJax from https://www.mathjax.org before deployment. The default value is: +# - in case of MathJax version 2: https://cdn.jsdelivr.net/npm/mathjax@2 +# - in case of MathJax version 3: https://cdn.jsdelivr.net/npm/mathjax@3 # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax # extension names that should be enabled during MathJax rendering. For example +# for MathJax version 2 (see https://docs.mathjax.org/en/v2.7-latest/tex.html +# #tex-and-latex-extensions): # MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# For example for MathJax version 3 (see +# http://docs.mathjax.org/en/latest/input/tex/extensions/index.html): +# MATHJAX_EXTENSIONS = ams # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_EXTENSIONS = # The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces # of code that will be used on startup of the MathJax code. See the MathJax site -# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an +# (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. For an # example see the documentation. # This tag requires that the tag USE_MATHJAX is set to YES. @@ -1544,7 +1772,7 @@ MATHJAX_CODEFILE = SEARCHENGINE = NO # When the SERVER_BASED_SEARCH tag is enabled the search engine will be -# implemented using a web server instead of a web client using Javascript. There +# implemented using a web server instead of a web client using JavaScript. There # are two flavors of web server based searching depending on the EXTERNAL_SEARCH # setting. When disabled, doxygen will generate a PHP script for searching and # an index file used by the script. When EXTERNAL_SEARCH is enabled the indexing @@ -1563,7 +1791,8 @@ SERVER_BASED_SEARCH = NO # # Doxygen ships with an example indexer (doxyindexer) and search engine # (doxysearch.cgi) which are based on the open source search engine library -# Xapian (see: http://xapian.org/). +# Xapian (see: +# https://xapian.org/). # # See the section "External Indexing and Searching" for details. # The default value is: NO. @@ -1576,8 +1805,9 @@ EXTERNAL_SEARCH = NO # # Doxygen ships with an example indexer (doxyindexer) and search engine # (doxysearch.cgi) which are based on the open source search engine library -# Xapian (see: http://xapian.org/). See the section "External Indexing and -# Searching" for details. +# Xapian (see: +# https://xapian.org/). See the section "External Indexing and Searching" for +# details. # This tag requires that the tag SEARCHENGINE is set to YES. SEARCHENGINE_URL = @@ -1628,21 +1858,35 @@ LATEX_OUTPUT = latex # The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be # invoked. # -# Note that when enabling USE_PDFLATEX this option is only used for generating -# bitmaps for formulas in the HTML output, but not in the Makefile that is -# written to the output directory. -# The default file is: latex. +# Note that when not enabling USE_PDFLATEX the default is latex when enabling +# USE_PDFLATEX the default is pdflatex and when in the later case latex is +# chosen this is overwritten by pdflatex. For specific output languages the +# default can have been set differently, this depends on the implementation of +# the output language. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_CMD_NAME = latex # The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate # index for LaTeX. +# Note: This tag is used in the Makefile / make.bat. +# See also: LATEX_MAKEINDEX_CMD for the part in the generated output file +# (.tex). # The default file is: makeindex. # This tag requires that the tag GENERATE_LATEX is set to YES. MAKEINDEX_CMD_NAME = makeindex +# The LATEX_MAKEINDEX_CMD tag can be used to specify the command name to +# generate index for LaTeX. In case there is no backslash (\) as first character +# it will be automatically added in the LaTeX code. +# Note: This tag is used in the generated output file (.tex). +# See also: MAKEINDEX_CMD_NAME for the part in the Makefile / make.bat. +# The default value is: makeindex. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_MAKEINDEX_CMD = makeindex + # If the COMPACT_LATEX tag is set to YES, doxygen generates more compact LaTeX # documents. This may be useful for small projects and may help to save some # trees in general. @@ -1672,29 +1916,31 @@ PAPER_TYPE = a4wide EXTRA_PACKAGES = -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the -# generated LaTeX document. The header should contain everything until the first -# chapter. If it is left blank doxygen will generate a standard header. See -# section "Doxygen usage" for information on how to let doxygen write the -# default header to a separate file. +# The LATEX_HEADER tag can be used to specify a user-defined LaTeX header for +# the generated LaTeX document. The header should contain everything until the +# first chapter. If it is left blank doxygen will generate a standard header. It +# is highly recommended to start with a default header using +# doxygen -w latex new_header.tex new_footer.tex new_stylesheet.sty +# and then modify the file new_header.tex. See also section "Doxygen usage" for +# information on how to generate the default header that doxygen normally uses. # -# Note: Only use a user-defined header if you know what you are doing! The -# following commands have a special meaning inside the header: $title, -# $datetime, $date, $doxygenversion, $projectname, $projectnumber, -# $projectbrief, $projectlogo. Doxygen will replace $title with the empty -# string, for the replacement values of the other commands the user is referred -# to HTML_HEADER. +# Note: Only use a user-defined header if you know what you are doing! +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. The following +# commands have a special meaning inside the header (and footer): For a +# description of the possible markers and block names see the documentation. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_HEADER = -# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the -# generated LaTeX document. The footer should contain everything after the last -# chapter. If it is left blank doxygen will generate a standard footer. See +# The LATEX_FOOTER tag can be used to specify a user-defined LaTeX footer for +# the generated LaTeX document. The footer should contain everything after the +# last chapter. If it is left blank doxygen will generate a standard footer. See # LATEX_HEADER for more information on how to generate a default footer and what -# special commands can be used inside the footer. -# -# Note: Only use a user-defined footer if you know what you are doing! +# special commands can be used inside the footer. See also section "Doxygen +# usage" for information on how to generate the default footer that doxygen +# normally uses. Note: Only use a user-defined footer if you know what you are +# doing! # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_FOOTER = @@ -1727,9 +1973,11 @@ LATEX_EXTRA_FILES = PDF_HYPERLINKS = YES -# If the USE_PDFLATEX tag is set to YES, doxygen will use pdflatex to generate -# the PDF file directly from the LaTeX files. Set this option to YES, to get a -# higher quality PDF documentation. +# If the USE_PDFLATEX tag is set to YES, doxygen will use the engine as +# specified with LATEX_CMD_NAME to generate the PDF file directly from the LaTeX +# files. Set this option to YES, to get a higher quality PDF documentation. +# +# See also section LATEX_CMD_NAME for selecting the engine. # The default value is: YES. # This tag requires that the tag GENERATE_LATEX is set to YES. @@ -1737,8 +1985,7 @@ USE_PDFLATEX = YES # If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode # command to the generated LaTeX files. This will instruct LaTeX to keep running -# if errors occur, instead of asking the user for help. This option is also used -# when generating formulas in HTML. +# if errors occur, instead of asking the user for help. # The default value is: NO. # This tag requires that the tag GENERATE_LATEX is set to YES. @@ -1751,19 +1998,9 @@ LATEX_BATCHMODE = NO LATEX_HIDE_INDICES = NO -# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source -# code with syntax highlighting in the LaTeX output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. -# This tag requires that the tag GENERATE_LATEX is set to YES. - -LATEX_SOURCE_CODE = NO - # The LATEX_BIB_STYLE tag can be used to specify the style to use for the # bibliography, e.g. plainnat, or ieeetr. See -# http://en.wikipedia.org/wiki/BibTeX and \cite for more info. +# https://en.wikipedia.org/wiki/BibTeX and \cite for more info. # The default value is: plain. # This tag requires that the tag GENERATE_LATEX is set to YES. @@ -1777,6 +2014,14 @@ LATEX_BIB_STYLE = plain LATEX_TIMESTAMP = NO +# The LATEX_EMOJI_DIRECTORY tag is used to specify the (relative or absolute) +# path from which the emoji images will be read. If a relative path is entered, +# it will be relative to the LATEX_OUTPUT directory. If left blank the +# LATEX_OUTPUT directory will be used. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_EMOJI_DIRECTORY = + #--------------------------------------------------------------------------- # Configuration options related to the RTF output #--------------------------------------------------------------------------- @@ -1816,9 +2061,9 @@ COMPACT_RTF = NO RTF_HYPERLINKS = NO -# Load stylesheet definitions from file. Syntax is similar to doxygen's config -# file, i.e. a series of assignments. You only have to provide replacements, -# missing definitions are set to their default value. +# Load stylesheet definitions from file. Syntax is similar to doxygen's +# configuration file, i.e. a series of assignments. You only have to provide +# replacements, missing definitions are set to their default value. # # See also section "Doxygen usage" for information on how to generate the # default style sheet that doxygen normally uses. @@ -1827,22 +2072,12 @@ RTF_HYPERLINKS = NO RTF_STYLESHEET_FILE = # Set optional variables used in the generation of an RTF document. Syntax is -# similar to doxygen's config file. A template extensions file can be generated -# using doxygen -e rtf extensionFile. +# similar to doxygen's configuration file. A template extensions file can be +# generated using doxygen -e rtf extensionFile. # This tag requires that the tag GENERATE_RTF is set to YES. RTF_EXTENSIONS_FILE = -# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code -# with syntax highlighting in the RTF output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. -# This tag requires that the tag GENERATE_RTF is set to YES. - -RTF_SOURCE_CODE = NO - #--------------------------------------------------------------------------- # Configuration options related to the man page output #--------------------------------------------------------------------------- @@ -1914,6 +2149,13 @@ XML_OUTPUT = xml XML_PROGRAMLISTING = YES +# If the XML_NS_MEMB_FILE_SCOPE tag is set to YES, doxygen will include +# namespace members in file scope as well, matching the HTML output. +# The default value is: NO. +# This tag requires that the tag GENERATE_XML is set to YES. + +XML_NS_MEMB_FILE_SCOPE = NO + #--------------------------------------------------------------------------- # Configuration options related to the DOCBOOK output #--------------------------------------------------------------------------- @@ -1932,23 +2174,14 @@ GENERATE_DOCBOOK = NO DOCBOOK_OUTPUT = docbook -# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the -# program listings (including syntax highlighting and cross-referencing -# information) to the DOCBOOK output. Note that enabling this will significantly -# increase the size of the DOCBOOK output. -# The default value is: NO. -# This tag requires that the tag GENERATE_DOCBOOK is set to YES. - -DOCBOOK_PROGRAMLISTING = NO - #--------------------------------------------------------------------------- # Configuration options for the AutoGen Definitions output #--------------------------------------------------------------------------- # If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an -# AutoGen Definitions (see http://autogen.sf.net) file that captures the -# structure of the code including all documentation. Note that this feature is -# still experimental and incomplete at the moment. +# AutoGen Definitions (see http://autogen.sourceforge.net/) file that captures +# the structure of the code including all documentation. Note that this feature +# is still experimental and incomplete at the moment. # The default value is: NO. GENERATE_AUTOGEN_DEF = NO @@ -2027,7 +2260,8 @@ SEARCH_INCLUDES = YES # The INCLUDE_PATH tag can be used to specify one or more directories that # contain include files that are not input files but should be processed by the -# preprocessor. +# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of +# RECURSIVE has no effect here. # This tag requires that the tag SEARCH_INCLUDES is set to YES. INCLUDE_PATH = @@ -2048,7 +2282,8 @@ INCLUDE_FILE_PATTERNS = # recursively expanded use the := operator instead of the = operator. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. -PREDEFINED = "DOXYGEN_SHOULD_SKIP_THIS=1" "DOXYGEN_DOCUMENTATION_BUILD" +PREDEFINED = DOXYGEN_SHOULD_SKIP_THIS=1 \ + DOXYGEN_DOCUMENTATION_BUILD # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this # tag can be used to specify a list of macro names that should be expanded. The @@ -2115,34 +2350,10 @@ EXTERNAL_GROUPS = YES EXTERNAL_PAGES = YES -# The PERL_PATH should be the absolute path and name of the perl script -# interpreter (i.e. the result of 'which perl'). -# The default file (with absolute path) is: /usr/bin/perl. - -PERL_PATH = /usr/bin/perl - #--------------------------------------------------------------------------- # Configuration options related to the dot tool #--------------------------------------------------------------------------- -# If the CLASS_DIAGRAMS tag is set to YES, doxygen will generate a class diagram -# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to -# NO turns the diagrams off. Note that this option also works with HAVE_DOT -# disabled, but it is recommended to install and use dot, since it yields more -# powerful graphs. -# The default value is: YES. - -CLASS_DIAGRAMS = YES - -# You can define message sequence charts within doxygen comments using the \msc -# command. Doxygen will then run the mscgen tool (see: -# http://www.mcternan.me.uk/mscgen/)) to produce the chart and insert it in the -# documentation. The MSCGEN_PATH tag allows you to specify the directory where -# the mscgen tool resides. If left empty the tool is assumed to be found in the -# default search path. - -MSCGEN_PATH = - # You can include diagrams made with dia in doxygen documentation. Doxygen will # then run dia to produce the diagram and insert it in the documentation. The # DIA_PATH tag allows you to specify the directory where the dia binary resides. @@ -2161,7 +2372,7 @@ HIDE_UNDOC_RELATIONS = YES # http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent # Bell Labs. The other options in this section have no effect if this option is # set to NO -# The default value is: YES. +# The default value is: NO. HAVE_DOT = NO @@ -2199,11 +2410,14 @@ DOT_FONTSIZE = 10 DOT_FONTPATH = -# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for -# each documented class showing the direct and indirect inheritance relations. -# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO. +# If the CLASS_GRAPH tag is set to YES (or GRAPH) then doxygen will generate a +# graph for each documented class showing the direct and indirect inheritance +# relations. In case HAVE_DOT is set as well dot will be used to draw the graph, +# otherwise the built-in generator will be used. If the CLASS_GRAPH tag is set +# to TEXT the direct and indirect inheritance relations will be shown as texts / +# links. +# Possible values are: NO, YES, TEXT and GRAPH. # The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. CLASS_GRAPH = YES @@ -2217,7 +2431,8 @@ CLASS_GRAPH = YES COLLABORATION_GRAPH = YES # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for -# groups, showing the direct groups dependencies. +# groups, showing the direct groups dependencies. See also the chapter Grouping +# in the manual. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. @@ -2240,10 +2455,32 @@ UML_LOOK = NO # but if the number exceeds 15, the total amount of fields shown is limited to # 10. # Minimum value: 0, maximum value: 100, default value: 10. -# This tag requires that the tag HAVE_DOT is set to YES. +# This tag requires that the tag UML_LOOK is set to YES. UML_LIMIT_NUM_FIELDS = 10 +# If the DOT_UML_DETAILS tag is set to NO, doxygen will show attributes and +# methods without types and arguments in the UML graphs. If the DOT_UML_DETAILS +# tag is set to YES, doxygen will add type and arguments for attributes and +# methods in the UML graphs. If the DOT_UML_DETAILS tag is set to NONE, doxygen +# will not generate fields with class member information in the UML graphs. The +# class diagrams will look similar to the default class diagrams but using UML +# notation for the relationships. +# Possible values are: NO, YES and NONE. +# The default value is: NO. +# This tag requires that the tag UML_LOOK is set to YES. + +DOT_UML_DETAILS = NO + +# The DOT_WRAP_THRESHOLD tag can be used to set the maximum number of characters +# to display on a single line. If the actual line length exceeds this threshold +# significantly it will wrapped across multiple lines. Some heuristics are apply +# to avoid ugly line breaks. +# Minimum value: 0, maximum value: 1000, default value: 17. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_WRAP_THRESHOLD = 17 + # If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and # collaboration graphs will show the relations between templates and their # instances. @@ -2310,6 +2547,13 @@ GRAPHICAL_HIERARCHY = YES DIRECTORY_GRAPH = YES +# The DIR_GRAPH_MAX_DEPTH tag can be used to limit the maximum number of levels +# of child directories generated in directory dependency graphs by dot. +# Minimum value: 1, maximum value: 25, default value: 1. +# This tag requires that the tag DIRECTORY_GRAPH is set to YES. + +DIR_GRAPH_MAX_DEPTH = 1 + # The DOT_IMAGE_FORMAT tag can be used to set the image format of the images # generated by dot. For an explanation of the image formats see the section # output formats in the documentation of the dot tool (Graphviz (see: @@ -2317,9 +2561,7 @@ DIRECTORY_GRAPH = YES # Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order # to make the SVG files visible in IE 9+ (other browsers do not have this # requirement). -# Possible values are: png, png:cairo, png:cairo:cairo, png:cairo:gd, png:gd, -# png:gd:gd, jpg, jpg:cairo, jpg:cairo:gd, jpg:gd, jpg:gd:gd, gif, gif:cairo, -# gif:cairo:gd, gif:gd, gif:gd:gd, svg, png:gd, png:gd:gd, png:cairo, +# Possible values are: png, jpg, gif, svg, png:gd, png:gd:gd, png:cairo, # png:cairo:gd, png:cairo:cairo, png:cairo:gdiplus, png:gdiplus and # png:gdiplus:gdiplus. # The default value is: png. @@ -2365,13 +2607,18 @@ MSCFILE_DIRS = DIAFILE_DIRS = # When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the -# path where java can find the plantuml.jar file. If left blank, it is assumed -# PlantUML is not used or called during a preprocessing step. Doxygen will -# generate a warning when it encounters a \startuml command in this case and -# will not generate output for the diagram. +# path where java can find the plantuml.jar file or to the filename of jar file +# to be used. If left blank, it is assumed PlantUML is not used or called during +# a preprocessing step. Doxygen will generate a warning when it encounters a +# \startuml command in this case and will not generate output for the diagram. PLANTUML_JAR_PATH = +# When using plantuml, the PLANTUML_CFG_FILE tag can be used to specify a +# configuration file for plantuml. + +PLANTUML_CFG_FILE = + # When using plantuml, the specified paths are searched for files specified by # the !include statement in a plantuml block. @@ -2425,14 +2672,18 @@ DOT_MULTI_TARGETS = NO # If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page # explaining the meaning of the various boxes and arrows in the dot generated # graphs. +# Note: This tag requires that UML_LOOK isn't set, i.e. the doxygen internal +# graphical representation for inheritance and collaboration diagrams is used. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. GENERATE_LEGEND = YES -# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate dot +# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate # files that are used to generate the various graphs. +# +# Note: This setting is not only used for dot files but also for msc temporary +# files. # The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. DOT_CLEANUP = YES diff --git a/rocprim/include/rocprim/device/detail/device_config_helper.hpp b/rocprim/include/rocprim/device/detail/device_config_helper.hpp index dcdd71984..d5bf29c06 100644 --- a/rocprim/include/rocprim/device/detail/device_config_helper.hpp +++ b/rocprim/include/rocprim/device/detail/device_config_helper.hpp @@ -1257,7 +1257,7 @@ struct search_config_params } // namespace detail -/// \brief Configuration of device-level find_end +/// \brief Configuration of device-level search/find_end. /// /// \tparam BlockSize number of threads in a block. /// \tparam ItemsPerThread number of items processed by each thread. diff --git a/rocprim/include/rocprim/device/device_adjacent_find.hpp b/rocprim/include/rocprim/device/device_adjacent_find.hpp index 4506dcbd4..1ed0fc4bb 100644 --- a/rocprim/include/rocprim/device/device_adjacent_find.hpp +++ b/rocprim/include/rocprim/device/device_adjacent_find.hpp @@ -215,9 +215,9 @@ hipError_t adjacent_find_impl(void* const temporary_storage, /// * Streams in graph capture mode are supported. /// /// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `adjacent_find_config`. -/// \tparam InputIteratorType [inferred] Random-access iterator type of the input range. Must meet the +/// \tparam InputIterator [inferred] Random-access iterator type of the input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam OutputIteratorType [inferred] Random-access iterator type of the output index. Must meet the +/// \tparam OutputIterator [inferred] Random-access iterator type of the output index. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. /// \tparam BinaryPred [inferred] Boolean binary operation function object that will be applied to /// consecutive items to check whether they are equal or not. The signature of the function should be equivalent @@ -226,7 +226,7 @@ hipError_t adjacent_find_impl(void* const temporary_storage, /// const &, but the function object must not modify the object passed to it. /// The operator must meet the C++ named requirement \p BinaryPredicate. /// The default operation used is rocprim::equal_to, where \p T is the type of the elements -/// in the input range obtained with std::iterator_traits::value_type>. +/// in the input range obtained with std::iterator_traits::value_type>. /// /// \param [in] temporary_storage Pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to diff --git a/rocprim/include/rocprim/device/device_search_n.hpp b/rocprim/include/rocprim/device/device_search_n.hpp index 168b4513b..8d6c8edbb 100644 --- a/rocprim/include/rocprim/device/device_search_n.hpp +++ b/rocprim/include/rocprim/device/device_search_n.hpp @@ -106,6 +106,9 @@ hipError_t search_n(void* temporary_storage, debug_synchronous); } +/// @} +// end of group devicemodule + END_ROCPRIM_NAMESPACE #endif // ROCPRIM_DEVICE_DEVICE_SEARCH_N_HPP_ From 3cf1ea077bb4a9366812ab7c4020fc8ad26e387b Mon Sep 17 00:00:00 2001 From: Milo Lurati Date: Thu, 5 Dec 2024 12:38:53 +0000 Subject: [PATCH 34/84] Performance improvements with shuffled warp exchange --- rocprim/include/rocprim/config.hpp | 3 + .../include/rocprim/warp/warp_exchange.hpp | 551 +++++++++++++++++- test/rocprim/test_warp_exchange.cpp | 18 +- 3 files changed, 542 insertions(+), 30 deletions(-) diff --git a/rocprim/include/rocprim/config.hpp b/rocprim/include/rocprim/config.hpp index 393297317..cc69d95e0 100644 --- a/rocprim/include/rocprim/config.hpp +++ b/rocprim/include/rocprim/config.hpp @@ -146,6 +146,9 @@ #define ROCPRIM_WARP_SIZE_64 64u #define ROCPRIM_MAX_WARP_SIZE ROCPRIM_WARP_SIZE_64 +/// Quad size (group of 4 threads) +#define ROCPRIM_QUAD_SIZE 4u + #if(defined(_MSC_VER) && !defined(__clang__)) || (defined(__GNUC__) && !defined(__clang__)) #define ROCPRIM_UNROLL #define ROCPRIM_NO_UNROLL diff --git a/rocprim/include/rocprim/warp/warp_exchange.hpp b/rocprim/include/rocprim/warp/warp_exchange.hpp index d46f65b0e..c8e2dbd31 100644 --- a/rocprim/include/rocprim/warp/warp_exchange.hpp +++ b/rocprim/include/rocprim/warp/warp_exchange.hpp @@ -21,6 +21,8 @@ #ifndef ROCPRIM_WARP_WARP_EXCHANGE_HPP_ #define ROCPRIM_WARP_WARP_EXCHANGE_HPP_ +#include +#include #include #include "../config.hpp" @@ -150,10 +152,123 @@ class warp_exchange std::make_integer_sequence{}); } + template + ROCPRIM_DEVICE ROCPRIM_INLINE + U warp_rotate_right(const U& input, const unsigned int n) + { + const int lane_id = ::rocprim::lane_id(); + + // Calculate which lane to get data from + // For right rotation by n in width w: + // if we're at position p, we want data from (p + n) % w + const int warp_base = (lane_id / Width) * Width; + const int logical_lane = lane_id % Width; + const int src_lane = warp_base + ((logical_lane - n + Width) % Width); + return ::rocprim::warp_shuffle(input, src_lane, Width); + } + + template + ROCPRIM_DEVICE ROCPRIM_INLINE + U warp_rotate_left(const U& input, const unsigned int n) + { + const int lane_id = ::rocprim::lane_id(); + + // Calculate which lane to get data from + // For left rotation by n in width w: + // if we're at position p, we want data from (p - n + w) % w + const int warp_base = (lane_id / Width) * Width; + const int logical_lane = lane_id % Width; + const int src_lane = warp_base + ((logical_lane + n) % Width); + return ::rocprim::warp_shuffle(input, src_lane, Width); + } + + // Conditions for blocked to striped and striped to blocked + struct conditions + { + static constexpr bool is_equal_size = WarpSize == ItemsPerThread; + + static constexpr bool is_quad_compatible_bs + = ItemsPerThread % ROCPRIM_QUAD_SIZE == 0 + && ItemsPerThread % (WarpSize / ROCPRIM_QUAD_SIZE) == 0; + + static constexpr bool is_quad_compatible_sb + = ItemsPerThread % ROCPRIM_QUAD_SIZE == 0 + && ItemsPerThread % (WarpSize / ROCPRIM_QUAD_SIZE) == 0 + // this config is not performant for the DPP quad_perm implementation + && !(WarpSize == 64 && ItemsPerThread == 32); + + static constexpr bool warp_divide_items = ItemsPerThread % WarpSize == 0; + + static constexpr bool items_divide_warp = WarpSize % ItemsPerThread == 0; + }; + + enum class ImplementationType + { + Unknown = 0, + EqualSize = 1, + QuadCompatible = 2, + WarpDivideItems = 3, + ItemsDivideWarp = 4 + }; + + template + struct implementation_selector_bs + { + static constexpr ImplementationType value + = conditions::is_equal_size ? ImplementationType::EqualSize + : conditions::is_quad_compatible_bs ? ImplementationType::QuadCompatible + : conditions::warp_divide_items ? ImplementationType::WarpDivideItems + : conditions::items_divide_warp ? ImplementationType::ItemsDivideWarp + : ImplementationType::Unknown; + }; + + template + struct implementation_selector_sb + { + static constexpr ImplementationType value + = conditions::is_equal_size ? ImplementationType::EqualSize + : conditions::is_quad_compatible_sb ? ImplementationType::QuadCompatible + : conditions::warp_divide_items ? ImplementationType::WarpDivideItems + : conditions::items_divide_warp ? ImplementationType::ItemsDivideWarp + : ImplementationType::Unknown; + }; + + // Rearrangement of items for blocked to striped and striped to blocked + // algorithms + template + ROCPRIM_DEVICE ROCPRIM_INLINE + void rearrange_items(unsigned int flat_id, + U (&input)[ItemsPerThread], + U (&output)[ItemsPerThread]) + { + constexpr unsigned int ipt_div_width = ItemsPerThread / Width; + + unsigned int m = flat_id % Width; + unsigned int n = 0; + ROCPRIM_UNROLL + while(n < ItemsPerThread) + { + ROCPRIM_UNROLL + for(unsigned int j = 0; j < ipt_div_width; j++) + { + unsigned int dst_idx = BlockedToStriped ? n : m + j * Width; + unsigned int src_idx = BlockedToStriped ? m + j * Width : n; + + output[dst_idx] = input[src_idx]; + + n++; + } + m = (m - 1) % Width; + } + } + + // Case 1: Equal size template - ROCPRIM_DEVICE ROCPRIM_INLINE void - blocked_striped_shuffle_efficient_impl(const T (&input)[ItemsPerThread], - U (&output)[ItemsPerThread]) + ROCPRIM_DEVICE ROCPRIM_INLINE + typename std::enable_if::value + == ImplementationType::EqualSize>::type + blocked_to_striped_shuffle_impl(const T (&input)[ItemsPerThread], + U (&output)[ItemsPerThread]) { static constexpr bool IS_ARCH_WARP = WarpSize == ::rocprim::device_warp_size(); const unsigned int flat_lane_id = ::rocprim::detail::logical_lane_id(); @@ -172,10 +287,210 @@ class warp_exchange } } + // Case 2: Quad compatible + // Works only when ItemsPerThread % ROCPRIM_QUAD_SIZE == 0 && + // ItemsPerThread % (WarpSize / ROCPRIM_QUAD_SIZE) == 0 + // + // FIRST PART: Going from blocked to striped at the quad level using DPP quad_perm and + // item rearrangements + // + // The following is an example with IPT = 8. The patter can be extended to any multiple of + // 4 items per thread. In the first rotation, we will have IPT/4 consecutive times the rotation + // pattern x0-x1-x2-x3. In the item rearrangement, items are rearranged to be in ascending order. + // In the second rotation, each rotation will happen IPT/4 times before decreasing by one until 0. + // + // First quad-level rotation Item rearrangement + // ---------------------------- ------------------ + // | 0 8 16 24 | | 0 8 16 24 | + // | 1 9 17 25 | x1 0x93 | 25 1 9 17 | + // | 2 10 18 26 | x2 0x4E | 18 26 2 10 | + // | 3 11 19 27 | x3 0x39 | 11 19 27 3 | + // | 4 12 20 28 | | 4 12 20 28 | + // | 5 13 21 29 | x1 0x93 | 29 5 13 21 | + // | 6 14 22 30 | x2 0x4E | 22 30 6 14 | + // | 7 15 23 31 | x3 0x39 | 15 23 31 7 | + + // Second quad-level rotation Goal Order + // --------------------------- ------------------ + // | 0 1 2 3 | | 0 1 2 3 | + // | 4 5 6 7 | | 4 5 6 7 | + // | 11 8 9 10 | x3 0x93 | 8 9 10 11 | + // | 15 12 13 14 | x3 0x93 | 12 13 14 15 | + // | 18 19 16 17 | x2 0x4E | 16 17 18 19 | + // | 22 23 20 21 | x2 0x4E | 20 21 22 23 | + // | 25 26 27 24 | x1 0x39 | 24 25 26 27 | + // | 29 30 31 28 | x1 0x39 | 28 29 30 31 | + + // SECOND PART: Going from blocked to striped at the warp level using shuffle and item rearrangements. + // We follow the same permutations as in part one but each rotation is mutiplied by 4. We basically + // abstract one item to a stripe of 4 items and follow the same steps. + template + ROCPRIM_DEVICE ROCPRIM_INLINE + typename std::enable_if::value + == ImplementationType::QuadCompatible>::type + blocked_to_striped_shuffle_impl(const T (&input)[ItemsPerThread], + U (&output)[ItemsPerThread]) + { + constexpr unsigned int NUM_QUADS = WarpSize / ROCPRIM_QUAD_SIZE; + constexpr unsigned int IPT_DIV_QUAD_SIZE = ItemsPerThread / ROCPRIM_QUAD_SIZE; + + const unsigned int flat_id = ::rocprim::detail::logical_lane_id(); + + U values1[ItemsPerThread]; + U values2[ItemsPerThread]; + U values3[ItemsPerThread]; + + ROCPRIM_UNROLL + for(unsigned int i = 0; i < ItemsPerThread; i++) + { + values1[i] = input[i]; + } + + // First quad-level permutations + ROCPRIM_UNROLL + for(unsigned int i = 1; i < ItemsPerThread; i++) + { + unsigned int i_mod4 = i % 4; + switch(i_mod4) + { + case 1: // quad_perm:[3,0,1,2] + values1[i] = ::rocprim::detail::warp_move_dpp(values1[i]); + break; + case 2: // quad_perm:[2,3,0,1] + values1[i] = ::rocprim::detail::warp_move_dpp(values1[i]); + break; + case 3: // quad_perm:[1,2,3,0] + values1[i] = ::rocprim::detail::warp_move_dpp(values1[i]); + break; + } + } + + rearrange_items(flat_id, values1, values2); + + // Second quad-level permutations + unsigned int i = IPT_DIV_QUAD_SIZE; + unsigned int quad_perm_index = 2; + while(i < ItemsPerThread) + { + ROCPRIM_UNROLL + for(unsigned int j = 0; j < IPT_DIV_QUAD_SIZE; j++) + { + switch(quad_perm_index) + { + case 2: // quad_perm:[1,2,3,0] + values2[i] = ::rocprim::detail::warp_move_dpp(values2[i]); + break; + case 1: // quad_perm:[2,3,0,1] + values2[i] = ::rocprim::detail::warp_move_dpp(values2[i]); + break; + case 0: // quad_perm:[3,0,1,2] + values2[i] = ::rocprim::detail::warp_move_dpp(values2[i]); + break; + } + i++; + } + quad_perm_index--; + } + + if ROCPRIM_IF_CONSTEXPR(WarpSize > ROCPRIM_QUAD_SIZE) + { + // First warp rotation + ROCPRIM_UNROLL + for(unsigned int i = 1; i < ItemsPerThread; i++) + { + unsigned int i_mod_quad_warp = i % NUM_QUADS; + if(i_mod_quad_warp != 0) + { + const unsigned int total_rotation = i_mod_quad_warp * ROCPRIM_QUAD_SIZE; + values2[i] = warp_rotate_right(values2[i], total_rotation); + } + } + + rearrange_items(flat_id / ROCPRIM_QUAD_SIZE, values2, values3); + + // Second warp rotation + constexpr unsigned int items_per_quad_warp = ItemsPerThread / NUM_QUADS; + unsigned int remaining_rotations = NUM_QUADS - 1; + ROCPRIM_UNROLL + for(unsigned int i = items_per_quad_warp; i < ItemsPerThread; i += items_per_quad_warp) + { + ROCPRIM_UNROLL + for(unsigned int j = 0; j < items_per_quad_warp && (i + j) < ItemsPerThread; j++) + { + const unsigned int rotation = remaining_rotations * ROCPRIM_QUAD_SIZE; + values3[i + j] = warp_rotate_right(values3[i + j], rotation); + } + remaining_rotations--; + } + } + + ROCPRIM_UNROLL + for(unsigned int i = 0; i < ItemsPerThread; i++) + { + output[i] = values3[i]; + } + } + + // Case 3: Warp divides items + // Similar logic of case 2 but only at a warp level using only shuffle + template + ROCPRIM_DEVICE ROCPRIM_INLINE + typename std::enable_if::value + == ImplementationType::WarpDivideItems>::type + blocked_to_striped_shuffle_impl(const T (&input)[ItemsPerThread], + U (&output)[ItemsPerThread]) + { + const unsigned int flat_id = ::rocprim::detail::logical_lane_id(); + constexpr unsigned int ipt_div_warp = ItemsPerThread / WarpSize; + U values1[ItemsPerThread]; + U values2[ItemsPerThread]; + + ROCPRIM_UNROLL + for(unsigned int i = 0; i < ItemsPerThread; i++) + { + values1[i] = input[i]; + } + + // First warp rotation + ROCPRIM_UNROLL + for(unsigned int i = 1; i < ItemsPerThread; i++) + { + const unsigned int rotations = i % WarpSize; + if(rotations != 0) + { + values1[i] = warp_rotate_right(values1[i], rotations); + } + } + + rearrange_items(flat_id, values1, values2); + + // Second warp rotation + unsigned int rotations = WarpSize - 1; + ROCPRIM_UNROLL + for(unsigned int i = 0; i < ItemsPerThread; i += ipt_div_warp) + { + ROCPRIM_UNROLL + for(unsigned int j = 0; j < ipt_div_warp; j++) + { + values2[i] = warp_rotate_right(values2[i], rotations); + rotations--; + } + } + + ROCPRIM_UNROLL + for(unsigned int i = 0; i < ItemsPerThread; i++) + { + output[i] = values2[i]; + } + } + + // Case 4: Items divide warp template - ROCPRIM_DEVICE ROCPRIM_INLINE void + ROCPRIM_DEVICE ROCPRIM_INLINE + typename std::enable_if::value + == ImplementationType::ItemsDivideWarp>::type blocked_to_striped_shuffle_impl(const T (&input)[ItemsPerThread], - U (&output)[ItemsPerThread]) + U (&output)[ItemsPerThread]) { const unsigned int flat_id = ::rocprim::detail::logical_lane_id(); U work_array[ItemsPerThread]; @@ -204,10 +519,191 @@ class warp_exchange } } + // Case 1: Equal size + template + ROCPRIM_DEVICE ROCPRIM_INLINE + typename std::enable_if::value + == ImplementationType::EqualSize>::type + striped_to_blocked_shuffle_impl(const T (&input)[ItemsPerThread], + U (&output)[ItemsPerThread]) + { + blocked_to_striped_shuffle_impl(input, output); + } + + // Case 2: Quad compatible + // Works only when ItemsPerThread % ROCPRIM_QUAD_SIZE == 0 && + // ItemsPerThread % (WarpSize / ROCPRIM_QUAD_SIZE) == 0 + // + // The logic of this implementation is the inverse of blocked to striped. + // Check comments of blocked to striped case 2 for more details. + template + ROCPRIM_DEVICE ROCPRIM_INLINE + typename std::enable_if::value + == ImplementationType::QuadCompatible>::type + striped_to_blocked_shuffle_impl(const T (&input)[ItemsPerThread], + U (&output)[ItemsPerThread]) + { + constexpr unsigned int NUM_QUADS = WarpSize / ROCPRIM_QUAD_SIZE; + constexpr unsigned int IPT_DIV_QUAD_SIZE = ItemsPerThread / ROCPRIM_QUAD_SIZE; + + const unsigned int flat_id = ::rocprim::detail::logical_lane_id(); + + U values1[ItemsPerThread]; + U values2[ItemsPerThread]; + U values3[ItemsPerThread]; + + ROCPRIM_UNROLL + for(unsigned int i = 0; i < ItemsPerThread; i++) + { + values1[i] = input[i]; + values2[i] = input[i]; + } + + if ROCPRIM_IF_CONSTEXPR(WarpSize > ROCPRIM_QUAD_SIZE) + { + // First warp rotation + constexpr unsigned int items_per_quad_warp = ItemsPerThread / NUM_QUADS; + unsigned int remaining_rotations = NUM_QUADS - 1; + ROCPRIM_UNROLL + for(unsigned int i = items_per_quad_warp; i < ItemsPerThread; i += items_per_quad_warp) + { + ROCPRIM_UNROLL + for(unsigned int j = 0; j < items_per_quad_warp && (i + j) < ItemsPerThread; j++) + { + const unsigned int rotation = remaining_rotations * ROCPRIM_QUAD_SIZE; + values1[i + j] = warp_rotate_left(values1[i + j], rotation); + } + remaining_rotations--; + } + + rearrange_items(flat_id / ROCPRIM_QUAD_SIZE, values1, values2); + + // Second warp rotation + ROCPRIM_UNROLL + for(unsigned int i = 1; i < ItemsPerThread; i++) + { + unsigned int i_mod_quad_warp = i % NUM_QUADS; + if(i_mod_quad_warp != 0) + { + const unsigned int total_rotation = i_mod_quad_warp * ROCPRIM_QUAD_SIZE; + values2[i] = warp_rotate_left(values2[i], total_rotation); + } + } + } + + // First quad-level permutations + unsigned int i = IPT_DIV_QUAD_SIZE; + unsigned int quad_perm_index = 0; + while(i < ItemsPerThread) + { + ROCPRIM_UNROLL + for(unsigned int j = 0; j < IPT_DIV_QUAD_SIZE; j++) + { + switch(quad_perm_index) + { + case 2: // quad_perm:[1,2,3,0] + values2[i] = ::rocprim::detail::warp_move_dpp(values2[i]); + break; + case 1: // quad_perm:[2,3,0,1] + values2[i] = ::rocprim::detail::warp_move_dpp(values2[i]); + break; + case 0: // quad_perm:[3,0,1,2] + values2[i] = ::rocprim::detail::warp_move_dpp(values2[i]); + break; + } + i++; + } + quad_perm_index++; + } + + rearrange_items(flat_id, values2, values3); + + // Second quad-level permutations + ROCPRIM_UNROLL + for(unsigned int i = 1; i < ItemsPerThread; i++) + { + unsigned int i_mod4 = i % 4; + switch(i_mod4) + { + case 3: // quad_perm:[3,0,1,2] + values3[i] = ::rocprim::detail::warp_move_dpp(values3[i]); + break; + case 2: // quad_perm:[2,3,0,1] + values3[i] = ::rocprim::detail::warp_move_dpp(values3[i]); + break; + case 1: // quad_perm:[1,2,3,0] + values3[i] = ::rocprim::detail::warp_move_dpp(values3[i]); + break; + } + } + + ROCPRIM_UNROLL + for(unsigned int i = 0; i < ItemsPerThread; i++) + { + output[i] = values3[i]; + } + } + + // Case 3: Warp divides items + // Similar logic of case 2 but only at a warp level using only shuffle + template + ROCPRIM_DEVICE ROCPRIM_INLINE + typename std::enable_if::value + == ImplementationType::WarpDivideItems>::type + striped_to_blocked_shuffle_impl(const T (&input)[ItemsPerThread], + U (&output)[ItemsPerThread]) + { + const unsigned int flat_id = ::rocprim::detail::logical_lane_id(); + constexpr unsigned int ipt_div_warp = ItemsPerThread / WarpSize; + U values1[ItemsPerThread]; + U values2[ItemsPerThread]; + + ROCPRIM_UNROLL + for(unsigned int i = 0; i < ItemsPerThread; i++) + { + values1[i] = input[i]; + } + + // First warp rotation + unsigned int rotations = WarpSize - 1; + ROCPRIM_UNROLL + for(unsigned int i = 0; i < ItemsPerThread; i += ipt_div_warp) + { + ROCPRIM_UNROLL + for(unsigned int j = 0; j < ipt_div_warp; j++) + { + values1[i] = warp_rotate_left(values1[i], rotations); + rotations--; + } + } + + rearrange_items(flat_id, values1, values2); + + // Second warp rotation + ROCPRIM_UNROLL + for(unsigned int i = 1; i < ItemsPerThread; i++) + { + const unsigned int rotations = i % WarpSize; + if(rotations != 0) + { + values2[i] = warp_rotate_left(values2[i], rotations); + } + } + + ROCPRIM_UNROLL + for(unsigned int i = 0; i < ItemsPerThread; i++) + { + output[i] = values2[i]; + } + } + + // Case 4: Items divide warp template - ROCPRIM_DEVICE ROCPRIM_INLINE void + ROCPRIM_DEVICE ROCPRIM_INLINE + typename std::enable_if::value + == ImplementationType::ItemsDivideWarp>::type striped_to_blocked_shuffle_impl(const T (&input)[ItemsPerThread], - U (&output)[ItemsPerThread]) + U (&output)[ItemsPerThread]) { const unsigned int flat_id = ::rocprim::detail::logical_lane_id(); U work_array[ItemsPerThread]; @@ -307,7 +803,11 @@ class warp_exchange /// across the warp, using warp shuffle operations. /// Uses an optimized implementation for when WarpSize is equal to ItemsPerThread. /// Caution: this API is experimental. Performance might not be consistent. - /// ItemsPerThread must be a divisor of WarpSize. + /// One of these following conditions must be satisfied: + /// 1. WarpSize is equal to ItemsPerThread + /// 2. ItemsPerThread % ROCPRIM_QUAD_SIZE == 0 && ItemsPerThread % (WarpSize / ROCPRIM_QUAD_SIZE) == 0 + /// 3. ItemsPerThread is divisible by WarpSize + /// 4. WarpSize is divisible by ItemsPerThread /// /// \tparam U - [inferred] the output type. /// @@ -338,16 +838,11 @@ class warp_exchange U (&output)[ItemsPerThread]) { static_assert( - WarpSize % ItemsPerThread == 0, - "ItemsPerThread must be a divisor of WarpSize to use blocked_to_striped_shuffle"); - if(WarpSize == ItemsPerThread) - { - blocked_striped_shuffle_efficient_impl(input, output); - } - else - { - blocked_to_striped_shuffle_impl(input, output); - } + conditions::is_equal_size || conditions::is_quad_compatible_bs + || conditions::warp_divide_items || conditions::items_divide_warp, + "Input constraints violated. Please see documentation for allowed configurations."); + + blocked_to_striped_shuffle_impl(input, output); } /// \brief Transposes a striped arrangement of items to a blocked arrangement @@ -411,7 +906,11 @@ class warp_exchange /// across the warp, using warp shuffle operations. /// Uses an optimized implementation for when WarpSize is equal to ItemsPerThread. /// Caution: this API is experimental. Performance might not be consistent. - /// ItemsPerThread must be a divisor of WarpSize. + /// One of these following conditions must be satisfied: + /// 1. WarpSize is equal to ItemsPerThread + /// 2. ItemsPerThread % ROCPRIM_QUAD_SIZE == 0 && ItemsPerThread % (WarpSize / ROCPRIM_QUAD_SIZE) == 0 + /// 3. ItemsPerThread is divisible by WarpSize + /// 4. WarpSize is divisible by ItemsPerThread /// /// \tparam U - [inferred] the output type. /// @@ -442,17 +941,11 @@ class warp_exchange U (&output)[ItemsPerThread]) { static_assert( - WarpSize % ItemsPerThread == 0, - "ItemsPerThread must be a divisor of WarpSize to use striped_to_blocked_shuffle"); + conditions::is_equal_size || conditions::is_quad_compatible_sb + || conditions::warp_divide_items || conditions::items_divide_warp, + "Input constraints violated. Please see documentation for allowed configurations."); - if(WarpSize == ItemsPerThread) - { - blocked_striped_shuffle_efficient_impl(input, output); - } - else - { - striped_to_blocked_shuffle_impl(input, output); - } + striped_to_blocked_shuffle_impl(input, output); } /// \brief Orders \p input values according to ranks using temporary storage, diff --git a/test/rocprim/test_warp_exchange.cpp b/test/rocprim/test_warp_exchange.cpp index 04bdf3e77..b6998a80a 100644 --- a/test/rocprim/test_warp_exchange.cpp +++ b/test/rocprim/test_warp_exchange.cpp @@ -183,6 +183,14 @@ using WarpExchangeTestParams = ::testing::Types< //Params, Params, + Params, + Params, + Params, + Params, + Params, + Params, + Params, + Params, Params, Params, @@ -208,7 +216,15 @@ using WarpExchangeTestParams = ::testing::Types< Params, Params, //Params, - Params>; + Params, + + Params, + Params, + Params, + Params, + Params, + Params, + Params>; template __device__ auto warp_exchange_test(T* d_input, T* d_output) From ce92984fa241fcd020ab882c36e032739705613b Mon Sep 17 00:00:00 2001 From: Nara Prasetya Date: Thu, 5 Dec 2024 15:00:54 +0000 Subject: [PATCH 35/84] log more information during configure --- cmake/Summary.cmake | 60 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 54 insertions(+), 6 deletions(-) diff --git a/cmake/Summary.cmake b/cmake/Summary.cmake index 58c286b32..9c368d6e1 100644 --- a/cmake/Summary.cmake +++ b/cmake/Summary.cmake @@ -21,6 +21,43 @@ # SOFTWARE. function(print_configuration_summary) + find_package(Git) + if(GIT_FOUND) + execute_process( + COMMAND ${GIT_EXECUTABLE} show --format=%H --no-patch + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + OUTPUT_VARIABLE COMMIT_HASH + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + execute_process( + COMMAND ${GIT_EXECUTABLE} show --format=%s --no-patch + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + OUTPUT_VARIABLE COMMIT_SUBJECT + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + endif() + + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} --version + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + OUTPUT_VARIABLE CMAKE_CXX_COMPILER_VERBOSE_DETAILS + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + find_program(UNAME_EXECUTABLE uname) + if(UNAME_EXECUTABLE) + execute_process( + COMMAND ${UNAME_EXECUTABLE} -a + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + OUTPUT_VARIABLE LINUX_KERNEL_DETAILS + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + endif() + + string(REPLACE "\n" ";" CMAKE_CXX_COMPILER_VERBOSE_DETAILS "${CMAKE_CXX_COMPILER_VERBOSE_DETAILS}") + list(TRANSFORM CMAKE_CXX_COMPILER_VERBOSE_DETAILS PREPEND "-- ") + string(REPLACE ";" "\n" CMAKE_CXX_COMPILER_VERBOSE_DETAILS "${CMAKE_CXX_COMPILER_VERBOSE_DETAILS}") + message(STATUS "") message(STATUS "******** Summary ********") message(STATUS "General:") @@ -49,10 +86,21 @@ function(print_configuration_summary) message(STATUS " Device targets : ${GPU_TARGETS}") endif() message(STATUS "") - message(STATUS " ONLY_INSTALL : ${ONLY_INSTALL}") - message(STATUS " BUILD_TEST : ${BUILD_TEST}") - message(STATUS " BUILD_BENCHMARK : ${BUILD_BENCHMARK}") - message(STATUS " BUILD_NAIVE_BENCHMARK : ${BUILD_NAIVE_BENCHMARK}") - message(STATUS " BUILD_EXAMPLE : ${BUILD_EXAMPLE}") - message(STATUS " BUILD_DOCS : ${BUILD_DOCS}") + message(STATUS " ONLY_INSTALL : ${ONLY_INSTALL}") + message(STATUS " BUILD_TEST : ${BUILD_TEST}") + message(STATUS " BUILD_BENCHMARK : ${BUILD_BENCHMARK}") + message(STATUS " BUILD_NAIVE_BENCHMARK : ${BUILD_NAIVE_BENCHMARK}") + message(STATUS " BUILD_EXAMPLE : ${BUILD_EXAMPLE}") + message(STATUS " BUILD_DOCS : ${BUILD_DOCS}") + message(STATUS "") + message(STATUS "Detailed:") + message(STATUS " C++ compiler details : \n${CMAKE_CXX_COMPILER_VERBOSE_DETAILS}") + if(GIT_FOUND) + message(STATUS " Commit : ${COMMIT_HASH}") + message(STATUS " ${COMMIT_SUBJECT}") + endif() + if(UNAME_EXECUTABLE) + message(STATUS " Unix name : ${LINUX_KERNEL_DETAILS}") + endif() + endfunction() From bac2c82e1f469e54bcdb2eff334fded130d61247 Mon Sep 17 00:00:00 2001 From: Sander Bos Date: Mon, 9 Dec 2024 13:22:29 +0000 Subject: [PATCH 36/84] Update documentation generation instructions --- README.md | 130 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 80 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 4c7088284..b58914679 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # rocPRIM > [!NOTE] -> The published documentation is available at [rocPRIM](https://rocm.docs.amd.com/projects/rocPRIM/en/latest/) in an organized, easy-to-read format, with search and a table of contents. The documentation source files reside in the `docs` folder of this repository. As with all ROCm projects, the documentation is open source. For more information on contributing to the documentation, see [Contribute to ROCm documentation](https://rocm.docs.amd.com/en/latest/contribute/contributing.html). +> The published rocPRIM documentation is available [here](https://rocm.docs.amd.com/projects/rocPRIM/en/latest/) in an organized, easy-to-read format, with search and a table of contents. The documentation source files reside in the `docs` folder of this repository. As with all ROCm projects, the documentation is open source. For more information on contributing to the documentation, see [Contribute to ROCm documentation](https://rocm.docs.amd.com/en/latest/contribute/contributing.html). rocPRIM is a header-only library that provides HIP parallel primitives. You can use this library to develop performant GPU-accelerated code on AMD ROCm platforms. @@ -28,55 +28,6 @@ Optional: * Required only for benchmarks. Building benchmarks is off by default. * This is automatically downloaded and built by the CMake script. -## Documentation - -Documentation for rocPRIM is available at -[https://rocm.docs.amd.com/projects/rocPRIM/en/latest/](https://rocm.docs.amd.com/projects/rocPRIM/en/latest/). - -### Build documentation locally - -```shell -# Change directory to rocPRIM docs -cd rocPRIM; cd docs - -# Install documentation dependencies -python3 -m pip install -r sphinx/requirements.txt - -# Build the documentation -python3 -m sphinx -T -E -b html -d _build/doctrees -D language=en . _build/html - -# To serve the HTML docs locally -cd _build/html -python3 -m http.server -``` - -### Build documentation via CMake - -Install [rocm-cmake](https://github.com/ROCm/rocm-cmake/) - -```shell -# Change directory to rocPRIM -cd rocPRIM - -# Install documentation dependencies -python3 -m pip install -r docs/sphinx/requirements.txt - -# Set C++ compiler -# This example uses hipcc and assumes it is at the path /usr/bin -export CXX=hipcc -export PATH=/usr/bin:$PATH - -# Configure the project -cmake -S . -B ./build -D BUILD_DOCS=ON - -# Build the documentation -cmake --build ./build --target doc - -# To serve the HTML docs locally -cd ./build/docs/html -python3 -m http.server -``` - ## Build and install You can build and install rocPRIM on Linux or Windows. @@ -326,6 +277,85 @@ based on the input types and the target architecture from the stream used. [ROCm](https://rocm.docs.amd.com/en/latest/) environment, hipCUB uses the rocPRIM library as a backend; on CUDA platforms, it uses CUB as a backend. +## Building the documentation locally + +### Requirements + +#### Doxygen + +The build system uses Doxygen [version 1.9.4](https://github.com/doxygen/doxygen/releases/tag/Release_1_9_4). You can try using a newer version, but that might cause issues. + +After you have downloaded Doxygen version 1.9.4: + +```shell +# Add doxygen to your PATH +echo 'export PATH=/bin:$PATH' >> ~/.bashrc + +# Apply the updated .bashrc +source ~/.bashrc + +# Confirm that you are using version 1.9.4 +doxygen --version +``` + +#### Python + +The build system uses Python version 3.10. You can try using a newer version, but that might cause issues. + +You can install Python 3.10 alongside your other Python versions using [pyenv](https://github.com/pyenv/pyenv?tab=readme-ov-file#installation): + +```shell +# Install Python 3.10 +pyenv install 3.10 + +# Create a Python 3.10 virtual environment +pyenv virtualenv 3.10 venv_rocprim + +# Activate the virtual environment +pyenv activate venv_rocprim +``` + +### Building + +After cloning this repository, and `cd`ing into it: + +```shell +# Install Python dependencies +python3 -m pip install -r docs/sphinx/requirements.txt + +# Build the documentation +python3 -m sphinx -T -E -b html -d docs/_build/doctrees -D language=en docs docs/_build/html +``` + +You can then open `docs/_build/html/index.html` in your browser to view the documentation. + +### Build documentation via CMake + +Install [rocm-cmake](https://github.com/ROCm/rocm-cmake/) + +```shell +# Change directory to rocPRIM +cd rocPRIM + +# Install documentation dependencies +python3 -m pip install -r docs/sphinx/requirements.txt + +# Set C++ compiler +# This example uses hipcc and assumes it is at the path /usr/bin +export CXX=hipcc +export PATH=/usr/bin:$PATH + +# Configure the project +cmake -S . -B ./build -D BUILD_DOCS=ON + +# Build the documentation +cmake --build ./build --target doc + +# To serve the HTML docs locally +cd ./build/docs/html +python3 -m http.server +``` + ## Support You can report bugs and feature requests through our GitHub From 6d3f6f19b922b23a4eb6be91a7f63d34107f0020 Mon Sep 17 00:00:00 2001 From: Cenxuan Tian Date: Tue, 10 Dec 2024 08:37:28 +0000 Subject: [PATCH 37/84] Resolve "Replace hipMalloc, hipMallocHelper, and hipFree in benchmarks and tests with device_ptr" Part 4 --- .../internal/test_internal_merge_path.cpp | 11 +- test/rocprim/test_device_search.cpp | 93 +-- .../test_device_segmented_radix_sort.hpp | 650 ++++++------------ test/rocprim/test_device_segmented_reduce.cpp | 136 ++-- test/rocprim/test_device_select.cpp | 191 ++--- test/rocprim/test_device_transform.cpp | 137 ++-- test/rocprim/test_intrinsics.cpp | 437 +++++------- .../test_temporary_storage_partitioning.cpp | 66 +- 8 files changed, 604 insertions(+), 1117 deletions(-) diff --git a/test/rocprim/internal/test_internal_merge_path.cpp b/test/rocprim/internal/test_internal_merge_path.cpp index 9b0873444..87e53b0d1 100644 --- a/test/rocprim/internal/test_internal_merge_path.cpp +++ b/test/rocprim/internal/test_internal_merge_path.cpp @@ -2,6 +2,7 @@ #include "../../common_test_header.hpp" #include "../test_utils_assertions.hpp" #include "../test_utils_data_generation.hpp" +#include "rocprim/test_utils_device_ptr.hpp" #include #include @@ -25,17 +26,13 @@ void serial_merge(std::vector& input, { static_assert(IPT >= N, "Kernel must be launched such that all items can be processed!"); - size_t num_bytes = sizeof(T) * N; - T* device_data; - - HIP_CHECK(hipMalloc(&device_data, num_bytes)); - HIP_CHECK(hipMemcpy(device_data, input.data(), num_bytes, hipMemcpyHostToDevice)); + test_utils::device_ptr device_data(input); merge_kernel - <<<1, 1>>>(device_data, rocprim::detail::range_t<>{0, mid, mid, N}, compare_function); + <<<1, 1>>>(device_data.get(), rocprim::detail::range_t<>{0, mid, mid, N}, compare_function); HIP_CHECK(hipGetLastError()); - HIP_CHECK(hipMemcpy(output.data(), device_data, num_bytes, hipMemcpyDeviceToHost)); + output = device_data.load(); } TEST(RocprimInternalMergePathTests, Basic) diff --git a/test/rocprim/test_device_search.cpp b/test/rocprim/test_device_search.cpp index 33735f5f8..52cafc901 100644 --- a/test/rocprim/test_device_search.cpp +++ b/test/rocprim/test_device_search.cpp @@ -21,6 +21,7 @@ // SOFTWARE. // required test headers +#include "../rocprim/test_utils_device_ptr.hpp" #include "indirect_iterator.hpp" #include "test_utils_assertions.hpp" #include "test_utils_custom_float_type.hpp" @@ -180,40 +181,23 @@ TYPED_TEST(RocprimDeviceSearchTests, Search) keys.assign(input.begin() + pattern, input.end()); } - value_type* d_input; - key_type* d_keys; - index_type* d_output; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(*d_input))); - - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_keys, keys.size() * sizeof(*d_keys))); - - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, sizeof(*d_output))); - - HIP_CHECK(hipMemcpy(d_input, - input.data(), - input.size() * sizeof(*d_input), - hipMemcpyHostToDevice)); - - HIP_CHECK(hipMemcpy(d_keys, - keys.data(), - keys.size() * sizeof(*d_keys), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_keys(keys); + test_utils::device_ptr d_output(1); const auto input_it - = test_utils::wrap_in_indirect_iterator(d_input); + = test_utils::wrap_in_indirect_iterator(d_input.get()); const auto keys_it - = test_utils::wrap_in_indirect_iterator(d_keys); + = test_utils::wrap_in_indirect_iterator(d_keys.get()); const auto output_keys - = test_utils::wrap_in_indirect_iterator(d_output); + = test_utils::wrap_in_indirect_iterator(d_output.get()); // compare function compare_function compare_op; // temp storage size_t temp_storage_size_bytes; - void* d_temp_storage = nullptr; + // Get size of d_temp_storage HIP_CHECK(rocprim::search(nullptr, temp_storage_size_bytes, @@ -229,9 +213,7 @@ TYPED_TEST(RocprimDeviceSearchTests, Search) // temp_storage_size_bytes must be >0 ASSERT_GT(temp_storage_size_bytes, 0); - // allocate temporary storage - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) @@ -240,7 +222,7 @@ TYPED_TEST(RocprimDeviceSearchTests, Search) } // Run - HIP_CHECK(rocprim::search(d_temp_storage, + HIP_CHECK(rocprim::search(d_temp_storage.get(), temp_storage_size_bytes, input_it, keys_it, @@ -259,9 +241,8 @@ TYPED_TEST(RocprimDeviceSearchTests, Search) HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); - index_type output; // Copy output to host - HIP_CHECK(hipMemcpy(&output, d_output, sizeof(*d_output), hipMemcpyDeviceToHost)); + const auto output = d_output.load()[0]; index_type expected = std::search(input.begin(), input.end(), keys.begin(), keys.end(), compare_op) @@ -269,11 +250,6 @@ TYPED_TEST(RocprimDeviceSearchTests, Search) ASSERT_EQ(output, expected); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_keys)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if(TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -344,41 +320,24 @@ TYPED_TEST(RocprimDeviceSearchTests, SearchRepetition) std::copy(keys.begin(), keys.end(), input.begin() + i * key_size); } - value_type* d_input; - key_type* d_keys; - index_type* d_output; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(*d_input))); - - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys, keys.size() * sizeof(*d_keys))); - - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, sizeof(*d_output))); - - HIP_CHECK(hipMemcpy(d_input, - input.data(), - input.size() * sizeof(*d_input), - hipMemcpyHostToDevice)); - - HIP_CHECK(hipMemcpy(d_keys, - keys.data(), - keys.size() * sizeof(*d_keys), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_keys(keys); + test_utils::device_ptr d_output(1); const auto input_it - = test_utils::wrap_in_indirect_iterator(d_input); + = test_utils::wrap_in_indirect_iterator(d_input.get()); // compare function compare_function compare_op; // temp storage size_t temp_storage_size_bytes; - void* d_temp_storage = nullptr; // Get size of d_temp_storage HIP_CHECK(rocprim::search(nullptr, temp_storage_size_bytes, input_it, - d_keys, - d_output, + d_keys.get(), + d_output.get(), input.size(), keys.size(), compare_op, @@ -388,8 +347,7 @@ TYPED_TEST(RocprimDeviceSearchTests, SearchRepetition) // temp_storage_size_bytes must be >0 ASSERT_GT(temp_storage_size_bytes, 0); - // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) @@ -398,11 +356,11 @@ TYPED_TEST(RocprimDeviceSearchTests, SearchRepetition) } // Run - HIP_CHECK(rocprim::search(d_temp_storage, + HIP_CHECK(rocprim::search(d_temp_storage.get(), temp_storage_size_bytes, input_it, - d_keys, - d_output, + d_keys.get(), + d_output.get(), input.size(), keys.size(), compare_op, @@ -417,9 +375,7 @@ TYPED_TEST(RocprimDeviceSearchTests, SearchRepetition) HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); - index_type output; - // Copy output to host - HIP_CHECK(hipMemcpy(&output, d_output, sizeof(*d_output), hipMemcpyDeviceToHost)); + const auto output = d_output.load()[0]; index_type expected = std::search(input.begin(), input.end(), keys.begin(), keys.end(), compare_op) @@ -427,11 +383,6 @@ TYPED_TEST(RocprimDeviceSearchTests, SearchRepetition) ASSERT_EQ(output, expected); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_keys)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if(TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); diff --git a/test/rocprim/test_device_segmented_radix_sort.hpp b/test/rocprim/test_device_segmented_radix_sort.hpp index 3daa8f99e..d26f88970 100644 --- a/test/rocprim/test_device_segmented_radix_sort.hpp +++ b/test/rocprim/test_device_segmented_radix_sort.hpp @@ -24,6 +24,7 @@ #define TEST_DEVICE_SEGMENTED_RADIX_SORT_HPP_ #include "../common_test_header.hpp" +#include "../rocprim/test_utils_device_ptr.hpp" // required rocprim headers #include @@ -157,23 +158,10 @@ inline void sort_keys() } offsets.push_back(size); - key_type* d_keys_input; - key_type* d_keys_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_input, size * sizeof(key_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_output, size * sizeof(key_type))); - HIP_CHECK(hipMemcpy(d_keys_input, - keys_input.data(), - size * sizeof(key_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_keys_input(keys_input); + test_utils::device_ptr d_keys_output(size); - offset_type* d_offsets; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_offsets, - (segments_count + 1) * sizeof(offset_type))); - HIP_CHECK(hipMemcpy(d_offsets, - offsets.data(), - (segments_count + 1) * sizeof(offset_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_offsets(offsets); // Calculate expected results on host std::vector expected(keys_input); @@ -188,31 +176,29 @@ inline void sort_keys() size_t temporary_storage_bytes = 0; HIP_CHECK(rocprim::segmented_radix_sort_keys(nullptr, temporary_storage_bytes, - d_keys_input, - d_keys_output, + d_keys_input.get(), + d_keys_output.get(), size, segments_count, - d_offsets, - d_offsets + 1, + d_offsets.get(), + d_offsets.get() + 1, start_bit, end_bit)); ASSERT_GT(temporary_storage_bytes, 0U); - void* d_temporary_storage; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); if(descending) { - HIP_CHECK(rocprim::segmented_radix_sort_keys_desc(d_temporary_storage, + HIP_CHECK(rocprim::segmented_radix_sort_keys_desc(d_temporary_storage.get(), temporary_storage_bytes, - d_keys_input, - d_keys_output, + d_keys_input.get(), + d_keys_output.get(), size, segments_count, - d_offsets, - d_offsets + 1, + d_offsets.get(), + d_offsets.get() + 1, start_bit, end_bit, stream, @@ -220,30 +206,21 @@ inline void sort_keys() } else { - HIP_CHECK(rocprim::segmented_radix_sort_keys(d_temporary_storage, + HIP_CHECK(rocprim::segmented_radix_sort_keys(d_temporary_storage.get(), temporary_storage_bytes, - d_keys_input, - d_keys_output, + d_keys_input.get(), + d_keys_output.get(), size, segments_count, - d_offsets, - d_offsets + 1, + d_offsets.get(), + d_offsets.get() + 1, start_bit, end_bit, stream, debug_synchronous)); } - std::vector keys_output(size); - HIP_CHECK(hipMemcpy(keys_output.data(), - d_keys_output, - size * sizeof(key_type), - hipMemcpyDeviceToHost)); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_keys_output)); - HIP_CHECK(hipFree(d_offsets)); + const auto keys_output = d_keys_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(keys_output, expected)); } @@ -288,78 +265,57 @@ inline void sort_keys_empty_data() offsets[0] = 0; offsets[1] = 0; - key_type* d_keys; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys, size * sizeof(key_type))); - HIP_CHECK(hipMemcpy(d_keys, - keys_input.data(), - size * sizeof(key_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_keys(keys_input); - offset_type* d_offsets; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_offsets, - (segments_count + 1) * sizeof(offset_type))); - HIP_CHECK(hipMemcpy(d_offsets, - offsets.data(), - (segments_count + 1) * sizeof(offset_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_offsets(offsets); size_t temporary_storage_bytes = 0; HIP_CHECK(rocprim::segmented_radix_sort_keys(nullptr, temporary_storage_bytes, - d_keys, - d_keys, + d_keys.get(), + d_keys.get(), size, segments_count, - d_offsets, - d_offsets + 1, + d_offsets.get(), + d_offsets.get() + 1, start_bit, end_bit)); ASSERT_GT(temporary_storage_bytes, 0U); - void* d_temporary_storage; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); if(descending) { - HIP_CHECK(rocprim::segmented_radix_sort_pairs_desc(d_temporary_storage, - temporary_storage_bytes, - d_keys, - d_keys, - size, - segments_count, - d_offsets, - d_offsets + 1, - start_bit, - end_bit, - stream)); + HIP_CHECK( + rocprim::segmented_radix_sort_pairs_desc(d_temporary_storage.get(), + temporary_storage_bytes, + d_keys.get(), + d_keys.get(), + size, + segments_count, + d_offsets.get(), + d_offsets.get() + 1, + start_bit, + end_bit, + stream)); } else { - HIP_CHECK(rocprim::segmented_radix_sort_pairs(d_temporary_storage, + HIP_CHECK(rocprim::segmented_radix_sort_pairs(d_temporary_storage.get(), temporary_storage_bytes, - d_keys, - d_keys, + d_keys.get(), + d_keys.get(), size, segments_count, - d_offsets, - d_offsets + 1, + d_offsets.get(), + d_offsets.get() + 1, start_bit, end_bit, stream)); } - std::vector keys_output(size); - HIP_CHECK(hipMemcpy(keys_output.data(), - d_keys, - size * sizeof(key_type), - hipMemcpyDeviceToHost)); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_keys)); - HIP_CHECK(hipFree(d_offsets)); + const auto keys_output = d_keys.load(); // Output should not have changed ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(keys_output, keys_input)); @@ -405,22 +361,10 @@ inline void sort_keys_large_segments() offsets[1] = static_cast(size / 2); offsets[2] = static_cast(size); - key_type* d_keys_input; - key_type* d_keys_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_input, size * sizeof(key_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_output, size * sizeof(key_type))); - HIP_CHECK(hipMemcpy(d_keys_input, - keys_input.data(), - size * sizeof(key_type), - hipMemcpyHostToDevice)); - - offset_type* d_offsets; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_offsets, - (segments_count + 1) * sizeof(offset_type))); - HIP_CHECK(hipMemcpy(d_offsets, - offsets.data(), - (segments_count + 1) * sizeof(offset_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_keys_input(keys_input); + test_utils::device_ptr d_keys_output(size); + + test_utils::device_ptr d_offsets(offsets); // Calculate expected results on host std::vector expected(keys_input); @@ -435,60 +379,49 @@ inline void sort_keys_large_segments() size_t temporary_storage_bytes = 0; HIP_CHECK(rocprim::segmented_radix_sort_keys(nullptr, temporary_storage_bytes, - d_keys_input, - d_keys_output, + d_keys_input.get(), + d_keys_output.get(), size, segments_count, - d_offsets, - d_offsets + 1, + d_offsets.get(), + d_offsets.get() + 1, start_bit, end_bit)); ASSERT_GT(temporary_storage_bytes, 0U); - void* d_temporary_storage; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); if(descending) { - HIP_CHECK(rocprim::segmented_radix_sort_keys_desc(d_temporary_storage, + HIP_CHECK(rocprim::segmented_radix_sort_keys_desc(d_temporary_storage.get(), temporary_storage_bytes, - d_keys_input, - d_keys_output, + d_keys_input.get(), + d_keys_output.get(), size, segments_count, - d_offsets, - d_offsets + 1, + d_offsets.get(), + d_offsets.get() + 1, start_bit, end_bit, stream)); } else { - HIP_CHECK(rocprim::segmented_radix_sort_keys(d_temporary_storage, + HIP_CHECK(rocprim::segmented_radix_sort_keys(d_temporary_storage.get(), temporary_storage_bytes, - d_keys_input, - d_keys_output, + d_keys_input.get(), + d_keys_output.get(), size, segments_count, - d_offsets, - d_offsets + 1, + d_offsets.get(), + d_offsets.get() + 1, start_bit, end_bit, stream)); } - std::vector keys_output(size); - HIP_CHECK(hipMemcpy(keys_output.data(), - d_keys_output, - size * sizeof(key_type), - hipMemcpyDeviceToHost)); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_keys_output)); - HIP_CHECK(hipFree(d_offsets)); + const auto keys_output = d_keys_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(keys_output, expected)); } @@ -563,34 +496,11 @@ inline void sort_keys_unspecified_ranges() } } - key_type* d_keys_input; - key_type* d_keys_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_input, size * sizeof(key_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_output, size * sizeof(key_type))); - HIP_CHECK(hipMemcpy(d_keys_input, - keys_input.data(), - size * sizeof(key_type), - hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_keys_output, - keys_input.data(), - size * sizeof(key_type), - hipMemcpyHostToDevice)); - - offset_type* d_offsets_begin; - offset_type* d_offsets_end; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_offsets_begin, - segments_count * sizeof(offset_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_offsets_end, - segments_count * sizeof(offset_type))); - HIP_CHECK(hipMemcpy(d_offsets_begin, - begin_offsets.data(), - segments_count * sizeof(offset_type), - hipMemcpyHostToDevice)); - - HIP_CHECK(hipMemcpy(d_offsets_end, - end_offsets.data(), - segments_count * sizeof(offset_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_keys_input(keys_input); + test_utils::device_ptr d_keys_output(keys_input); + + test_utils::device_ptr d_offsets_begin(begin_offsets); + test_utils::device_ptr d_offsets_end(end_offsets); // Calculate expected results on host std::vector expected(keys_input); @@ -605,61 +515,49 @@ inline void sort_keys_unspecified_ranges() size_t temporary_storage_bytes = 0; HIP_CHECK(rocprim::segmented_radix_sort_keys(nullptr, temporary_storage_bytes, - d_keys_input, - d_keys_output, + d_keys_input.get(), + d_keys_output.get(), size, segments_count, - d_offsets_begin, - d_offsets_end, + d_offsets_begin.get(), + d_offsets_end.get(), start_bit, end_bit)); ASSERT_GT(temporary_storage_bytes, 0U); - void* d_temporary_storage; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); if(descending) { - HIP_CHECK(rocprim::segmented_radix_sort_keys_desc(d_temporary_storage, + HIP_CHECK(rocprim::segmented_radix_sort_keys_desc(d_temporary_storage.get(), temporary_storage_bytes, - d_keys_input, - d_keys_output, + d_keys_input.get(), + d_keys_output.get(), size, segments_count, - d_offsets_begin, - d_offsets_end, + d_offsets_begin.get(), + d_offsets_end.get(), start_bit, end_bit, stream)); } else { - HIP_CHECK(rocprim::segmented_radix_sort_keys(d_temporary_storage, + HIP_CHECK(rocprim::segmented_radix_sort_keys(d_temporary_storage.get(), temporary_storage_bytes, - d_keys_input, - d_keys_output, + d_keys_input.get(), + d_keys_output.get(), size, segments_count, - d_offsets_begin, - d_offsets_end, + d_offsets_begin.get(), + d_offsets_end.get(), start_bit, end_bit, stream)); } - std::vector keys_output(size); - HIP_CHECK(hipMemcpy(keys_output.data(), - d_keys_output, - size * sizeof(key_type), - hipMemcpyDeviceToHost)); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_keys_output)); - HIP_CHECK(hipFree(d_offsets_begin)); - HIP_CHECK(hipFree(d_offsets_end)); + const auto keys_output = d_keys_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(keys_output, expected)); } @@ -725,34 +623,13 @@ inline void sort_pairs() std::vector values_input(size); test_utils::iota(values_input.begin(), values_input.end(), 0); - key_type* d_keys_input; - key_type* d_keys_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_input, size * sizeof(key_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_output, size * sizeof(key_type))); - HIP_CHECK(hipMemcpy(d_keys_input, - keys_input.data(), - size * sizeof(key_type), - hipMemcpyHostToDevice)); - - value_type* d_values_input; - value_type* d_values_output; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_values_input, size * sizeof(value_type))); - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_values_output, size * sizeof(value_type))); - HIP_CHECK(hipMemcpy(d_values_input, - values_input.data(), - size * sizeof(value_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_keys_input(keys_input); + test_utils::device_ptr d_keys_output(size); + + test_utils::device_ptr d_values_input(values_input); + test_utils::device_ptr d_values_output(size); - offset_type* d_offsets; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_offsets, - (segments_count + 1) * sizeof(offset_type))); - HIP_CHECK(hipMemcpy(d_offsets, - offsets.data(), - (segments_count + 1) * sizeof(offset_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_offsets(offsets); using key_value = std::pair; @@ -780,79 +657,62 @@ inline void sort_pairs() values_expected[i] = expected[i].second; } - void* d_temporary_storage = nullptr; size_t temporary_storage_bytes = 0; - HIP_CHECK(rocprim::segmented_radix_sort_pairs(d_temporary_storage, + HIP_CHECK(rocprim::segmented_radix_sort_pairs(nullptr, temporary_storage_bytes, - d_keys_input, - d_keys_output, - d_values_input, - d_values_output, + d_keys_input.get(), + d_keys_output.get(), + d_values_input.get(), + d_values_output.get(), size, segments_count, - d_offsets, - d_offsets + 1, + d_offsets.get(), + d_offsets.get() + 1, start_bit, end_bit)); ASSERT_GT(temporary_storage_bytes, 0U); - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); if(descending) { - HIP_CHECK(rocprim::segmented_radix_sort_pairs_desc(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_keys_output, - d_values_input, - d_values_output, - size, - segments_count, - d_offsets, - d_offsets + 1, - start_bit, - end_bit, - stream, - debug_synchronous)); + HIP_CHECK( + rocprim::segmented_radix_sort_pairs_desc(d_temporary_storage.get(), + temporary_storage_bytes, + d_keys_input.get(), + d_keys_output.get(), + d_values_input.get(), + d_values_output.get(), + size, + segments_count, + d_offsets.get(), + d_offsets.get() + 1, + start_bit, + end_bit, + stream, + debug_synchronous)); } else { - HIP_CHECK(rocprim::segmented_radix_sort_pairs(d_temporary_storage, + HIP_CHECK(rocprim::segmented_radix_sort_pairs(d_temporary_storage.get(), temporary_storage_bytes, - d_keys_input, - d_keys_output, - d_values_input, - d_values_output, + d_keys_input.get(), + d_keys_output.get(), + d_values_input.get(), + d_values_output.get(), size, segments_count, - d_offsets, - d_offsets + 1, + d_offsets.get(), + d_offsets.get() + 1, start_bit, end_bit, stream, debug_synchronous)); } - std::vector keys_output(size); - HIP_CHECK(hipMemcpy(keys_output.data(), - d_keys_output, - size * sizeof(key_type), - hipMemcpyDeviceToHost)); - - std::vector values_output(size); - HIP_CHECK(hipMemcpy(values_output.data(), - d_values_output, - size * sizeof(value_type), - hipMemcpyDeviceToHost)); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_values_input)); - HIP_CHECK(hipFree(d_keys_output)); - HIP_CHECK(hipFree(d_values_output)); - HIP_CHECK(hipFree(d_offsets)); + const auto keys_output = d_keys_output.load(); + const auto values_output = d_values_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(keys_output, keys_expected)); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(values_output, values_expected)); @@ -933,49 +793,15 @@ inline void sort_pairs_unspecified_ranges() } } - key_type* d_keys_input; - key_type* d_keys_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_input, size * sizeof(key_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_output, size * sizeof(key_type))); - HIP_CHECK(hipMemcpy(d_keys_input, - keys_input.data(), - size * sizeof(key_type), - hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_keys_output, - keys_input.data(), - size * sizeof(key_type), - hipMemcpyHostToDevice)); - - value_type* d_values_input; - value_type* d_values_output; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_values_input, size * sizeof(value_type))); - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_values_output, size * sizeof(value_type))); - HIP_CHECK(hipMemcpy(d_values_input, - values_input.data(), - size * sizeof(value_type), - hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_values_output, - values_input.data(), - size * sizeof(value_type), - hipMemcpyHostToDevice)); - - offset_type* d_offsets_begin; - offset_type* d_offsets_end; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_offsets_begin, - segments_count * sizeof(offset_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_offsets_end, - segments_count * sizeof(offset_type))); - HIP_CHECK(hipMemcpy(d_offsets_begin, - begin_offsets.data(), - segments_count * sizeof(offset_type), - hipMemcpyHostToDevice)); - - HIP_CHECK(hipMemcpy(d_offsets_end, - end_offsets.data(), - segments_count * sizeof(offset_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_keys_input(keys_input); + test_utils::device_ptr d_keys_output(keys_input); + + test_utils::device_ptr d_values_input(values_input); + test_utils::device_ptr d_values_output(values_input); + + test_utils::device_ptr d_offsets_begin(begin_offsets); + test_utils::device_ptr d_offsets_end(end_offsets); + using key_value = std::pair; // Calculate expected results on host @@ -998,75 +824,57 @@ inline void sort_pairs_unspecified_ranges() size_t temporary_storage_bytes = 0; HIP_CHECK(rocprim::segmented_radix_sort_pairs(nullptr, temporary_storage_bytes, - d_keys_input, - d_keys_output, - d_values_input, - d_values_output, + d_keys_input.get(), + d_keys_output.get(), + d_values_input.get(), + d_values_output.get(), size, segments_count, - d_offsets_begin, - d_offsets_end, + d_offsets_begin.get(), + d_offsets_end.get(), start_bit, end_bit)); ASSERT_GT(temporary_storage_bytes, 0U); - void* d_temporary_storage; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); if(descending) { - HIP_CHECK(rocprim::segmented_radix_sort_pairs_desc(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_keys_output, - d_values_input, - d_values_output, - size, - segments_count, - d_offsets_begin, - d_offsets_end, - start_bit, - end_bit, - stream)); + HIP_CHECK( + rocprim::segmented_radix_sort_pairs_desc(d_temporary_storage.get(), + temporary_storage_bytes, + d_keys_input.get(), + d_keys_output.get(), + d_values_input.get(), + d_values_output.get(), + size, + segments_count, + d_offsets_begin.get(), + d_offsets_end.get(), + start_bit, + end_bit, + stream)); } else { - HIP_CHECK(rocprim::segmented_radix_sort_pairs(d_temporary_storage, + HIP_CHECK(rocprim::segmented_radix_sort_pairs(d_temporary_storage.get(), temporary_storage_bytes, - d_keys_input, - d_keys_output, - d_values_input, - d_values_output, + d_keys_input.get(), + d_keys_output.get(), + d_values_input.get(), + d_values_output.get(), size, segments_count, - d_offsets_begin, - d_offsets_end, + d_offsets_begin.get(), + d_offsets_end.get(), start_bit, end_bit, stream)); } - std::vector keys_output(size); - HIP_CHECK(hipMemcpy(keys_output.data(), - d_keys_output, - size * sizeof(key_type), - hipMemcpyDeviceToHost)); - - std::vector values_output(size); - HIP_CHECK(hipMemcpy(values_output.data(), - d_values_output, - size * sizeof(value_type), - hipMemcpyDeviceToHost)); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_values_input)); - HIP_CHECK(hipFree(d_keys_output)); - HIP_CHECK(hipFree(d_values_output)); - HIP_CHECK(hipFree(d_offsets_begin)); - HIP_CHECK(hipFree(d_offsets_end)); + const auto keys_output = d_keys_output.load(); + const auto values_output = d_values_output.load(); for(size_t i = 0; i < size; i++) { @@ -1132,23 +940,10 @@ inline void sort_keys_double_buffer() } offsets.push_back(size); - key_type* d_keys_input; - key_type* d_keys_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_input, size * sizeof(key_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_output, size * sizeof(key_type))); - HIP_CHECK(hipMemcpy(d_keys_input, - keys_input.data(), - size * sizeof(key_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_keys_input(keys_input); + test_utils::device_ptr d_keys_output(size); - offset_type* d_offsets; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_offsets, - (segments_count + 1) * sizeof(offset_type))); - HIP_CHECK(hipMemcpy(d_offsets, - offsets.data(), - (segments_count + 1) * sizeof(offset_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_offsets(offsets); // Calculate expected results on host std::vector expected(keys_input); @@ -1160,7 +955,7 @@ inline void sort_keys_double_buffer() test_utils::key_comparator()); } - rocprim::double_buffer d_keys(d_keys_input, d_keys_output); + rocprim::double_buffer d_keys(d_keys_input.get(), d_keys_output.get()); size_t temporary_storage_bytes = 0; HIP_CHECK(rocprim::segmented_radix_sort_keys(nullptr, @@ -1168,26 +963,24 @@ inline void sort_keys_double_buffer() d_keys, size, segments_count, - d_offsets, - d_offsets + 1, + d_offsets.get(), + d_offsets.get() + 1, start_bit, end_bit)); ASSERT_GT(temporary_storage_bytes, 0U); - void* d_temporary_storage; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); if(descending) { - HIP_CHECK(rocprim::segmented_radix_sort_keys_desc(d_temporary_storage, + HIP_CHECK(rocprim::segmented_radix_sort_keys_desc(d_temporary_storage.get(), temporary_storage_bytes, d_keys, size, segments_count, - d_offsets, - d_offsets + 1, + d_offsets.get(), + d_offsets.get() + 1, start_bit, end_bit, stream, @@ -1195,13 +988,13 @@ inline void sort_keys_double_buffer() } else { - HIP_CHECK(rocprim::segmented_radix_sort_keys(d_temporary_storage, + HIP_CHECK(rocprim::segmented_radix_sort_keys(d_temporary_storage.get(), temporary_storage_bytes, d_keys, size, segments_count, - d_offsets, - d_offsets + 1, + d_offsets.get(), + d_offsets.get() + 1, start_bit, end_bit, stream, @@ -1214,11 +1007,6 @@ inline void sort_keys_double_buffer() size * sizeof(key_type), hipMemcpyDeviceToHost)); - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_keys_output)); - HIP_CHECK(hipFree(d_offsets)); - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(keys_output, expected)); } } @@ -1283,34 +1071,13 @@ inline void sort_pairs_double_buffer() std::vector values_input(size); test_utils::iota(values_input.begin(), values_input.end(), 0); - key_type* d_keys_input; - key_type* d_keys_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_input, size * sizeof(key_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_output, size * sizeof(key_type))); - HIP_CHECK(hipMemcpy(d_keys_input, - keys_input.data(), - size * sizeof(key_type), - hipMemcpyHostToDevice)); - - value_type* d_values_input; - value_type* d_values_output; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_values_input, size * sizeof(value_type))); - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_values_output, size * sizeof(value_type))); - HIP_CHECK(hipMemcpy(d_values_input, - values_input.data(), - size * sizeof(value_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_keys_input(keys_input); + test_utils::device_ptr d_keys_output(size); - offset_type* d_offsets; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_offsets, - (segments_count + 1) * sizeof(offset_type))); - HIP_CHECK(hipMemcpy(d_offsets, - offsets.data(), - (segments_count + 1) * sizeof(offset_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_values_input(values_input); + test_utils::device_ptr d_values_output(size); + + test_utils::device_ptr d_offsets(offsets); using key_value = std::pair; @@ -1338,52 +1105,52 @@ inline void sort_pairs_double_buffer() values_expected[i] = expected[i].second; } - rocprim::double_buffer d_keys(d_keys_input, d_keys_output); - rocprim::double_buffer d_values(d_values_input, d_values_output); + rocprim::double_buffer d_keys(d_keys_input.get(), d_keys_output.get()); + rocprim::double_buffer d_values(d_values_input.get(), + d_values_output.get()); - void* d_temporary_storage = nullptr; size_t temporary_storage_bytes = 0; - HIP_CHECK(rocprim::segmented_radix_sort_pairs(d_temporary_storage, + HIP_CHECK(rocprim::segmented_radix_sort_pairs(nullptr, temporary_storage_bytes, d_keys, d_values, size, segments_count, - d_offsets, - d_offsets + 1, + d_offsets.get(), + d_offsets.get() + 1, start_bit, end_bit)); ASSERT_GT(temporary_storage_bytes, 0U); - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); if(descending) { - HIP_CHECK(rocprim::segmented_radix_sort_pairs_desc(d_temporary_storage, - temporary_storage_bytes, - d_keys, - d_values, - size, - segments_count, - d_offsets, - d_offsets + 1, - start_bit, - end_bit, - stream, - debug_synchronous)); + HIP_CHECK( + rocprim::segmented_radix_sort_pairs_desc(d_temporary_storage.get(), + temporary_storage_bytes, + d_keys, + d_values, + size, + segments_count, + d_offsets.get(), + d_offsets.get() + 1, + start_bit, + end_bit, + stream, + debug_synchronous)); } else { - HIP_CHECK(rocprim::segmented_radix_sort_pairs(d_temporary_storage, + HIP_CHECK(rocprim::segmented_radix_sort_pairs(d_temporary_storage.get(), temporary_storage_bytes, d_keys, d_values, size, segments_count, - d_offsets, - d_offsets + 1, + d_offsets.get(), + d_offsets.get() + 1, start_bit, end_bit, stream, @@ -1402,13 +1169,6 @@ inline void sort_pairs_double_buffer() size * sizeof(value_type), hipMemcpyDeviceToHost)); - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_keys_output)); - HIP_CHECK(hipFree(d_values_input)); - HIP_CHECK(hipFree(d_values_output)); - HIP_CHECK(hipFree(d_offsets)); - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(keys_output, keys_expected)); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(values_output, values_expected)); } diff --git a/test/rocprim/test_device_segmented_reduce.cpp b/test/rocprim/test_device_segmented_reduce.cpp index ad3ebdc4a..6fd35160c 100644 --- a/test/rocprim/test_device_segmented_reduce.cpp +++ b/test/rocprim/test_device_segmented_reduce.cpp @@ -21,6 +21,7 @@ // SOFTWARE. #include "../common_test_header.hpp" +#include "../rocprim/test_utils_device_ptr.hpp" // required rocprim headers #include @@ -128,7 +129,16 @@ typedef ::testing::Types< SegmentedReduceParamsList(half, float, plus, 0, 10, 300, false), SegmentedReduceParamsList(bfloat16, float, plus, 0, 10, 300, false), // Test with graphs - SegmentedReduceParams, 0, 0, 1000, false, bra::default_algorithm, false, true>> + SegmentedReduceParams, + 0, + 0, + 1000, + false, + bra::default_algorithm, + false, + true>> Params; #undef plus @@ -176,7 +186,7 @@ TYPED_TEST(RocprimDeviceSegmentedReduce, Reduce) SCOPED_TRACE(testing::Message() << "with size = " << size); hipStream_t stream = 0; // default - if (TestFixture::params::use_graphs) + if(TestFixture::params::use_graphs) { // Default stream does not support hipGraph stream capture, so create one HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); @@ -227,46 +237,27 @@ TYPED_TEST(RocprimDeviceSegmentedReduce, Reduce) continue; } - input_type* d_values_input; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_values_input, size * sizeof(input_type))); - HIP_CHECK(hipMemcpy(d_values_input, - values_input.data(), - size * sizeof(input_type), - hipMemcpyHostToDevice)); - - offset_type* d_offsets; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_offsets, - (segments_count + 1) * sizeof(offset_type))); - HIP_CHECK(hipMemcpy(d_offsets, - offsets.data(), - (segments_count + 1) * sizeof(offset_type), - hipMemcpyHostToDevice)); - - output_type* d_aggregates_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_aggregates_output, - segments_count * sizeof(output_type))); - - size_t temporary_storage_bytes; + test_utils::device_ptr d_values_input(values_input); + test_utils::device_ptr d_offsets(offsets); + test_utils::device_ptr d_aggregates_output(segments_count); + + size_t temp_storage_bytes; HIP_CHECK(rocprim::segmented_reduce(nullptr, - temporary_storage_bytes, - d_values_input, - d_aggregates_output, + temp_storage_bytes, + d_values_input.get(), + d_aggregates_output.get(), segments_count, - d_offsets, - d_offsets + 1, + d_offsets.get(), + d_offsets.get() + 1, reduce_op, init, stream, debug_synchronous)); - ASSERT_GT(temporary_storage_bytes, 0); + ASSERT_GT(temp_storage_bytes, 0); - void* d_temporary_storage; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_bytes); test_utils::GraphHelper gHelper; if(TestFixture::params::use_graphs) @@ -275,37 +266,27 @@ TYPED_TEST(RocprimDeviceSegmentedReduce, Reduce) } HIP_CHECK(rocprim::segmented_reduce( - d_temporary_storage, - temporary_storage_bytes, - d_values_input, - test_utils::wrap_in_identity_iterator(d_aggregates_output), + d_temp_storage.get(), + temp_storage_bytes, + d_values_input.get(), + test_utils::wrap_in_identity_iterator( + d_aggregates_output.get()), segments_count, - d_offsets, - d_offsets + 1, + d_offsets.get(), + d_offsets.get() + 1, reduce_op, init, stream, debug_synchronous)); - if(TestFixture::params::use_graphs) { gHelper.createAndLaunchGraph(stream); } - HIP_CHECK(hipFree(d_temporary_storage)); - - std::vector aggregates_output(segments_count); - HIP_CHECK(hipMemcpy(aggregates_output.data(), - d_aggregates_output, - segments_count * sizeof(output_type), - hipMemcpyDeviceToHost)); + const auto aggregates_output = d_aggregates_output.load(); - HIP_CHECK(hipFree(d_values_input)); - HIP_CHECK(hipFree(d_offsets)); - HIP_CHECK(hipFree(d_aggregates_output)); - - if (TestFixture::params::use_graphs) + if(TestFixture::params::use_graphs) { gHelper.cleanupGraphHelper(); HIP_CHECK(hipStreamDestroy(stream)); @@ -363,8 +344,8 @@ void testLargeIndices() std::vector aggregates_expected; std::vector offsets; - int num_segments = 0; - size_t offset = 0; + int segments_count = 0; + size_t offset = 0; while(offset < size) { const size_t segment_length = segment_length_dis(gen); @@ -374,43 +355,35 @@ void testLargeIndices() T aggregate = reduce_op(init, gauss_sum(end) - gauss_sum(offset)); aggregates_expected.push_back(aggregate); - num_segments++; + segments_count++; offset += segment_length; } offsets.push_back(size); // Device inputs - const Iterator values_input{0}; - T* d_offsets = nullptr; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_offsets, sizeof(T) * (num_segments + 1))); - HIP_CHECK(hipMemcpy(d_offsets, - offsets.data(), - sizeof(T) * (num_segments + 1), - hipMemcpyHostToDevice)); + const Iterator values_input{0}; + test_utils::device_ptr d_offsets(offsets); // Device outputs - T* d_aggregates_output = nullptr; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_aggregates_output, sizeof(T) * num_segments)); + test_utils::device_ptr d_aggregates_output(segments_count); // temp storage size_t temp_storage_size_bytes = 0; - void* d_temp_storage = nullptr; // Get size of d_temp_storage HIP_CHECK(rocprim::segmented_reduce(nullptr, temp_storage_size_bytes, values_input, - d_aggregates_output, - num_segments, - d_offsets, - d_offsets + 1, + d_aggregates_output.get(), + segments_count, + d_offsets.get(), + d_offsets.get() + 1, reduce_op, init, stream, debug_synchronous)); // Allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); HIP_CHECK(hipDeviceSynchronize()); test_utils::GraphHelper gHelper; @@ -420,19 +393,18 @@ void testLargeIndices() } // Run - HIP_CHECK(rocprim::segmented_reduce(d_temp_storage, + HIP_CHECK(rocprim::segmented_reduce(d_temp_storage.get(), temp_storage_size_bytes, values_input, - d_aggregates_output, - num_segments, - d_offsets, - d_offsets + 1, + d_aggregates_output.get(), + segments_count, + d_offsets.get(), + d_offsets.get() + 1, reduce_op, init, stream, debug_synchronous)); - if(use_graphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -442,20 +414,12 @@ void testLargeIndices() HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - std::vector aggregates_output(num_segments); - HIP_CHECK(hipMemcpy(aggregates_output.data(), - d_aggregates_output, - sizeof(T) * num_segments, - hipMemcpyDeviceToHost)); + const auto aggregates_output = d_aggregates_output.load(); HIP_CHECK(hipDeviceSynchronize()); SCOPED_TRACE(testing::Message() << "with seed = " << seed); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(aggregates_output, aggregates_expected)); - HIP_CHECK(hipFree(d_offsets)); - HIP_CHECK(hipFree(d_temp_storage)); - HIP_CHECK(hipFree(d_aggregates_output)); - if(use_graphs) { gHelper.cleanupGraphHelper(); diff --git a/test/rocprim/test_device_select.cpp b/test/rocprim/test_device_select.cpp index 4f30d675a..c5f4c7b9c 100644 --- a/test/rocprim/test_device_select.cpp +++ b/test/rocprim/test_device_select.cpp @@ -147,7 +147,7 @@ TYPED_TEST(RocprimDeviceSelectTests, Flagged) // allocate temporary storage test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); - test_utils::GraphHelper gHelper;; + test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) { gHelper.startStreamCapture(stream); @@ -173,11 +173,11 @@ TYPED_TEST(RocprimDeviceSelectTests, Flagged) HIP_CHECK(hipDeviceSynchronize()); // Check if number of selected value is as expected - unsigned int selected_count_output = d_selected_count_output.load()[0]; + const auto selected_count_output = d_selected_count_output.load()[0]; ASSERT_EQ(selected_count_output, expected.size()); // Check if output values are as expected - std::vector output = d_output.load(); + const auto output = d_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected, expected.size())); if(TestFixture::use_graphs) @@ -272,7 +272,7 @@ TYPED_TEST(RocprimDeviceSelectTests, SelectOp) // allocate temporary storage test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); - test_utils::GraphHelper gHelper;; + test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) { gHelper.startStreamCapture(stream); @@ -298,11 +298,11 @@ TYPED_TEST(RocprimDeviceSelectTests, SelectOp) HIP_CHECK(hipDeviceSynchronize()); // Check if number of selected value is as expected - unsigned int selected_count_output = d_selected_count_output.load()[0]; + const auto selected_count_output = d_selected_count_output.load()[0]; ASSERT_EQ(selected_count_output, expected.size()); // Check if output values are as expected - std::vector output = d_output.load(); + const auto output = d_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected, expected.size())); if(TestFixture::use_graphs) @@ -350,19 +350,10 @@ TYPED_TEST(RocprimDeviceSelectTests, SelectFlagged) std::vector input = test_utils::get_random_data(size, 1, 100, seed_value); std::vector flags = test_utils::get_random_data(size, 0, 1, seed_value); - T* d_input; - F* d_flags; - U* d_output; - unsigned int* d_selected_count_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_flags, flags.size() * sizeof(F))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, input.size() * sizeof(U))); - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_selected_count_output, sizeof(unsigned int))); - HIP_CHECK( - hipMemcpy(d_input, input.data(), input.size() * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK( - hipMemcpy(d_flags, flags.data(), flags.size() * sizeof(F), hipMemcpyHostToDevice)); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_flags(flags); + test_utils::device_ptr d_output(input.size()); + test_utils::device_ptr d_selected_count_output(1); // Calculate expected results on host std::vector expected; @@ -381,10 +372,10 @@ TYPED_TEST(RocprimDeviceSelectTests, SelectFlagged) HIP_CHECK(rocprim::select( nullptr, temp_storage_size_bytes, - d_input, - d_flags, - test_utils::wrap_in_identity_iterator(d_output), - d_selected_count_output, + d_input.get(), + d_flags.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), + d_selected_count_output.get(), input.size(), select_op(), stream, @@ -396,8 +387,7 @@ TYPED_TEST(RocprimDeviceSelectTests, SelectFlagged) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - void* d_temp_storage = nullptr; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) @@ -407,12 +397,12 @@ TYPED_TEST(RocprimDeviceSelectTests, SelectFlagged) // Run HIP_CHECK(rocprim::select( - d_temp_storage, + d_temp_storage.get(), temp_storage_size_bytes, - d_input, - d_flags, - test_utils::wrap_in_identity_iterator(d_output), - d_selected_count_output, + d_input.get(), + d_flags.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), + d_selected_count_output.get(), input.size(), select_op(), stream, @@ -426,27 +416,13 @@ TYPED_TEST(RocprimDeviceSelectTests, SelectFlagged) HIP_CHECK(hipDeviceSynchronize()); // Check if number of selected value is as expected - unsigned int selected_count_output = 0; - HIP_CHECK(hipMemcpy(&selected_count_output, - d_selected_count_output, - sizeof(unsigned int), - hipMemcpyDeviceToHost)); + const auto selected_count_output = d_selected_count_output.load()[0]; ASSERT_EQ(selected_count_output, expected.size()); // Check if output values are as expected - std::vector output(input.size()); - HIP_CHECK(hipMemcpy(output.data(), - d_output, - output.size() * sizeof(U), - hipMemcpyDeviceToHost)); + const auto output = d_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected, expected.size())); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_flags)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_selected_count_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if(TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -551,7 +527,7 @@ TYPED_TEST(RocprimDeviceSelectTests, Unique) // allocate temporary storage test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); - test_utils::GraphHelper gHelper;; + test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) { gHelper.startStreamCapture(stream); @@ -577,11 +553,11 @@ TYPED_TEST(RocprimDeviceSelectTests, Unique) HIP_CHECK(hipDeviceSynchronize()); // Check if number of selected value is as expected - unsigned int selected_count_output = d_selected_count_output.load()[0]; + const auto selected_count_output = d_selected_count_output.load()[0]; ASSERT_EQ(selected_count_output, expected.size()); // Check if output values are as expected - std::vector output = d_output.load(); + const auto output = d_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected, expected.size())); if(TestFixture::use_graphs) @@ -715,7 +691,7 @@ void testUniqueGuardedOperator() // allocate temporary storage test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); - test_utils::GraphHelper gHelper;; + test_utils::GraphHelper gHelper; if(UseGraphs) { gHelper.startStreamCapture(stream); @@ -741,11 +717,11 @@ void testUniqueGuardedOperator() HIP_CHECK(hipDeviceSynchronize()); // Check if number of selected value is as expected - unsigned int selected_count_output = d_selected_count_output.load()[0]; + const auto selected_count_output = d_selected_count_output.load()[0]; ASSERT_EQ(selected_count_output, expected.size()); // Check if output values are as expected - auto output = d_output.load(); + const auto output = d_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected, expected.size())); if(UseGraphs) @@ -919,7 +895,7 @@ TYPED_TEST(RocprimDeviceUniqueByKeyTests, UniqueByKey) // allocate temporary storage test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); - test_utils::GraphHelper gHelper;; + test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) { gHelper.startStreamCapture(stream); @@ -949,12 +925,12 @@ TYPED_TEST(RocprimDeviceUniqueByKeyTests, UniqueByKey) HIP_CHECK(hipDeviceSynchronize()); // Check if number of selected value is as expected - auto selected_count_output = d_selected_count_output.load()[0]; + const auto selected_count_output = d_selected_count_output.load()[0]; ASSERT_EQ(selected_count_output, expected_keys.size()); // Check if outputs are as expected - auto output_keys = d_keys_output.load(); - auto output_values = d_values_output.load(); + const auto output_keys = d_keys_output.load(); + const auto output_values = d_values_output.load(); ASSERT_NO_FATAL_FAILURE( test_utils::assert_eq(output_keys, expected_keys, expected_keys.size())); ASSERT_NO_FATAL_FAILURE( @@ -1077,7 +1053,7 @@ TYPED_TEST(RocprimDeviceUniqueByKeyTests, UniqueByKeyAlias) // allocate temporary storage test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); - test_utils::GraphHelper gHelper;; + test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) { gHelper.startStreamCapture(stream); @@ -1107,12 +1083,12 @@ TYPED_TEST(RocprimDeviceUniqueByKeyTests, UniqueByKeyAlias) HIP_CHECK(hipDeviceSynchronize()); // Check if number of selected value is as expected - auto selected_count_output = d_selected_count_output.load()[0]; + const auto selected_count_output = d_selected_count_output.load()[0]; ASSERT_EQ(selected_count_output, expected_keys.size()); // Check if outputs are as expected - auto output_keys = d_keys_input.load(); - auto output_values = d_values_input.load(); + const auto output_keys = d_keys_input.load(); + const auto output_values = d_values_input.load(); ASSERT_NO_FATAL_FAILURE( test_utils::assert_eq(output_keys, expected_keys, expected_keys.size())); ASSERT_NO_FATAL_FAILURE( @@ -1190,12 +1166,10 @@ TEST_P(RocprimDeviceSelectLargeInputTests, LargeInputFlagged) return 0; }); - size_t selected_count_output = 0; test_utils::device_ptr d_selected_count_output(1); size_t expected_output_size = rocprim::detail::ceiling_div(size, flag_selector); - std::vector output(expected_output_size); test_utils::device_ptr d_output(expected_output_size); // Calculate expected results on host @@ -1223,7 +1197,7 @@ TEST_P(RocprimDeviceSelectLargeInputTests, LargeInputFlagged) // temp_storage_size_bytes must be >0 ASSERT_GT(temp_storage_size_bytes, 0); test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); - test_utils::GraphHelper gHelper;; + test_utils::GraphHelper gHelper; if(use_graphs) { gHelper.startStreamCapture(stream); @@ -1248,11 +1222,11 @@ TEST_P(RocprimDeviceSelectLargeInputTests, LargeInputFlagged) HIP_CHECK(hipDeviceSynchronize()); // Check if number of selected value is as expected - selected_count_output = d_selected_count_output.load()[0]; + const auto selected_count_output = d_selected_count_output.load()[0]; ASSERT_EQ(selected_count_output, expected_output_size); // Check if output values are as expected - output = d_output.load(); + const auto output = d_output.load(); ASSERT_NO_FATAL_FAILURE( test_utils::assert_eq(output, expected_output, expected_output.size())); @@ -1310,33 +1284,25 @@ TEST_P(RocprimDeviceSelectLargeInputTests, LargeInputSelectOp) // Generate data auto input_iota = rocprim::make_counting_iterator(std::size_t{0}); - size_t selected_count_output = 0; - size_t* d_selected_count_output; + test_utils::device_ptr d_selected_count_output(1); size_t expected_output_size = selected_input; - size_t* d_output; - std::vector output(expected_output_size); + test_utils::device_ptr d_output(expected_output_size); // Calculate expected results on host std::vector expected_output(expected_output_size); std::iota(expected_output.begin(), expected_output.end(), 0); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, - sizeof(d_output[0]) * expected_output_size)); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_selected_count_output, - sizeof(d_selected_count_output[0]))); - // temp storage size_t temp_storage_size_bytes; - void* d_temp_storage = nullptr; // Get size of d_temp_storage - HIP_CHECK(rocprim::select(d_temp_storage, + HIP_CHECK(rocprim::select(nullptr, temp_storage_size_bytes, input_iota, - d_output, - d_selected_count_output, + d_output.get(), + d_selected_count_output.get(), size, select_op, stream, @@ -1348,7 +1314,7 @@ TEST_P(RocprimDeviceSelectLargeInputTests, LargeInputSelectOp) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper; if(use_graphs) @@ -1357,11 +1323,11 @@ TEST_P(RocprimDeviceSelectLargeInputTests, LargeInputSelectOp) } // Run - HIP_CHECK(rocprim::select(d_temp_storage, + HIP_CHECK(rocprim::select(d_temp_storage.get(), temp_storage_size_bytes, input_iota, - d_output, - d_selected_count_output, + d_output.get(), + d_selected_count_output.get(), size, select_op, stream, @@ -1375,25 +1341,15 @@ TEST_P(RocprimDeviceSelectLargeInputTests, LargeInputSelectOp) HIP_CHECK(hipDeviceSynchronize()); // Check if number of selected value is as expected - HIP_CHECK(hipMemcpy(&selected_count_output, - d_selected_count_output, - sizeof(size_t), - hipMemcpyDeviceToHost)); + const auto selected_count_output = d_selected_count_output.load()[0]; ASSERT_EQ(selected_count_output, expected_output_size); // Check if output values are as expected - HIP_CHECK(hipMemcpy(output.data(), - d_output, - sizeof(output[0]) * expected_output_size, - hipMemcpyDeviceToHost)); + const auto output = d_output.load(); ASSERT_NO_FATAL_FAILURE( test_utils::assert_eq(output, expected_output, expected_output.size())); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_selected_count_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if(use_graphs) { gHelper.cleanupGraphHelper(); @@ -1440,34 +1396,26 @@ TEST_P(RocprimDeviceSelectLargeInputTests, LargeInputSelectFlagged) auto flags_it = rocprim::make_counting_iterator(size_t(0)); - size_t selected_count_output = 0; - size_t* d_selected_count_output; + test_utils::device_ptr d_selected_count_output(1); size_t expected_output_size = selected_flags; - size_t* d_output; - std::vector output(expected_output_size); + test_utils::device_ptr d_output(expected_output_size); // Calculate expected results on host std::vector expected_output(expected_output_size); std::iota(expected_output.begin(), expected_output.end(), 0); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, - sizeof(d_output[0]) * expected_output_size)); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_selected_count_output, - sizeof(d_selected_count_output[0]))); - // temp storage size_t temp_storage_size_bytes; - void* d_temp_storage = nullptr; // Get size of d_temp_storage - HIP_CHECK(rocprim::select(d_temp_storage, + HIP_CHECK(rocprim::select(nullptr, temp_storage_size_bytes, input_begin, flags_it, - d_output, - d_selected_count_output, + d_output.get(), + d_selected_count_output.get(), size, select_op, stream, @@ -1479,7 +1427,7 @@ TEST_P(RocprimDeviceSelectLargeInputTests, LargeInputSelectFlagged) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper; if(use_graphs) @@ -1488,12 +1436,12 @@ TEST_P(RocprimDeviceSelectLargeInputTests, LargeInputSelectFlagged) } // Run - HIP_CHECK(rocprim::select(d_temp_storage, + HIP_CHECK(rocprim::select(d_temp_storage.get(), temp_storage_size_bytes, input_begin, flags_it, - d_output, - d_selected_count_output, + d_output.get(), + d_selected_count_output.get(), size, select_op, stream, @@ -1507,25 +1455,14 @@ TEST_P(RocprimDeviceSelectLargeInputTests, LargeInputSelectFlagged) HIP_CHECK(hipDeviceSynchronize()); // Check if number of selected value is as expected - HIP_CHECK(hipMemcpy(&selected_count_output, - d_selected_count_output, - sizeof(size_t), - hipMemcpyDeviceToHost)); + const auto selected_count_output = d_selected_count_output.load()[0]; ASSERT_EQ(selected_count_output, expected_output_size); // Check if output values are as expected - HIP_CHECK(hipMemcpy(output.data(), - d_output, - sizeof(output[0]) * expected_output_size, - hipMemcpyDeviceToHost)); - + const auto output = d_output.load(); ASSERT_NO_FATAL_FAILURE( test_utils::assert_eq(output, expected_output, expected_output.size())); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_selected_count_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if(use_graphs) { gHelper.cleanupGraphHelper(); @@ -1587,7 +1524,7 @@ TEST_P(RocprimDeviceSelectLargeInputTests, LargeInputUnique) ASSERT_GT(temp_storage_size_bytes, 0); test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); - test_utils::GraphHelper gHelper;; + test_utils::GraphHelper gHelper; if(use_graphs) { gHelper.startStreamCapture(stream); @@ -1608,10 +1545,10 @@ TEST_P(RocprimDeviceSelectLargeInputTests, LargeInputUnique) gHelper.createAndLaunchGraph(stream); } - auto unique_count_output = d_unique_count_output.load()[0]; + const auto unique_count_output = d_unique_count_output.load()[0]; ASSERT_EQ(unique_count_output, expected_output_size); - auto output = d_output.load(); + const auto output = d_output.load(); ASSERT_NO_FATAL_FAILURE( test_utils::assert_eq(output, expected_output, expected_output.size())); diff --git a/test/rocprim/test_device_transform.cpp b/test/rocprim/test_device_transform.cpp index 5e8106818..6afa8327e 100644 --- a/test/rocprim/test_device_transform.cpp +++ b/test/rocprim/test_device_transform.cpp @@ -32,20 +32,18 @@ #include "test_utils_types.hpp" // Params for tests -template< - class InputType, - class OutputType = InputType, - bool UseIdentityIterator = false, - unsigned int SizeLimit = ROCPRIM_GRID_SIZE_LIMIT, - bool UseGraphs = false -> +template struct DeviceTransformParams { - using input_type = InputType; - using output_type = OutputType; - static constexpr bool use_identity_iterator = UseIdentityIterator; - static constexpr size_t size_limit = SizeLimit; - static constexpr bool use_graphs = UseGraphs; + using input_type = InputType; + using output_type = OutputType; + static constexpr bool use_identity_iterator = UseIdentityIterator; + static constexpr size_t size_limit = SizeLimit; + static constexpr bool use_graphs = UseGraphs; }; // --------------------------------------------------------- @@ -56,12 +54,12 @@ template class RocprimDeviceTransformTests : public ::testing::Test { public: - using input_type = typename Params::input_type; - using output_type = typename Params::output_type; - static constexpr bool use_identity_iterator = Params::use_identity_iterator; - static constexpr bool debug_synchronous = false; - static constexpr size_t size_limit = Params::size_limit; - static constexpr bool use_graphs = Params::use_graphs; + using input_type = typename Params::input_type; + using output_type = typename Params::output_type; + static constexpr bool use_identity_iterator = Params::use_identity_iterator; + static constexpr bool debug_synchronous = false; + static constexpr size_t size_limit = Params::size_limit; + static constexpr bool use_graphs = Params::use_graphs; }; using custom_short2 = test_utils::custom_test_type; @@ -86,17 +84,19 @@ typedef ::testing::Types, DeviceTransformParams> RocprimDeviceTransformTestsParams; -template -struct size_limit_config { +template +struct size_limit_config +{ using type = rocprim::transform_config<256, 16, SizeLimit>; }; -template <> -struct size_limit_config { +template<> +struct size_limit_config +{ using type = rocprim::default_config; }; -template +template using size_limit_config_t = typename size_limit_config::type; TYPED_TEST_SUITE(RocprimDeviceTransformTests, RocprimDeviceTransformTestsParams); @@ -104,8 +104,9 @@ TYPED_TEST_SUITE(RocprimDeviceTransformTests, RocprimDeviceTransformTestsParams) template struct transform { - __device__ __host__ inline - T operator()(const T& a) const + __device__ __host__ + inline T + operator()(const T& a) const { return rocprim::plus()(a, T(5)); } @@ -117,20 +118,21 @@ TYPED_TEST(RocprimDeviceTransformTests, Transform) SCOPED_TRACE(testing::Message() << "with device_id = " << device_id); HIP_CHECK(hipSetDevice(device_id)); - using T = typename TestFixture::input_type; - using U = typename TestFixture::output_type; + using T = typename TestFixture::input_type; + using U = typename TestFixture::output_type; static constexpr bool use_identity_iterator = TestFixture::use_identity_iterator; - using Config = size_limit_config_t; + using Config = size_limit_config_t; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { - unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); for(auto size : test_utils::get_sizes(seed_value)) { hipStream_t stream = 0; // default - if (TestFixture::use_graphs) + if(TestFixture::use_graphs) { // Default stream does not support hipGraph stream capture, so create one HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); @@ -140,10 +142,9 @@ TYPED_TEST(RocprimDeviceTransformTests, Transform) // Generate data std::vector input = test_utils::get_random_data(size, 1, 100, seed_value); - std::vector output(input.size(), (U)0); test_utils::device_ptr d_input(input); - test_utils::device_ptr d_output(output.size()); + test_utils::device_ptr d_output(input.size()); // Calculate expected results on host std::vector expected(input.size()); @@ -173,15 +174,15 @@ TYPED_TEST(RocprimDeviceTransformTests, Transform) HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - output = d_output.load(); + const auto output = d_output.load(); // Check if output values are as expected ASSERT_NO_FATAL_FAILURE( test_utils::assert_near(output, expected, test_utils::precision)); - if (TestFixture::use_graphs) + if(TestFixture::use_graphs) { - gHelper.cleanupGraphHelper(); + gHelper.cleanupGraphHelper(); HIP_CHECK(hipStreamDestroy(stream)); } } @@ -191,8 +192,9 @@ TYPED_TEST(RocprimDeviceTransformTests, Transform) template struct binary_transform { - __device__ __host__ inline - constexpr U operator()(const T1& a, const T2& b) const + __device__ __host__ + inline constexpr U + operator()(const T1& a, const T2& b) const { return a + b; } @@ -204,22 +206,23 @@ TYPED_TEST(RocprimDeviceTransformTests, BinaryTransform) SCOPED_TRACE(testing::Message() << "with device_id = " << device_id); HIP_CHECK(hipSetDevice(device_id)); - using T1 = typename TestFixture::input_type; - using T2 = typename TestFixture::input_type; - using U = typename TestFixture::output_type; + using T1 = typename TestFixture::input_type; + using T2 = typename TestFixture::input_type; + using U = typename TestFixture::output_type; static constexpr bool use_identity_iterator = TestFixture::use_identity_iterator; - const bool debug_synchronous = TestFixture::debug_synchronous; - using Config = size_limit_config_t; + const bool debug_synchronous = TestFixture::debug_synchronous; + using Config = size_limit_config_t; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { - unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); for(auto size : test_utils::get_sizes(seed_value)) { hipStream_t stream = 0; // default - if (TestFixture::use_graphs) + if(TestFixture::use_graphs) { // Default stream does not support hipGraph stream capture, so create one HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); @@ -230,18 +233,18 @@ TYPED_TEST(RocprimDeviceTransformTests, BinaryTransform) // Generate data std::vector input1 = test_utils::get_random_data(size, 1, 100, seed_value); std::vector input2 = test_utils::get_random_data(size, 1, 100, seed_value); - std::vector output(input1.size(), (U)0); test_utils::device_ptr d_input1(input1); test_utils::device_ptr d_input2(input2); - test_utils::device_ptr d_output(output.size()); + test_utils::device_ptr d_output(input1.size()); // Calculate expected results on host std::vector expected(input1.size()); - std::transform( - input1.begin(), input1.end(), input2.begin(), - expected.begin(), binary_transform() - ); + std::transform(input1.begin(), + input1.end(), + input2.begin(), + expected.begin(), + binary_transform()); test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) @@ -268,15 +271,15 @@ TYPED_TEST(RocprimDeviceTransformTests, BinaryTransform) HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - output = d_output.load(); + const auto output = d_output.load(); // Check if output values are as expected ASSERT_NO_FATAL_FAILURE( test_utils::assert_near(output, expected, test_utils::precision)); - if (TestFixture::use_graphs) + if(TestFixture::use_graphs) { - gHelper.cleanupGraphHelper(); + gHelper.cleanupGraphHelper(); HIP_CHECK(hipStreamDestroy(stream)); } } @@ -318,7 +321,7 @@ void testLargeIndices() const bool debug_synchronous = false; hipStream_t stream = 0; // default - if (UseGraphs) + if(UseGraphs) { // Default stream does not support hipGraph stream capture, so create one HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); @@ -334,16 +337,15 @@ void testLargeIndices() { SCOPED_TRACE(testing::Message() << "with size = " << size); - const InputIterator input {0}; + const InputIterator input{0}; const OutputIterator output; - bool flags[2] = {false, false}; - bool* d_flag = nullptr; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_flag, sizeof(T))); - HIP_CHECK(hipMemcpy(d_flag, flags, sizeof(flags), hipMemcpyHostToDevice)); + // Using char instead of bool here, since C++ vectors pack bools in single bits + std::vector flags = {false, false}; + test_utils::device_ptr d_flag(flags); const auto expected = test_utils::get_random_value(0, size - 1, seed_value); - const auto limit = ROCPRIM_GRID_SIZE_LIMIT; + const auto limit = ROCPRIM_GRID_SIZE_LIMIT; const auto expected_above_limit = size - 1 > limit ? test_utils::get_random_value(limit, size - 1, seed_value) : size - 1; @@ -352,7 +354,7 @@ void testLargeIndices() SCOPED_TRACE(testing::Message() << "expected_above_limit = " << expected_above_limit); const auto flag_expected - = flag_expected_op_t{d_flag, expected, expected_above_limit}; + = flag_expected_op_t{(bool*)d_flag.get(), expected, expected_above_limit}; test_utils::GraphHelper gHelper; if(UseGraphs) @@ -364,7 +366,6 @@ void testLargeIndices() HIP_CHECK( rocprim::transform(input, output, size, flag_expected, stream, debug_synchronous)); - if(UseGraphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -374,16 +375,14 @@ void testLargeIndices() HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - HIP_CHECK(hipMemcpy(flags, d_flag, sizeof(flags), hipMemcpyDeviceToHost)); + flags = d_flag.load(); HIP_CHECK(hipDeviceSynchronize()); ASSERT_TRUE(flags[0]); ASSERT_TRUE(flags[1]); - - HIP_CHECK(hipFree(d_flag)); - if (UseGraphs) - gHelper.cleanupGraphHelper(); + if(UseGraphs) + gHelper.cleanupGraphHelper(); } } diff --git a/test/rocprim/test_intrinsics.cpp b/test/rocprim/test_intrinsics.cpp index d2f4c45cc..25f85182a 100644 --- a/test/rocprim/test_intrinsics.cpp +++ b/test/rocprim/test_intrinsics.cpp @@ -21,6 +21,7 @@ // SOFTWARE. #include "../common_test_header.hpp" +#include "../rocprim/test_utils_device_ptr.hpp" // required rocprim headers #include @@ -38,7 +39,8 @@ using max_lane_mask_type = uint64_t; constexpr static max_lane_mask_type all_lanes_active = std::numeric_limits::max(); -ROCPRIM_HOST_DEVICE bool is_lane_active(const max_lane_mask_type mask, const unsigned int lane) +ROCPRIM_HOST_DEVICE +bool is_lane_active(const max_lane_mask_type mask, const unsigned int lane) { return (mask >> lane) & 1; } @@ -46,9 +48,9 @@ ROCPRIM_HOST_DEVICE bool is_lane_active(const max_lane_mask_type mask, const uns // Custom structure struct custom_notaligned { - short i; + short i; double d; - float f; + float f; unsigned int u; ROCPRIM_HOST_DEVICE constexpr custom_notaligned() : i(123), d(1234), f(12345), u(12345){}; @@ -59,17 +61,16 @@ struct custom_notaligned }; ROCPRIM_HOST_DEVICE -inline bool operator==(const custom_notaligned& lhs, - const custom_notaligned& rhs) +inline bool + operator==(const custom_notaligned& lhs, const custom_notaligned& rhs) { - return lhs.i == rhs.i && lhs.d == rhs.d - && lhs.f == rhs.f &&lhs.u == rhs.u; + return lhs.i == rhs.i && lhs.d == rhs.d && lhs.f == rhs.f && lhs.u == rhs.u; } // Custom structure aligned to 16 bytes struct alignas(16) custom_16aligned { - int i; + int i; unsigned int u; float f; @@ -92,7 +93,8 @@ struct test_type_helper /// Returns the 'zero' value for this type: In this representation, all of the type's /// bits should be initialized to 0. This function is here because some rocPRIM intrinsic operations /// zero-initialize a type like this. - ROCPRIM_HOST_DEVICE constexpr static T zero() + ROCPRIM_HOST_DEVICE + constexpr static T zero() { return 0; } @@ -101,7 +103,8 @@ struct test_type_helper /// to touch inactive lanes. In the relevant test scenarios, an output value is first initialized with /// this 'initialized' value, so that these values can later be checked if they were really unaltered after /// the intrinsic operation. - ROCPRIM_HOST_DEVICE constexpr static T uninitialized() + ROCPRIM_HOST_DEVICE + constexpr static T uninitialized() { return 0xAA; } @@ -119,12 +122,14 @@ struct test_type_helper template<> struct test_type_helper { - ROCPRIM_HOST_DEVICE constexpr static custom_notaligned zero() + ROCPRIM_HOST_DEVICE + constexpr static custom_notaligned zero() { return {0, 0, 0, 0}; } - ROCPRIM_HOST_DEVICE constexpr static custom_notaligned uninitialized() + ROCPRIM_HOST_DEVICE + constexpr static custom_notaligned uninitialized() { return custom_notaligned(0xAA, 0xAA, 0xAA, 0xAA); } @@ -149,12 +154,14 @@ struct test_type_helper template<> struct test_type_helper { - ROCPRIM_HOST_DEVICE constexpr static custom_16aligned zero() + ROCPRIM_HOST_DEVICE + constexpr static custom_16aligned zero() { return {0, 0, 0}; } - ROCPRIM_HOST_DEVICE constexpr static custom_16aligned uninitialized() + ROCPRIM_HOST_DEVICE + constexpr static custom_16aligned uninitialized() { return custom_16aligned(0xAA, 0xAA, 0xAA); } @@ -232,10 +239,11 @@ enum class shuffle_test_type }; template -__global__ void shuffle_kernel(T* data, - const unsigned int delta, - const unsigned int width, - const max_lane_mask_type active_lanes) +__global__ +void shuffle_kernel(T* data, + const unsigned int delta, + const unsigned int width, + const max_lane_mask_type active_lanes) { const unsigned int index = blockIdx.x * blockDim.x + threadIdx.x; @@ -306,17 +314,16 @@ void test_shuffle() } }; - T* d_data; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_data, size * sizeof(T))); + test_utils::device_ptr d_data(size); - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { - unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); // Generate input - auto input = test_type_helper::get_random_data(size, seed_value); - std::vector output(input.size()); + auto input = test_type_helper::get_random_data(size, seed_value); for(const auto active_lanes : active_lanes_tests(device_id)) { @@ -342,10 +349,7 @@ void test_shuffle() shuffle(expected.data(), input.data(), delta, logical_warp_size, active_lanes); // Writing to device memory - HIP_CHECK(hipMemcpy(d_data, - input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice)); + d_data.store(input); // Launching kernel hipLaunchKernelGGL(HIP_KERNEL_NAME(shuffle_kernel), @@ -353,7 +357,7 @@ void test_shuffle() dim3(hardware_warp_size), 0, 0, - d_data, + d_data.get(), delta, logical_warp_size, active_lanes); @@ -361,18 +365,13 @@ void test_shuffle() HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK(hipMemcpy(output.data(), - d_data, - output.size() * sizeof(T), - hipMemcpyDeviceToHost)); + const auto output = d_data.load(); test_utils::assert_eq(output, expected); } } } } - - HIP_CHECK(hipFree(d_data)); } TYPED_TEST(RocprimIntrinsicsTests, ShuffleUp) @@ -391,16 +390,13 @@ TYPED_TEST(RocprimIntrinsicsTests, ShuffleXor) } template -__global__ -__launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) +__global__ __launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) void shuffle_index_kernel(T* data, int* src_lanes, unsigned int width) { const unsigned int index = (blockIdx.x * blockDim.x) + threadIdx.x; - T value = data[index]; - value = rocprim::warp_shuffle( - value, src_lanes[threadIdx.x/width], width - ); - data[index] = value; + T value = data[index]; + value = rocprim::warp_shuffle(value, src_lanes[threadIdx.x / width], width); + data[index] = value; } TYPED_TEST(RocprimIntrinsicsTests, ShuffleIndex) @@ -414,31 +410,19 @@ TYPED_TEST(RocprimIntrinsicsTests, ShuffleIndex) HIP_CHECK(::rocprim::host_warp_size(device_id, hardware_warp_size)); const size_t size = hardware_warp_size; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { - unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); // Generate input - auto input = test_type_helper::get_random_data(size, seed_value); - std::vector output(input.size()); - - T* device_data; - int * device_src_lanes; - HIP_CHECK( - test_common_utils::hipMallocHelper( - &device_data, - input.size() * sizeof(typename decltype(input)::value_type) - ) - ); - HIP_CHECK( - test_common_utils::hipMallocHelper( - &device_src_lanes, - hardware_warp_size * sizeof(int) - ) - ); - - for(unsigned int i = hardware_warp_size; i > 1; i = i/2) + auto input = test_type_helper::get_random_data(size, seed_value); + + test_utils::device_ptr device_data(input.size()); + test_utils::device_ptr device_src_lanes(hardware_warp_size); + + for(unsigned int i = hardware_warp_size; i > 1; i = i / 2) { const unsigned int logical_warp_size = i; SCOPED_TRACE(testing::Message() << "where logical_warp_size = " << i); @@ -451,62 +435,47 @@ TYPED_TEST(RocprimIntrinsicsTests, ShuffleIndex) // Calculate expected results on host std::vector expected(size, test_type_helper::zero()); - for(size_t j = 0; j < input.size()/logical_warp_size; j++) + for(size_t j = 0; j < input.size() / logical_warp_size; j++) { int src_lane = src_lanes[j]; for(size_t k = 0; k < logical_warp_size; k++) { size_t index = k + logical_warp_size * j; - if(src_lane >= int(logical_warp_size) || src_lane < 0) src_lane = index; + if(src_lane >= int(logical_warp_size) || src_lane < 0) + src_lane = index; expected[index] = input[src_lane + logical_warp_size * j]; } } // Writing to device memory - HIP_CHECK( - hipMemcpy( - device_data, input.data(), - input.size() * sizeof(typename decltype(input)::value_type), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - device_src_lanes, src_lanes.data(), - src_lanes.size() * sizeof(typename decltype(src_lanes)::value_type), - hipMemcpyHostToDevice - ) - ); + device_data.store(input); + device_src_lanes.store(src_lanes); // Launching kernel - hipLaunchKernelGGL( - HIP_KERNEL_NAME(shuffle_index_kernel), - dim3(1), dim3(hardware_warp_size), 0, 0, - device_data, device_src_lanes, logical_warp_size - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(shuffle_index_kernel), + dim3(1), + dim3(hardware_warp_size), + 0, + 0, + device_data.get(), + device_src_lanes.get(), + logical_warp_size); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK( - hipMemcpy( - output.data(), device_data, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + const auto output = device_data.load(); for(size_t j = 0; j < output.size(); j++) { ASSERT_EQ(output[j], expected[j]) << "where index = " << j; } } - HIP_CHECK(hipFree(device_data)); - HIP_CHECK(hipFree(device_src_lanes)); } } -__global__ void lane_id_kernel(unsigned int* data) +__global__ +void lane_id_kernel(unsigned int* data) { const unsigned int index = blockIdx.x * blockDim.x + threadIdx.x; data[index] = rocprim::lane_id(); @@ -521,26 +490,23 @@ TEST(RocprimIntrinsicsTests, LaneId) unsigned int hardware_warp_size; HIP_CHECK(::rocprim::host_warp_size(device_id, hardware_warp_size)); - const size_t warps_per_block = 4; - const size_t block_size = warps_per_block * hardware_warp_size; - const size_t blocks = 2; - const size_t size = blocks * block_size; + const size_t warps_per_block = 4; + const size_t block_size = warps_per_block * hardware_warp_size; + const size_t blocks = 2; + const size_t size = blocks * block_size; SCOPED_TRACE(testing::Message() << "with hardware_warp_size = " << hardware_warp_size); - unsigned int* d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, size * sizeof(unsigned int))); + test_utils::device_ptr d_output(size); hipLaunchKernelGGL(lane_id_kernel, dim3(blocks), dim3(block_size), 0, hipStreamDefault, - d_output); + d_output.get()); HIP_CHECK(hipGetLastError()); - std::vector h_output(size); - HIP_CHECK( - hipMemcpy(h_output.data(), d_output, size * sizeof(unsigned int), hipMemcpyDeviceToHost)); + const auto h_output = d_output.load(); size_t i = 0; for(size_t block = 0; block < blocks; ++block) @@ -554,14 +520,13 @@ TEST(RocprimIntrinsicsTests, LaneId) } } } - - HIP_CHECK(hipFree(d_output)); } -__global__ void masked_bit_count_kernel(unsigned int* out, - const max_lane_mask_type* in, - const unsigned int add, - const max_lane_mask_type active_lanes) +__global__ +void masked_bit_count_kernel(unsigned int* out, + const max_lane_mask_type* in, + const unsigned int add, + const max_lane_mask_type active_lanes) { const unsigned int out_index = blockIdx.x * blockDim.x + threadIdx.x; const unsigned int in_index = out_index / rocprim::device_warp_size(); @@ -582,23 +547,20 @@ TEST(RocprimIntrinsicsTests, MaskedBitCount) unsigned int hardware_warp_size; HIP_CHECK(::rocprim::host_warp_size(device_id, hardware_warp_size)); - const size_t warps_per_block = 4; - const size_t block_size = warps_per_block * hardware_warp_size; - const size_t blocks = 2; - const size_t in_size = blocks * warps_per_block; - const size_t out_size = blocks * block_size; - const size_t n_add = 2; + const size_t warps_per_block = 4; + const size_t block_size = warps_per_block * hardware_warp_size; + const size_t blocks = 2; + const size_t in_size = blocks * warps_per_block; + const size_t out_size = blocks * block_size; + const size_t n_add = 2; SCOPED_TRACE(testing::Message() << "with hardware_warp_size = " << hardware_warp_size); - max_lane_mask_type* d_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, in_size * sizeof(max_lane_mask_type))); + test_utils::device_ptr d_input(in_size); - unsigned int* d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, out_size * sizeof(unsigned int))); + test_utils::device_ptr d_output(out_size); std::vector expected(out_size); - std::vector output(out_size); for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { @@ -611,10 +573,7 @@ TEST(RocprimIntrinsicsTests, MaskedBitCount) 0, all_lanes_active, seed_value); - HIP_CHECK(hipMemcpy(d_input, - input.data(), - in_size * sizeof(max_lane_mask_type), - hipMemcpyHostToDevice)); + d_input.store(input); auto adds = test_utils::get_random_data(n_add, @@ -653,24 +612,18 @@ TEST(RocprimIntrinsicsTests, MaskedBitCount) dim3(block_size), 0, hipStreamDefault, - d_output, - d_input, + d_output.get(), + d_input.get(), add, active_lanes); HIP_CHECK(hipGetLastError()); - HIP_CHECK(hipMemcpy(output.data(), - d_output, - out_size * sizeof(unsigned int), - hipMemcpyDeviceToHost)); + const auto output = d_output.load(); test_utils::assert_eq(output, expected); } } } - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); } enum class warp_any_all_test_type @@ -680,11 +633,12 @@ enum class warp_any_all_test_type }; template -__global__ void warp_any_all_kernel(unsigned int* out, - const max_lane_mask_type* in, - max_lane_mask_type active_lanes) +__global__ +void warp_any_all_kernel(unsigned int* out, + const max_lane_mask_type* in, + max_lane_mask_type active_lanes) { - const unsigned int index = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int index = blockIdx.x * blockDim.x + threadIdx.x; const unsigned int predicate = (in[index / rocprim::device_warp_size()] >> rocprim::lane_id()) & 1; @@ -710,21 +664,18 @@ void warp_any_all_test() unsigned int hardware_warp_size; HIP_CHECK(::rocprim::host_warp_size(device_id, hardware_warp_size)); - const size_t warps_per_block = 4; - const size_t block_size = warps_per_block * hardware_warp_size; - const size_t blocks = 2; - const size_t in_size = blocks * warps_per_block; - const size_t out_size = blocks * block_size; + const size_t warps_per_block = 4; + const size_t block_size = warps_per_block * hardware_warp_size; + const size_t blocks = 2; + const size_t in_size = blocks * warps_per_block; + const size_t out_size = blocks * block_size; SCOPED_TRACE(testing::Message() << "with hardware_warp_size = " << hardware_warp_size); - max_lane_mask_type* d_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, in_size * sizeof(max_lane_mask_type))); + test_utils::device_ptr d_input(in_size); - unsigned int* d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, out_size * sizeof(unsigned int))); + test_utils::device_ptr d_output(out_size); - std::vector output(out_size); std::vector expected(out_size); for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { @@ -771,32 +722,23 @@ void warp_any_all_test() } } - HIP_CHECK(hipMemcpy(d_input, - input.data(), - in_size * sizeof(max_lane_mask_type), - hipMemcpyHostToDevice)); + d_input.store(input); hipLaunchKernelGGL(HIP_KERNEL_NAME(warp_any_all_kernel), dim3(blocks), dim3(block_size), 0, hipStreamDefault, - d_output, - d_input, + d_output.get(), + d_input.get(), active_lanes); HIP_CHECK(hipGetLastError()); - HIP_CHECK(hipMemcpy(output.data(), - d_output, - out_size * sizeof(unsigned int), - hipMemcpyDeviceToHost)); + const auto output = d_output.load(); test_utils::assert_eq(output, expected); } } - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); } TEST(RocprimIntrinsicsTests, WarpAny) @@ -810,11 +752,12 @@ TEST(RocprimIntrinsicsTests, WarpAll) } template -__global__ void warp_permute_kernel(T* out, - const T* in, - const unsigned int* indices, - const unsigned int logical_warp_size, - const max_lane_mask_type active_lanes) +__global__ +void warp_permute_kernel(T* out, + const T* in, + const unsigned int* indices, + const unsigned int logical_warp_size, + const max_lane_mask_type active_lanes) { const unsigned int index = blockIdx.x * blockDim.x + threadIdx.x; @@ -838,21 +781,17 @@ TYPED_TEST(RocprimIntrinsicsTests, WarpPermute) unsigned int hardware_warp_size; HIP_CHECK(::rocprim::host_warp_size(device_id, hardware_warp_size)); - const size_t warps_per_block = 4; - const size_t block_size = warps_per_block * hardware_warp_size; - const size_t blocks = 2; - const size_t size = blocks * block_size; + const size_t warps_per_block = 4; + const size_t block_size = warps_per_block * hardware_warp_size; + const size_t blocks = 2; + const size_t size = blocks * block_size; SCOPED_TRACE(testing::Message() << "with hardware_warp_size = " << hardware_warp_size); - T* d_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, size * sizeof(T))); - - T* d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, size * sizeof(T))); + test_utils::device_ptr d_input(size); + test_utils::device_ptr d_output(size); - unsigned int* d_indices; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_indices, size * sizeof(unsigned int))); + test_utils::device_ptr d_indices(size); const auto permute = [&](T* out, const T* in, @@ -881,7 +820,6 @@ TYPED_TEST(RocprimIntrinsicsTests, WarpPermute) } }; - std::vector output(size); std::vector expected(size); std::vector indices(size); @@ -922,43 +860,35 @@ TYPED_TEST(RocprimIntrinsicsTests, WarpPermute) logical_warp_size, active_lanes); - HIP_CHECK( - hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_indices, - indices.data(), - size * sizeof(unsigned int), - hipMemcpyHostToDevice)); + d_input.store(input); + d_indices.store(indices); hipLaunchKernelGGL(HIP_KERNEL_NAME(warp_permute_kernel), dim3(blocks), dim3(block_size), 0, hipStreamDefault, - d_output, - d_input, - d_indices, + d_output.get(), + d_input.get(), + d_indices.get(), logical_warp_size, active_lanes); HIP_CHECK(hipGetLastError()); - HIP_CHECK( - hipMemcpy(output.data(), d_output, size * sizeof(T), hipMemcpyDeviceToHost)); + const auto output = d_output.load(); test_utils::assert_eq(output, expected); } } } - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_indices)); } template -__global__ void match_any_kernel(max_lane_mask_type* output, - const unsigned int* input, - max_lane_mask_type active_lanes, - max_lane_mask_type lane_predicates) +__global__ +void match_any_kernel(max_lane_mask_type* output, + const unsigned int* input, + max_lane_mask_type active_lanes, + max_lane_mask_type lane_predicates) { const unsigned int index = blockIdx.x * blockDim.x + threadIdx.x; @@ -978,21 +908,17 @@ TEST(RocprimIntrinsicsTests, MatchAny) unsigned int hardware_warp_size; HIP_CHECK(::rocprim::host_warp_size(device_id, hardware_warp_size)); - const size_t warps_per_block = 4; - const size_t block_size = warps_per_block * hardware_warp_size; - const size_t blocks = 2; - const size_t size = blocks * block_size; - constexpr unsigned int label_bits = 3; + const size_t warps_per_block = 4; + const size_t block_size = warps_per_block * hardware_warp_size; + const size_t blocks = 2; + const size_t size = blocks * block_size; + constexpr unsigned int label_bits = 3; SCOPED_TRACE(testing::Message() << "with hardware_warp_size = " << hardware_warp_size); - unsigned int* d_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, size * sizeof(unsigned int))); - - max_lane_mask_type* d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, size * sizeof(max_lane_mask_type))); + test_utils::device_ptr d_input(size); + test_utils::device_ptr d_output(size); std::vector expected(size); - std::vector output(size); for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { @@ -1048,38 +974,31 @@ TEST(RocprimIntrinsicsTests, MatchAny) } } - HIP_CHECK(hipMemcpy(d_input, - input.data(), - size * sizeof(unsigned int), - hipMemcpyHostToDevice)); + d_input.store(input); hipLaunchKernelGGL(HIP_KERNEL_NAME(match_any_kernel), dim3(blocks), dim3(block_size), 0, hipStreamDefault, - d_output, - d_input, + d_output.get(), + d_input.get(), active_lanes, lane_predicates); HIP_CHECK(hipGetLastError()); - HIP_CHECK(hipMemcpy(output.data(), - d_output, - size * sizeof(max_lane_mask_type), - hipMemcpyDeviceToHost)); + const auto output = d_output.load(); test_utils::assert_eq(output, expected); } } } - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); } -__global__ void - ballot_kernel(max_lane_mask_type* output, const unsigned int* input, max_lane_mask_type active_lanes) +__global__ +void ballot_kernel(max_lane_mask_type* output, + const unsigned int* input, + max_lane_mask_type active_lanes) { const unsigned int index = blockIdx.x * blockDim.x + threadIdx.x; @@ -1099,20 +1018,16 @@ TEST(RocprimIntrinsicsTests, Ballot) unsigned int hardware_warp_size; HIP_CHECK(::rocprim::host_warp_size(device_id, hardware_warp_size)); - const size_t warps_per_block = 4; - const size_t block_size = warps_per_block * hardware_warp_size; - const size_t blocks = 2; - const size_t size = blocks * block_size; + const size_t warps_per_block = 4; + const size_t block_size = warps_per_block * hardware_warp_size; + const size_t blocks = 2; + const size_t size = blocks * block_size; SCOPED_TRACE(testing::Message() << "with hardware_warp_size = " << hardware_warp_size); - unsigned int* d_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, size * sizeof(unsigned int))); - - max_lane_mask_type* d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, size * sizeof(max_lane_mask_type))); + test_utils::device_ptr d_input(size); + test_utils::device_ptr d_output(size); std::vector expected(size); - std::vector output(size); for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { @@ -1148,37 +1063,29 @@ TEST(RocprimIntrinsicsTests, Ballot) } } - HIP_CHECK(hipMemcpy(d_input, - input.data(), - size * sizeof(unsigned int), - hipMemcpyHostToDevice)); + d_input.store(input); hipLaunchKernelGGL(ballot_kernel, dim3(blocks), dim3(block_size), 0, hipStreamDefault, - d_output, - d_input, + d_output.get(), + d_input.get(), active_lanes); HIP_CHECK(hipGetLastError()); - HIP_CHECK(hipMemcpy(output.data(), - d_output, - size * sizeof(max_lane_mask_type), - hipMemcpyDeviceToHost)); + const auto output = d_output.load(); test_utils::assert_eq(output, expected); } } - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); } -__global__ void group_elect_kernel(max_lane_mask_type* output, - max_lane_mask_type* input, - size_t warps_per_block) +__global__ +void group_elect_kernel(max_lane_mask_type* output, + max_lane_mask_type* input, + size_t warps_per_block) { const unsigned int input_index = blockIdx.x * blockDim.x + threadIdx.x; @@ -1202,14 +1109,12 @@ TEST(RocprimIntrinsicsTests, GroupElect) const size_t warps_per_block = 4; const size_t block_size = warps_per_block * hardware_warp_size; const size_t blocks = 48; + const size_t size = blocks * block_size; const size_t number_of_warps = blocks * warps_per_block; SCOPED_TRACE(testing::Message() << "with hardware_warp_size = " << hardware_warp_size); - max_lane_mask_type* d_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, blocks * block_size * sizeof(*d_input))); - - max_lane_mask_type* d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, number_of_warps * sizeof(*d_output))); + test_utils::device_ptr d_input(size); + test_utils::device_ptr d_output(number_of_warps); std::vector output; output.reserve(number_of_warps); @@ -1220,7 +1125,7 @@ TEST(RocprimIntrinsicsTests, GroupElect) = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); - std::vector input(blocks * block_size, 0); + std::vector input(size, 0); std::vector> warp_histograms(blocks * warps_per_block); auto input_it = input.begin(); @@ -1256,29 +1161,22 @@ TEST(RocprimIntrinsicsTests, GroupElect) output.assign(number_of_warps, 0); - HIP_CHECK(hipMemcpy(d_input, - input.data(), - blocks * block_size * sizeof(*d_input), - hipMemcpyHostToDevice)); - - HIP_CHECK(hipMemset(d_output, 0, number_of_warps * sizeof(*d_output))); + d_input.store(input); + d_output.store(output); hipLaunchKernelGGL(HIP_KERNEL_NAME(group_elect_kernel), dim3(blocks), dim3(block_size), 0, hipStreamDefault, - d_output, - d_input, + d_output.get(), + d_input.get(), warps_per_block); HIP_CHECK(hipGetLastError()); - HIP_CHECK(hipMemcpy(output.data(), - d_output, - number_of_warps * sizeof(output[0]), - hipMemcpyDeviceToHost)); + output = d_output.load(); - for(size_t i = 0; i < blocks * block_size; ++i) + for(size_t i = 0; i < size; ++i) { const auto group_mask = input[i]; const auto warp_output = output[i / hardware_warp_size]; @@ -1293,7 +1191,4 @@ TEST(RocprimIntrinsicsTests, GroupElect) } } } - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); } diff --git a/test/rocprim/test_temporary_storage_partitioning.cpp b/test/rocprim/test_temporary_storage_partitioning.cpp index b93f247f2..836473fe8 100644 --- a/test/rocprim/test_temporary_storage_partitioning.cpp +++ b/test/rocprim/test_temporary_storage_partitioning.cpp @@ -20,6 +20,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +#include "../rocprim/test_utils_device_ptr.hpp" #include "common_test_header.hpp" #include @@ -58,15 +59,12 @@ TEST(RocprimTemporaryStoragePartitioningTests, Basic) ASSERT_EQ(result, hipSuccess); ASSERT_EQ(storage_size, size); - void* temporary_storage; - HIP_CHECK(test_common_utils::hipMallocHelper(&temporary_storage, storage_size)); + test_utils::device_ptr temporary_storage(storage_size); - result = partition(temporary_storage, storage_size); + result = partition(temporary_storage.get(), storage_size); ASSERT_EQ(result, hipSuccess); ASSERT_EQ(storage_size, size); - ASSERT_EQ(test_allocation, temporary_storage); - - HIP_CHECK(hipFree(temporary_storage)); + ASSERT_EQ(test_allocation, (decltype(test_allocation))temporary_storage.get()); } TEST(RocprimTemporaryStoragePartitioningTests, ZeroSizePartition) @@ -89,14 +87,11 @@ TEST(RocprimTemporaryStoragePartitioningTests, ZeroSizePartition) ASSERT_EQ(result, hipSuccess); ASSERT_NE(storage_size, 0); - void* temporary_storage; - HIP_CHECK(test_common_utils::hipMallocHelper(&temporary_storage, storage_size)); + test_utils::device_ptr temporary_storage(storage_size); - result = partition(temporary_storage, storage_size); + result = partition(temporary_storage.get(), storage_size); ASSERT_EQ(result, hipSuccess); ASSERT_EQ(test_allocation, nullptr); - - HIP_CHECK(hipFree(temporary_storage)); } TEST(RocprimTemporaryStoragePartitioningTests, ZeroSizePartitionInsufficientAllocation) @@ -116,14 +111,11 @@ TEST(RocprimTemporaryStoragePartitioningTests, ZeroSizePartitionInsufficientAllo ASSERT_EQ(result, hipSuccess); ASSERT_NE(storage_size, 0); - void* temporary_storage; - HIP_CHECK(test_common_utils::hipMallocHelper(&temporary_storage, storage_size)); + test_utils::device_ptr temporary_storage(storage_size); storage_size = 0; - result = partition(temporary_storage, storage_size); + result = partition(temporary_storage.get(), storage_size); ASSERT_EQ(result, hipErrorInvalidValue); - - HIP_CHECK(hipFree(temporary_storage)); } TEST(RocprimTemporaryStoragePartitioningTests, Sequence) @@ -167,24 +159,22 @@ TEST(RocprimTemporaryStoragePartitioningTests, Sequence) const size_t expected_size = expected_offset_e + elements_e * sizeof(type_e); ASSERT_EQ(storage_size, expected_size); - void* temporary_storage; - HIP_CHECK(test_common_utils::hipMallocHelper(&temporary_storage, storage_size)); + test_utils::device_ptr temporary_storage(storage_size); - result = partition(temporary_storage, storage_size); + result = partition(temporary_storage.get(), storage_size); ASSERT_EQ(result, hipSuccess); - ASSERT_EQ(test_allocation_a, temporary_storage); - ASSERT_EQ(test_allocation_b, displace_ptr(temporary_storage, expected_offset_b)); - ASSERT_EQ(test_allocation_c, displace_ptr(temporary_storage, expected_offset_c)); + ASSERT_EQ(test_allocation_a, (decltype(test_allocation_a))temporary_storage.get()); + ASSERT_EQ(test_allocation_b, displace_ptr(temporary_storage.get(), expected_offset_b)); + ASSERT_EQ(test_allocation_c, displace_ptr(temporary_storage.get(), expected_offset_c)); ASSERT_EQ(test_allocation_d, nullptr); - ASSERT_EQ(test_allocation_e, displace_ptr(temporary_storage, expected_offset_e)); + ASSERT_EQ(test_allocation_e, displace_ptr(temporary_storage.get(), expected_offset_e)); + ASSERT_TRUE(is_aligned_to(displace_ptr(temporary_storage.get(), expected_offset_b), + layout_b.alignment)); + ASSERT_TRUE(is_aligned_to(displace_ptr(temporary_storage.get(), expected_offset_c), + layout_c.alignment)); ASSERT_TRUE( - is_aligned_to(displace_ptr(temporary_storage, expected_offset_b), layout_b.alignment)); - ASSERT_TRUE( - is_aligned_to(displace_ptr(temporary_storage, expected_offset_c), layout_c.alignment)); - ASSERT_TRUE(is_aligned_to(displace_ptr(temporary_storage, expected_offset_e), alignof(type_e))); - - HIP_CHECK(hipFree(temporary_storage)); + is_aligned_to(displace_ptr(temporary_storage.get(), expected_offset_e), alignof(type_e))); } TEST(RocprimTemporaryStoragePartitioningTests, MutuallyExclusive) @@ -226,21 +216,18 @@ TEST(RocprimTemporaryStoragePartitioningTests, MutuallyExclusive) = expected_offset_bc + std::max(elements_c * sizeof(type_c), layout_b.size); ASSERT_EQ(storage_size, expected_size); - void* temporary_storage; - HIP_CHECK(test_common_utils::hipMallocHelper(&temporary_storage, storage_size)); + test_utils::device_ptr temporary_storage(storage_size); - result = partition(temporary_storage, storage_size); + result = partition(temporary_storage.get(), storage_size); ASSERT_EQ(result, hipSuccess); - ASSERT_EQ(test_allocation_a, temporary_storage); - void* const bc = displace_ptr(temporary_storage, expected_offset_bc); + ASSERT_EQ(test_allocation_a, (decltype(test_allocation_a))temporary_storage.get()); + void* const bc = displace_ptr(temporary_storage.get(), expected_offset_bc); ASSERT_EQ(test_allocation_b, bc); ASSERT_EQ(test_allocation_c, bc); ASSERT_EQ(test_allocation_d, nullptr); ASSERT_TRUE(is_aligned_to(bc, layout_b.alignment)); ASSERT_TRUE(is_aligned_to(bc, alignof(type_c))); - - HIP_CHECK(hipFree(temporary_storage)); } TEST(RocprimTemporaryStoragePartitioningTests, InsufficientAllocation) @@ -266,12 +253,9 @@ TEST(RocprimTemporaryStoragePartitioningTests, InsufficientAllocation) ASSERT_EQ(result, hipSuccess); ASSERT_EQ(storage_size, size * 2); - void* temporary_storage; storage_size -= 1; - HIP_CHECK(test_common_utils::hipMallocHelper(&temporary_storage, storage_size)); + test_utils::device_ptr temporary_storage(storage_size); - result = partition(temporary_storage, storage_size); + result = partition(temporary_storage.get(), storage_size); ASSERT_EQ(result, hipErrorInvalidValue); - - HIP_CHECK(hipFree(temporary_storage)); } From b3d59dc609fc083b18e64a08dfc97fda490a5401 Mon Sep 17 00:00:00 2001 From: Cenxuan Tian Date: Tue, 10 Dec 2024 10:40:43 +0000 Subject: [PATCH 38/84] Resolve "search_n optimization" --- .../benchmark_device_search_n.parallel.hpp | 131 +++++- docs/device_ops/search_n.rst | 6 +- .../device/detail/device_config_helper.hpp | 2 +- .../rocprim/device/detail/device_search_n.hpp | 428 +++++++++--------- .../rocprim/device/device_search_n_config.hpp | 2 +- test/rocprim/test_device_search_n.cpp | 107 ++++- 6 files changed, 431 insertions(+), 245 deletions(-) diff --git a/benchmark/benchmark_device_search_n.parallel.hpp b/benchmark/benchmark_device_search_n.parallel.hpp index 6bec544ed..eeb8b1857 100644 --- a/benchmark/benchmark_device_search_n.parallel.hpp +++ b/benchmark/benchmark_device_search_n.parallel.hpp @@ -339,53 +339,134 @@ inline void add_one_benchmark_search_n(std::vector(_seed, - _stream, - _size_byte, - _size_byte, - 0); - auto start_from_mid - = new benchmark_search_n(_seed, - _stream, - _size_byte, - _size_byte / 2, - _size_byte / 2); + auto half = new benchmark_search_n(_seed, + _stream, + _size_byte, + _size_byte / 2, + _size_byte / 2); // small count test + auto small_count1 + = new benchmark_search_n(_seed, + _stream, + _size_byte, + 1, // count times + 1); + auto small_count2 + = new benchmark_search_n(_seed, + _stream, + _size_byte, + 1, // count times + 2); + auto small_count4 + = new benchmark_search_n(_seed, + _stream, + _size_byte, + 1, // count times + 4); auto small_count6 = new benchmark_search_n(_seed, _stream, _size_byte, 1, // count times 6); + auto small_count8 + = new benchmark_search_n(_seed, + _stream, + _size_byte, + 1, // count times + 8); + auto small_count10 + = new benchmark_search_n(_seed, + _stream, + _size_byte, + 1, // count times + 10); + auto small_count12 + = new benchmark_search_n(_seed, + _stream, + _size_byte, + 1, // count times + 12); + auto small_count36 + = new benchmark_search_n(_seed, + _stream, + _size_byte, + 1, // count times + 36); // mid count test - auto mid_count4095 + auto mid_count1023 = new benchmark_search_n(_seed, _stream, _size_byte, 1, // count times - 4095); - // big input - auto big_count6 + 1023); + auto mid_count2047 + = new benchmark_search_n(_seed, + _stream, + _size_byte, + 1, // count times + 2047); + + auto mid_count4095 = new benchmark_search_n(_seed, _stream, _size_byte, - 6, // count times - (size_t)-1); - std::vector bs = {start_from_0->bench_register(), - start_from_mid->bench_register(), - small_count6->bench_register(), - mid_count4095->bench_register(), - big_count6->bench_register()}; + 1, // count times + 4095); + // big input + auto big_count3 = new benchmark_search_n( + _seed, + _stream, + _size_byte, + 3, // count times + (size_t)-1); // block_size + auto big_count6 = new benchmark_search_n( + _seed, + _stream, + _size_byte, + 6, // count times + (size_t)-1); // block_size + std::vector bs = { + + small_count1->bench_register(), + small_count2->bench_register(), + small_count4->bench_register(), + small_count6->bench_register(), + small_count8->bench_register(), + small_count10->bench_register(), + small_count12->bench_register(), + small_count36->bench_register(), + + mid_count1023->bench_register(), + mid_count2047->bench_register(), + mid_count4095->bench_register(), + + big_count3->bench_register(), + big_count6->bench_register(), + half->bench_register()}; + destructors.emplace_back( [=]() { - delete start_from_0; - delete start_from_mid; + delete small_count1; + delete small_count2; + delete small_count4; delete small_count6; + delete small_count8; + delete small_count10; + delete small_count12; + delete small_count36; + + delete mid_count1023; + delete mid_count2047; delete mid_count4095; + + delete big_count3; delete big_count6; + + delete half; }); + benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); } diff --git a/docs/device_ops/search_n.rst b/docs/device_ops/search_n.rst index a214042f4..f615607a0 100644 --- a/docs/device_ops/search_n.rst +++ b/docs/device_ops/search_n.rst @@ -9,11 +9,11 @@ ******************************************************************** Configuring the kernel -======================== +====================== .. doxygenstruct:: rocprim::search_n_config search_n -======================== +======== -.. doxygenfunction:: rocprim::search_n(void* temporary_storage, size_t& storage_size, InputIterator input, OutputIterator output, const size_t size, const size_t count, const typename std::iterator_traits::value_type* value, const BinaryPredicate binary_predicate = BinaryPredicate(), const hipStream_t stream = static_cast(0), const bool debug_synchronous = false) +.. doxygenfunction:: rocprim::search_n(void* temporary_storage, size_t& storage_size,InputIterator input, OutputIterator output, const size_t size, const size_t count, const typename std::iterator_traits::value_type* value, const BinaryPredicate binary_predicate = BinaryPredicate(), const hipStream_t stream = static_cast(0), const bool debug_synchronous = false) \ No newline at end of file diff --git a/rocprim/include/rocprim/device/detail/device_config_helper.hpp b/rocprim/include/rocprim/device/detail/device_config_helper.hpp index d5bf29c06..4cdfc19a5 100644 --- a/rocprim/include/rocprim/device/detail/device_config_helper.hpp +++ b/rocprim/include/rocprim/device/detail/device_config_helper.hpp @@ -1293,7 +1293,7 @@ struct search_n_config : public detail::search_n_config_params #ifndef DOXYGEN_DOCUMENTATION_BUILD constexpr search_n_config() : detail::search_n_config_params{ - 6, {BlockSize, ItemsPerThread, 0} + 8, {BlockSize, ItemsPerThread, 0} } {} #endif diff --git a/rocprim/include/rocprim/device/detail/device_search_n.hpp b/rocprim/include/rocprim/device/detail/device_search_n.hpp index ec5386184..6a46a008f 100644 --- a/rocprim/include/rocprim/device/detail/device_search_n.hpp +++ b/rocprim/include/rocprim/device/detail/device_search_n.hpp @@ -34,164 +34,128 @@ BEGIN_ROCPRIM_NAMESPACE namespace detail { -inline void search_n_start_timer(std::chrono::steady_clock::time_point& start, - const bool debug_synchronous) -{ - if(debug_synchronous) - { - start = std::chrono::steady_clock::now(); - } -} - template struct search_n_impl_kernels { + static constexpr auto params = device_params(); + static constexpr auto block_size = params.kernel_config.block_size; + static constexpr auto items_per_thread = params.kernel_config.items_per_thread; + static constexpr auto items_per_block = block_size * items_per_thread; - template - static ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(1) void search_n_init_kernel(SizeType* __restrict__ output, - const SizeType target) - { - *output = target; - } - - /// \brief Supports all forms of search_n operations, - /// but the efficiency is insufficient when `items_per_block` is too large. static ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params() .kernel_config.block_size) void search_n_normal_kernel( InputIterator input, size_t* __restrict__ output, - const size_t size, - const size_t count, + const size_t size, + const size_t possible_head_exist_size, + const size_t count, const typename std::iterator_traits::value_type* value, const BinaryPredicate binary_predicate) { - constexpr auto params = device_params(); - constexpr auto block_size = params.kernel_config.block_size; - constexpr auto items_per_thread = params.kernel_config.items_per_thread; - constexpr auto items_per_block = block_size * items_per_thread; const size_t this_thread_start_idx = (block_id<0>() * items_per_block) + (items_per_thread * block_thread_id<0>()); - // TODO: This could cause load imbalance among threads - // So maybe there is a better way to do this - if(size < count + this_thread_start_idx) - { // not able to find a sequence equal to or longer than count + // Not able to find a sequence equal to or longer than count + if(this_thread_start_idx >= possible_head_exist_size) + { return; } - size_t remaining_count = count; - size_t sequence_start_idx = this_thread_start_idx; + size_t remaining_count = count; + size_t head = this_thread_start_idx; + const size_t this_thread_end_idx + = std::min(this_thread_start_idx + items_per_thread, size); - const size_t items_this_thread - = std::min(size - this_thread_start_idx, items_per_thread); for(size_t i = this_thread_start_idx; - sequence_start_idx - this_thread_start_idx < items_this_thread - && i + remaining_count <= size; + head < this_thread_end_idx && i + remaining_count <= size; ++i) { if(binary_predicate(input[i], *value)) { if(--remaining_count == 0) { - atomic_min(output, sequence_start_idx); + atomic_min(output, head); return; } } else { - remaining_count = count; - sequence_start_idx = i + 1; + remaining_count = count; + head = i + 1; } } } - static ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params() - .kernel_config.block_size) void - search_n_find_heads_kernel( - InputIterator input, - const size_t size, - const typename std::iterator_traits::value_type* value, - const BinaryPredicate binary_predicate, - size_t* __restrict__ unfiltered_heads, - const size_t group_size) + template + static ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(1) void search_n_init_kernel(SizeType* __restrict__ output, + const SizeType target) { - constexpr auto params = device_params(); - constexpr auto block_size = params.kernel_config.block_size; - constexpr auto items_per_thread = params.kernel_config.items_per_thread; - constexpr auto items_per_block = block_size * items_per_thread; + *output = target; + } + static ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(block_size) void search_n_find_heads_kernel( + InputIterator input, + const size_t input_size, + const size_t possible_head_exist_size, + const typename std::iterator_traits::value_type* value, + const BinaryPredicate binary_predicate, + size_t* __restrict__ unfiltered_heads, + const size_t group_size) + { const size_t this_thread_start_idx = (block_id<0>() * items_per_block) + (items_per_thread * block_thread_id<0>()); - const size_t items_this_thread - = std::min(this_thread_start_idx < size ? size - this_thread_start_idx : 0, - items_per_thread); - - for(size_t i = this_thread_start_idx; i < this_thread_start_idx + items_this_thread; i++) + const size_t this_thread_end_idx + = std::min(this_thread_start_idx + items_per_thread, possible_head_exist_size); + for(size_t i = this_thread_start_idx; i < this_thread_end_idx; i++) { if(binary_predicate(input[i], *value)) { if(i == 0) - { // is head // `size - i - 1` is the distance to the end - atomic_min(&(unfiltered_heads[i / group_size]), size - i - 1); + { + // This item is the first head + // `input_size - i - 1` is the distance to the end + atomic_min(&(unfiltered_heads[i / group_size]), input_size - i - 1); } else if(!binary_predicate(input[i - 1], *value)) - { // is head - atomic_min(&(unfiltered_heads[i / group_size]), size - i - 1); + { + // This item is head + atomic_min(&(unfiltered_heads[i / group_size]), input_size - i - 1); } } } } - static ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params() - .kernel_config.block_size) void - search_n_heads_filter_kernel(const size_t size, + static ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(block_size) void + search_n_heads_filter_kernel(const size_t input_size, const size_t count, - const size_t* __restrict__ heads, - const size_t heads_size, + const size_t* __restrict__ unfiltered_heads, + const size_t unfiltered_heads_size, size_t* __restrict__ filtered_heads, size_t* __restrict__ filtered_heads_size) { - constexpr auto params = device_params(); - constexpr auto block_size = params.kernel_config.block_size; - constexpr auto items_per_thread = params.kernel_config.items_per_thread; - constexpr auto items_per_block = block_size * items_per_thread; - const size_t this_thread_start_idx = (block_id<0>() * items_per_block) + (block_thread_id<0>() * items_per_thread); const size_t this_thread_end_idx - = std::min(items_per_thread + this_thread_start_idx, heads_size); + = std::min(items_per_thread + this_thread_start_idx, unfiltered_heads_size); for(size_t i = this_thread_start_idx; i < this_thread_end_idx; ++i) { - const auto cur_val = heads[i]; + const auto cur_val = unfiltered_heads[i]; + // This is not a valid head if(cur_val == (size_t)-1) - { // this is not a valid head + { continue; } - const size_t this_head = size - cur_val - 1; - if(i + 1 == heads_size) - { // head of last group - if(size - this_head < count) - { // cannot make it to count - continue; - } - } - else if(i + 2 == heads_size) - { // the head before last head (last group might be incomplete so, the head before last head can be invalid) - const auto next_val = heads[i + 1]; - if(((next_val != (size_t)-1) ? ((size - next_val - 1) - this_head - 1) - : (size - this_head)) - < count) - { // cannot make it to count - continue; - } - } - else - { // other heads - const auto next_val = heads[i + 1]; - if((next_val != (size_t)-1) && (((size - next_val - 1) - this_head - 1) < count)) - { // if next head is invalid, the limit of this head should the next head, else it is possible to make the sequence to count + const size_t this_head = input_size - cur_val - 1; + // Other heads + if(i + 1 < unfiltered_heads_size) + { + const auto next_val = unfiltered_heads[i + 1]; + // Check if this head is valid by calculating the distance between this head and the next head. + if((next_val != (size_t)-1) + && (((input_size - next_val - 1) - this_head - 1) < count)) + { continue; } } @@ -199,64 +163,70 @@ struct search_n_impl_kernels } } - static ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params() - .kernel_config.block_size) void - search_n_discard_heads_kernel( - InputIterator input, - const size_t size, - const size_t count, - const typename std::iterator_traits::value_type* value, - const BinaryPredicate binary_predicate, - size_t* __restrict__ heads, - size_t* num_heads) + static ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(block_size) void search_n_discard_heads_kernel( + InputIterator input, + const size_t input_size, + const size_t count, + const typename std::iterator_traits::value_type* value, + const BinaryPredicate binary_predicate, + size_t* __restrict__ filtered_heads, + size_t* filtered_heads_size) { - constexpr auto params = device_params(); - constexpr auto block_size = params.kernel_config.block_size; - constexpr auto items_per_thread = params.kernel_config.items_per_thread; - constexpr auto items_per_block = block_size * items_per_thread; - - const size_t heads_size = *num_heads; + const size_t heads_size = *filtered_heads_size; + const auto block_idx = block_id<0>(); if(heads_size == 0) { - return; // should return + return; } - - size_t num_blocks_needed = ceiling_div(heads_size * count /*group_size*/, items_per_block); - if(block_id<0>() >= num_blocks_needed) + const size_t total_check_size = heads_size * count /*group_size*/; + const size_t num_blocks_needed = ceiling_div(total_check_size, items_per_block); + if(block_idx >= num_blocks_needed) { return; } const size_t this_thread_start_idx - = (block_id<0>() * items_per_block) + (block_thread_id<0>() * items_per_thread); - - for(size_t global_idx = this_thread_start_idx; - global_idx < this_thread_start_idx + items_per_block; + = (block_idx * items_per_block) + (block_thread_id<0>() * items_per_thread); + const size_t this_thread_end_idx + = std::min(items_per_thread + this_thread_start_idx, total_check_size); + // The `global_idx` is the index of item in the whole `input` + for(size_t global_idx = this_thread_start_idx; global_idx < this_thread_end_idx; global_idx++) { - const size_t g_id /*group id*/ = global_idx / count /*group_size*/; - if(g_id >= heads_size) + // The id of the group who contains the item on global_idx + const size_t group_id = global_idx / count /*group_size*/; + if(group_id >= heads_size) { return; } const size_t check_head - = heads[g_id] + 1; // the `head` is already checked, so we check the next value here + = filtered_heads[group_id] + + 1; // The `head` is already checked, so we check the next value here const size_t check_count = count - 1; const size_t idx = check_head + (global_idx % count); - if((idx >= size) || (idx >= (check_head + check_count))) + if((idx >= input_size) || (idx >= (check_head + check_count))) { return; } if(!binary_predicate(input[idx], *value)) { - heads[g_id] = size; + filtered_heads[group_id] = input_size; return; } } } }; +inline void search_n_start_timer(std::chrono::steady_clock::time_point& start, + const bool debug_synchronous) +{ + if(debug_synchronous) + { + start = std::chrono::steady_clock::now(); + } +} + template ROCPRIM_INLINE hipError_t search_n_impl(void* temporary_storage, @@ -275,8 +245,9 @@ hipError_t search_n_impl(void* temporary_storage, using config = wrapped_search_n_config; using search_n_kernels = search_n_impl_kernels; + // The `size` must greater than or equal to `count` if(count > size) - { // size must greater than or equal to count + { return hipErrorInvalidValue; } @@ -287,22 +258,25 @@ hipError_t search_n_impl(void* temporary_storage, const unsigned int block_size = params.kernel_config.block_size; const unsigned int items_per_thread = params.kernel_config.items_per_thread; const unsigned int items_per_block = block_size * items_per_thread; - const unsigned int num_blocks = ceiling_div(size, items_per_block); std::chrono::steady_clock::time_point start; size_t* tmp_output = reinterpret_cast(temporary_storage); + // Only the items on index that smaller than possible_head_exist_size are possible heads + const size_t possible_head_exist_size = size - count + 1; + + // To be consistent with std::search_n if(size == 0 || count <= 0) - { // to be consist to the std::search_n - // calculate size + { + // Calculate size if(tmp_output == nullptr) { storage_size = sizeof(size_t); return hipSuccess; } - // return end or begin + // Return end or begin search_n_start_timer(start, debug_synchronous); search_n_kernels::search_n_init_kernel<<<1, 1, 0, stream>>>(tmp_output, count <= 0 ? 0 : size); @@ -321,15 +295,17 @@ hipError_t search_n_impl(void* temporary_storage, return hipSuccess; } + const unsigned int num_blocks = ceiling_div(possible_head_exist_size, items_per_block); + // do `normal_search_n` search_n_start_timer(start, debug_synchronous); search_n_kernels::search_n_init_kernel<<<1, 1, 0, stream>>>(tmp_output, size); ROCPRIM_DETAIL_HIP_SYNC_AND_RETURN_ON_ERROR("search_n_init_kernel", 1, start); - // TODO: There can be overlapping between threads, this probably can be optimized search_n_kernels::search_n_normal_kernel<<>>( input, tmp_output, size, + possible_head_exist_size, count, value, binary_predicate); @@ -338,96 +314,126 @@ hipError_t search_n_impl(void* temporary_storage, transform(tmp_output, output, 1, identity(), stream, debug_synchronous)); return hipSuccess; } - else - { // the count is greater than params.threshold - // group_size is equal to `count` - const size_t num_groups = ceiling_div(size, count /*group_size*/); - size_t reduce_storage_size; - ROCPRIM_RETURN_ON_ERROR(reduce(nullptr, - reduce_storage_size, - reinterpret_cast(0), - output, - size, - num_groups, - minimum{}, - stream, - debug_synchronous)); - size_t front_size - = std::max(sizeof(size_t) + (sizeof(size_t) * num_groups), reduce_storage_size); - if(tmp_output == nullptr) - { - storage_size = front_size + (sizeof(size_t) * num_groups); - return hipSuccess; - } - const size_t num_blocks_for_heads_filter = ceiling_div(num_groups, items_per_block); - const size_t num_blocks_for_discard_heads - = ceiling_div(num_groups * count, items_per_block); - - auto unfiltered_heads = reinterpret_cast(reinterpret_cast(temporary_storage) - + sizeof(size_t)); - auto filtered_heads - = reinterpret_cast(reinterpret_cast(temporary_storage) + front_size); - - search_n_start_timer(start, debug_synchronous); - // initialization - ROCPRIM_RETURN_ON_ERROR(hipMemsetAsync(tmp_output, 0, sizeof(size_t), stream)); - ROCPRIM_RETURN_ON_ERROR( - hipMemsetAsync(unfiltered_heads, -1, sizeof(size_t) * num_groups * 2, stream)); + const size_t num_groups = ceiling_div(possible_head_exist_size, count /*group_size*/); + + size_t reduce_storage_size{}; + ROCPRIM_RETURN_ON_ERROR(reduce(nullptr, + reduce_storage_size, + reinterpret_cast(0), + output, + size, // Original value + num_groups, + minimum{}, + stream, + debug_synchronous)); + const size_t front_size + = std::max(sizeof(size_t) + (sizeof(size_t) * num_groups), reduce_storage_size); + if(tmp_output == nullptr) + { + storage_size = front_size + (sizeof(size_t) * num_groups); + return hipSuccess; + } - // find the thread heads of each group - search_n_kernels::search_n_find_heads_kernel<<>>( + // Prepare device variables + auto unfiltered_heads + = reinterpret_cast(reinterpret_cast(temporary_storage) + sizeof(size_t)); + auto filtered_heads + = reinterpret_cast(reinterpret_cast(temporary_storage) + front_size); + + // check if is capturing + hipStreamCaptureStatus is_capturing = hipStreamCaptureStatusNone; + ROCPRIM_RETURN_ON_ERROR(hipStreamIsCapturing(stream, &is_capturing)); + + search_n_start_timer(start, debug_synchronous); + // Initialization + ROCPRIM_RETURN_ON_ERROR(hipMemsetAsync(tmp_output, 0, sizeof(size_t), stream)); + ROCPRIM_RETURN_ON_ERROR( + hipMemsetAsync(unfiltered_heads, -1, sizeof(size_t) * num_groups * 2, stream)); + + // This function processes `possible_head_exist_size` items + const size_t num_blocks_for_find_heads = ceiling_div(possible_head_exist_size, items_per_block); + search_n_kernels:: + search_n_find_heads_kernel<<>>( input, size, + possible_head_exist_size, value, binary_predicate, unfiltered_heads, count /*group_size*/); - ROCPRIM_DETAIL_HIP_SYNC_AND_RETURN_ON_ERROR("search_n_find_heads_kernel", size, start); - - // filter heads - // move valid heads into filtered_heads, and set the size of filtered_heads to tmp_output - search_n_kernels:: - search_n_heads_filter_kernel<<>>( - size, - count, - unfiltered_heads, - num_groups, - filtered_heads, - tmp_output); - ROCPRIM_DETAIL_HIP_SYNC_AND_RETURN_ON_ERROR("search_n_heads_filter_kernel", - num_groups, - start); - - // check if any valid heads make a valid sequence - // max access time for each item is 1 - // TODO: num_blocks_for_discard_heads is actually graeter than the actural valid filtered_heads_size - // so the actural num_blocks_for_discard_heads needed is smaller than the current value - search_n_kernels:: - search_n_discard_heads_kernel<<>>( - input, - size, - count, - value, - binary_predicate, - filtered_heads, - tmp_output); // currently the tmp_output contains the actual size of filtered_heads - ROCPRIM_DETAIL_HIP_SYNC_AND_RETURN_ON_ERROR("search_n_discard_heads_kernel ", - num_groups, - start); - - // calculate the minimum valid head - ROCPRIM_RETURN_ON_ERROR(reduce(temporary_storage, - reduce_storage_size, - filtered_heads, - output, - size, // original value - num_groups, - minimum{}, - stream, - debug_synchronous)); - return hipSuccess; // no needs to call transform, return directly + ROCPRIM_DETAIL_HIP_SYNC_AND_RETURN_ON_ERROR("search_n_find_heads_kernel", + possible_head_exist_size, + start); + + // This function processes `num_groups` items + const size_t num_blocks_for_heads_filter = ceiling_div(num_groups, items_per_block); + search_n_kernels:: + search_n_heads_filter_kernel<<>>( + size, // It is just a value to decode the heads' index value + count, // It is just a value to determine whether a certain head is invalid + unfiltered_heads, // Input heads + num_groups, // Size of unfiltered_heads + filtered_heads, // Output + tmp_output); // Used to store the number of items in `filtered_heads` + ROCPRIM_DETAIL_HIP_SYNC_AND_RETURN_ON_ERROR("search_n_heads_filter_kernel", num_groups, start); + + size_t h_filtered_heads_size = 0; + if(is_capturing != hipStreamCaptureStatusActive) + { + ROCPRIM_RETURN_ON_ERROR(hipMemcpyAsync(&h_filtered_heads_size, + tmp_output, + sizeof(size_t), + hipMemcpyDeviceToHost, + stream)); + if(h_filtered_heads_size == 0) + { + // Return end + search_n_start_timer(start, debug_synchronous); + search_n_kernels::search_n_init_kernel<<<1, 1, 0, stream>>>(tmp_output, size); + ROCPRIM_DETAIL_HIP_SYNC_AND_RETURN_ON_ERROR("search_n_init_kernel", 1, start); + ROCPRIM_RETURN_ON_ERROR(transform(tmp_output, + output, + 1, + identity(), + stream, + debug_synchronous)); + return hipSuccess; + } + } + else + { + h_filtered_heads_size = num_groups; } + + // Max access time for each item is 1 + // So the actual num_blocks_for_discard_heads needed is smaller than the current value + const size_t num_blocks_for_discard_heads + = ceiling_div(h_filtered_heads_size * count, items_per_block); + search_n_kernels:: + search_n_discard_heads_kernel<<>>( + input, + size, + count, + value, + binary_predicate, + filtered_heads, + tmp_output); // Currently, `tmp_output` contains the actual size of `filtered_heads` + ROCPRIM_DETAIL_HIP_SYNC_AND_RETURN_ON_ERROR("search_n_discard_heads_kernel ", + h_filtered_heads_size, + start); + + // Calculate the minimum valid head + ROCPRIM_RETURN_ON_ERROR(reduce(temporary_storage, + reduce_storage_size, + filtered_heads, + output, + size, // Original value + h_filtered_heads_size, + minimum{}, + stream, + debug_synchronous)); + return hipSuccess; // No needs to call transform, return directly } } // namespace detail diff --git a/rocprim/include/rocprim/device/device_search_n_config.hpp b/rocprim/include/rocprim/device/device_search_n_config.hpp index 1aa14d039..0f5f00490 100644 --- a/rocprim/include/rocprim/device/device_search_n_config.hpp +++ b/rocprim/include/rocprim/device/device_search_n_config.hpp @@ -48,7 +48,7 @@ struct wrapped_search_n_config template struct architecture_config { - static constexpr search_n_config_params params = {6, kernel_config<256, 4>()}; + static constexpr search_n_config_params params = {8, kernel_config<256, 4>()}; }; }; diff --git a/test/rocprim/test_device_search_n.cpp b/test/rocprim/test_device_search_n.cpp index 32ae29507..1764e6493 100644 --- a/test/rocprim/test_device_search_n.cpp +++ b/test/rocprim/test_device_search_n.cpp @@ -389,6 +389,105 @@ TYPED_TEST(RocprimDeviceSearchNTests, MinCount) } } +TYPED_TEST(RocprimDeviceSearchNTests, SmallCount) +{ + int device_id = test_common_utils::obtain_device_from_ctest(); + + HIP_CHECK(hipSetDevice(device_id)); + + using input_type = typename TestFixture::input_type; + using output_type = typename TestFixture::output_type; + using op_type = typename TestFixture::op_type; + using config = typename TestFixture::config; + + constexpr bool debug_synchronous = TestFixture::debug_synchronous; + op_type op{}; + + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + { + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + + for(const auto size : test_utils::get_sizes(seed_value)) + { + hipStream_t stream = 0; // default + size_t count = 0; + size_t temp_storage_size; + input_type h_value{1}; + input_type h_noise{0}; + std::vector h_input(size, h_noise); + output_type h_output; + test_utils::device_ptr d_input(h_input); + test_utils::device_ptr d_value(&h_value, 1); + test_utils::device_ptr d_output(1); + + if(size > 0 && size - 1 > 0) + { + count = 1; + size_t random_idx = test_utils::get_random_value(0, size - 1, ++seed_value); + h_input[random_idx] = h_noise; + } + + SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); + SCOPED_TRACE(testing::Message() << "with count = " << count); + SCOPED_TRACE(testing::Message() << "with value = " << h_value); + + if ROCPRIM_IF_CONSTEXPR(TestFixture::use_graphs) + { + // Default stream does not support hipGraph stream capture, so create one + HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); + } + // get size + HIP_CHECK(rocprim::search_n(0, + temp_storage_size, + d_input.get(), + d_output.get(), + h_input.size(), + count, + nullptr)); + + test_utils::device_ptr d_temp_storage(temp_storage_size); + test_utils::GraphHelper gHelper; + if(TestFixture::use_graphs) + { + gHelper.startStreamCapture(stream); + } + HIP_CHECK(rocprim::search_n(d_temp_storage.get(), + temp_storage_size, + d_input.get(), + d_output.get(), + h_input.size(), + count, + d_value.get(), + op, + stream, + debug_synchronous)); + + if ROCPRIM_IF_CONSTEXPR(TestFixture::use_graphs) + { + gHelper.createAndLaunchGraph(stream); + } + + HIP_CHECK(hipGetLastError()); + HIP_CHECK(hipStreamSynchronize(stream)); + + const auto expected + = std::search_n(h_input.cbegin(), h_input.cend(), count, h_value, op) + - h_input.cbegin(); + + h_output = d_output.load()[0]; + + ASSERT_EQ(h_output, expected); + + if ROCPRIM_IF_CONSTEXPR(TestFixture::use_graphs) + { + gHelper.cleanupGraphHelper(); + HIP_CHECK(hipStreamDestroy(stream)); + } + } + } +} + TYPED_TEST(RocprimDeviceSearchNTests, StartFromBegin) { int device_id = test_common_utils::obtain_device_from_ctest(); @@ -967,7 +1066,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, NoiseTest_2block) nullptr)); test_utils::device_ptr d_temp_storage(temp_storage_size); - test_utils::GraphHelper gHelper; + test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) { gHelper.startStreamCapture(stream); @@ -1087,7 +1186,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, NoiseTest_3block) nullptr)); test_utils::device_ptr d_temp_storage(temp_storage_size); - test_utils::GraphHelper gHelper; + test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) { gHelper.startStreamCapture(stream); @@ -1209,7 +1308,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, MultiResult1) nullptr)); test_utils::device_ptr d_temp_storage(temp_storage_size); - test_utils::GraphHelper gHelper; + test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) { gHelper.startStreamCapture(stream); @@ -1325,7 +1424,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, MultiResult2) nullptr)); test_utils::device_ptr d_temp_storage(temp_storage_size); - test_utils::GraphHelper gHelper; + test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) { gHelper.startStreamCapture(stream); From c8466437d41dad8c2b5d5b666e9574f0d8dc5889 Mon Sep 17 00:00:00 2001 From: Nick Breed Date: Wed, 11 Dec 2024 07:11:10 +0000 Subject: [PATCH 39/84] CI changes to speedup development --- .gitlab-ci.yml | 48 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6c58d71e6..a9321eea4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,6 +42,10 @@ stages: - test - benchmark +workflow: + rules: + - if: $CI_MERGE_REQUEST_LABELS !~ /CI Skip/ + variables: PACKAGE_DIR: $BUILD_DIR/package AUTOTUNE_CONFIG_DIR: ${CI_PROJECT_DIR}/autotune_config @@ -111,7 +115,6 @@ copyright-date: -D BUILD_EXAMPLE=ON -D BUILD_BENCHMARK=ON -D GPU_TARGETS=$GPU_TARGETS - -D AMDGPU_TEST_TARGETS=$GPU_TARGETS -D CMAKE_C_COMPILER_LAUNCHER=phc_sccache_c -D CMAKE_CXX_COMPILER_LAUNCHER=phc_sccache_cxx -D CMAKE_CXX_STANDARD=14 @@ -182,7 +185,6 @@ build:cmake-minimum-apt: -D CMAKE_CXX_COMPILER_LAUNCHER=phc_sccache_cxx -D BUILD_EXAMPLE=ON -D GPU_TARGETS=$GPU_TARGETS - -D AMDGPU_TEST_TARGETS=$GPU_TARGETS -D CMAKE_CXX_STANDARD="$BUILD_VERSION" -S $CI_PROJECT_DIR -B $BUILD_DIR @@ -286,6 +288,14 @@ build:windows: - $CI_PROJECT_DIR/build/.ninja_log - $CI_PROJECT_DIR/build/CTestTestfile.cmake expire_in: 2 weeks + rules: + - if: $CI_MERGE_REQUEST_TITLE !~ /Draft:/ || $CI_MERGE_REQUEST_LABELS =~ /windows/ + when: always + - if: $CI_MERGE_REQUEST_TITLE =~ /Draft:/ && + $CI_MERGE_REQUEST_LABELS !~ /windows/ && + $BUILD_TYPE == "Release" + when: always + - when: never autotune:build: stage: autotune @@ -334,12 +344,14 @@ autotune:build: - $BUILD_DIR/deps/googlebenchmark/ expire_in: 1 week -test: +.test:common: stage: test + tags: + - rocm + - $GPU extends: - .cmake-minimum - .rules:test - - .gpus:rocm needs: - job: build:cmake-minimum parallel: @@ -359,10 +371,36 @@ test: - HSA_ENABLE_SDMA=0 ctest --output-on-failure --repeat-until-fail 2 - --tests-regex "hip|$GPU_TARGET" --resource-spec-file ./resources.json --parallel $PARALLEL_JOBS +test:any-gpu: + variables: + GPU: "" + PARALLEL_JOBS: 1 + extends: + - .test:common + rules: + - if: $CI_MERGE_REQUEST_TITLE =~ /Draft:/ && + $CI_MERGE_REQUEST_LABELS !~ /Arch::/ + when: always + - when: never + +test:label-arch: + extends: + - .gpus:rocm + - .test:common + - .rules:arch-labels + +test:all-gpus: + extends: + - .gpus:rocm + - .test:common + rules: + - if: '$CI_MERGE_REQUEST_TITLE !~ /Draft:/' + when: always + - when: never + .test-windows-base: stage: test extends: From e4b3c54b72774cce757273b4e466d6ab965e2981 Mon Sep 17 00:00:00 2001 From: Sander Bos Date: Tue, 10 Dec 2024 13:54:10 +0000 Subject: [PATCH 40/84] Remove select_arch and select_arch_case --- .../include/rocprim/device/config_types.hpp | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/rocprim/include/rocprim/device/config_types.hpp b/rocprim/include/rocprim/device/config_types.hpp index 2569aa3c4..ee3c75290 100644 --- a/rocprim/include/rocprim/device/config_types.hpp +++ b/rocprim/include/rocprim/device/config_types.hpp @@ -132,24 +132,6 @@ struct limit_block_size -struct select_arch_case -{ - static constexpr unsigned int arch = Arch; - using type = T; -}; - -template -struct select_arch - : std::conditional< - Case::arch == TargetArch, - extract_type, - select_arch - >::type { }; - -template -struct select_arch : extract_type { }; - template using default_or_custom_config = typename std::conditional< From bb5a0b59f6d7df9b7b907ca7acffbdff98808cef Mon Sep 17 00:00:00 2001 From: Nara Prasetya Date: Fri, 13 Dec 2024 13:43:41 +0000 Subject: [PATCH 41/84] fix missing equality operator and function constness --- CHANGELOG.md | 6 +-- .../include/rocprim/types/key_value_pair.hpp | 50 +++++++++++-------- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13a8d6ad8..f22b6665b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,14 +7,12 @@ Full documentation for rocPRIM is available at [https://rocm.docs.amd.com/projec ### Added * Added gfx950 support. +* Added missing operator `rocprim::key_value_pair::operator==`. ### Resolved issues * Fixed device radix sort not returning the correct required temporary storage when a double buffer contains `nullptr`. - -### Resolved issues - -* Fixed the warp-level reduction `rocprim::warp_reduce.reduce` DPP implementation to avoid undefined intermediate values during the reduction. +* Fixed constness of equality operators (`==` and `!=`) in `rocprim::key_value_pair`. ## rocPRIM 3.4.0 for ROCm 6.4.0 diff --git a/rocprim/include/rocprim/types/key_value_pair.hpp b/rocprim/include/rocprim/types/key_value_pair.hpp index 55e2028f2..14e5f54cd 100644 --- a/rocprim/include/rocprim/types/key_value_pair.hpp +++ b/rocprim/include/rocprim/types/key_value_pair.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -29,42 +29,48 @@ BEGIN_ROCPRIM_NAMESPACE /// \brief Convenience struct that allows you to store key-value pairs as a composite entity. -template< - class Key_, - class Value_ -> +template struct key_value_pair { - #ifndef DOXYGEN_SHOULD_SKIP_THIS - using Key = Key_; +#ifndef DOXYGEN_SHOULD_SKIP_THIS + using Key = Key_; using Value = Value_; - #endif +#endif - using key_type = Key_; ///< the key's type - using value_type = Value_; ///< the value's type + /// The key's type. + using key_type = Key_; + /// The value's type. + using value_type = Value_; - key_type key; ///< the stored key - value_type value; ///< the stored value + /// The stored key. + key_type key; + /// The stored value. + value_type value; - ROCPRIM_HOST_DEVICE inline - key_value_pair() = default; + ROCPRIM_HOST_DEVICE ROCPRIM_INLINE key_value_pair() = default; - ROCPRIM_HOST_DEVICE inline - ~key_value_pair() = default; + ROCPRIM_HOST_DEVICE ROCPRIM_INLINE ~key_value_pair() = default; /// \brief Constructs a key-value pair using the supplied values. - ROCPRIM_HOST_DEVICE inline - key_value_pair(const key_type key, const value_type value) : key(key), value(value) - { - } + ROCPRIM_HOST_DEVICE ROCPRIM_INLINE key_value_pair(const key_type key, const value_type value) + : key(key), value(value) + {} /// \brief Returns true if the given key-value pair is not equal to this one. /// Otherwise returns false. - ROCPRIM_HOST_DEVICE inline - bool operator !=(const key_value_pair& kvb) + ROCPRIM_HOST_DEVICE ROCPRIM_INLINE + bool operator!=(const key_value_pair& kvb) const { return (key != kvb.key) || (value != kvb.value); } + + /// \brief Returns true if the given key-value pair is equal to this one. + /// Otherwise returns false. + ROCPRIM_HOST_DEVICE ROCPRIM_INLINE + bool operator==(const key_value_pair& kvb) const + { + return (key == kvb.key) && (value == kvb.value); + } }; END_ROCPRIM_NAMESPACE From b25ca66d8b0ee23ac65bf4027e52b1964bbf2976 Mon Sep 17 00:00:00 2001 From: Sander Bos Date: Fri, 13 Dec 2024 18:32:24 +0000 Subject: [PATCH 42/84] Print arguments and timing info in `batch_memcpy_func` when `debug_synchronous` is true --- .../device/detail/device_batch_memcpy.hpp | 161 +++++++----------- .../rocprim/device/device_partition.hpp | 15 -- .../include/rocprim/device/device_scan.hpp | 14 -- test/rocprim/test_device_batch_memcpy.cpp | 33 ++-- 4 files changed, 86 insertions(+), 137 deletions(-) diff --git a/rocprim/include/rocprim/device/detail/device_batch_memcpy.hpp b/rocprim/include/rocprim/device/detail/device_batch_memcpy.hpp index 669ad9a61..902545fab 100644 --- a/rocprim/include/rocprim/device/detail/device_batch_memcpy.hpp +++ b/rocprim/include/rocprim/device/detail/device_batch_memcpy.hpp @@ -1033,8 +1033,6 @@ ROCPRIM_INLINE static hipError_t batch_memcpy_func(void* temporary_ using BufferOffsetType = unsigned int; using BlockOffsetType = unsigned int; - hipError_t error = hipSuccess; - using batch_memcpy_impl_type = detail::batch_memcpy_impl>>( scan_state_buffer, scan_state_block, num_blocks); - error = hipGetLastError(); - if(error != hipSuccess) - { - return error; - } - if(debug_synchronous) - { - ROCPRIM_RETURN_ON_ERROR(hipStreamSynchronize(stream)); - } + ROCPRIM_DETAIL_HIP_SYNC_AND_RETURN_ON_ERROR("init_tile_state_kernel", num_blocks, start); // Launch batch_memcpy_non_blev_kernel. + start_timer(); batch_memcpy_impl_type:: non_blev_memcpy_kernel<<>>( buffers, @@ -1210,31 +1190,18 @@ ROCPRIM_INLINE static hipError_t batch_memcpy_func(void* temporary_ blev_buffers, scan_state_buffer, scan_state_block); - error = hipGetLastError(); - if(error != hipSuccess) - { - return error; - } - if(debug_synchronous) - { - ROCPRIM_RETURN_ON_ERROR(hipStreamSynchronize(stream)); - } + ROCPRIM_DETAIL_HIP_SYNC_AND_RETURN_ON_ERROR("non_blev_memcpy_kernel", num_copies, start); // Launch batch_memcpy_blev_kernel. + start_timer(); batch_memcpy_impl_type:: blev_memcpy_kernel<<>>( blev_buffers, scan_state_buffer, batch_memcpy_grid_size - 1); - error = hipGetLastError(); - if(error != hipSuccess) - { - return error; - } - if(debug_synchronous) - { - ROCPRIM_RETURN_ON_ERROR(hipStreamSynchronize(stream)); - } + ROCPRIM_DETAIL_HIP_SYNC_AND_RETURN_ON_ERROR("blev_memcpy_kernel", + batch_memcpy_grid_size - 1, + start); return hipSuccess; } diff --git a/rocprim/include/rocprim/device/device_partition.hpp b/rocprim/include/rocprim/device/device_partition.hpp index 0ba83cff7..c27d58732 100644 --- a/rocprim/include/rocprim/device/device_partition.hpp +++ b/rocprim/include/rocprim/device/device_partition.hpp @@ -90,18 +90,6 @@ ROCPRIM_KERNEL predicates...); } -#define ROCPRIM_DETAIL_HIP_SYNC(name, size, start) \ - if(debug_synchronous) \ - { \ - std::cout << name << "(" << size << ")"; \ - auto error = hipStreamSynchronize(stream); \ - if(error != hipSuccess) return error; \ - auto end = std::chrono::steady_clock::now(); \ - auto d = std::chrono::duration_cast>(end - start); \ - std::cout << " " << d.count() * 1000 << " ms" << '\n'; \ - } - - template>(end - start); \ - std::cout << " " << d.count() * 1000 << " ms" << '\n'; \ - } - template::wlev_size_threshold; constexpr int blev_min_size = rocprim::batch_memcpy_config<>::blev_size_threshold; @@ -411,7 +417,8 @@ TYPED_TEST(RocprimDeviceBatchMemcpyTests, SizeAndTypeVariation) d_buffer_dsts, d_buffer_sizes, num_buffers, - hipStreamDefault); + hipStreamDefault, + debug_synchronous); void* d_temp_storage = nullptr; @@ -516,7 +523,8 @@ TYPED_TEST(RocprimDeviceBatchMemcpyTests, SizeAndTypeVariation) output_src_it, d_buffer_sizes, num_buffers, - hipStreamDefault); + hipStreamDefault, + debug_synchronous); // Verify results. check_result(h_input_for_memcpy, @@ -570,11 +578,12 @@ struct GetRunLength unsigned int* d_offsets; }; -TEST(RocprimDeviceBatchMemcpyTests, IteratorTest) +TYPED_TEST(RocprimDeviceBatchMemcpyTests, IteratorTest) { // Create the data and copy it to the device. - const unsigned int num_ranges = 5; - const unsigned int num_outputs = 14; + const unsigned int num_ranges = 5; + const unsigned int num_outputs = 14; + constexpr bool debug_synchronous = TestFixture::debug_synchronous; std::vector h_data_in = {4, 2, 7, 3, 1}; // size should be num_ranges std::vector h_data_out(num_outputs, 0); // size should be num_outputs @@ -621,7 +630,8 @@ TEST(RocprimDeviceBatchMemcpyTests, IteratorTest) ptrs_out, sizes, num_ranges, - 0); + 0, + debug_synchronous); // Allocate temporary storage HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); @@ -633,7 +643,8 @@ TEST(RocprimDeviceBatchMemcpyTests, IteratorTest) ptrs_out, sizes, num_ranges, - 0); + 0, + debug_synchronous); // Copy results back to host and print HIP_CHECK( From 9ca710529f62135e66f1b9657659541f1e0b1a5b Mon Sep 17 00:00:00 2001 From: Cenxuan Tian Date: Mon, 16 Dec 2024 09:41:38 +0000 Subject: [PATCH 43/84] Resolve "Replace hipMalloc, hipMallocHelper, and hipFree in benchmarks and tests with device_ptr" Part 3 --- test/rocprim/test_device_merge.cpp | 197 ++------ test/rocprim/test_device_merge_sort.cpp | 196 +++----- test/rocprim/test_device_nth_element.cpp | 92 +--- test/rocprim/test_device_partial_sort.cpp | 70 +-- test/rocprim/test_device_partition.cpp | 114 +++-- test/rocprim/test_device_reduce.cpp | 341 +++++-------- test/rocprim/test_device_reduce_by_key.cpp | 200 ++------ .../rocprim/test_device_run_length_encode.cpp | 165 ++----- test/rocprim/test_device_scan.cpp | 450 ++++++------------ 9 files changed, 551 insertions(+), 1274 deletions(-) diff --git a/test/rocprim/test_device_merge.cpp b/test/rocprim/test_device_merge.cpp index 0d0a6fde2..ad534a7e4 100644 --- a/test/rocprim/test_device_merge.cpp +++ b/test/rocprim/test_device_merge.cpp @@ -22,6 +22,7 @@ // required test headers #include "../common_test_header.hpp" +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" // required rocprim headers @@ -170,40 +171,21 @@ TYPED_TEST(RocprimDeviceMergeTests, MergeKey) test_utils::out_of_bounds_flag out_of_bounds; - key_type * d_keys_input1; - key_type * d_keys_input2; - key_type * d_keys_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_input1, keys_input1.size() * sizeof(key_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_input2, keys_input2.size() * sizeof(key_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_output, keys_output.size() * sizeof(key_type))); - HIP_CHECK( - hipMemcpy( - d_keys_input1, keys_input1.data(), - keys_input1.size() * sizeof(key_type), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_keys_input2, keys_input2.data(), - keys_input2.size() * sizeof(key_type), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr d_keys_input1(keys_input1); + test_utils::device_ptr d_keys_input2(keys_input2); + test_utils::device_ptr d_keys_output(keys_output.size()); test_utils::bounds_checking_iterator d_keys_checking_output( - d_keys_output, + d_keys_output.get(), out_of_bounds.device_pointer(), size1 + size2); - // temp storage - size_t temp_storage_size_bytes; - void * d_temp_storage = nullptr; // Get size of d_temp_storage - HIP_CHECK(rocprim::merge(d_temp_storage, + size_t temp_storage_size_bytes; + HIP_CHECK(rocprim::merge(nullptr, temp_storage_size_bytes, - d_keys_input1, - d_keys_input2, + d_keys_input1.get(), + d_keys_input2.get(), d_keys_checking_output, keys_input1.size(), keys_input2.size(), @@ -215,7 +197,7 @@ TYPED_TEST(RocprimDeviceMergeTests, MergeKey) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) @@ -224,10 +206,10 @@ TYPED_TEST(RocprimDeviceMergeTests, MergeKey) } // Run - HIP_CHECK(rocprim::merge(d_temp_storage, + HIP_CHECK(rocprim::merge(d_temp_storage.get(), temp_storage_size_bytes, - d_keys_input1, - d_keys_input2, + d_keys_input1.get(), + d_keys_input2.get(), d_keys_checking_output, keys_input1.size(), keys_input2.size(), @@ -246,22 +228,11 @@ TYPED_TEST(RocprimDeviceMergeTests, MergeKey) ASSERT_FALSE(out_of_bounds.get()); // Copy keys_output to host - HIP_CHECK( - hipMemcpy( - keys_output.data(), d_keys_output, - keys_output.size() * sizeof(key_type), - hipMemcpyDeviceToHost - ) - ); + keys_output = d_keys_output.load(); // Check if keys_output values are as expected ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(keys_output, expected)); - HIP_CHECK(hipFree(d_keys_input1)); - HIP_CHECK(hipFree(d_keys_input2)); - HIP_CHECK(hipFree(d_keys_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if (TestFixture::use_graphs) gHelper.cleanupGraphHelper(); } @@ -351,68 +322,31 @@ TYPED_TEST(RocprimDeviceMergeTests, MergeKeyValue) test_utils::out_of_bounds_flag out_of_bounds; - key_type * d_keys_input1; - key_type * d_keys_input2; - key_type * d_keys_output; - value_type * d_values_input1; - value_type * d_values_input2; - value_type * d_values_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_input1, keys_input1.size() * sizeof(key_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_input2, keys_input2.size() * sizeof(key_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_output, keys_output.size() * sizeof(key_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_values_input1, values_input1.size() * sizeof(value_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_values_input2, values_input2.size() * sizeof(value_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_values_output, values_output.size() * sizeof(value_type))); - HIP_CHECK( - hipMemcpy( - d_keys_input1, keys_input1.data(), - keys_input1.size() * sizeof(key_type), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_keys_input2, keys_input2.data(), - keys_input2.size() * sizeof(key_type), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_values_input1, values_input1.data(), - values_input1.size() * sizeof(value_type), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_values_input2, values_input2.data(), - values_input2.size() * sizeof(value_type), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr d_keys_input1(keys_input1); + test_utils::device_ptr d_keys_input2(keys_input2); + test_utils::device_ptr d_keys_output(keys_output.size()); + test_utils::device_ptr d_values_input1(values_input1); + test_utils::device_ptr d_values_input2(values_input2); + test_utils::device_ptr d_values_output(values_output.size()); test_utils::bounds_checking_iterator d_keys_checking_output( - d_keys_output, + d_keys_output.get(), out_of_bounds.device_pointer(), - size1 + size2 - ); + size1 + size2); test_utils::bounds_checking_iterator d_values_checking_output( - d_values_output, + d_values_output.get(), out_of_bounds.device_pointer(), size1 + size2); - // temp storage - size_t temp_storage_size_bytes; - void * d_temp_storage = nullptr; // Get size of d_temp_storage - HIP_CHECK(rocprim::merge(d_temp_storage, + size_t temp_storage_size_bytes; + HIP_CHECK(rocprim::merge(nullptr, temp_storage_size_bytes, - d_keys_input1, - d_keys_input2, + d_keys_input1.get(), + d_keys_input2.get(), d_keys_checking_output, - d_values_input1, - d_values_input2, + d_values_input1.get(), + d_values_input2.get(), d_values_checking_output, keys_input1.size(), keys_input2.size(), @@ -424,7 +358,7 @@ TYPED_TEST(RocprimDeviceMergeTests, MergeKeyValue) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) @@ -433,13 +367,13 @@ TYPED_TEST(RocprimDeviceMergeTests, MergeKeyValue) } // Run - HIP_CHECK(rocprim::merge(d_temp_storage, + HIP_CHECK(rocprim::merge(d_temp_storage.get(), temp_storage_size_bytes, - d_keys_input1, - d_keys_input2, + d_keys_input1.get(), + d_keys_input2.get(), d_keys_checking_output, - d_values_input1, - d_values_input2, + d_values_input1.get(), + d_values_input2.get(), d_values_checking_output, keys_input1.size(), keys_input2.size(), @@ -457,20 +391,8 @@ TYPED_TEST(RocprimDeviceMergeTests, MergeKeyValue) ASSERT_FALSE(out_of_bounds.get()); - HIP_CHECK( - hipMemcpy( - keys_output.data(), d_keys_output, - keys_output.size() * sizeof(key_type), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK( - hipMemcpy( - values_output.data(), d_values_output, - values_output.size() * sizeof(value_type), - hipMemcpyDeviceToHost - ) - ); + keys_output = d_keys_output.load(); + values_output = d_values_output.load(); // Check if keys_output values are as expected std::vector expected_key(expected.size()); @@ -483,14 +405,6 @@ TYPED_TEST(RocprimDeviceMergeTests, MergeKeyValue) ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(keys_output, expected_key)); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(values_output, expected_value)); - HIP_CHECK(hipFree(d_keys_input1)); - HIP_CHECK(hipFree(d_keys_input2)); - HIP_CHECK(hipFree(d_keys_output)); - HIP_CHECK(hipFree(d_values_input1)); - HIP_CHECK(hipFree(d_values_input2)); - HIP_CHECK(hipFree(d_values_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if (TestFixture::use_graphs) gHelper.cleanupGraphHelper(); } @@ -520,18 +434,8 @@ void testMergeMismatchedIteratorTypes() std::vector expected_keys_output(2 * keys_input1.size()); std::iota(expected_keys_output.begin(), expected_keys_output.end(), 0); - int* d_keys_input1 = nullptr; - int* d_keys_output = nullptr; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_input1, - keys_input1.size() * sizeof(keys_input1[0]))); - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_keys_output, - expected_keys_output.size() * sizeof(keys_input1[0]))); - - HIP_CHECK(hipMemcpy(d_keys_input1, - keys_input1.data(), - keys_input1.size() * sizeof(keys_input1[0]), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_keys_input1(keys_input1); + test_utils::device_ptr d_keys_output(expected_keys_output.size()); const auto d_keys_input2 = rocprim::make_transform_iterator(rocprim::make_counting_iterator(0), @@ -549,9 +453,9 @@ void testMergeMismatchedIteratorTypes() size_t temp_storage_size_bytes = 0; HIP_CHECK(rocprim::merge(nullptr, temp_storage_size_bytes, - d_keys_input1, + d_keys_input1.get(), d_keys_input2, - d_keys_output, + d_keys_output.get(), keys_input1.size(), keys_input1.size(), rocprim::less{}, @@ -560,8 +464,7 @@ void testMergeMismatchedIteratorTypes() ASSERT_GT(temp_storage_size_bytes, 0); - void* d_temp_storage = nullptr; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper; if(UseGraphs) @@ -569,11 +472,11 @@ void testMergeMismatchedIteratorTypes() gHelper.startStreamCapture(stream); } - HIP_CHECK(rocprim::merge(d_temp_storage, + HIP_CHECK(rocprim::merge(d_temp_storage.get(), temp_storage_size_bytes, - d_keys_input1, + d_keys_input1.get(), d_keys_input2, - d_keys_output, + d_keys_output.get(), keys_input1.size(), keys_input1.size(), rocprim::less{}, @@ -585,18 +488,10 @@ void testMergeMismatchedIteratorTypes() gHelper.createAndLaunchGraph(stream); } - std::vector keys_output(expected_keys_output.size()); - HIP_CHECK(hipMemcpy(keys_output.data(), - d_keys_output, - keys_output.size() * sizeof(keys_output[0]), - hipMemcpyDeviceToHost)); + const auto keys_output = d_keys_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(keys_output, expected_keys_output)); - HIP_CHECK(hipFree(d_temp_storage)); - HIP_CHECK(hipFree(d_keys_output)); - HIP_CHECK(hipFree(d_keys_input1)); - if (UseGraphs) { gHelper.cleanupGraphHelper(); diff --git a/test/rocprim/test_device_merge_sort.cpp b/test/rocprim/test_device_merge_sort.cpp index e0e5b5c20..0ccc803db 100644 --- a/test/rocprim/test_device_merge_sort.cpp +++ b/test/rocprim/test_device_merge_sort.cpp @@ -30,6 +30,7 @@ // required test headers #include "test_utils_custom_float_type.hpp" +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" // Params for tests @@ -118,28 +119,16 @@ TYPED_TEST(RocprimDeviceSortTests, SortKey) in_place = !in_place; // Generate data - std::vector input = test_utils::get_random_data(size, -100, 100, seed_value); // float16 can't exceed 65504 - std::vector output(size); + std::vector input + = test_utils::get_random_data(size, + -100, + 100, + seed_value); // float16 can't exceed 65504 - key_type * d_input; - key_type * d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(key_type))); - if(in_place) - { - d_output = d_input; - } - else - { - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(key_type))); - } - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - input.size() * sizeof(key_type), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output_alloc; + d_output_alloc.resize(in_place ? 0 : size); + test_utils::device_ptr& d_output = in_place ? d_input : d_output_alloc; // compare function compare_function compare_op; @@ -148,24 +137,22 @@ TYPED_TEST(RocprimDeviceSortTests, SortKey) std::vector expected(input); std::stable_sort(expected.begin(), expected.end(), compare_op); - // temp storage - size_t temp_storage_size_bytes; - void * d_temp_storage = nullptr; // Get size of d_temp_storage - HIP_CHECK( - rocprim::merge_sort( - d_temp_storage, temp_storage_size_bytes, - d_input, d_output, input.size(), - compare_op, stream, debug_synchronous - ) - ); + size_t temp_storage_size_bytes; + HIP_CHECK(rocprim::merge_sort(nullptr, + temp_storage_size_bytes, + d_input.get(), + d_output.get(), + input.size(), + compare_op, + stream, + debug_synchronous)); // temp_storage_size_bytes must be >0 ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) @@ -174,13 +161,14 @@ TYPED_TEST(RocprimDeviceSortTests, SortKey) } // Run - HIP_CHECK( - rocprim::merge_sort( - d_temp_storage, temp_storage_size_bytes, - d_input, d_output, input.size(), - compare_op, stream, debug_synchronous - ) - ); + HIP_CHECK(rocprim::merge_sort(d_temp_storage.get(), + temp_storage_size_bytes, + d_input.get(), + d_output.get(), + input.size(), + compare_op, + stream, + debug_synchronous)); if(TestFixture::use_graphs) { @@ -191,25 +179,11 @@ TYPED_TEST(RocprimDeviceSortTests, SortKey) HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - output.size() * sizeof(key_type), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + const auto output = d_output.load(); // Check if output values are as expected ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); - HIP_CHECK(hipFree(d_input)); - if(!in_place) - { - HIP_CHECK(hipFree(d_output)); - } - HIP_CHECK(hipFree(d_temp_storage)); - if (TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -257,48 +231,17 @@ TYPED_TEST(RocprimDeviceSortTests, SortKeyValue) std::vector values_input(size); test_utils::iota(values_input.begin(), values_input.end(), 0); - std::vector keys_output(size); - std::vector values_output(size); - - key_type * d_keys_input; - key_type * d_keys_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_input, keys_input.size() * sizeof(key_type))); - if(in_place) - { - d_keys_output = d_keys_input; - } - else - { - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_output, keys_output.size() * sizeof(key_type))); - } - HIP_CHECK( - hipMemcpy( - d_keys_input, keys_input.data(), - keys_input.size() * sizeof(key_type), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_keys_input(keys_input); + test_utils::device_ptr d_keys_output_alloc; + d_keys_output_alloc.resize(in_place ? 0 : size); + test_utils::device_ptr& d_keys_output + = in_place ? d_keys_input : d_keys_output_alloc; - value_type * d_values_input; - value_type * d_values_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_values_input, values_input.size() * sizeof(value_type))); - if(in_place) - { - d_values_output = d_values_input; - } - else - { - HIP_CHECK(test_common_utils::hipMallocHelper(&d_values_output, values_output.size() * sizeof(value_type))); - } - HIP_CHECK( - hipMemcpy( - d_values_input, values_input.data(), - values_input.size() * sizeof(value_type), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_values_input(values_input); + test_utils::device_ptr d_values_output_alloc; + d_values_output_alloc.resize(in_place ? 0 : size); + test_utils::device_ptr& d_values_output + = in_place ? d_values_input : d_values_output_alloc; // compare function compare_function compare_op; @@ -315,16 +258,14 @@ TYPED_TEST(RocprimDeviceSortTests, SortKeyValue) [compare_op](const key_value& a, const key_value& b) { return compare_op(a.first, b.first); }); - // temp storage - size_t temp_storage_size_bytes; - void * d_temp_storage = nullptr; // Get size of d_temp_storage - HIP_CHECK(rocprim::merge_sort(d_temp_storage, + size_t temp_storage_size_bytes; + HIP_CHECK(rocprim::merge_sort(nullptr, temp_storage_size_bytes, - d_keys_input, - d_keys_output, - d_values_input, - d_values_output, + d_keys_input.get(), + d_keys_output.get(), + d_values_input.get(), + d_values_output.get(), keys_input.size(), compare_op, stream, @@ -334,8 +275,7 @@ TYPED_TEST(RocprimDeviceSortTests, SortKeyValue) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) @@ -344,14 +284,16 @@ TYPED_TEST(RocprimDeviceSortTests, SortKeyValue) } // Run - HIP_CHECK( - rocprim::merge_sort( - d_temp_storage, temp_storage_size_bytes, - d_keys_input, d_keys_output, - d_values_input, d_values_output, keys_input.size(), - compare_op, stream, debug_synchronous - ) - ); + HIP_CHECK(rocprim::merge_sort(d_temp_storage.get(), + temp_storage_size_bytes, + d_keys_input.get(), + d_keys_output.get(), + d_values_input.get(), + d_values_output.get(), + keys_input.size(), + compare_op, + stream, + debug_synchronous)); if(TestFixture::use_graphs) { @@ -362,21 +304,8 @@ TYPED_TEST(RocprimDeviceSortTests, SortKeyValue) HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - HIP_CHECK( - hipMemcpy( - keys_output.data(), d_keys_output, - keys_output.size() * sizeof(key_type), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK( - hipMemcpy( - values_output.data(), d_values_output, - values_output.size() * sizeof(value_type), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + const auto keys_output = d_keys_output.load(); + const auto values_output = d_values_output.load(); // Check if output values are as expected std::vector expected_key(expected.size()); @@ -390,15 +319,6 @@ TYPED_TEST(RocprimDeviceSortTests, SortKeyValue) ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(keys_output, expected_key)); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(values_output, expected_value)); - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_values_input)); - if(!in_place) - { - HIP_CHECK(hipFree(d_keys_output)); - HIP_CHECK(hipFree(d_values_output)); - } - HIP_CHECK(hipFree(d_temp_storage)); - if (TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); diff --git a/test/rocprim/test_device_nth_element.cpp b/test/rocprim/test_device_nth_element.cpp index 6359de486..4d4199227 100644 --- a/test/rocprim/test_device_nth_element.cpp +++ b/test/rocprim/test_device_nth_element.cpp @@ -26,6 +26,7 @@ #include "test_utils_custom_float_type.hpp" #include "test_utils_custom_test_types.hpp" #include "test_utils_data_generation.hpp" +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" #include "../common_test_header.hpp" @@ -224,40 +225,22 @@ TYPED_TEST(RocprimDeviceNthelementTests, NthelementKey) test_utils::generate_limits::max(), seed_value); - std::vector output(size); - - key_type* d_input; - key_type* d_output; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(*d_input))); - if(in_place) - { - d_output = d_input; - } - else - { - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, - output.size() * sizeof(*d_output))); - } - - HIP_CHECK(hipMemcpy(d_input, - input.data(), - input.size() * sizeof(*d_input), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output_alloc; + d_output_alloc.resize(in_place ? 0 : size); + test_utils::device_ptr& d_output = in_place ? d_input : d_output_alloc; const auto input_it - = test_utils::wrap_in_indirect_iterator(d_input); + = test_utils::wrap_in_indirect_iterator(d_input.get()); // compare function compare_function compare_op; - // temp storage - size_t temp_storage_size_bytes; - void* d_temp_storage = nullptr; // Get size of d_temp_storage + size_t temp_storage_size_bytes; if(in_place) { - HIP_CHECK(rocprim::nth_element(d_temp_storage, + HIP_CHECK(rocprim::nth_element(nullptr, temp_storage_size_bytes, input_it, nth_element, @@ -268,10 +251,10 @@ TYPED_TEST(RocprimDeviceNthelementTests, NthelementKey) } else { - HIP_CHECK(rocprim::nth_element(d_temp_storage, + HIP_CHECK(rocprim::nth_element(nullptr, temp_storage_size_bytes, input_it, - d_output, + d_output.get(), nth_element, input.size(), compare_op, @@ -283,7 +266,7 @@ TYPED_TEST(RocprimDeviceNthelementTests, NthelementKey) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::use_graphs) @@ -294,7 +277,7 @@ TYPED_TEST(RocprimDeviceNthelementTests, NthelementKey) if(in_place) { // Run - HIP_CHECK(rocprim::nth_element(d_temp_storage, + HIP_CHECK(rocprim::nth_element(d_temp_storage.get(), temp_storage_size_bytes, input_it, nth_element, @@ -306,10 +289,10 @@ TYPED_TEST(RocprimDeviceNthelementTests, NthelementKey) else { // Run - HIP_CHECK(rocprim::nth_element(d_temp_storage, + HIP_CHECK(rocprim::nth_element(d_temp_storage.get(), temp_storage_size_bytes, input_it, - d_output, + d_output.get(), nth_element, input.size(), compare_op, @@ -326,23 +309,13 @@ TYPED_TEST(RocprimDeviceNthelementTests, NthelementKey) HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - HIP_CHECK(hipMemcpy(output.data(), - d_output, - output.size() * sizeof(*d_output), - hipMemcpyDeviceToHost)); + const auto output = d_output.load(); if(size > 0) { compare(input, output, nth_element, compare_op); } - HIP_CHECK(hipFree(d_input)); - if(!in_place) - { - HIP_CHECK(hipFree(d_output)); - } - HIP_CHECK(hipFree(d_temp_storage)); - if(TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -375,27 +348,18 @@ TEST(RocprimNthelementKeySameTests, NthelementKeySame) // Generate data std::vector input(size, 8); - std::vector output(size); - key_type* d_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(*d_input))); - - HIP_CHECK(hipMemcpy(d_input, - input.data(), - input.size() * sizeof(*d_input), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_input(input); // compare function compare_function compare_op; - // temp storage - size_t temp_storage_size_bytes; - void* d_temp_storage = nullptr; // Get size of d_temp_storage - HIP_CHECK(rocprim::nth_element(d_temp_storage, + size_t temp_storage_size_bytes; + HIP_CHECK(rocprim::nth_element(nullptr, temp_storage_size_bytes, - d_input, - d_input, + d_input.get(), + d_input.get(), nth_element, input.size(), compare_op, @@ -406,13 +370,13 @@ TEST(RocprimNthelementKeySameTests, NthelementKeySame) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); // Run - HIP_CHECK(rocprim::nth_element(d_temp_storage, + HIP_CHECK(rocprim::nth_element(d_temp_storage.get(), temp_storage_size_bytes, - d_input, - d_input, + d_input.get(), + d_input.get(), nth_element, input.size(), compare_op, @@ -422,15 +386,9 @@ TEST(RocprimNthelementKeySameTests, NthelementKeySame) HIP_CHECK(hipGetLastError()); // Copy output to host - HIP_CHECK(hipMemcpy(output.data(), - d_input, - output.size() * sizeof(*d_input), - hipMemcpyDeviceToHost)); + const auto output = d_input.load(); // Check if the values are the same ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(input, output)); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_temp_storage)); } } diff --git a/test/rocprim/test_device_partial_sort.cpp b/test/rocprim/test_device_partial_sort.cpp index 2c9a3162f..0a7a4295d 100644 --- a/test/rocprim/test_device_partial_sort.cpp +++ b/test/rocprim/test_device_partial_sort.cpp @@ -27,6 +27,7 @@ #include "test_utils_custom_float_type.hpp" #include "test_utils_custom_test_types.hpp" #include "test_utils_data_generation.hpp" +#include "test_utils_device_ptr.hpp" #include "test_utils_sort_comparator.hpp" #include "test_utils_types.hpp" @@ -228,24 +229,18 @@ TYPED_TEST(RocprimDevicePartialSortTests, PartialSort) test_utils::generate_limits::max(), seed_value); - key_type* d_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, size * sizeof(key_type))); - HIP_CHECK(hipMemcpy(d_input, - input.data(), - size * sizeof(key_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_input(input); - key_type* d_output; - d_output = d_input; + test_utils::device_ptr& d_output = d_input; const auto input_it - = test_utils::wrap_in_indirect_iterator(d_input); + = test_utils::wrap_in_indirect_iterator(d_input.get()); compare_function compare_op; decomposer decomposer_op; // Allocate temporary storage - size_t temp_storage_size_bytes{}; + size_t temp_storage_size_bytes; HIP_CHECK(rocprim::partial_sort(nullptr, temp_storage_size_bytes, input_it, @@ -257,16 +252,15 @@ TYPED_TEST(RocprimDevicePartialSortTests, PartialSort) decomposer_op)); ASSERT_GT(temp_storage_size_bytes, 0); - void* d_temp_storage{}; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::use_graphs) { gHelper.startStreamCapture(stream); } - HIP_CHECK(rocprim::partial_sort(d_temp_storage, + HIP_CHECK(rocprim::partial_sort(d_temp_storage.get(), temp_storage_size_bytes, input_it, middle, @@ -282,17 +276,10 @@ TYPED_TEST(RocprimDevicePartialSortTests, PartialSort) gHelper.createAndLaunchGraph(stream); } - std::vector output(size); - HIP_CHECK(hipMemcpy(output.data(), - d_output, - size * sizeof(key_type), - hipMemcpyDeviceToHost)); + const auto output = d_output.load(); compare_partial_sort(input, output, middle, compare_op); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_temp_storage)); - if(TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -434,23 +421,11 @@ TYPED_TEST(RocprimDevicePartialSortTests, PartialSortCopy) test_utils::generate_limits::max(), seed_value + 1); - key_type* d_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, size * sizeof(key_type))); - HIP_CHECK(hipMemcpy(d_input, - input.data(), - size * sizeof(key_type), - hipMemcpyHostToDevice)); - - key_type* d_output; - - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, size * sizeof(key_type))); - HIP_CHECK(hipMemcpy(d_output, - output_original.data(), - size * sizeof(key_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(output_original); const auto input_it = test_utils::wrap_in_indirect_iterator( - test_utils::wrap_in_const(d_input)); + test_utils::wrap_in_const(d_input.get())); compare_function compare_op; decomposer decomposer_op; @@ -461,7 +436,7 @@ TYPED_TEST(RocprimDevicePartialSortTests, PartialSortCopy) HIP_CHECK(rocprim::partial_sort_copy(nullptr, temp_storage_size_bytes, input_it, - d_output, + d_output.get(), middle, size, compare_op, @@ -470,9 +445,8 @@ TYPED_TEST(RocprimDevicePartialSortTests, PartialSortCopy) decomposer_op)); ASSERT_GT(temp_storage_size_bytes, 0); - void* d_temp_storage{}; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::use_graphs) @@ -480,10 +454,10 @@ TYPED_TEST(RocprimDevicePartialSortTests, PartialSortCopy) gHelper.startStreamCapture(stream); } - HIP_CHECK(rocprim::partial_sort_copy(d_temp_storage, + HIP_CHECK(rocprim::partial_sort_copy(d_temp_storage.get(), temp_storage_size_bytes, input_it, - d_output, + d_output.get(), middle, size, compare_op, @@ -497,18 +471,10 @@ TYPED_TEST(RocprimDevicePartialSortTests, PartialSortCopy) gHelper.createAndLaunchGraph(stream); } - std::vector output(size); - HIP_CHECK(hipMemcpy(output.data(), - d_output, - size * sizeof(key_type), - hipMemcpyDeviceToHost)); + const auto output = d_output.load(); compare_partial_sort_copy(input, output, output_original, middle, compare_op); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if(TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); diff --git a/test/rocprim/test_device_partition.cpp b/test/rocprim/test_device_partition.cpp index 951cb769d..8268acce7 100644 --- a/test/rocprim/test_device_partition.cpp +++ b/test/rocprim/test_device_partition.cpp @@ -412,7 +412,7 @@ TYPED_TEST(RocprimDevicePartitionTests, Predicate) // Check if number of selected value is as expected_selected unsigned int selected_count_output = d_selected_count_output.load()[0]; ASSERT_EQ(selected_count_output, expected_selected.size()); - + // Check if output values are as expected_selected const auto output = d_output.load(); @@ -576,7 +576,7 @@ struct LessOp { } ROCPRIM_HOST_DEVICE bool operator()(const T& val) const { - return val < pivot_; + return val < pivot_; } private: T pivot_; @@ -597,7 +597,7 @@ TYPED_TEST(RocprimDevicePartitionTests, PredicateThreeWay) // Default stream does not support hipGraph stream capture, so create one HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - + const std::vector> limit_pairs{ { static_cast(30), static_cast(60) }, // all sections may contain items { static_cast(0), static_cast(60) }, // first section is empty @@ -999,7 +999,7 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartition) auto param = GetParam(); const size_t modulo = std::get<0>(param); const bool use_graphs = std::get<1>(param); - + hipStream_t stream = 0; // default if (use_graphs) { @@ -1023,26 +1023,29 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartition) const auto input_iterator = rocprim::make_counting_iterator(static_cast(0)); const modulo_predicate predicate{modulo}; - unsigned int* d_incorrect_flag{}; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_incorrect_flag, sizeof(*d_incorrect_flag))); - HIP_CHECK(hipMemsetAsync(d_incorrect_flag, 0, sizeof(*d_incorrect_flag), stream)); + test_utils::device_ptr d_incorrect_flag(1); + + HIP_CHECK(hipMemsetAsync(d_incorrect_flag.get(), 0, sizeof(unsigned int), stream)); + const auto output_checker_it - = check_modulo_iterator(modulo, size, d_incorrect_flag); + = check_modulo_iterator(modulo, size, d_incorrect_flag.get()); - size_t* d_count_output{}; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_count_output, sizeof(*d_count_output))); + test_utils::device_ptr d_count_output(1); + + size_t temporary_storage_size; - size_t temporary_storage_size{}; HIP_CHECK(rocprim::partition(nullptr, temporary_storage_size, input_iterator, output_checker_it, - d_count_output, + d_count_output.get(), size, predicate, stream, debug_synchronous)); + ASSERT_NE(0, temporary_storage_size); + test_utils::device_ptr d_temporary_storage(temporary_storage_size); test_utils::GraphHelper gHelper;; @@ -1055,7 +1058,7 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartition) temporary_storage_size, input_iterator, output_checker_it, - d_count_output, + d_count_output.get(), size, predicate, stream, @@ -1068,7 +1071,7 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartition) size_t count_output{}; HIP_CHECK(hipMemcpyWithStream(&count_output, - d_count_output, + d_count_output.get(), sizeof(count_output), hipMemcpyDeviceToHost, stream)); @@ -1078,16 +1081,13 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartition) unsigned int incorrect_flag{}; HIP_CHECK(hipMemcpyWithStream(&incorrect_flag, - d_incorrect_flag, + d_incorrect_flag.get(), sizeof(incorrect_flag), hipMemcpyDeviceToHost, stream)); ASSERT_EQ(incorrect_flag, 0); - HIP_CHECK(hipFree(d_count_output)); - HIP_CHECK(hipFree(d_incorrect_flag)); - if(use_graphs) { gHelper.cleanupGraphHelper(); @@ -1130,37 +1130,35 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartitionTwoWay) const auto input_iterator = rocprim::make_counting_iterator(static_cast(0)); const modulo_predicate predicate{modulo}; - unsigned int* d_incorrect_select_flag{}; - unsigned int* d_incorrect_reject_flag{}; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_incorrect_select_flag, - sizeof(*d_incorrect_select_flag))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_incorrect_reject_flag, - sizeof(*d_incorrect_reject_flag))); - HIP_CHECK( - hipMemsetAsync(d_incorrect_select_flag, 0, sizeof(*d_incorrect_select_flag), stream)); - HIP_CHECK( - hipMemsetAsync(d_incorrect_reject_flag, 0, sizeof(*d_incorrect_reject_flag), stream)); + test_utils::device_ptr d_incorrect_select_flag(1); + test_utils::device_ptr d_incorrect_reject_flag(1); + + HIP_CHECK(hipMemsetAsync(d_incorrect_select_flag.get(), 0, sizeof(unsigned int), stream)); + HIP_CHECK(hipMemsetAsync(d_incorrect_reject_flag.get(), 0, sizeof(unsigned int), stream)); + const auto select_checker_it - = check_modulo_iterator(modulo, size, d_incorrect_select_flag); + = check_modulo_iterator(modulo, size, d_incorrect_select_flag.get()); const auto reject_checker_it - = check_modulo_iterator(modulo, size, d_incorrect_reject_flag); + = check_modulo_iterator(modulo, + size, + d_incorrect_reject_flag.get()); - size_t* d_count_output{}; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_count_output, sizeof(*d_count_output))); + test_utils::device_ptr d_count_output(1); - size_t temporary_storage_size{}; + size_t temporary_storage_size; HIP_CHECK(rocprim::partition_two_way(nullptr, temporary_storage_size, input_iterator, select_checker_it, reject_checker_it, - d_count_output, + d_count_output.get(), size, predicate, stream, debug_synchronous)); ASSERT_NE(0, temporary_storage_size); + test_utils::device_ptr d_temporary_storage(temporary_storage_size); test_utils::GraphHelper gHelper;; @@ -1174,7 +1172,7 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartitionTwoWay) input_iterator, select_checker_it, reject_checker_it, - d_count_output, + d_count_output.get(), size, predicate, stream, @@ -1187,7 +1185,7 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartitionTwoWay) size_t count_output{}; HIP_CHECK(hipMemcpyWithStream(&count_output, - d_count_output, + d_count_output.get(), sizeof(count_output), hipMemcpyDeviceToHost, stream)); @@ -1197,13 +1195,13 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartitionTwoWay) unsigned int incorrect_select_flag{}; HIP_CHECK(hipMemcpyWithStream(&incorrect_select_flag, - d_incorrect_select_flag, + d_incorrect_select_flag.get(), sizeof(incorrect_select_flag), hipMemcpyDeviceToHost, stream)); unsigned int incorrect_reject_flag{}; HIP_CHECK(hipMemcpyWithStream(&incorrect_reject_flag, - d_incorrect_reject_flag, + d_incorrect_reject_flag.get(), sizeof(incorrect_reject_flag), hipMemcpyDeviceToHost, stream)); @@ -1211,10 +1209,6 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartitionTwoWay) ASSERT_EQ(incorrect_select_flag, 0); ASSERT_EQ(incorrect_reject_flag, 0); - HIP_CHECK(hipFree(d_count_output)); - HIP_CHECK(hipFree(d_incorrect_select_flag)); - HIP_CHECK(hipFree(d_incorrect_reject_flag)); - if(use_graphs) { gHelper.cleanupGraphHelper(); @@ -1232,7 +1226,7 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartitionThreeWay) static constexpr bool debug_synchronous = false; auto param = GetParam(); const bool use_graphs = std::get<1>(param); - + hipStream_t stream = 0; // default if (use_graphs) { @@ -1260,32 +1254,35 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartitionThreeWay) const auto predicate_a = modulo_predicate{modulo_a}; const auto predicate_b = modulo_predicate{modulo_b}; - unsigned int* d_incorrect_flag{}; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_incorrect_flag, sizeof(*d_incorrect_flag))); - HIP_CHECK(hipMemsetAsync(d_incorrect_flag, 0, sizeof(*d_incorrect_flag), stream)); + test_utils::device_ptr d_incorrect_flag(1); + + HIP_CHECK(hipMemsetAsync(d_incorrect_flag.get(), 0, sizeof(unsigned int), stream)); + const auto output_checker_a - = check_modulo_iterator(modulo_a, size, d_incorrect_flag); + = check_modulo_iterator(modulo_a, size, d_incorrect_flag.get()); const auto output_checker_b - = check_modulo_iterator(modulo_b, size, d_incorrect_flag); + = check_modulo_iterator(modulo_b, size, d_incorrect_flag.get()); + const auto unselected_output = rocprim::make_discard_iterator(); - size_t* d_count_output{}; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_count_output, 2 * sizeof(*d_count_output))); + test_utils::device_ptr d_count_output(2); - size_t temporary_storage_size{}; + size_t temporary_storage_size; HIP_CHECK(rocprim::partition_three_way(nullptr, temporary_storage_size, input_iterator, output_checker_a, output_checker_b, unselected_output, - d_count_output, + d_count_output.get(), size, predicate_a, predicate_b, stream, debug_synchronous)); + ASSERT_NE(0, temporary_storage_size); + test_utils::device_ptr d_temporary_storage(temporary_storage_size); test_utils::GraphHelper gHelper; @@ -1300,7 +1297,7 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartitionThreeWay) output_checker_a, output_checker_b, unselected_output, - d_count_output, + d_count_output.get(), size, predicate_a, predicate_b, @@ -1314,7 +1311,7 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartitionThreeWay) size_t count_output[2]{}; HIP_CHECK(hipMemcpyWithStream(&count_output, - d_count_output, + d_count_output.get(), sizeof(count_output), hipMemcpyDeviceToHost, stream)); @@ -1328,16 +1325,13 @@ TEST_P(RocprimDevicePartitionLargeInputTests, LargeInputPartitionThreeWay) unsigned int incorrect_flag{}; HIP_CHECK(hipMemcpyWithStream(&incorrect_flag, - d_incorrect_flag, + d_incorrect_flag.get(), sizeof(incorrect_flag), hipMemcpyDeviceToHost, stream)); ASSERT_EQ(incorrect_flag, 0); - HIP_CHECK(hipFree(d_count_output)); - HIP_CHECK(hipFree(d_incorrect_flag)); - if(use_graphs) { gHelper.cleanupGraphHelper(); @@ -1375,12 +1369,12 @@ TEST(RocprimDevicePartitionBlockSizeTests, BlockSize) HIP_CHECK(hipSetDevice(device_id)); // Create a large struct to test with. It must be big enough that when - // we the use default block size (defined in rocprim::default_select_config + // we the use default block size (defined in rocprim::default_select_config // struct as 256), giving one instance to each thread will cause us to hit // 32 KiB of shared memory (the limit enforced by the rocprim::limit_block_size // struct's boolean template parameter). Since the device_partition algorithm also // uses some shared memory to store state, this will cause the total usage to exceed - // the 32 KiB limit. If everything's working correctly, this should be detected in + // the 32 KiB limit. If everything's working correctly, this should be detected in // the limit_block_size's template logic, and it should reduce the block size. const size_t test_obj_size = 128; // Choose 128, since 256 * 128 = 2^15 bytes (32 KiB). struct TestObject diff --git a/test/rocprim/test_device_reduce.cpp b/test/rocprim/test_device_reduce.cpp index 8514f1c5f..9b63c813c 100644 --- a/test/rocprim/test_device_reduce.cpp +++ b/test/rocprim/test_device_reduce.cpp @@ -29,6 +29,7 @@ #include // required test headers +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" using bra = ::rocprim::block_reduce_algorithm; @@ -167,25 +168,23 @@ TYPED_TEST(RocprimDeviceReduceTests, ReduceEmptyInput) HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - U * d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, sizeof(U))); + test_utils::device_ptr d_output(1); const U initial_value = U(1234); - size_t temp_storage_size_bytes; // Get size of d_temp_storage - HIP_CHECK( - rocprim::reduce( - nullptr, temp_storage_size_bytes, - rocprim::make_constant_iterator(T(345)), - d_output, - initial_value, - 0, rocprim::minimum(), stream, debug_synchronous - ) - ); - - void * d_temp_storage = nullptr; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + size_t temp_storage_size_bytes; + HIP_CHECK(rocprim::reduce(nullptr, + temp_storage_size_bytes, + rocprim::make_constant_iterator(T(345)), + d_output.get(), + initial_value, + 0, + rocprim::minimum(), + stream, + debug_synchronous)); + + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) @@ -194,17 +193,16 @@ TYPED_TEST(RocprimDeviceReduceTests, ReduceEmptyInput) } // Run - HIP_CHECK( - rocprim::reduce( - d_temp_storage, temp_storage_size_bytes, - rocprim::make_constant_iterator(T(345)), - d_output, - initial_value, - 0, rocprim::minimum(), stream, debug_synchronous - ) - ); - - + HIP_CHECK(rocprim::reduce(d_temp_storage.get(), + temp_storage_size_bytes, + rocprim::make_constant_iterator(T(345)), + d_output.get(), + initial_value, + 0, + rocprim::minimum(), + stream, + debug_synchronous)); + if(TestFixture::use_graphs) { gHelper.createAndLaunchGraph(stream); @@ -212,18 +210,9 @@ TYPED_TEST(RocprimDeviceReduceTests, ReduceEmptyInput) HIP_CHECK(hipDeviceSynchronize()); - U output; - HIP_CHECK( - hipMemcpy( - &output, d_output, - sizeof(U), - hipMemcpyDeviceToHost - ) - ); - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, initial_value)); + const auto output = d_output.load()[0]; - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_temp_storage)); + ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, initial_value)); if (TestFixture::use_graphs) { @@ -273,20 +262,9 @@ TYPED_TEST(RocprimDeviceReduceTests, ReduceSum) // Generate data std::vector input = test_utils::get_random_data(size, 0, 100, seed_value); - std::vector output(1, U(0)); - T * d_input; - U * d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(U))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(1); // Calculate expected results on host U expected = test_utils::host_reduce(input.begin(), input.end(), rocprim::plus()); @@ -294,15 +272,13 @@ TYPED_TEST(RocprimDeviceReduceTests, ReduceSum) if(size == 0) expected = U(); - // temp storage - size_t temp_storage_size_bytes; - void * d_temp_storage = nullptr; // Get size of d_temp_storage + size_t temp_storage_size_bytes; HIP_CHECK(rocprim::reduce( - d_temp_storage, + nullptr, temp_storage_size_bytes, - d_input, - test_utils::wrap_in_identity_iterator(d_output), + d_input.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), input.size(), rocprim::plus(), stream, @@ -312,8 +288,7 @@ TYPED_TEST(RocprimDeviceReduceTests, ReduceSum) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) @@ -322,16 +297,16 @@ TYPED_TEST(RocprimDeviceReduceTests, ReduceSum) } // Run - HIP_CHECK( - rocprim::reduce( - d_temp_storage, temp_storage_size_bytes, - d_input, - test_utils::wrap_in_identity_iterator(d_output), - input.size(), rocprim::plus(), stream, debug_synchronous - ) - ); - - + HIP_CHECK(rocprim::reduce( + d_temp_storage.get(), + temp_storage_size_bytes, + d_input.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), + input.size(), + rocprim::plus(), + stream, + debug_synchronous)); + if(TestFixture::use_graphs) { gHelper.createAndLaunchGraph(stream); @@ -341,23 +316,12 @@ TYPED_TEST(RocprimDeviceReduceTests, ReduceSum) HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - output.size() * sizeof(U), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + const auto output = d_output.load(); // Check if output values are as expected ASSERT_NO_FATAL_FAILURE( test_utils::assert_near(output[0], expected, test_utils::precision * size)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if (TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -422,20 +386,9 @@ TYPED_TEST(RocprimDeviceReduceTests, ReduceArgMinimum) input[i].key = (int)i; input[i].value = test_utils::get_random_data(1, 1, 100, seed_value)[0]; } - std::vector output(1); - - key_value * d_input; - key_value * d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(key_value))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(key_value))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - input.size() * sizeof(key_value), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(1); arg_min reduce_op; const key_value max(std::numeric_limits::max(), test_utils::numeric_limits::max()); @@ -447,25 +400,24 @@ TYPED_TEST(RocprimDeviceReduceTests, ReduceArgMinimum) expected = reduce_op(expected, input[i]); } - // temp storage - size_t temp_storage_size_bytes; - void * d_temp_storage = nullptr; // Get size of d_temp_storage - HIP_CHECK( - rocprim::reduce( - d_temp_storage, temp_storage_size_bytes, - d_input, - test_utils::wrap_in_identity_iterator(d_output), - max, input.size(), reduce_op, stream, debug_synchronous - ) - ); + size_t temp_storage_size_bytes; + HIP_CHECK(rocprim::reduce( + nullptr, + temp_storage_size_bytes, + d_input.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), + max, + input.size(), + reduce_op, + stream, + debug_synchronous)); // temp_storage_size_bytes must be >0 ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) @@ -474,16 +426,17 @@ TYPED_TEST(RocprimDeviceReduceTests, ReduceArgMinimum) } // Run - HIP_CHECK( - rocprim::reduce( - d_temp_storage, temp_storage_size_bytes, - d_input, - test_utils::wrap_in_identity_iterator(d_output), - max, input.size(), reduce_op, stream, debug_synchronous - ) - ); - - + HIP_CHECK(rocprim::reduce( + d_temp_storage.get(), + temp_storage_size_bytes, + d_input.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), + max, + input.size(), + reduce_op, + stream, + debug_synchronous)); + if(TestFixture::use_graphs) { gHelper.createAndLaunchGraph(stream); @@ -493,23 +446,12 @@ TYPED_TEST(RocprimDeviceReduceTests, ReduceArgMinimum) HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - output.size() * sizeof(key_value), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + const auto output = d_output.load(); // Check if output values are as expected test_utils::assert_eq(output[0].key, expected.key); test_utils::assert_eq(output[0].value, expected.value); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if (TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -549,25 +491,21 @@ void testLargeIndices() const Iterator input {0}; - T* d_output = nullptr; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, sizeof(T))); + test_utils::device_ptr d_output(1); - // temp storage - size_t temp_storage_size_bytes = 0; - void* d_temp_storage = nullptr; // Get size of d_temp_storage + size_t temp_storage_size_bytes; HIP_CHECK(rocprim::reduce(nullptr, temp_storage_size_bytes, input, - d_output, + d_output.get(), size, rocprim::plus{}, stream, debug_synchronous)); // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper; if(use_graphs) @@ -576,16 +514,15 @@ void testLargeIndices() } // Run - HIP_CHECK(rocprim::reduce(d_temp_storage, + HIP_CHECK(rocprim::reduce(d_temp_storage.get(), temp_storage_size_bytes, input, - d_output, + d_output.get(), size, - rocprim::plus {}, + rocprim::plus{}, stream, debug_synchronous)); - if(use_graphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -595,9 +532,7 @@ void testLargeIndices() HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - T output = 0; - HIP_CHECK(hipMemcpy(&output, d_output, sizeof(T), hipMemcpyDeviceToHost)); - HIP_CHECK(hipDeviceSynchronize()); + const auto output = d_output.load()[0]; // Sum of numbers 0 to n - 1 is n(n - 1) / 2, note that this is correct even in case of overflow // The division is not integer division but either n or n - 1 has to be even. @@ -605,9 +540,6 @@ void testLargeIndices() ASSERT_EQ(output, expected_output); - HIP_CHECK(hipFree(d_temp_storage)); - HIP_CHECK(hipFree(d_output)); - if(use_graphs) { gHelper.cleanupGraphHelper(); @@ -677,28 +609,20 @@ TYPED_TEST(RocprimDeviceReducePrecisionTests, ReduceSumInputEqualExponentFunctio // Generate data std::vector input(size, lowest); - std::vector output(1, (U)0); - - T* d_input; - U* d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(U))); - HIP_CHECK( - hipMemcpy(d_input, input.data(), input.size() * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); + + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(1); // Calculate expected results on host mathematically (instead of using reduce on host) U expected = static_cast(static_cast(size) * static_cast(lowest)); - // temp storage - size_t temp_storage_size_bytes; - void* d_temp_storage = nullptr; // Get size of d_temp_storage + size_t temp_storage_size_bytes; HIP_CHECK(rocprim::reduce( - d_temp_storage, + nullptr, temp_storage_size_bytes, - d_input, - test_utils::wrap_in_identity_iterator(d_output), + d_input.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), input.size(), rocprim::plus(), stream, @@ -708,8 +632,7 @@ TYPED_TEST(RocprimDeviceReducePrecisionTests, ReduceSumInputEqualExponentFunctio ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) @@ -719,16 +642,15 @@ TYPED_TEST(RocprimDeviceReducePrecisionTests, ReduceSumInputEqualExponentFunctio // Run HIP_CHECK(rocprim::reduce( - d_temp_storage, + d_temp_storage.get(), temp_storage_size_bytes, - d_input, - test_utils::wrap_in_identity_iterator(d_output), + d_input.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), input.size(), rocprim::plus(), stream, debug_synchronous)); - if(TestFixture::use_graphs) { gHelper.createAndLaunchGraph(stream); @@ -738,17 +660,11 @@ TYPED_TEST(RocprimDeviceReducePrecisionTests, ReduceSumInputEqualExponentFunctio HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - HIP_CHECK( - hipMemcpy(output.data(), d_output, output.size() * sizeof(U), hipMemcpyDeviceToHost)); - HIP_CHECK(hipDeviceSynchronize()); + const auto output = d_output.load(); // Check if output values are as expected ASSERT_NO_FATAL_FAILURE(test_utils::assert_near(output[0], expected, precision)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if (TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -789,20 +705,9 @@ TYPED_TEST(RocprimDeviceReduceTests, ReduceMinimum) // Generate data std::vector input = test_utils::get_random_data(size, 1, 100, seed_value); - std::vector output(1, U(0)); - T * d_input; - U * d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(U))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(1); // reduce function binary_op_type min_op; @@ -816,22 +721,22 @@ TYPED_TEST(RocprimDeviceReduceTests, ReduceMinimum) // Get size of d_temp_storage size_t temp_storage_size_bytes; - void * d_temp_storage = nullptr; - HIP_CHECK( - rocprim::reduce( - d_temp_storage, temp_storage_size_bytes, - d_input, - test_utils::wrap_in_identity_iterator(d_output), - test_utils::numeric_limits::max(), input.size(), rocprim::minimum(), stream, TestFixture::debug_synchronous - ) - ); + HIP_CHECK(rocprim::reduce( + nullptr, + temp_storage_size_bytes, + d_input.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), + test_utils::numeric_limits::max(), + input.size(), + rocprim::minimum(), + stream, + TestFixture::debug_synchronous)); // temp_storage_size_bytes must be >0 ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) @@ -840,25 +745,25 @@ TYPED_TEST(RocprimDeviceReduceTests, ReduceMinimum) } // Run - HIP_CHECK( - rocprim::reduce( - d_temp_storage, temp_storage_size_bytes, - d_input, - test_utils::wrap_in_identity_iterator(d_output), - test_utils::numeric_limits::max(), input.size(), rocprim::minimum(), stream, TestFixture::debug_synchronous - ) - ); + HIP_CHECK(rocprim::reduce( + d_temp_storage.get(), + temp_storage_size_bytes, + d_input.get(), + test_utils::wrap_in_identity_iterator(d_output.get()), + test_utils::numeric_limits::max(), + input.size(), + rocprim::minimum(), + stream, + TestFixture::debug_synchronous)); // Copy output to host - HIP_CHECK( - hipMemcpyAsync( - output.data(), d_output, - output.size() * sizeof(U), - hipMemcpyDeviceToHost, stream - ) - ); - - + std::vector output(1, U(0)); + HIP_CHECK(hipMemcpyAsync(output.data(), + d_output.get(), + output.size() * sizeof(U), + hipMemcpyDeviceToHost, + stream)); + if(TestFixture::use_graphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -874,10 +779,6 @@ TYPED_TEST(RocprimDeviceReduceTests, ReduceMinimum) ? 0 : std::max(test_utils::precision, test_utils::precision))); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if (TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); diff --git a/test/rocprim/test_device_reduce_by_key.cpp b/test/rocprim/test_device_reduce_by_key.cpp index ae9dc7a56..278fe186c 100644 --- a/test/rocprim/test_device_reduce_by_key.cpp +++ b/test/rocprim/test_device_reduce_by_key.cpp @@ -32,6 +32,7 @@ // required test headers #include "rocprim/types.hpp" #include "test_utils_custom_test_types.hpp" +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" template d_keys_input(keys_input); + test_utils::device_ptr d_values_input(values_input); - key_type * d_unique_output; - aggregate_type * d_aggregates_output; - unsigned int * d_unique_count_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_unique_output, unique_count_expected * sizeof(key_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_aggregates_output, unique_count_expected * sizeof(aggregate_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_unique_count_output, sizeof(unsigned int))); + test_utils::device_ptr d_unique_output(unique_count_expected); + test_utils::device_ptr d_aggregates_output(unique_count_expected); + test_utils::device_ptr d_unique_count_output(1); size_t temporary_storage_bytes; - HIP_CHECK((invoke_reduce_by_key( nullptr, temporary_storage_bytes, - d_keys_input, - d_values_input, + d_keys_input.get(), + d_values_input.get(), size, - test_utils::wrap_in_identity_iterator(d_unique_output), - test_utils::wrap_in_identity_iterator(d_aggregates_output), - d_unique_count_output, + test_utils::wrap_in_identity_iterator(d_unique_output.get()), + test_utils::wrap_in_identity_iterator( + d_aggregates_output.get()), + d_unique_count_output.get(), reduce_op, key_compare_op, stream, @@ -293,8 +275,7 @@ TYPED_TEST(RocprimDeviceReduceByKey, ReduceByKey) ASSERT_GT(temporary_storage_bytes, 0); - void * d_temporary_storage; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); test_utils::GraphHelper gHelper; if(TestFixture::params::use_graphs) @@ -302,57 +283,27 @@ TYPED_TEST(RocprimDeviceReduceByKey, ReduceByKey) gHelper.startStreamCapture(stream); } - HIP_CHECK((invoke_reduce_by_key(d_temporary_storage, + HIP_CHECK((invoke_reduce_by_key(d_temporary_storage.get(), temporary_storage_bytes, - d_keys_input, - d_values_input, + d_keys_input.get(), + d_values_input.get(), size, - d_unique_output, - d_aggregates_output, - d_unique_count_output, + d_unique_output.get(), + d_aggregates_output.get(), + d_unique_count_output.get(), reduce_op, key_compare_op, stream, debug_synchronous))); - if(TestFixture::params::use_graphs) { gHelper.createAndLaunchGraph(stream); } - HIP_CHECK(hipFree(d_temporary_storage)); - - std::vector unique_output(unique_count_expected); - std::vector aggregates_output(unique_count_expected); - std::vector unique_count_output(1); - HIP_CHECK( - hipMemcpy( - unique_output.data(), d_unique_output, - unique_count_expected * sizeof(key_type), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK( - hipMemcpy( - aggregates_output.data(), d_aggregates_output, - unique_count_expected * sizeof(aggregate_type), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK( - hipMemcpy( - unique_count_output.data(), d_unique_count_output, - sizeof(unsigned int), - hipMemcpyDeviceToHost - ) - ); - - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_values_input)); - HIP_CHECK(hipFree(d_unique_output)); - HIP_CHECK(hipFree(d_aggregates_output)); - HIP_CHECK(hipFree(d_unique_count_output)); + const auto unique_output = d_unique_output.load(); + const auto aggregates_output = d_aggregates_output.load(); + const auto unique_count_output = d_unique_count_output.load(); if (TestFixture::params::use_graphs) { @@ -399,7 +350,7 @@ void large_indices_reduce_by_key() // out: 0 1 1 2 2 2 2 3 3 3 3 3 3 3 3 4 auto d_keys_input = rocprim::make_transform_iterator( rocprim::make_counting_iterator(key_type(1)), - [] ROCPRIM_DEVICE(key_type i) + [](key_type i) { // for i > 0, returns the position of the most significant set bit, // which is equal to the floor of log2 @@ -412,26 +363,19 @@ void large_indices_reduce_by_key() // the count is value of the last key plus one as the value of the first key is zero unsigned int unique_count_expected = log2(size) + 1; - key_type* d_unique_output; - aggregate_type* d_aggregates_output; - unsigned int* d_unique_count_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_unique_output, - unique_count_expected * sizeof(key_type))); - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_aggregates_output, - unique_count_expected * sizeof(aggregate_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_unique_count_output, sizeof(unsigned int))); + test_utils::device_ptr d_unique_output(unique_count_expected); + test_utils::device_ptr d_aggregates_output(unique_count_expected); + test_utils::device_ptr d_unique_count_output(1); size_t temporary_storage_bytes; - HIP_CHECK(invoke_reduce_by_key(nullptr, temporary_storage_bytes, d_keys_input, d_values_input, size, - d_unique_output, - d_aggregates_output, - d_unique_count_output, + d_unique_output.get(), + d_aggregates_output.get(), + d_unique_count_output.get(), reduce_op, key_compare_op, stream, @@ -439,9 +383,7 @@ void large_indices_reduce_by_key() ASSERT_GT(temporary_storage_bytes, 0); - void* d_temporary_storage; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); test_utils::GraphHelper gHelper; if(use_graphs) @@ -449,46 +391,27 @@ void large_indices_reduce_by_key() gHelper.startStreamCapture(stream); } - HIP_CHECK(invoke_reduce_by_key(d_temporary_storage, + HIP_CHECK(invoke_reduce_by_key(d_temporary_storage.get(), temporary_storage_bytes, d_keys_input, d_values_input, size, - d_unique_output, - d_aggregates_output, - d_unique_count_output, + d_unique_output.get(), + d_aggregates_output.get(), + d_unique_count_output.get(), reduce_op, key_compare_op, stream, debug_synchronous)); - if(use_graphs) { gHelper.createAndLaunchGraph(stream); } - HIP_CHECK(hipFree(d_temporary_storage)); - - std::vector unique_output(unique_count_expected); - std::vector aggregates_output(unique_count_expected); - std::vector unique_count_output(1); - HIP_CHECK(hipMemcpy(unique_output.data(), - d_unique_output, - unique_count_expected * sizeof(key_type), - hipMemcpyDeviceToHost)); - HIP_CHECK(hipMemcpy(aggregates_output.data(), - d_aggregates_output, - unique_count_expected * sizeof(aggregate_type), - hipMemcpyDeviceToHost)); - HIP_CHECK(hipMemcpy(unique_count_output.data(), - d_unique_count_output, - sizeof(unsigned int), - hipMemcpyDeviceToHost)); - - HIP_CHECK(hipFree(d_unique_output)); - HIP_CHECK(hipFree(d_aggregates_output)); - HIP_CHECK(hipFree(d_unique_count_output)); + const auto unique_output = d_unique_output.load(); + const auto aggregates_output = d_aggregates_output.load(); + const auto unique_count_output = d_unique_count_output.load(); if(use_graphs) { @@ -572,9 +495,7 @@ void large_segment_count_reduce_by_key() auto d_unique_output = rocprim::make_discard_iterator(); auto d_aggregates_output = rocprim::make_discard_iterator(); - size_t* d_unique_count_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_unique_count_output, - sizeof(*d_unique_count_output))); + test_utils::device_ptr d_unique_count_output(1); size_t temporary_storage_bytes; HIP_CHECK(invoke_reduce_by_key(nullptr, @@ -584,7 +505,7 @@ void large_segment_count_reduce_by_key() size, d_unique_output, d_aggregates_output, - d_unique_count_output, + d_unique_count_output.get(), reduce_op, key_compare_op, stream, @@ -592,9 +513,7 @@ void large_segment_count_reduce_by_key() ASSERT_GT(temporary_storage_bytes, 0); - void* d_temporary_storage; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); test_utils::GraphHelper gHelper; if(use_graphs) @@ -602,34 +521,25 @@ void large_segment_count_reduce_by_key() gHelper.startStreamCapture(stream); } - HIP_CHECK(invoke_reduce_by_key(d_temporary_storage, + HIP_CHECK(invoke_reduce_by_key(d_temporary_storage.get(), temporary_storage_bytes, d_keys_input, d_values_input, size, d_unique_output, d_aggregates_output, - d_unique_count_output, + d_unique_count_output.get(), reduce_op, key_compare_op, stream, debug_synchronous)); - if(use_graphs) { gHelper.createAndLaunchGraph(stream); } - HIP_CHECK(hipFree(d_temporary_storage)); - - size_t unique_count_output; - HIP_CHECK(hipMemcpy(&unique_count_output, - d_unique_count_output, - sizeof(unique_count_output), - hipMemcpyDeviceToHost)); - - HIP_CHECK(hipFree(d_unique_count_output)); + const auto unique_count_output = d_unique_count_output.load()[0]; ASSERT_EQ(unique_count_output, unique_count_expected); @@ -702,11 +612,9 @@ TEST(RocprimDeviceReduceByKey, ReduceByNonEqualKeys) auto d_unique_output = rocprim::make_discard_iterator(); auto d_aggregates_output = rocprim::make_discard_iterator(); - size_t* d_unique_count_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_unique_count_output, sizeof(size_t))); + test_utils::device_ptr d_unique_count_output(1); size_t temporary_storage_bytes; - HIP_CHECK(invoke_reduce_by_key(nullptr, temporary_storage_bytes, d_keys_input, @@ -714,7 +622,7 @@ TEST(RocprimDeviceReduceByKey, ReduceByNonEqualKeys) size, d_unique_output, d_aggregates_output, - d_unique_count_output, + d_unique_count_output.get(), reduce_op, key_compare_op, stream, @@ -722,32 +630,22 @@ TEST(RocprimDeviceReduceByKey, ReduceByNonEqualKeys) ASSERT_GT(temporary_storage_bytes, 0); - void* d_temporary_storage; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); - HIP_CHECK(invoke_reduce_by_key(d_temporary_storage, + HIP_CHECK(invoke_reduce_by_key(d_temporary_storage.get(), temporary_storage_bytes, d_keys_input, d_values_input, size, d_unique_output, d_aggregates_output, - d_unique_count_output, + d_unique_count_output.get(), reduce_op, key_compare_op, stream, debug_synchronous)); - HIP_CHECK(hipFree(d_temporary_storage)); - - size_t unique_count_output; - HIP_CHECK(hipMemcpy(&unique_count_output, - d_unique_count_output, - sizeof(unique_count_output), - hipMemcpyDeviceToHost)); - - HIP_CHECK(hipFree(d_unique_count_output)); + const auto unique_count_output = d_unique_count_output.load()[0]; ASSERT_EQ(unique_count_output, unique_count_expected); } diff --git a/test/rocprim/test_device_run_length_encode.cpp b/test/rocprim/test_device_run_length_encode.cpp index a2d07e268..22d758fef 100644 --- a/test/rocprim/test_device_run_length_encode.cpp +++ b/test/rocprim/test_device_run_length_encode.cpp @@ -30,6 +30,7 @@ #include "rocprim/block/block_scan.hpp" #include "rocprim/device/detail/lookback_scan_state.hpp" #include "rocprim/types.hpp" +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" template d_input(input); - key_type * d_unique_output; - count_type * d_counts_output; - count_type * d_runs_count_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_unique_output, runs_count_expected * sizeof(key_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_counts_output, runs_count_expected * sizeof(count_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_runs_count_output, sizeof(count_type))); + test_utils::device_ptr d_unique_output(runs_count_expected); + test_utils::device_ptr d_counts_output(runs_count_expected); + test_utils::device_ptr d_runs_count_output(1); size_t temporary_storage_bytes = 0; HIP_CHECK(rocprim::run_length_encode( nullptr, temporary_storage_bytes, - d_input, + d_input.get(), size, - test_utils::wrap_in_identity_iterator(d_unique_output), - test_utils::wrap_in_identity_iterator(d_counts_output), - test_utils::wrap_in_identity_iterator(d_runs_count_output), + test_utils::wrap_in_identity_iterator(d_unique_output.get()), + test_utils::wrap_in_identity_iterator(d_counts_output.get()), + test_utils::wrap_in_identity_iterator( + d_runs_count_output.get()), stream, debug_synchronous)); ASSERT_GT(temporary_storage_bytes, 0U); - void * d_temporary_storage; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); test_utils::GraphHelper gHelper; gHelper.startStreamCapture(stream); HIP_CHECK(rocprim::run_length_encode( - d_temporary_storage, + d_temporary_storage.get(), temporary_storage_bytes, - d_input, + d_input.get(), size, - test_utils::wrap_in_identity_iterator(d_unique_output), - test_utils::wrap_in_identity_iterator(d_counts_output), - test_utils::wrap_in_identity_iterator(d_runs_count_output), + test_utils::wrap_in_identity_iterator(d_unique_output.get()), + test_utils::wrap_in_identity_iterator(d_counts_output.get()), + test_utils::wrap_in_identity_iterator( + d_runs_count_output.get()), stream, debug_synchronous)); @@ -234,38 +224,9 @@ TYPED_TEST(RocprimDeviceRunLengthEncode, Encode) HIP_CHECK(hipDeviceSynchronize()); - HIP_CHECK(hipFree(d_temporary_storage)); - - std::vector unique_output(runs_count_expected); - std::vector counts_output(runs_count_expected); - std::vector runs_count_output(1); - - HIP_CHECK( - hipMemcpy( - unique_output.data(), d_unique_output, - runs_count_expected * sizeof(key_type), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK( - hipMemcpy( - counts_output.data(), d_counts_output, - runs_count_expected * sizeof(count_type), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK( - hipMemcpy( - runs_count_output.data(), d_runs_count_output, - sizeof(count_type), - hipMemcpyDeviceToHost - ) - ); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_unique_output)); - HIP_CHECK(hipFree(d_counts_output)); - HIP_CHECK(hipFree(d_runs_count_output)); + const auto unique_output = d_unique_output.load(); + const auto counts_output = d_counts_output.load(); + const auto runs_count_output = d_runs_count_output.load(); gHelper.cleanupGraphHelper(); HIP_CHECK(hipStreamDestroy(stream)); @@ -362,53 +323,45 @@ TYPED_TEST(RocprimDeviceRunLengthEncode, NonTrivialRuns) offset += key_count; } - key_type * d_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, size * sizeof(key_type))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - size * sizeof(key_type), - hipMemcpyHostToDevice - ) - ); - - offset_type * d_offsets_output; - count_type * d_counts_output; - count_type * d_runs_count_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_offsets_output, std::max(1, runs_count_expected) * sizeof(offset_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_counts_output, std::max(1, runs_count_expected) * sizeof(count_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_runs_count_output, sizeof(count_type))); + test_utils::device_ptr d_input(input); - size_t temporary_storage_bytes = 0; + test_utils::device_ptr d_offsets_output( + std::max(1, runs_count_expected)); + test_utils::device_ptr d_counts_output( + std::max(1, runs_count_expected)); + test_utils::device_ptr d_runs_count_output(1); + size_t temporary_storage_bytes; HIP_CHECK(rocprim::run_length_encode_non_trivial_runs( nullptr, temporary_storage_bytes, - d_input, + d_input.get(), size, - test_utils::wrap_in_identity_iterator(d_offsets_output), - test_utils::wrap_in_identity_iterator(d_counts_output), - test_utils::wrap_in_identity_iterator(d_runs_count_output), + test_utils::wrap_in_identity_iterator( + d_offsets_output.get()), + test_utils::wrap_in_identity_iterator(d_counts_output.get()), + test_utils::wrap_in_identity_iterator( + d_runs_count_output.get()), stream, debug_synchronous)); ASSERT_GT(temporary_storage_bytes, 0U); - void * d_temporary_storage; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); test_utils::GraphHelper gHelper; gHelper.startStreamCapture(stream); HIP_CHECK(rocprim::run_length_encode_non_trivial_runs( - d_temporary_storage, + d_temporary_storage.get(), temporary_storage_bytes, - d_input, + d_input.get(), size, - test_utils::wrap_in_identity_iterator(d_offsets_output), - test_utils::wrap_in_identity_iterator(d_counts_output), - test_utils::wrap_in_identity_iterator(d_runs_count_output), + test_utils::wrap_in_identity_iterator( + d_offsets_output.get()), + test_utils::wrap_in_identity_iterator(d_counts_output.get()), + test_utils::wrap_in_identity_iterator( + d_runs_count_output.get()), stream, debug_synchronous)); @@ -416,41 +369,15 @@ TYPED_TEST(RocprimDeviceRunLengthEncode, NonTrivialRuns) HIP_CHECK(hipDeviceSynchronize()); - HIP_CHECK(hipFree(d_temporary_storage)); - - std::vector offsets_output(runs_count_expected); - std::vector counts_output(runs_count_expected); - std::vector runs_count_output(1); + std::vector offsets_output; + std::vector counts_output; + const auto runs_count_output = d_runs_count_output.load(); if(runs_count_expected > 0) { - HIP_CHECK( - hipMemcpy( - offsets_output.data(), d_offsets_output, - runs_count_expected * sizeof(offset_type), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK( - hipMemcpy( - counts_output.data(), d_counts_output, - runs_count_expected * sizeof(count_type), - hipMemcpyDeviceToHost - ) - ); + offsets_output = d_offsets_output.load(); + counts_output = d_counts_output.load(); } - HIP_CHECK( - hipMemcpy( - runs_count_output.data(), d_runs_count_output, - sizeof(count_type), - hipMemcpyDeviceToHost - ) - ); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_offsets_output)); - HIP_CHECK(hipFree(d_counts_output)); - HIP_CHECK(hipFree(d_runs_count_output)); gHelper.cleanupGraphHelper(); HIP_CHECK(hipStreamDestroy(stream)); diff --git a/test/rocprim/test_device_scan.cpp b/test/rocprim/test_device_scan.cpp index 03831c46f..56deecc84 100644 --- a/test/rocprim/test_device_scan.cpp +++ b/test/rocprim/test_device_scan.cpp @@ -31,6 +31,7 @@ #include // required test headers +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" #include @@ -257,15 +258,12 @@ TYPED_TEST(RocprimDeviceScanTests, InclusiveScanEmptyInput) HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - U * d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, sizeof(U))); + test_utils::device_ptr d_output(1); test_utils::out_of_bounds_flag out_of_bounds; - test_utils::bounds_checking_iterator d_checking_output( - d_output, - out_of_bounds.device_pointer(), - 0 - ); + test_utils::bounds_checking_iterator d_checking_output(d_output.get(), + out_of_bounds.device_pointer(), + 0); // scan function scan_op_type scan_op; @@ -274,11 +272,9 @@ TYPED_TEST(RocprimDeviceScanTests, InclusiveScanEmptyInput) = rocprim::make_transform_iterator(rocprim::make_constant_iterator(T(345)), [](T in) { return static_cast(in); }); - // temp storage - size_t temp_storage_size_bytes; - void * d_temp_storage = nullptr; // Get size of d_temp_storage - HIP_CHECK(invoke_inclusive_scan(d_temp_storage, + size_t temp_storage_size_bytes; + HIP_CHECK(invoke_inclusive_scan(nullptr, temp_storage_size_bytes, input_iterator, d_checking_output, @@ -288,7 +284,7 @@ TYPED_TEST(RocprimDeviceScanTests, InclusiveScanEmptyInput) debug_synchronous)); // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::use_graphs) @@ -297,7 +293,7 @@ TYPED_TEST(RocprimDeviceScanTests, InclusiveScanEmptyInput) } // Run - HIP_CHECK(invoke_inclusive_scan(d_temp_storage, + HIP_CHECK(invoke_inclusive_scan(d_temp_storage.get(), temp_storage_size_bytes, input_iterator, d_checking_output, @@ -306,7 +302,6 @@ TYPED_TEST(RocprimDeviceScanTests, InclusiveScanEmptyInput) stream, debug_synchronous)); - if(TestFixture::use_graphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -317,9 +312,6 @@ TYPED_TEST(RocprimDeviceScanTests, InclusiveScanEmptyInput) ASSERT_FALSE(out_of_bounds.get()); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if (TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -381,20 +373,9 @@ TYPED_TEST(RocprimDeviceScanTests, InclusiveScan) // Generate data std::vector input = test_utils::get_random_data(size, 1, 10, seed_value); - std::vector output(input.size(), U{0}); - - T * d_input; - U * d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(U))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(input.size()); // scan function scan_op_type scan_op; @@ -407,18 +388,16 @@ TYPED_TEST(RocprimDeviceScanTests, InclusiveScan) ); auto input_iterator - = rocprim::make_transform_iterator(d_input, + = rocprim::make_transform_iterator(d_input.get(), [](T in) { return static_cast(in); }); - // temp storage - size_t temp_storage_size_bytes; - void * d_temp_storage = nullptr; // Get size of d_temp_storage + size_t temp_storage_size_bytes; HIP_CHECK((invoke_inclusive_scan( - d_temp_storage, + nullptr, temp_storage_size_bytes, input_iterator, - test_utils::wrap_in_identity_iterator(d_output), + test_utils::wrap_in_identity_iterator(d_output.get()), input.size(), scan_op, stream, @@ -428,8 +407,7 @@ TYPED_TEST(RocprimDeviceScanTests, InclusiveScan) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::use_graphs) @@ -439,16 +417,15 @@ TYPED_TEST(RocprimDeviceScanTests, InclusiveScan) // Run HIP_CHECK((invoke_inclusive_scan( - d_temp_storage, + d_temp_storage.get(), temp_storage_size_bytes, input_iterator, - test_utils::wrap_in_identity_iterator(d_output), + test_utils::wrap_in_identity_iterator(d_output.get()), input.size(), scan_op, stream, TestFixture::debug_synchronous))); - if(TestFixture::use_graphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -458,23 +435,12 @@ TYPED_TEST(RocprimDeviceScanTests, InclusiveScan) HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - output.size() * sizeof(U), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + const auto output = d_output.load(); // Check if output values are as expected ASSERT_NO_FATAL_FAILURE( test_utils::assert_near(output, expected, single_op_precision * size)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if (TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -540,20 +506,9 @@ TYPED_TEST(RocprimDeviceScanTests, ExclusiveScan) // Generate data std::vector input = test_utils::get_random_data(size, 1, 10, seed_value); - std::vector output(input.size()); - - T * d_input; - U * d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(U))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(input.size()); // scan function scan_op_type scan_op; @@ -568,18 +523,16 @@ TYPED_TEST(RocprimDeviceScanTests, ExclusiveScan) ); auto input_iterator - = rocprim::make_transform_iterator(d_input, + = rocprim::make_transform_iterator(d_input.get(), [](T in) { return static_cast(in); }); - // temp storage - size_t temp_storage_size_bytes; - void * d_temp_storage = nullptr; // Get size of d_temp_storage + size_t temp_storage_size_bytes; HIP_CHECK((invoke_exclusive_scan( - d_temp_storage, + nullptr, temp_storage_size_bytes, input_iterator, - test_utils::wrap_in_identity_iterator(d_output), + test_utils::wrap_in_identity_iterator(d_output.get()), initial_value, input.size(), scan_op, @@ -590,8 +543,7 @@ TYPED_TEST(RocprimDeviceScanTests, ExclusiveScan) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::use_graphs) @@ -601,17 +553,16 @@ TYPED_TEST(RocprimDeviceScanTests, ExclusiveScan) // Run HIP_CHECK((invoke_exclusive_scan( - d_temp_storage, + d_temp_storage.get(), temp_storage_size_bytes, input_iterator, - test_utils::wrap_in_identity_iterator(d_output), + test_utils::wrap_in_identity_iterator(d_output.get()), initial_value, input.size(), scan_op, stream, debug_synchronous))); - if(TestFixture::use_graphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -621,23 +572,12 @@ TYPED_TEST(RocprimDeviceScanTests, ExclusiveScan) HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - output.size() * sizeof(U), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + const auto output = d_output.load(); // Check if output values are as expected ASSERT_NO_FATAL_FAILURE( test_utils::assert_near(output, expected, single_op_precision * size)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if (TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -698,6 +638,7 @@ TYPED_TEST(RocprimDeviceScanTests, InclusiveScanByKey) // Generate data std::vector input = test_utils::get_random_data(size, 0, 9, seed_value); + std::vector keys; if(use_unique_keys) { @@ -708,29 +649,10 @@ TYPED_TEST(RocprimDeviceScanTests, InclusiveScanByKey) { keys = test_utils::get_random_data(size, 0, 3, seed_value); } - std::vector output(input.size(), U{0}); - - T * d_input; - K * d_keys; - U * d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys, keys.size() * sizeof(K))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(U))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_keys, keys.data(), - keys.size() * sizeof(K), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_keys(keys); + test_utils::device_ptr d_output(input.size()); // scan function scan_op_type scan_op; @@ -746,18 +668,16 @@ TYPED_TEST(RocprimDeviceScanTests, InclusiveScanByKey) ); auto input_iterator - = rocprim::make_transform_iterator(d_input, + = rocprim::make_transform_iterator(d_input.get(), [](T in) { return static_cast(in); }); - // temp storage - size_t temp_storage_size_bytes; - void * d_temp_storage = nullptr; // Get size of d_temp_storage - HIP_CHECK((invoke_inclusive_scan_by_key(d_temp_storage, + size_t temp_storage_size_bytes; + HIP_CHECK((invoke_inclusive_scan_by_key(nullptr, temp_storage_size_bytes, - d_keys, + d_keys.get(), input_iterator, - d_output, + d_output.get(), input.size(), scan_op, keys_compare_op, @@ -768,8 +688,7 @@ TYPED_TEST(RocprimDeviceScanTests, InclusiveScanByKey) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::use_graphs) @@ -778,18 +697,17 @@ TYPED_TEST(RocprimDeviceScanTests, InclusiveScanByKey) } // Run - HIP_CHECK((invoke_inclusive_scan_by_key(d_temp_storage, + HIP_CHECK((invoke_inclusive_scan_by_key(d_temp_storage.get(), temp_storage_size_bytes, - d_keys, + d_keys.get(), input_iterator, - d_output, + d_output.get(), input.size(), scan_op, keys_compare_op, stream, debug_synchronous))); - if(TestFixture::use_graphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -799,24 +717,12 @@ TYPED_TEST(RocprimDeviceScanTests, InclusiveScanByKey) HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - output.size() * sizeof(U), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + const auto output = d_output.load(); // Check if output values are as expected ASSERT_NO_FATAL_FAILURE( test_utils::assert_near(output, expected, single_op_precision * size)); - HIP_CHECK(hipFree(d_keys)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if (TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -889,29 +795,10 @@ TYPED_TEST(RocprimDeviceScanTests, ExclusiveScanByKey) { keys = test_utils::get_random_data(size, 0, 3, seed_value); } - std::vector output(input.size(), U{0}); - - T * d_input; - K * d_keys; - U * d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys, keys.size() * sizeof(K))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(U))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - input.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_keys, keys.data(), - keys.size() * sizeof(K), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_keys(keys); + test_utils::device_ptr d_output(input.size()); // scan function scan_op_type scan_op; @@ -928,18 +815,16 @@ TYPED_TEST(RocprimDeviceScanTests, ExclusiveScanByKey) ); auto input_iterator - = rocprim::make_transform_iterator(d_input, + = rocprim::make_transform_iterator(d_input.get(), [](T in) { return static_cast(in); }); - // temp storage - size_t temp_storage_size_bytes; - void * d_temp_storage = nullptr; // Get size of d_temp_storage - HIP_CHECK((invoke_exclusive_scan_by_key(d_temp_storage, + size_t temp_storage_size_bytes; + HIP_CHECK((invoke_exclusive_scan_by_key(nullptr, temp_storage_size_bytes, - d_keys, + d_keys.get(), input_iterator, - d_output, + d_output.get(), initial_value, input.size(), scan_op, @@ -951,8 +836,7 @@ TYPED_TEST(RocprimDeviceScanTests, ExclusiveScanByKey) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::use_graphs) @@ -961,11 +845,11 @@ TYPED_TEST(RocprimDeviceScanTests, ExclusiveScanByKey) } // Run - HIP_CHECK((invoke_exclusive_scan_by_key(d_temp_storage, + HIP_CHECK((invoke_exclusive_scan_by_key(d_temp_storage.get(), temp_storage_size_bytes, - d_keys, + d_keys.get(), input_iterator, - d_output, + d_output.get(), initial_value, input.size(), scan_op, @@ -973,7 +857,6 @@ TYPED_TEST(RocprimDeviceScanTests, ExclusiveScanByKey) stream, debug_synchronous))); - if(TestFixture::use_graphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -983,24 +866,12 @@ TYPED_TEST(RocprimDeviceScanTests, ExclusiveScanByKey) HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - output.size() * sizeof(U), - hipMemcpyDeviceToHost - ) - ); - HIP_CHECK(hipDeviceSynchronize()); + const auto output = d_output.load(); // Check if output values are as expected ASSERT_NO_FATAL_FAILURE( test_utils::assert_near(output, expected, single_op_precision * size)); - HIP_CHECK(hipFree(d_keys)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if (TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -1111,33 +982,26 @@ void testLargeIndicesInclusiveScan() SCOPED_TRACE(testing::Message() << "with starting point = " << *input_begin); - T output; - T * d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, sizeof(T))); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_output(1); - OutputIterator output_it{d_output, size - 1}; - - // temp storage - size_t temp_storage_size_bytes; - void* d_temp_storage = nullptr; + OutputIterator output_it{d_output.get(), size - 1}; // Get temporary array size - HIP_CHECK( - rocprim::inclusive_scan( - d_temp_storage, temp_storage_size_bytes, - input_begin, output_it, size, + size_t temp_storage_size_bytes; + HIP_CHECK(rocprim::inclusive_scan(nullptr, + temp_storage_size_bytes, + input_begin, + output_it, + size, ::rocprim::plus(), - stream, debug_synchronous - ) - ); + stream, + debug_synchronous)); // temp_storage_size_bytes must be >0 ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(UseGraphs) @@ -1146,16 +1010,15 @@ void testLargeIndicesInclusiveScan() } // Run - HIP_CHECK( - rocprim::inclusive_scan( - d_temp_storage, temp_storage_size_bytes, - input_begin, output_it, size, + HIP_CHECK(rocprim::inclusive_scan(d_temp_storage.get(), + temp_storage_size_bytes, + input_begin, + output_it, + size, ::rocprim::plus(), - stream, debug_synchronous - ) - ); + stream, + debug_synchronous)); - if(UseGraphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -1165,8 +1028,7 @@ void testLargeIndicesInclusiveScan() HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - HIP_CHECK(hipMemcpy(&output, d_output, sizeof(T), hipMemcpyDeviceToHost)); - HIP_CHECK(hipDeviceSynchronize()); + const auto output = d_output.load()[0]; // Sum of 'size' increasing numbers starting at 'n' is size * (2n + size - 1) // The division is not integer division but either (size) or (2n + size - 1) has to be even. @@ -1177,9 +1039,6 @@ void testLargeIndicesInclusiveScan() ASSERT_EQ(output, expected_output); - HIP_CHECK(hipFree(d_temp_storage)); - HIP_CHECK(hipFree(d_output)); - if(UseGraphs) { gHelper.cleanupGraphHelper(); @@ -1239,34 +1098,27 @@ void testLargeIndicesExclusiveScan() SCOPED_TRACE(testing::Message() << "with starting point = " << *input_begin); SCOPED_TRACE(testing::Message() << "with initial value = " << initial_value); - T output; - T* d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, sizeof(T))); - HIP_CHECK(hipDeviceSynchronize()); - - OutputIterator output_it{d_output, size - 1}; + test_utils::device_ptr d_output(1); - // temp storage - size_t temp_storage_size_bytes; - void * d_temp_storage = nullptr; + OutputIterator output_it{d_output.get(), size - 1}; // Get temporary array size - HIP_CHECK( - rocprim::exclusive_scan( - d_temp_storage, temp_storage_size_bytes, - input_begin, output_it, - initial_value, size, + size_t temp_storage_size_bytes; + HIP_CHECK(rocprim::exclusive_scan(nullptr, + temp_storage_size_bytes, + input_begin, + output_it, + initial_value, + size, ::rocprim::plus(), - stream, debug_synchronous - ) - ); + stream, + debug_synchronous)); // temp_storage_size_bytes must be >0 ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(UseGraphs) @@ -1275,17 +1127,16 @@ void testLargeIndicesExclusiveScan() } // Run - HIP_CHECK( - rocprim::exclusive_scan( - d_temp_storage, temp_storage_size_bytes, - input_begin, output_it, - initial_value, size, + HIP_CHECK(rocprim::exclusive_scan(d_temp_storage.get(), + temp_storage_size_bytes, + input_begin, + output_it, + initial_value, + size, ::rocprim::plus(), - stream, debug_synchronous - ) - ); + stream, + debug_synchronous)); - if(UseGraphs) { gHelper.createAndLaunchGraph(stream, true, false); @@ -1295,8 +1146,7 @@ void testLargeIndicesExclusiveScan() HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - HIP_CHECK(hipMemcpy(&output, d_output, sizeof(T), hipMemcpyDeviceToHost)); - HIP_CHECK(hipDeviceSynchronize()); + const auto output = d_output.load()[0]; // Sum of 'size' - 1 increasing numbers starting at 'n' is (size - 1) * (2n + size - 2) // The division is not integer division but either (size - 1) or (2n + size - 2) has to be even. @@ -1310,9 +1160,6 @@ void testLargeIndicesExclusiveScan() ASSERT_EQ(output, expected_output); - HIP_CHECK(hipFree(d_temp_storage)); - HIP_CHECK(hipFree(d_output)); - if(UseGraphs) { gHelper.cleanupGraphHelper(); @@ -1491,9 +1338,8 @@ void large_indices_scan_by_key_test(ScanByKeyFun scan_by_key_fun) const auto size = test_utils::get_large_sizes(seed_value).back(); SCOPED_TRACE(testing::Message() << "with size = " << size); - unsigned int* d_incorrect_flag; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_incorrect_flag, sizeof(*d_incorrect_flag))); - HIP_CHECK(hipMemset(d_incorrect_flag, 0, sizeof(*d_incorrect_flag))); + test_utils::device_ptr d_incorrect_flag(1); + HIP_CHECK(hipMemset(d_incorrect_flag.get(), 0, sizeof(unsigned int))); const size_t run_length = test_utils::get_random_value(1, 10000, seed_value); SCOPED_TRACE(testing::Message() << "with run_length = " << run_length); @@ -1504,20 +1350,20 @@ void large_indices_scan_by_key_test(ScanByKeyFun scan_by_key_fun) const auto values_input = rocprim::counting_iterator(0); size_t temp_storage_size_bytes; - void* d_temp_storage = nullptr; - HIP_CHECK(scan_by_key_fun(d_temp_storage, + HIP_CHECK(scan_by_key_fun(nullptr, temp_storage_size_bytes, keys_input, values_input, run_length, - d_incorrect_flag, + d_incorrect_flag.get(), size, stream, debug_synchronous, seed_value)); ASSERT_GT(temp_storage_size_bytes, 0); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper;; if(UseGraphs) @@ -1525,18 +1371,17 @@ void large_indices_scan_by_key_test(ScanByKeyFun scan_by_key_fun) gHelper.startStreamCapture(stream); } - HIP_CHECK(scan_by_key_fun(d_temp_storage, + HIP_CHECK(scan_by_key_fun(d_temp_storage.get(), temp_storage_size_bytes, keys_input, values_input, run_length, - d_incorrect_flag, + d_incorrect_flag.get(), size, stream, debug_synchronous, seed_value)); - if(UseGraphs) { gHelper.createAndLaunchGraph(stream); @@ -1544,17 +1389,10 @@ void large_indices_scan_by_key_test(ScanByKeyFun scan_by_key_fun) HIP_CHECK(hipGetLastError()); - unsigned int incorrect_flag; - HIP_CHECK(hipMemcpy(&incorrect_flag, - d_incorrect_flag, - sizeof(incorrect_flag), - hipMemcpyDeviceToHost)); + const auto incorrect_flag = d_incorrect_flag.load()[0]; ASSERT_EQ(0, incorrect_flag); - HIP_CHECK(hipFree(d_temp_storage)); - HIP_CHECK(hipFree(d_incorrect_flag)); - if (UseGraphs) { gHelper.cleanupGraphHelper(); @@ -1715,24 +1553,11 @@ TYPED_TEST(RocprimDeviceScanFutureTests, ExclusiveScan) const std::vector future_input = test_utils::get_random_data(future_size, 1, 10, ~seed_value); const std::vector input = test_utils::get_random_data(size, 1, 10, seed_value); - std::vector output(input.size()); - - T* d_input; - U* d_output; - T* d_future_input; - T* d_initial_value; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(U))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_future_input, - future_input.size() * sizeof(T))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_initial_value, sizeof(T))); - HIP_CHECK( - hipMemcpy(d_input, input.data(), input.size() * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_future_input, - future_input.data(), - future_input.size() * sizeof(T), - hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); + + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(input.size()); + test_utils::device_ptr d_future_input(future_input); + test_utils::device_ptr d_initial_value(1); // scan function scan_op_type scan_op; @@ -1744,25 +1569,23 @@ TYPED_TEST(RocprimDeviceScanFutureTests, ExclusiveScan) test_utils::host_exclusive_scan( input.begin(), input.end(), initial_value, expected.begin(), scan_op); - const auto future_iter - = test_utils::wrap_in_identity_iterator(d_initial_value); + const auto future_iter = test_utils::wrap_in_identity_iterator( + d_initial_value.get()); const auto future_initial_value = rocprim::future_value>{ future_iter}; auto input_iterator - = rocprim::make_transform_iterator(d_input, + = rocprim::make_transform_iterator(d_input.get(), [](T in) { return static_cast(in); }); - // temp storage - size_t temp_storage_size_bytes; - char* d_temp_storage = nullptr; // Get size of d_temp_storage + size_t temp_storage_size_bytes; HIP_CHECK((invoke_exclusive_scan( nullptr, temp_storage_size_bytes, input_iterator, - test_utils::wrap_in_identity_iterator(d_output), + test_utils::wrap_in_identity_iterator(d_output.get()), future_initial_value, input.size(), scan_op, @@ -1773,13 +1596,17 @@ TYPED_TEST(RocprimDeviceScanFutureTests, ExclusiveScan) ASSERT_GT(temp_storage_size_bytes, 0); size_t temp_storage_reduce = 0; - HIP_CHECK(rocprim::reduce( - nullptr, temp_storage_reduce, d_future_input, d_initial_value, 2048, rocprim::plus(), stream)); + HIP_CHECK(rocprim::reduce(nullptr, + temp_storage_reduce, + d_future_input.get(), + d_initial_value.get(), + 2048, + rocprim::plus(), + stream)); // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper( - &d_temp_storage, temp_storage_size_bytes + temp_storage_reduce)); - HIP_CHECK(hipDeviceSynchronize()); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes + + temp_storage_reduce); test_utils::GraphHelper gHelper;; if(TestFixture::use_graphs) @@ -1788,20 +1615,20 @@ TYPED_TEST(RocprimDeviceScanFutureTests, ExclusiveScan) } // Fill initial value on the device - HIP_CHECK(rocprim::reduce(d_temp_storage + temp_storage_size_bytes, + HIP_CHECK(rocprim::reduce(d_temp_storage.get() + temp_storage_size_bytes, temp_storage_reduce, - d_future_input, - d_initial_value, + d_future_input.get(), + d_initial_value.get(), 2048, rocprim::plus(), stream)); // Run HIP_CHECK((invoke_exclusive_scan( - d_temp_storage, + d_temp_storage.get(), temp_storage_size_bytes, input_iterator, - test_utils::wrap_in_identity_iterator(d_output), + test_utils::wrap_in_identity_iterator(d_output.get()), future_initial_value, input.size(), scan_op, @@ -1809,26 +1636,17 @@ TYPED_TEST(RocprimDeviceScanFutureTests, ExclusiveScan) debug_synchronous))); HIP_CHECK(hipGetLastError()); - if(TestFixture::use_graphs) { gHelper.createAndLaunchGraph(stream); } // Copy output to host - HIP_CHECK(hipMemcpy( - output.data(), d_output, output.size() * sizeof(U), hipMemcpyDeviceToHost)); - HIP_CHECK(hipDeviceSynchronize()); + const auto output = d_output.load(); // Check if output values are as expected ASSERT_NO_FATAL_FAILURE(test_utils::assert_near(output, expected, precision)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_future_input)); - HIP_CHECK(hipFree(d_initial_value)); - HIP_CHECK(hipFree(d_temp_storage)); - if (TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); From 4d8aac5014a52e8523e9d102ec92161eb15140e3 Mon Sep 17 00:00:00 2001 From: Cenxuan Tian Date: Mon, 16 Dec 2024 16:02:25 +0000 Subject: [PATCH 44/84] Resolve "Replace hipMalloc, hipMallocHelper, and hipFree in benchmarks and tests with device_ptr" Part 1 --- test/hip/test_hip_async_copy.cpp | 128 ++-- ...test_block_adjacent_difference.kernels.hpp | 642 +++++++----------- test/rocprim/test_block_discontinuity.cpp.in | 1 + .../test_block_discontinuity.kernels.hpp | 144 +--- test/rocprim/test_block_exchange.cpp | 3 +- test/rocprim/test_block_exchange.kernels.hpp | 266 ++------ test/rocprim/test_block_load_store.cpp | 3 +- test/rocprim/test_block_load_store.hpp | 145 ++-- test/rocprim/test_block_reduce.hpp | 183 +++-- test/rocprim/test_utils_device_ptr.hpp | 8 +- 10 files changed, 601 insertions(+), 922 deletions(-) diff --git a/test/hip/test_hip_async_copy.cpp b/test/hip/test_hip_async_copy.cpp index 20ea96b68..fcaf5e6e7 100644 --- a/test/hip/test_hip_async_copy.cpp +++ b/test/hip/test_hip_async_copy.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2023-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,24 +20,22 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +#include "../rocprim/test_utils_device_ptr.hpp" #include "common_test_header.hpp" -using test_common_utils::hipMallocHelper; - namespace { -template +template struct PinnedAllocator { using value_type = T; PinnedAllocator() = default; - template + template constexpr PinnedAllocator(const PinnedAllocator&) noexcept - { - } + {} T* allocate(const size_t size) { @@ -64,11 +62,9 @@ struct PinnedAllocator std::vector get_sizes() { - std::vector sizes{ - 1, 10, 53, 211, 1024, 2345, 4096, 34567, - (1 << 16) - 1220, (1 << 22) - 76543 - }; - if (!test_common_utils::use_hmm()) + std::vector + sizes{1, 10, 53, 211, 1024, 2345, 4096, 34567, (1 << 16) - 1220, (1 << 22) - 76543}; + if(!test_common_utils::use_hmm()) { sizes.insert(sizes.begin(), 0); } @@ -86,26 +82,26 @@ void increment_kernel(T* const d_input, const size_t size) } } -} +} // namespace class HipAsyncCopyTests : public ::testing::Test { protected: - using T = int; - using vector_type = std::vector>; - static constexpr int seed = 543897; + using T = int; + using vector_type = std::vector>; + static constexpr int seed = 543897; static constexpr unsigned int block_size = 1024; - std::vector sizes; - std::vector inputs; - std::vector expecteds; - std::vector d_inputs; - std::vector outputs; - std::vector streams; + std::vector sizes; + std::vector inputs; + std::vector expecteds; + std::vector> d_inputs; + std::vector outputs; + std::vector streams; void SetUp() override { - std::default_random_engine prng(seed); + std::default_random_engine prng(seed); std::uniform_int_distribution dist; sizes = get_sizes(); @@ -118,10 +114,12 @@ class HipAsyncCopyTests : public ::testing::Test for(size_t i = 0; i < sizes.size(); i++) { const auto size = sizes[i]; - std::generate_n(std::back_inserter(inputs[i]), size, [&](){ return dist(prng); }); - std::transform(inputs[i].begin(), inputs[i].end(), std::back_inserter(expecteds[i]), - [](const auto& val) { return val + static_cast(1); }); - HIP_CHECK(hipMallocHelper(d_inputs.data() + i, size * sizeof(T))); + std::generate_n(std::back_inserter(inputs[i]), size, [&]() { return dist(prng); }); + std::transform(inputs[i].begin(), + inputs[i].end(), + std::back_inserter(expecteds[i]), + [](const auto& val) { return val + static_cast(1); }); + d_inputs[i].resize(size); outputs[i].resize(size); if(i == 0) { @@ -142,7 +140,6 @@ class HipAsyncCopyTests : public ::testing::Test { HIP_CHECK(hipStreamDestroy(streams[i])); } - HIP_CHECK(hipFree(d_inputs[i])); } } }; @@ -152,7 +149,11 @@ TEST_F(HipAsyncCopyTests, AsyncCopyDepthFirst) for(size_t i = 0; i < sizes.size(); i++) { const auto size_bytes = sizes[i] * sizeof(T); - HIP_CHECK(hipMemcpyAsync(d_inputs[i], inputs[i].data(), size_bytes, hipMemcpyHostToDevice, streams[i])); + HIP_CHECK(hipMemcpyAsync(d_inputs[i].get(), + inputs[i].data(), + size_bytes, + hipMemcpyHostToDevice, + streams[i])); const unsigned int grid_size = (sizes[i] + block_size - 1) / block_size; if(sizes[i] > 0) { @@ -161,11 +162,15 @@ TEST_F(HipAsyncCopyTests, AsyncCopyDepthFirst) dim3(block_size), 0, streams[i], - d_inputs[i], + d_inputs[i].get(), sizes[i]); HIP_CHECK(hipGetLastError()); } - HIP_CHECK(hipMemcpyAsync(outputs[i].data(), d_inputs[i], size_bytes, hipMemcpyDeviceToHost, streams[i])); + HIP_CHECK(hipMemcpyAsync(outputs[i].data(), + d_inputs[i].get(), + size_bytes, + hipMemcpyDeviceToHost, + streams[i])); } HIP_CHECK(hipDeviceSynchronize()); ASSERT_EQ(expecteds, outputs); @@ -176,7 +181,11 @@ TEST_F(HipAsyncCopyTests, AsyncCopyBreadthFirst) for(size_t i = 0; i < sizes.size(); i++) { const auto size_bytes = sizes[i] * sizeof(T); - HIP_CHECK(hipMemcpyAsync(d_inputs[i], inputs[i].data(), size_bytes, hipMemcpyHostToDevice, streams[i])); + HIP_CHECK(hipMemcpyAsync(d_inputs[i].get(), + inputs[i].data(), + size_bytes, + hipMemcpyHostToDevice, + streams[i])); } for(size_t i = 0; i < sizes.size(); i++) { @@ -188,7 +197,7 @@ TEST_F(HipAsyncCopyTests, AsyncCopyBreadthFirst) dim3(block_size), 0, streams[i], - d_inputs[i], + d_inputs[i].get(), sizes[i]); HIP_CHECK(hipGetLastError()); } @@ -196,7 +205,11 @@ TEST_F(HipAsyncCopyTests, AsyncCopyBreadthFirst) for(size_t i = 0; i < sizes.size(); i++) { const auto size_bytes = sizes[i] * sizeof(T); - HIP_CHECK(hipMemcpyAsync(outputs[i].data(), d_inputs[i], size_bytes, hipMemcpyDeviceToHost, streams[i])); + HIP_CHECK(hipMemcpyAsync(outputs[i].data(), + d_inputs[i].get(), + size_bytes, + hipMemcpyDeviceToHost, + streams[i])); } HIP_CHECK(hipDeviceSynchronize()); ASSERT_EQ(expecteds, outputs); @@ -208,40 +221,43 @@ TEST(HipAsyncCopyTestsExtra, StreamInStruct) { hipStream_t stream; }; - using T = int; - using vector_type = std::vector>; - static constexpr int seed = 543897; + using T = int; + using vector_type = std::vector>; + static constexpr int seed = 543897; static constexpr unsigned int block_size = 1024; - const size_t size = get_sizes().back(); - std::default_random_engine prng(seed); + const size_t size = get_sizes().back(); + std::default_random_engine prng(seed); std::uniform_int_distribution dist; StreamWrapper stream_wrapper; HIP_CHECK(hipStreamCreateWithFlags(&stream_wrapper.stream, hipStreamNonBlocking)); vector_type input; - std::generate_n(std::back_inserter(input), size, [&](){ return dist(prng); }); + std::generate_n(std::back_inserter(input), size, [&]() { return dist(prng); }); vector_type expected; - std::transform(input.begin(), input.end(), std::back_inserter(expected), - [](const auto& val){ return val + static_cast(1); }); - - T* d_input{}; - const auto size_bytes = size * sizeof(T); - HIP_CHECK(hipMallocHelper(&d_input, size_bytes)); - HIP_CHECK(hipMemcpyAsync(d_input, input.data(), size_bytes, hipMemcpyHostToDevice, stream_wrapper.stream)); - - const unsigned int grid_size = (size + block_size - 1) / block_size; - hipLaunchKernelGGL( - HIP_KERNEL_NAME(increment_kernel), - dim3(grid_size), dim3(block_size), 0, stream_wrapper.stream, - d_input, size - ); + std::transform(input.begin(), + input.end(), + std::back_inserter(expected), + [](const auto& val) { return val + static_cast(1); }); + + test_utils::device_ptr d_input(input); + const unsigned int grid_size = (size + block_size - 1) / block_size; + hipLaunchKernelGGL(HIP_KERNEL_NAME(increment_kernel), + dim3(grid_size), + dim3(block_size), + 0, + stream_wrapper.stream, + d_input.get(), + size); HIP_CHECK(hipGetLastError()); vector_type output(size); - HIP_CHECK(hipMemcpyAsync(output.data(), d_input, size_bytes, hipMemcpyDeviceToHost, stream_wrapper.stream)); + HIP_CHECK(hipMemcpyAsync(output.data(), + d_input.get(), + size * sizeof(T), + hipMemcpyDeviceToHost, + stream_wrapper.stream)); HIP_CHECK(hipStreamSynchronize(stream_wrapper.stream)); - HIP_CHECK(hipFree(d_input)); HIP_CHECK(hipStreamDestroy(stream_wrapper.stream)); ASSERT_EQ(output, expected); diff --git a/test/rocprim/test_block_adjacent_difference.kernels.hpp b/test/rocprim/test_block_adjacent_difference.kernels.hpp index cf55dca18..bee409992 100644 --- a/test/rocprim/test_block_adjacent_difference.kernels.hpp +++ b/test/rocprim/test_block_adjacent_difference.kernels.hpp @@ -24,16 +24,17 @@ #define TEST_BLOCK_ADJACENT_DIFFERENCE_KERNELS_HPP_ #include "test_utils.hpp" +#include "test_utils_device_ptr.hpp" // Host (CPU) implementaions of the wrapping function that allows to pass 3 args -template +template auto apply(FlagOp flag_op, const T& a, const T& b, unsigned int b_index) -> decltype(flag_op(b, a, b_index)) { return flag_op(b, a, b_index); } -template +template auto apply(FlagOp flag_op, const T& a, const T& b, unsigned int) -> decltype(flag_op(b, a)) { return flag_op(b, a); @@ -42,31 +43,29 @@ auto apply(FlagOp flag_op, const T& a, const T& b, unsigned int) -> decltype(fla template struct test_op { - __host__ __device__ T operator()(const T& a, const T& b) const + __host__ __device__ + T operator()(const T& a, const T& b) const { return (b + b) - a; } }; -template< - class Type, - class FlagType, - class FlagOpType, - unsigned int BlockSize, - unsigned int ItemsPerThread -> -__global__ -__launch_bounds__(BlockSize, ROCPRIM_DEFAULT_MIN_WARPS_PER_EU) -void flag_heads_kernel(Type* device_input, long long* device_heads) +template +__global__ __launch_bounds__(BlockSize, ROCPRIM_DEFAULT_MIN_WARPS_PER_EU) void flag_heads_kernel( + Type* device_input, long long* device_heads) { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int items_per_block = BlockSize * ItemsPerThread; - const unsigned int block_offset = blockIdx.x * items_per_block; + const unsigned int block_offset = blockIdx.x * items_per_block; Type input[ItemsPerThread]; rocprim::block_load_direct_blocked(lid, device_input + block_offset, input); - rocprim::block_adjacent_difference adjacent_difference; + rocprim::block_adjacent_difference adjacent_difference; __shared__ typename decltype(adjacent_difference)::storage_type storage; FlagType head_flags[ItemsPerThread]; @@ -91,35 +90,35 @@ void flag_heads_kernel(Type* device_input, long long* device_heads) rocprim::block_store_direct_blocked(lid, device_heads + block_offset, head_flags); } -template< - typename T, - typename Output, - typename StorageType, - typename BinaryFunction, - unsigned int BlockSize, - unsigned int ItemsPerThread -> -__global__ -__launch_bounds__(BlockSize, ROCPRIM_DEFAULT_MIN_WARPS_PER_EU) -void subtract_left_kernel(const T* input, StorageType* output) +template +__global__ __launch_bounds__(BlockSize, ROCPRIM_DEFAULT_MIN_WARPS_PER_EU) void subtract_left_kernel( + const T* input, StorageType* output) { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int items_per_block = BlockSize * ItemsPerThread; - const unsigned int block_offset = blockIdx.x * items_per_block; + const unsigned int block_offset = blockIdx.x * items_per_block; T thread_items[ItemsPerThread]; rocprim::block_load_direct_blocked(lid, input + block_offset, thread_items); rocprim::block_adjacent_difference adjacent_difference; - __shared__ typename decltype(adjacent_difference)::storage_type storage; + __shared__ typename decltype(adjacent_difference)::storage_type storage; Output thread_output[ItemsPerThread]; if(blockIdx.x % 2 == 1) { const T tile_predecessor_item = input[block_offset - 1]; - adjacent_difference.subtract_left( - thread_items, thread_output, BinaryFunction{}, tile_predecessor_item, storage); + adjacent_difference.subtract_left(thread_items, + thread_output, + BinaryFunction{}, + tile_predecessor_item, + storage); } else { @@ -129,27 +128,27 @@ void subtract_left_kernel(const T* input, StorageType* output) rocprim::block_store_direct_blocked(lid, output + block_offset, thread_output); } -template< - typename T, - typename Output, - typename StorageType, - typename BinaryFunction, - unsigned int BlockSize, - unsigned int ItemsPerThread -> -__global__ -__launch_bounds__(BlockSize, ROCPRIM_DEFAULT_MIN_WARPS_PER_EU) -void subtract_left_partial_kernel(const T* input, unsigned int* tile_sizes, StorageType* output) +template +__global__ __launch_bounds__( + BlockSize, + ROCPRIM_DEFAULT_MIN_WARPS_PER_EU) void subtract_left_partial_kernel(const T* input, + unsigned int* tile_sizes, + StorageType* output) { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int items_per_block = BlockSize * ItemsPerThread; - const unsigned int block_offset = blockIdx.x * items_per_block; + const unsigned int block_offset = blockIdx.x * items_per_block; T thread_items[ItemsPerThread]; rocprim::block_load_direct_blocked(lid, input + block_offset, thread_items); rocprim::block_adjacent_difference adjacent_difference; - __shared__ typename decltype(adjacent_difference)::storage_type storage; + __shared__ typename decltype(adjacent_difference)::storage_type storage; Output thread_output[ItemsPerThread]; @@ -159,49 +158,53 @@ void subtract_left_partial_kernel(const T* input, unsigned int* tile_sizes, Stor const T tile_predecessor_item = input[block_offset - 1]; adjacent_difference.subtract_left_partial(thread_items, thread_output, - BinaryFunction {}, + BinaryFunction{}, tile_predecessor_item, tile_size, storage); } else { - adjacent_difference.subtract_left_partial( - thread_items, thread_output, BinaryFunction {}, tile_size, storage); + adjacent_difference.subtract_left_partial(thread_items, + thread_output, + BinaryFunction{}, + tile_size, + storage); } rocprim::block_store_direct_blocked(lid, output + block_offset, thread_output); } -template< - typename T, - typename Output, - typename StorageType, - typename BinaryFunction, - unsigned int BlockSize, - unsigned int ItemsPerThread -> -__global__ -__launch_bounds__(BlockSize, ROCPRIM_DEFAULT_MIN_WARPS_PER_EU) -void subtract_right_kernel(const T* input, StorageType* output) +template +__global__ __launch_bounds__( + BlockSize, ROCPRIM_DEFAULT_MIN_WARPS_PER_EU) void subtract_right_kernel(const T* input, + StorageType* output) { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int items_per_block = BlockSize * ItemsPerThread; - const unsigned int block_offset = blockIdx.x * items_per_block; + const unsigned int block_offset = blockIdx.x * items_per_block; T thread_items[ItemsPerThread]; rocprim::block_load_direct_blocked(lid, input + block_offset, thread_items); rocprim::block_adjacent_difference adjacent_difference; - __shared__ typename decltype(adjacent_difference)::storage_type storage; + __shared__ typename decltype(adjacent_difference)::storage_type storage; Output thread_output[ItemsPerThread]; if(blockIdx.x % 2 == 0) { const T tile_successor_item = input[block_offset + items_per_block]; - adjacent_difference.subtract_right( - thread_items, thread_output, BinaryFunction{}, tile_successor_item, storage); + adjacent_difference.subtract_right(thread_items, + thread_output, + BinaryFunction{}, + tile_successor_item, + storage); } else { @@ -211,56 +214,56 @@ void subtract_right_kernel(const T* input, StorageType* output) rocprim::block_store_direct_blocked(lid, output + block_offset, thread_output); } -template< - typename T, - typename Output, - typename StorageType, - typename BinaryFunction, - unsigned int BlockSize, - unsigned int ItemsPerThread -> -__global__ -__launch_bounds__(BlockSize, ROCPRIM_DEFAULT_MIN_WARPS_PER_EU) -void subtract_right_partial_kernel(const T* input, unsigned int* tile_sizes, StorageType* output) +template +__global__ __launch_bounds__( + BlockSize, + ROCPRIM_DEFAULT_MIN_WARPS_PER_EU) void subtract_right_partial_kernel(const T* input, + unsigned int* tile_sizes, + StorageType* output) { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int items_per_block = BlockSize * ItemsPerThread; - const unsigned int block_offset = blockIdx.x * items_per_block; + const unsigned int block_offset = blockIdx.x * items_per_block; T thread_items[ItemsPerThread]; rocprim::block_load_direct_blocked(lid, input + block_offset, thread_items); rocprim::block_adjacent_difference adjacent_difference; - __shared__ typename decltype(adjacent_difference)::storage_type storage; + __shared__ typename decltype(adjacent_difference)::storage_type storage; Output thread_output[ItemsPerThread]; const unsigned int tile_size = tile_sizes[blockIdx.x]; - adjacent_difference.subtract_right_partial( - thread_items, thread_output, BinaryFunction {}, tile_size, storage); + adjacent_difference.subtract_right_partial(thread_items, + thread_output, + BinaryFunction{}, + tile_size, + storage); rocprim::block_store_direct_blocked(lid, output + block_offset, thread_output); } -template< - class Type, - class FlagType, - class FlagOpType, - unsigned int BlockSize, - unsigned int ItemsPerThread -> -__global__ -__launch_bounds__(BlockSize, ROCPRIM_DEFAULT_MIN_WARPS_PER_EU) -void flag_tails_kernel(Type* device_input, long long* device_tails) +template +__global__ __launch_bounds__(BlockSize, ROCPRIM_DEFAULT_MIN_WARPS_PER_EU) void flag_tails_kernel( + Type* device_input, long long* device_tails) { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int items_per_block = BlockSize * ItemsPerThread; - const unsigned int block_offset = blockIdx.x * items_per_block; + const unsigned int block_offset = blockIdx.x * items_per_block; Type input[ItemsPerThread]; rocprim::block_load_direct_blocked(lid, device_input + block_offset, input); - rocprim::block_adjacent_difference adjacent_difference; + rocprim::block_adjacent_difference adjacent_difference; __shared__ typename decltype(adjacent_difference)::storage_type storage; FlagType tail_flags[ItemsPerThread]; @@ -285,25 +288,25 @@ void flag_tails_kernel(Type* device_input, long long* device_tails) rocprim::block_store_direct_blocked(lid, device_tails + block_offset, tail_flags); } -template< - class Type, - class FlagType, - class FlagOpType, - unsigned int BlockSize, - unsigned int ItemsPerThread -> -__global__ -__launch_bounds__(BlockSize, ROCPRIM_DEFAULT_MIN_WARPS_PER_EU) -void flag_heads_and_tails_kernel(Type* device_input, long long* device_heads, long long* device_tails) +template +__global__ __launch_bounds__( + BlockSize, + ROCPRIM_DEFAULT_MIN_WARPS_PER_EU) void flag_heads_and_tails_kernel(Type* device_input, + long long* device_heads, + long long* device_tails) { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int items_per_block = BlockSize * ItemsPerThread; - const unsigned int block_offset = blockIdx.x * items_per_block; + const unsigned int block_offset = blockIdx.x * items_per_block; Type input[ItemsPerThread]; rocprim::block_load_direct_blocked(lid, device_input + block_offset, input); - rocprim::block_adjacent_difference adjacent_difference; + rocprim::block_adjacent_difference adjacent_difference; __shared__ typename decltype(adjacent_difference)::storage_type storage; FlagType head_flags[ItemsPerThread]; @@ -357,16 +360,13 @@ void flag_heads_and_tails_kernel(Type* device_input, long long* device_heads, lo rocprim::block_store_direct_blocked(lid, device_tails + block_offset, tail_flags); } -template< - class Type, - class FlagType, - class FlagOpType, - unsigned int Method, - unsigned int BlockSize, - unsigned int ItemsPerThread -> -auto test_block_adjacent_difference() --> typename std::enable_if::type +template +auto test_block_adjacent_difference() -> typename std::enable_if::type { using type = Type; // std::vector is a special case that will cause an error in hipMemcpy @@ -379,13 +379,13 @@ auto test_block_adjacent_difference() || std::is_same::value, float, FlagType>::type>::type; - using flag_type = FlagType; - using flag_op_type = FlagOpType; - static constexpr size_t block_size = BlockSize; + using flag_type = FlagType; + using flag_op_type = FlagOpType; + static constexpr size_t block_size = BlockSize; static constexpr size_t items_per_thread = ItemsPerThread; - static constexpr size_t items_per_block = block_size * items_per_thread; - static constexpr size_t size = items_per_block * 20; - static constexpr size_t grid_size = size / items_per_block; + static constexpr size_t items_per_block = block_size * items_per_thread; + static constexpr size_t size = items_per_block * 20; + static constexpr size_t grid_size = size / items_per_block; SCOPED_TRACE(testing::Message() << "items_per_block = " << items_per_block); SCOPED_TRACE(testing::Message() << "size = " << size); @@ -397,18 +397,18 @@ auto test_block_adjacent_difference() return; } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { - unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); // Generate data std::vector input = test_utils::get_random_data(size, 0, 10, seed_value); - std::vector heads(size); // Calculate expected results on host std::vector expected_heads(size); - flag_op_type flag_op; + flag_op_type flag_op; for(size_t bi = 0; bi < size / items_per_block; bi++) { for(size_t ii = 0; ii < items_per_block; ii++) @@ -425,70 +425,47 @@ auto test_block_adjacent_difference() } else { - expected_heads[i] = apply(flag_op, input[i - 1], input[i], ii); + expected_heads[i] + = apply(flag_op, input[i - 1], input[i], ii); } } } // Preparing Device - type* device_input; - HIP_CHECK(hipMalloc(reinterpret_cast(&device_input), input.size() * sizeof(typename decltype(input)::value_type))); - long long* device_heads; - HIP_CHECK(hipMalloc(reinterpret_cast(&device_heads), heads.size() * sizeof(typename decltype(heads)::value_type))); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(type), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_heads(size); // Running kernel hipLaunchKernelGGL( HIP_KERNEL_NAME( - flag_heads_kernel< - type, flag_type, flag_op_type, - block_size, items_per_thread - > - ), - dim3(grid_size), dim3(block_size), 0, 0, - device_input, device_heads - ); + flag_heads_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_input.get(), + device_heads.get()); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Reading results - HIP_CHECK( - hipMemcpy( - heads.data(), device_heads, - heads.size() * sizeof(typename decltype(heads)::value_type), - hipMemcpyDeviceToHost - ) - ); + const auto heads = device_heads.load(); // Validating results for(size_t i = 0; i < size; i++) { ASSERT_EQ(heads[i], expected_heads[i]); } - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_heads)); } - } -template< - class Type, - class FlagType, - class FlagOpType, - unsigned int Method, - unsigned int BlockSize, - unsigned int ItemsPerThread -> -auto test_block_adjacent_difference() --> typename std::enable_if::type +template +auto test_block_adjacent_difference() -> typename std::enable_if::type { using type = Type; // std::vector is a special case that will cause an error in hipMemcpy @@ -501,13 +478,13 @@ auto test_block_adjacent_difference() || std::is_same::value, float, FlagType>::type>::type; - using flag_type = FlagType; - using flag_op_type = FlagOpType; - static constexpr size_t block_size = BlockSize; + using flag_type = FlagType; + using flag_op_type = FlagOpType; + static constexpr size_t block_size = BlockSize; static constexpr size_t items_per_thread = ItemsPerThread; - static constexpr size_t items_per_block = block_size * items_per_thread; - static constexpr size_t size = items_per_block * 20; - static constexpr size_t grid_size = size / items_per_block; + static constexpr size_t items_per_block = block_size * items_per_thread; + static constexpr size_t size = items_per_block * 20; + static constexpr size_t grid_size = size / items_per_block; SCOPED_TRACE(testing::Message() << "items_per_block = " << items_per_block); SCOPED_TRACE(testing::Message() << "size = " << size); @@ -519,18 +496,18 @@ auto test_block_adjacent_difference() return; } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { - unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); // Generate data std::vector input = test_utils::get_random_data(size, 0, 10, seed_value); - std::vector tails(size); // Calculate expected results on host std::vector expected_tails(size); - flag_op_type flag_op; + flag_op_type flag_op; for(size_t bi = 0; bi < size / items_per_block; bi++) { for(size_t ii = 0; ii < items_per_block; ii++) @@ -547,70 +524,49 @@ auto test_block_adjacent_difference() } else { - expected_tails[i] = apply(flag_op, input[i], input[i + 1], ii + 1); + expected_tails[i] = apply(flag_op, + input[i], + input[i + 1], + ii + 1); } } } // Preparing Device - type* device_input; - HIP_CHECK(hipMalloc(reinterpret_cast(&device_input), input.size() * sizeof(typename decltype(input)::value_type))); - long long* device_tails; - HIP_CHECK(hipMalloc(reinterpret_cast(&device_tails), tails.size() * sizeof(typename decltype(tails)::value_type))); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(type), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_tails(size); // Running kernel hipLaunchKernelGGL( HIP_KERNEL_NAME( - flag_tails_kernel< - type, flag_type, flag_op_type, - block_size, items_per_thread - > - ), - dim3(grid_size), dim3(block_size), 0, 0, - device_input, device_tails - ); + flag_tails_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_input.get(), + device_tails.get()); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Reading results - HIP_CHECK( - hipMemcpy( - tails.data(), device_tails, - tails.size() * sizeof(typename decltype(tails)::value_type), - hipMemcpyDeviceToHost - ) - ); + const auto tails = device_tails.load(); // Validating results for(size_t i = 0; i < size; i++) { ASSERT_EQ(tails[i], expected_tails[i]); } - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_tails)); } - } -template< - class Type, - class FlagType, - class FlagOpType, - unsigned int Method, - unsigned int BlockSize, - unsigned int ItemsPerThread -> -auto test_block_adjacent_difference() --> typename std::enable_if::type +template +auto test_block_adjacent_difference() -> typename std::enable_if::type { using type = Type; // std::vector is a special case that will cause an error in hipMemcpy @@ -623,13 +579,13 @@ auto test_block_adjacent_difference() || std::is_same::value, float, FlagType>::type>::type; - using flag_type = FlagType; - using flag_op_type = FlagOpType; - static constexpr size_t block_size = BlockSize; + using flag_type = FlagType; + using flag_op_type = FlagOpType; + static constexpr size_t block_size = BlockSize; static constexpr size_t items_per_thread = ItemsPerThread; - static constexpr size_t items_per_block = block_size * items_per_thread; - static constexpr size_t size = items_per_block * 20; - static constexpr size_t grid_size = size / items_per_block; + static constexpr size_t items_per_block = block_size * items_per_thread; + static constexpr size_t size = items_per_block * 20; + static constexpr size_t grid_size = size / items_per_block; SCOPED_TRACE(testing::Message() << "items_per_block = " << items_per_block); SCOPED_TRACE(testing::Message() << "size = " << size); @@ -641,20 +597,19 @@ auto test_block_adjacent_difference() return; } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { - unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); // Generate data std::vector input = test_utils::get_random_data(size, 0, 10, seed_value); - std::vector heads(size); - std::vector tails(size); // Calculate expected results on host std::vector expected_heads(size); std::vector expected_tails(size); - flag_op_type flag_op; + flag_op_type flag_op; for(size_t bi = 0; bi < size / items_per_block; bi++) { for(size_t ii = 0; ii < items_per_block; ii++) @@ -671,7 +626,8 @@ auto test_block_adjacent_difference() } else { - expected_heads[i] = apply(flag_op, input[i - 1], input[i], ii); + expected_heads[i] + = apply(flag_op, input[i - 1], input[i], ii); } if(ii == items_per_block - 1) { @@ -684,79 +640,53 @@ auto test_block_adjacent_difference() } else { - expected_tails[i] = apply(flag_op, input[i], input[i + 1], ii + 1); + expected_tails[i] = apply(flag_op, + input[i], + input[i + 1], + ii + 1); } } } // Preparing Device - type* device_input; - HIP_CHECK(hipMalloc(reinterpret_cast(&device_input), input.size() * sizeof(typename decltype(input)::value_type))); - long long* device_heads; - HIP_CHECK(hipMalloc(reinterpret_cast(&device_heads), tails.size() * sizeof(typename decltype(heads)::value_type))); - long long* device_tails; - HIP_CHECK(hipMalloc(reinterpret_cast(&device_tails), tails.size() * sizeof(typename decltype(tails)::value_type))); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(type), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_heads(size); + test_utils::device_ptr device_tails(size); // Running kernel - hipLaunchKernelGGL( - HIP_KERNEL_NAME( - flag_heads_and_tails_kernel< - type, flag_type, flag_op_type, - block_size, items_per_thread - > - ), - dim3(grid_size), dim3(block_size), 0, 0, - device_input, device_heads, device_tails - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(flag_heads_and_tails_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_input.get(), + device_heads.get(), + device_tails.get()); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Reading results - HIP_CHECK( - hipMemcpy( - heads.data(), device_heads, - heads.size() * sizeof(typename decltype(heads)::value_type), - hipMemcpyDeviceToHost - ) - ); - - HIP_CHECK( - hipMemcpy( - tails.data(), device_tails, - tails.size() * sizeof(typename decltype(tails)::value_type), - hipMemcpyDeviceToHost - ) - ); - + const auto heads = device_heads.load(); + const auto tails = device_tails.load(); // Validating results for(size_t i = 0; i < size; i++) { ASSERT_EQ(heads[i], expected_heads[i]); ASSERT_EQ(tails[i], expected_tails[i]); } - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_heads)); - HIP_CHECK(hipFree(device_tails)); } - - } -template +template auto test_block_adjacent_difference() -> typename std::enable_if::type { using stored_type = std::conditional_t::value, int, Output>; @@ -787,8 +717,7 @@ auto test_block_adjacent_difference() -> typename std::enable_if::t SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); // Generate data - const std::vector input = test_utils::get_random_data(size, 0, 10, seed_value); - std::vector output(size); + const std::vector input = test_utils::get_random_data(size, 0, 10, seed_value); // Calculate expected results on host std::vector expected(size); @@ -810,12 +739,8 @@ auto test_block_adjacent_difference() -> typename std::enable_if::t } // Preparing Device - T* d_input; - stored_type* d_output; - HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(input[0]))); - HIP_CHECK(hipMalloc(&d_output, output.size() * sizeof(output[0]))); - HIP_CHECK(hipMemcpy( - d_input, input.data(), input.size() * sizeof(input[0]), hipMemcpyHostToDevice)); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(size); // Running kernel hipLaunchKernelGGL(HIP_KERNEL_NAME(subtract_left_kernel typename std::enable_if::t dim3(block_size), 0, 0, - d_input, - d_output); + d_input.get(), + d_output.get()); HIP_CHECK(hipGetLastError()); // Reading results - HIP_CHECK(hipMemcpy( - output.data(), d_output, output.size() * sizeof(output[0]), hipMemcpyDeviceToHost)); + const auto output = d_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_near( output, expected, std::max(test_utils::precision, test_utils::precision))); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); } } -template +template auto test_block_adjacent_difference() -> typename std::enable_if::type { using stored_type = std::conditional_t::value, int, Output>; @@ -882,8 +803,7 @@ auto test_block_adjacent_difference() -> typename std::enable_if::t SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); // Generate data - const std::vector input = test_utils::get_random_data(size, 0, 10, seed_value); - std::vector output(size); + const std::vector input = test_utils::get_random_data(size, 0, 10, seed_value); // Calculate expected results on host std::vector expected(size); @@ -905,12 +825,8 @@ auto test_block_adjacent_difference() -> typename std::enable_if::t } // Preparing Device - T* d_input; - stored_type* d_output; - HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(input[0]))); - HIP_CHECK(hipMalloc(&d_output, output.size() * sizeof(output[0]))); - HIP_CHECK(hipMemcpy( - d_input, input.data(), input.size() * sizeof(input[0]), hipMemcpyHostToDevice)); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(size); // Running kernel hipLaunchKernelGGL(HIP_KERNEL_NAME(subtract_right_kernel typename std::enable_if::t dim3(block_size), 0, 0, - d_input, - d_output); + d_input.get(), + d_output.get()); HIP_CHECK(hipGetLastError()); // Reading results - HIP_CHECK(hipMemcpy( - output.data(), d_output, output.size() * sizeof(output[0]), hipMemcpyDeviceToHost)); + const auto output = d_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_near( output, expected, std::max(test_utils::precision, test_utils::precision))); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); } } -template +template auto test_block_adjacent_difference() -> typename std::enable_if::type { using stored_type = std::conditional_t::value, int, Output>; @@ -977,8 +889,7 @@ auto test_block_adjacent_difference() -> typename std::enable_if::t SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); // Generate data - const std::vector input = test_utils::get_random_data(size, 0, 10, seed_value); - std::vector output(size); + const std::vector input = test_utils::get_random_data(size, 0, 10, seed_value); const std::vector tile_sizes = test_utils::get_random_data(grid_size, 0, items_per_block, seed_value); @@ -1010,18 +921,9 @@ auto test_block_adjacent_difference() -> typename std::enable_if::t } // Preparing Device - T* d_input; - unsigned int* d_tile_sizes; - stored_type* d_output; - HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(input[0]))); - HIP_CHECK(hipMalloc(&d_tile_sizes, tile_sizes.size() * sizeof(tile_sizes[0]))); - HIP_CHECK(hipMalloc(&d_output, output.size() * sizeof(output[0]))); - HIP_CHECK(hipMemcpy( - d_input, input.data(), input.size() * sizeof(input[0]), hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_tile_sizes, - tile_sizes.data(), - tile_sizes.size() * sizeof(tile_sizes[0]), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_tile_sizes(tile_sizes); + test_utils::device_ptr d_output(size); // Running kernel hipLaunchKernelGGL(HIP_KERNEL_NAME(subtract_left_partial_kernel typename std::enable_if::t dim3(block_size), 0, 0, - d_input, - d_tile_sizes, - d_output); + d_input.get(), + d_tile_sizes.get(), + d_output.get()); HIP_CHECK(hipGetLastError()); // Reading results - HIP_CHECK(hipMemcpy( - output.data(), d_output, output.size() * sizeof(output[0]), hipMemcpyDeviceToHost)); + const auto output = d_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_near( output, expected, std::max(test_utils::precision, test_utils::precision))); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_tile_sizes)); - HIP_CHECK(hipFree(d_output)); } } -template +template auto test_block_adjacent_difference() -> typename std::enable_if::type { using stored_type = std::conditional_t::value, int, Output>; @@ -1090,8 +987,7 @@ auto test_block_adjacent_difference() -> typename std::enable_if::t SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); // Generate data - const std::vector input = test_utils::get_random_data(size, 0, 10, seed_value); - std::vector output(size); + const std::vector input = test_utils::get_random_data(size, 0, 10, seed_value); const std::vector tile_sizes = test_utils::get_random_data(grid_size, 0, items_per_block, seed_value); @@ -1123,18 +1019,9 @@ auto test_block_adjacent_difference() -> typename std::enable_if::t } // Preparing Device - T* d_input; - unsigned int* d_tile_sizes; - stored_type* d_output; - HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(input[0]))); - HIP_CHECK(hipMalloc(&d_tile_sizes, tile_sizes.size() * sizeof(tile_sizes[0]))); - HIP_CHECK(hipMalloc(&d_output, output.size() * sizeof(output[0]))); - HIP_CHECK(hipMemcpy( - d_input, input.data(), input.size() * sizeof(input[0]), hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_tile_sizes, - tile_sizes.data(), - tile_sizes.size() * sizeof(tile_sizes[0]), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_tile_sizes(tile_sizes); + test_utils::device_ptr d_output(size); // Running kernel hipLaunchKernelGGL(HIP_KERNEL_NAME(subtract_right_partial_kernel typename std::enable_if::t dim3(block_size), 0, 0, - d_input, - d_tile_sizes, - d_output); + d_input.get(), + d_tile_sizes.get(), + d_output.get()); HIP_CHECK(hipGetLastError()); // Reading results - HIP_CHECK(hipMemcpy( - output.data(), d_output, output.size() * sizeof(output[0]), hipMemcpyDeviceToHost)); + const auto output = d_output.load(); using is_add_op = test_utils::is_add_operator; // clang-format off @@ -1165,10 +1051,6 @@ auto test_block_adjacent_difference() -> typename std::enable_if::t ? 0 : test_utils::precision)); // clang-format on - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_tile_sizes)); - HIP_CHECK(hipFree(d_output)); } } diff --git a/test/rocprim/test_block_discontinuity.cpp.in b/test/rocprim/test_block_discontinuity.cpp.in index 239ced550..f857407d0 100644 --- a/test/rocprim/test_block_discontinuity.cpp.in +++ b/test/rocprim/test_block_discontinuity.cpp.in @@ -30,6 +30,7 @@ // required test headers #include "test_utils_types.hpp" +#include "test_utils_device_ptr.hpp" // kernel definitions #include "test_block_discontinuity.kernels.hpp" diff --git a/test/rocprim/test_block_discontinuity.kernels.hpp b/test/rocprim/test_block_discontinuity.kernels.hpp index 15dfd5aa4..81556fec0 100644 --- a/test/rocprim/test_block_discontinuity.kernels.hpp +++ b/test/rocprim/test_block_discontinuity.kernels.hpp @@ -238,7 +238,6 @@ auto test_block_discontinuity() // Generate data std::vector input = test_utils::get_random_data(size, 0, 10, seed_value); - std::vector heads(size); // Calculate expected results on host std::vector expected_heads(size); @@ -261,50 +260,30 @@ auto test_block_discontinuity() } // Preparing Device - type* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_input, input.size() * sizeof(typename decltype(input)::value_type))); - long long* device_heads; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_heads, heads.size() * sizeof(typename decltype(heads)::value_type))); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(type), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_heads(size); // Running kernel hipLaunchKernelGGL( HIP_KERNEL_NAME( - flag_heads_kernel< - type, flag_type, flag_op_type, - block_size, items_per_thread - > - ), - dim3(grid_size), dim3(block_size), 0, 0, - device_input, device_heads - ); + flag_heads_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_input.get(), + device_heads.get()); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Reading results - HIP_CHECK( - hipMemcpy( - heads.data(), device_heads, - heads.size() * sizeof(typename decltype(heads)::value_type), - hipMemcpyDeviceToHost - ) - ); + const auto heads = device_heads.load(); // Validating results for(size_t i = 0; i < size; i++) { ASSERT_EQ(heads[i], expected_heads[i]); } - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_heads)); } } @@ -356,7 +335,6 @@ auto test_block_discontinuity() // Generate data std::vector input = test_utils::get_random_data(size, 0, 10, seed_value); - std::vector tails(size); // Calculate expected results on host std::vector expected_tails(size); @@ -379,50 +357,30 @@ auto test_block_discontinuity() } // Preparing Device - type* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_input, input.size() * sizeof(typename decltype(input)::value_type))); - long long* device_tails; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_tails, tails.size() * sizeof(typename decltype(tails)::value_type))); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(type), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_tails(size); // Running kernel hipLaunchKernelGGL( HIP_KERNEL_NAME( - flag_tails_kernel< - type, flag_type, flag_op_type, - block_size, items_per_thread - > - ), - dim3(grid_size), dim3(block_size), 0, 0, - device_input, device_tails - ); + flag_tails_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_input.get(), + device_tails.get()); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Reading results - HIP_CHECK( - hipMemcpy( - tails.data(), device_tails, - tails.size() * sizeof(typename decltype(tails)::value_type), - hipMemcpyDeviceToHost - ) - ); + const auto tails = device_tails.load(); // Validating results for(size_t i = 0; i < size; i++) { ASSERT_EQ(tails[i], expected_tails[i]); } - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_tails)); } } @@ -474,8 +432,6 @@ auto test_block_discontinuity() // Generate data std::vector input = test_utils::get_random_data(size, 0, 10, seed_value); - std::vector heads(size); - std::vector tails(size); // Calculate expected results on host std::vector expected_heads(size); @@ -510,51 +466,29 @@ auto test_block_discontinuity() } // Preparing Device - type* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_input, input.size() * sizeof(typename decltype(input)::value_type))); - long long* device_heads; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_heads, tails.size() * sizeof(typename decltype(heads)::value_type))); - long long* device_tails; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_tails, tails.size() * sizeof(typename decltype(tails)::value_type))); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(type), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_heads(size); + test_utils::device_ptr device_tails(size); // Running kernel - hipLaunchKernelGGL( - HIP_KERNEL_NAME( - flag_heads_and_tails_kernel< - type, flag_type, flag_op_type, - block_size, items_per_thread - > - ), - dim3(grid_size), dim3(block_size), 0, 0, - device_input, device_heads, device_tails - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(flag_heads_and_tails_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_input.get(), + device_heads.get(), + device_tails.get()); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Reading results - HIP_CHECK( - hipMemcpy( - heads.data(), device_heads, - heads.size() * sizeof(typename decltype(heads)::value_type), - hipMemcpyDeviceToHost - ) - ); - - HIP_CHECK( - hipMemcpy( - tails.data(), device_tails, - tails.size() * sizeof(typename decltype(tails)::value_type), - hipMemcpyDeviceToHost - ) - ); + const auto heads = device_heads.load(); + const auto tails = device_tails.load(); // Validating results for(size_t i = 0; i < size; i++) @@ -562,10 +496,6 @@ auto test_block_discontinuity() ASSERT_EQ(heads[i], expected_heads[i]); ASSERT_EQ(tails[i], expected_tails[i]); } - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_heads)); - HIP_CHECK(hipFree(device_tails)); } } diff --git a/test/rocprim/test_block_exchange.cpp b/test/rocprim/test_block_exchange.cpp index 90b5754a5..b003265fc 100644 --- a/test/rocprim/test_block_exchange.cpp +++ b/test/rocprim/test_block_exchange.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2021 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -28,6 +28,7 @@ #include // required test headers +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" // kernel definitions diff --git a/test/rocprim/test_block_exchange.kernels.hpp b/test/rocprim/test_block_exchange.kernels.hpp index 17cba8ab3..dfce23141 100644 --- a/test/rocprim/test_block_exchange.kernels.hpp +++ b/test/rocprim/test_block_exchange.kernels.hpp @@ -198,7 +198,6 @@ auto test_block_exchange(int /*device_id*/) -> typename std::enable_if input(size); std::vector expected(size); - std::vector output(size, (output_type)0); // Calculate input and expected results on host std::vector values(size); @@ -223,42 +222,26 @@ auto test_block_exchange(int /*device_id*/) -> typename std::enable_if device_input(input); + test_utils::device_ptr device_output(size); // Running kernel constexpr unsigned int grid_size = (size / items_per_block); hipLaunchKernelGGL( - HIP_KERNEL_NAME(blocked_to_striped_kernel), - dim3(grid_size), dim3(block_size), 0, 0, - device_input, device_output - ); + HIP_KERNEL_NAME( + blocked_to_striped_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_input.get(), + device_output.get()); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Reading results - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(typename decltype(output)::value_type), - hipMemcpyDeviceToHost - ) - ); - + const auto output = device_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); } template typename std::enable_if input(size); std::vector expected(size); - std::vector output(size, output_type(0)); // Calculate input and expected results on host std::vector values(size); @@ -312,42 +294,26 @@ auto test_block_exchange(int /*device_id*/) -> typename std::enable_if device_input(input); + test_utils::device_ptr device_output(size); // Running kernel constexpr unsigned int grid_size = (size / items_per_block); hipLaunchKernelGGL( - HIP_KERNEL_NAME(striped_to_blocked_kernel), - dim3(grid_size), dim3(block_size), 0, 0, - device_input, device_output - ); + HIP_KERNEL_NAME( + striped_to_blocked_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_input.get(), + device_output.get()); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Reading results - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(typename decltype(output)::value_type), - hipMemcpyDeviceToHost - ) - ); - + const auto output = device_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); } template typename std::enable_if: // Generate data std::vector input(size); std::vector expected(size); - std::vector output(size, output_type(0)); unsigned int current_device_warp_size; HIP_CHECK(::rocprim::host_warp_size(device_id, current_device_warp_size)); @@ -414,44 +379,26 @@ auto test_block_exchange(int device_id) -> typename std::enable_if: } // Preparing device - type* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_input, input.size() * sizeof(typename decltype(input)::value_type))); - output_type* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(type), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(size); // Running kernel constexpr unsigned int grid_size = (size / items_per_block); hipLaunchKernelGGL( - HIP_KERNEL_NAME(blocked_to_warp_striped_kernel< - type, output_type, items_per_block, items_per_thread - >), - dim3(grid_size), dim3(block_size), 0, 0, - device_input, device_output - ); + HIP_KERNEL_NAME( + blocked_to_warp_striped_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_input.get(), + device_output.get()); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Reading results - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(typename decltype(output)::value_type), - hipMemcpyDeviceToHost - ) - ); - + const auto output = device_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); } template typename std::enable_if: // Generate data std::vector input(size); std::vector expected(size); - std::vector output(size, output_type(0)); unsigned int current_device_warp_size; HIP_CHECK(::rocprim::host_warp_size(device_id, current_device_warp_size)); @@ -518,42 +464,26 @@ auto test_block_exchange(int device_id) -> typename std::enable_if: } // Preparing device - type* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_input, input.size() * sizeof(typename decltype(input)::value_type))); - output_type* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(type), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(size); // Running kernel constexpr unsigned int grid_size = (size / items_per_block); hipLaunchKernelGGL( - HIP_KERNEL_NAME(warp_striped_to_blocked_kernel), - dim3(grid_size), dim3(block_size), 0, 0, - device_input, device_output - ); + HIP_KERNEL_NAME( + warp_striped_to_blocked_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_input.get(), + device_output.get()); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Reading results - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(typename decltype(output)::value_type), - hipMemcpyDeviceToHost - ) - ); - + const auto output = device_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); } template typename std::enable_if input(size); - std::vector expected(size); - std::vector output(size, output_type(0)); + std::vector expected(size); std::vector ranks(size); // Calculate input and expected results on host @@ -615,53 +544,28 @@ auto test_block_exchange(int /*device_id*/) -> typename std::enable_if device_input(input); + test_utils::device_ptr device_output(size); + test_utils::device_ptr device_ranks(ranks); // Running kernel constexpr unsigned int grid_size = (size / items_per_block); hipLaunchKernelGGL( - HIP_KERNEL_NAME(scatter_to_blocked_kernel), - dim3(grid_size), dim3(block_size), 0, 0, - device_input, device_output, device_ranks - ); + HIP_KERNEL_NAME( + scatter_to_blocked_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_input.get(), + device_output.get(), + device_ranks.get()); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Reading results - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(typename decltype(output)::value_type), - hipMemcpyDeviceToHost - ) - ); - + const auto output = device_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); - HIP_CHECK(hipFree(device_ranks)); } template typename std::enable_if input(size); - std::vector expected(size); - std::vector output(size, output_type(0)); + std::vector expected(size); std::vector ranks(size); // Calculate input and expected results on host @@ -725,53 +628,28 @@ auto test_block_exchange(int /*device_id*/) -> typename std::enable_if device_input(input); + test_utils::device_ptr device_output(size); + test_utils::device_ptr device_ranks(ranks); // Running kernel constexpr unsigned int grid_size = (size / items_per_block); hipLaunchKernelGGL( - HIP_KERNEL_NAME(scatter_to_striped_kernel), - dim3(grid_size), dim3(block_size), 0, 0, - device_input, device_output, device_ranks - ); + HIP_KERNEL_NAME( + scatter_to_striped_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_input.get(), + device_output.get(), + device_ranks.get()); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Reading results - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(typename decltype(output)::value_type), - hipMemcpyDeviceToHost - ) - ); - + const auto output = device_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); - HIP_CHECK(hipFree(device_ranks)); } // Static for-loop diff --git a/test/rocprim/test_block_load_store.cpp b/test/rocprim/test_block_load_store.cpp index ecdb92bca..d9b533b97 100644 --- a/test/rocprim/test_block_load_store.cpp +++ b/test/rocprim/test_block_load_store.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2021 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -28,6 +28,7 @@ // required test headers #include "test_utils.hpp" +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" // kernel definitions diff --git a/test/rocprim/test_block_load_store.hpp b/test/rocprim/test_block_load_store.hpp index 71fbf9597..1a5a7f1f1 100644 --- a/test/rocprim/test_block_load_store.hpp +++ b/test/rocprim/test_block_load_store.hpp @@ -67,7 +67,6 @@ typed_test_def(suite_name, name_suffix, LoadStoreClass) // Generate data std::vector input = test_utils::get_random_data(size, -100, 100, seed_value); - std::vector output(input.size(), (Type)0); // Calculate expected results on host std::vector expected(input.size(), (Type)0); @@ -81,46 +80,26 @@ typed_test_def(suite_name, name_suffix, LoadStoreClass) } // Preparing device - Type* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_input, input.size() * sizeof(typename decltype(input)::value_type))); - Type* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(typename decltype(input)::value_type), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(size); // Running kernel hipLaunchKernelGGL( HIP_KERNEL_NAME( - load_store_kernel< - Type, load_method, store_method, - block_size, items_per_thread - > - ), - dim3(grid_size), dim3(block_size), 0, 0, - device_input, device_output - ); + load_store_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_input.get(), + device_output.get()); HIP_CHECK(hipGetLastError()); // Reading results from device - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(typename decltype(output)::value_type), - hipMemcpyDeviceToHost - ) - ); + const auto output = device_output.load(); // Validating results ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); } } @@ -183,55 +162,30 @@ typed_test_def(suite_name, name_suffix, LoadStoreClassValid) } // Preparing device - Type* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_input, input.size() * sizeof(typename decltype(input)::value_type))); - Type* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(typename decltype(input)::value_type), - hipMemcpyHostToDevice - ) - ); - + test_utils::device_ptr device_input(input); // Have to initialize output for unvalid data to make sure they are not changed - HIP_CHECK( - hipMemcpy( - device_output, output.data(), - output.size() * sizeof(typename decltype(output)::value_type), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_output(output); // Running kernel - hipLaunchKernelGGL( - HIP_KERNEL_NAME( - load_store_valid_kernel< - Type, load_method, store_method, - block_size, items_per_thread - > - ), - dim3(grid_size), dim3(block_size), 0, 0, - device_input, device_output, valid - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(load_store_valid_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_input.get(), + device_output.get(), + valid); HIP_CHECK(hipGetLastError()); // Reading results from device - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(typename decltype(output)::value_type), - hipMemcpyDeviceToHost - ) - ); + output = device_output.load(); // Validating results ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); } } @@ -278,7 +232,6 @@ typed_test_def(suite_name, name_suffix, LoadStoreClassDefault) // Generate data std::vector input = test_utils::get_random_data(size, -100, 100, seed_value); - std::vector output(input.size(), (Type)0); // Calculate expected results on host std::vector expected(input.size(), _default); @@ -295,45 +248,29 @@ typed_test_def(suite_name, name_suffix, LoadStoreClassDefault) } // Preparing device - Type* device_input; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_input, input.size() * sizeof(typename decltype(input)::value_type))); - Type* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - - HIP_CHECK( - hipMemcpy( - device_input, input.data(), - input.size() * sizeof(typename decltype(input)::value_type), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(size); // Running kernel - hipLaunchKernelGGL( - HIP_KERNEL_NAME( - load_store_valid_default_kernel< - Type, load_method, store_method, - block_size, items_per_thread - > - ), - dim3(grid_size), dim3(block_size), 0, 0, - device_input, device_output, valid, _default - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(load_store_valid_default_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_input.get(), + device_output.get(), + valid, + _default); HIP_CHECK(hipGetLastError()); // Reading results from device - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(typename decltype(output)::value_type), - hipMemcpyDeviceToHost - ) - ); + const auto output = device_output.load(); // Validating results ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); } } diff --git a/test/rocprim/test_block_reduce.hpp b/test/rocprim/test_block_reduce.hpp index e7f736e70..105514d7f 100644 --- a/test/rocprim/test_block_reduce.hpp +++ b/test/rocprim/test_block_reduce.hpp @@ -73,26 +73,39 @@ typed_test_def(suite_name_single, name_suffix, Reduce) } // Preparing device - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(T))); - T* device_output_reductions; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output_reductions, output_reductions.size() * sizeof(T))); - - static_run_algo::run( - output, output_reductions, expected_reductions, - device_output, device_output_reductions, grid_size, false - ); - static_run_algo::run( - output, output_reductions, expected_reductions, - device_output, device_output_reductions, grid_size, false - ); - static_run_algo::run( - output, output_reductions, expected_reductions, - device_output, device_output_reductions, grid_size, false - ); - - HIP_CHECK(hipFree(device_output)); - HIP_CHECK(hipFree(device_output_reductions)); + test_utils::device_ptr device_output(output.size()); + test_utils::device_ptr device_output_reductions(output_reductions.size()); + + static_run_algo::run(output, + output_reductions, + expected_reductions, + device_output.get(), + device_output_reductions.get(), + grid_size, + false); + static_run_algo::run(output, + output_reductions, + expected_reductions, + device_output.get(), + device_output_reductions.get(), + grid_size, + false); + static_run_algo::run(output, + output_reductions, + expected_reductions, + device_output.get(), + device_output_reductions.get(), + grid_size, + false); } } @@ -139,11 +152,8 @@ typed_test_def(suite_name_single, name_suffix, ReduceMultiplies) } // Preparing device - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(T))); - T* device_output_reductions; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output_reductions, - output_reductions.size() * sizeof(T))); + test_utils::device_ptr device_output(output.size()); + test_utils::device_ptr device_output_reductions(output_reductions.size()); static_run_algo::run(output, output_reductions, expected_reductions, - device_output, - device_output_reductions, + device_output.get(), + device_output_reductions.get(), grid_size, false); static_run_algo::run(output, output_reductions, expected_reductions, - device_output, - device_output_reductions, + device_output.get(), + device_output_reductions.get(), grid_size, false); @@ -172,13 +182,10 @@ typed_test_def(suite_name_single, name_suffix, ReduceMultiplies) binary_op_type>::run(output, output_reductions, expected_reductions, - device_output, - device_output_reductions, + device_output.get(), + device_output_reductions.get(), grid_size, false); - - HIP_CHECK(hipFree(device_output)); - HIP_CHECK(hipFree(device_output_reductions)); } } @@ -230,27 +237,40 @@ typed_test_def(suite_name_single, name_suffix, ReduceMultipliesExact) } // Preparing device - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(T))); - T* device_output_reductions; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output_reductions, output_reductions.size() * sizeof(T))); - - static_run_algo::run( - output, output_reductions, expected_reductions, - device_output, device_output_reductions, grid_size, true - ); - static_run_algo::run( - output, output_reductions, expected_reductions, - device_output, device_output_reductions, grid_size, true - ); - - static_run_algo::run( - output, output_reductions, expected_reductions, - device_output, device_output_reductions, grid_size, true - ); - - HIP_CHECK(hipFree(device_output)); - HIP_CHECK(hipFree(device_output_reductions)); + test_utils::device_ptr device_output(output.size()); + test_utils::device_ptr device_output_reductions(output_reductions.size()); + + static_run_algo::run(output, + output_reductions, + expected_reductions, + device_output.get(), + device_output_reductions.get(), + grid_size, + true); + static_run_algo::run(output, + output_reductions, + expected_reductions, + device_output.get(), + device_output_reductions.get(), + grid_size, + true); + + static_run_algo::run(output, + output_reductions, + expected_reductions, + device_output.get(), + device_output_reductions.get(), + grid_size, + true); } } @@ -303,26 +323,39 @@ typed_test_def(suite_name_single, name_suffix, ReduceValid) } // Preparing device - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(T))); - T* device_output_reductions; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output_reductions, output_reductions.size() * sizeof(T))); - - static_run_valid::run( - output, output_reductions, expected_reductions, - device_output, device_output_reductions, valid_items, grid_size - ); - static_run_valid::run( - output, output_reductions, expected_reductions, - device_output, device_output_reductions, valid_items, grid_size - ); - static_run_valid::run( - output, output_reductions, expected_reductions, - device_output, device_output_reductions, valid_items, grid_size - ); - - HIP_CHECK(hipFree(device_output)); - HIP_CHECK(hipFree(device_output_reductions)); + test_utils::device_ptr device_output(output.size()); + test_utils::device_ptr device_output_reductions(output_reductions.size()); + + static_run_valid::run(output, + output_reductions, + expected_reductions, + device_output.get(), + device_output_reductions.get(), + valid_items, + grid_size); + static_run_valid::run(output, + output_reductions, + expected_reductions, + device_output.get(), + device_output_reductions.get(), + valid_items, + grid_size); + static_run_valid::run(output, + output_reductions, + expected_reductions, + device_output.get(), + device_output_reductions.get(), + valid_items, + grid_size); } } diff --git a/test/rocprim/test_utils_device_ptr.hpp b/test/rocprim/test_utils_device_ptr.hpp index f9a570978..6f78ec8c4 100644 --- a/test/rocprim/test_utils_device_ptr.hpp +++ b/test/rocprim/test_utils_device_ptr.hpp @@ -60,8 +60,8 @@ class device_ptr }; /// \brief Construct by host vectors with the same sized value_type - template - explicit device_ptr(std::vector const& data) + template + explicit device_ptr(std::vector const& data) : device_raw_ptr_(nullptr), number_of_ele_(data.size()) { static_assert( @@ -185,8 +185,8 @@ class device_ptr } /// \brief Copy from host to device - template - void store(std::vector const& host_vec, size_type offset = 0) + template + void store(std::vector const& host_vec, size_type offset = 0) { static_assert( sizeof(InVecValueType) == sizeof(value_type), From f6d0082a6a96c9b014217fa4ecbf8783d73f11a9 Mon Sep 17 00:00:00 2001 From: Cenxuan Tian Date: Tue, 17 Dec 2024 10:57:11 +0000 Subject: [PATCH 45/84] Resolve "Replace hipMalloc, hipMallocHelper, and hipFree in benchmarks and tests with device_ptr" Part 2 --- test/rocprim/test_block_scan.cpp.in | 3 +- test/rocprim/test_block_scan.hpp | 198 ++++++----- test/rocprim/test_block_scan.kernels.hpp | 320 +++++------------- test/rocprim/test_block_shuffle.cpp | 3 +- test/rocprim/test_block_shuffle.hpp | 184 +++------- test/rocprim/test_block_sort.hpp | 72 +--- test/rocprim/test_block_sort_bitonic.cpp | 3 +- test/rocprim/test_block_sort_merge.cpp | 3 +- test/rocprim/test_block_sort_merge_stable.cpp | 3 +- .../test_device_adjacent_difference.cpp | 78 ++--- test/rocprim/test_device_batch_memcpy.cpp | 78 ++--- test/rocprim/test_device_binary_search.cpp | 169 +++------ test/rocprim/test_device_find_end.cpp | 95 ++---- test/rocprim/test_device_find_first_of.cpp | 72 ++-- test/rocprim/test_device_histogram.cpp | 302 +++++++---------- 15 files changed, 535 insertions(+), 1048 deletions(-) diff --git a/test/rocprim/test_block_scan.cpp.in b/test/rocprim/test_block_scan.cpp.in index e58a4ca41..d2099e6e5 100644 --- a/test/rocprim/test_block_scan.cpp.in +++ b/test/rocprim/test_block_scan.cpp.in @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2022 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -28,6 +28,7 @@ #include // required test headers +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" // kernel definitions diff --git a/test/rocprim/test_block_scan.hpp b/test/rocprim/test_block_scan.hpp index f713d4081..82d123740 100644 --- a/test/rocprim/test_block_scan.hpp +++ b/test/rocprim/test_block_scan.hpp @@ -72,20 +72,27 @@ typed_test_def(suite_name_single, name_suffix, InclusiveScan) } // Writing to device memory - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); + test_utils::device_ptr device_output(output.size()); static_run_algo::run( - output, output, expected, expected, - device_output, NULL, T(0), grid_size - ); + output, + output, + expected, + expected, + device_output.get(), + NULL, + T(0), + grid_size); static_run_algo::run( - output2, output2, expected, expected, - device_output, NULL, T(0), grid_size - ); - - HIP_CHECK(hipFree(device_output)); + output2, + output2, + expected, + expected, + device_output.get(), + NULL, + T(0), + grid_size); } } @@ -139,28 +146,28 @@ typed_test_def(suite_name_single, name_suffix, InclusiveScanReduce) } // Writing to device memory - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - T* device_output_reductions; - HIP_CHECK( - test_common_utils::hipMallocHelper( - &device_output_reductions, - output_reductions.size() * sizeof(typename decltype(output_reductions)::value_type) - ) - ); + test_utils::device_ptr device_output(output.size()); + test_utils::device_ptr device_output_reductions(output_reductions.size()); static_run_algo::run( - output, output_reductions, expected, expected_reductions, - device_output, device_output_reductions, T(0), grid_size - ); + output, + output_reductions, + expected, + expected_reductions, + device_output.get(), + device_output_reductions.get(), + T(0), + grid_size); static_run_algo::run( - output2, output_reductions, expected, expected_reductions, - device_output, device_output_reductions, T(0), grid_size - ); - - HIP_CHECK(hipFree(device_output)); - HIP_CHECK(hipFree(device_output_reductions)); + output2, + output_reductions, + expected, + expected_reductions, + device_output.get(), + device_output_reductions.get(), + T(0), + grid_size); } } @@ -215,28 +222,28 @@ typed_test_def(suite_name_single, name_suffix, InclusiveScanPrefixCallback) } // Writing to device memory - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - T* device_output_bp; - HIP_CHECK( - test_common_utils::hipMallocHelper( - &device_output_bp, - output_block_prefixes.size() * sizeof(typename decltype(output_block_prefixes)::value_type) - ) - ); + test_utils::device_ptr device_output(output.size()); + test_utils::device_ptr device_output_bp(output_block_prefixes.size()); static_run_algo::run( - output, output_block_prefixes, expected, expected_block_prefixes, - device_output, device_output_bp, block_prefix, grid_size - ); + output, + output_block_prefixes, + expected, + expected_block_prefixes, + device_output.get(), + device_output_bp.get(), + block_prefix, + grid_size); static_run_algo::run( - output2, output_block_prefixes, expected, expected_block_prefixes, - device_output, device_output_bp, block_prefix, grid_size - ); - - HIP_CHECK(hipFree(device_output)); - HIP_CHECK(hipFree(device_output_bp)); + output2, + output_block_prefixes, + expected, + expected_block_prefixes, + device_output.get(), + device_output_bp.get(), + block_prefix, + grid_size); } } @@ -289,20 +296,27 @@ typed_test_def(suite_name_single, name_suffix, ExclusiveScan) } // Writing to device memory - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); + test_utils::device_ptr device_output(output.size()); static_run_algo::run( - output, output, expected, expected, - device_output, NULL, init, grid_size - ); + output, + output, + expected, + expected, + device_output.get(), + NULL, + init, + grid_size); static_run_algo::run( - output2, output2, expected, expected, - device_output, NULL, init, grid_size - ); - - HIP_CHECK(hipFree(device_output)); + output2, + output2, + expected, + expected, + device_output.get(), + NULL, + init, + grid_size); } } @@ -366,28 +380,28 @@ typed_test_def(suite_name_single, name_suffix, ExclusiveScanReduce) } // Writing to device memory - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - T* device_output_reductions; - HIP_CHECK( - test_common_utils::hipMallocHelper( - &device_output_reductions, - output_reductions.size() * sizeof(typename decltype(output_reductions)::value_type) - ) - ); + test_utils::device_ptr device_output(output.size()); + test_utils::device_ptr device_output_reductions(output_reductions.size()); static_run_algo::run( - output, output_reductions, expected, expected_reductions, - device_output, device_output_reductions, init, grid_size - ); + output, + output_reductions, + expected, + expected_reductions, + device_output.get(), + device_output_reductions.get(), + init, + grid_size); static_run_algo::run( - output2, output_reductions, expected, expected_reductions, - device_output, device_output_reductions, init, grid_size - ); - - HIP_CHECK(hipFree(device_output)); - HIP_CHECK(hipFree(device_output_reductions)); + output2, + output_reductions, + expected, + expected_reductions, + device_output.get(), + device_output_reductions.get(), + init, + grid_size); } } @@ -451,28 +465,28 @@ typed_test_def(suite_name_single, name_suffix, ExclusiveScanPrefixCallback) } // Writing to device memory - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - T* device_output_bp; - HIP_CHECK( - test_common_utils::hipMallocHelper( - &device_output_bp, - output_block_prefixes.size() * sizeof(typename decltype(output_block_prefixes)::value_type) - ) - ); + test_utils::device_ptr device_output(output.size()); + test_utils::device_ptr device_output_bp(output_block_prefixes.size()); static_run_algo::run( - output, output_block_prefixes, expected, expected_block_prefixes, - device_output, device_output_bp, block_prefix, grid_size - ); + output, + output_block_prefixes, + expected, + expected_block_prefixes, + device_output.get(), + device_output_bp.get(), + block_prefix, + grid_size); static_run_algo::run( - output2, output_block_prefixes, expected, expected_block_prefixes, - device_output, device_output_bp, block_prefix, grid_size - ); - - HIP_CHECK(hipFree(device_output)); - HIP_CHECK(hipFree(device_output_bp)); + output2, + output_block_prefixes, + expected, + expected_block_prefixes, + device_output.get(), + device_output_bp.get(), + block_prefix, + grid_size); } } diff --git a/test/rocprim/test_block_scan.kernels.hpp b/test/rocprim/test_block_scan.kernels.hpp index acde7be9b..baea8f02e 100644 --- a/test/rocprim/test_block_scan.kernels.hpp +++ b/test/rocprim/test_block_scan.kernels.hpp @@ -509,39 +509,28 @@ auto test_block_scan_input_arrays() } // Writing to device memory - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - - HIP_CHECK( - hipMemcpy( - device_output, output.data(), - output.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_output(output); // Launching kernel - hipLaunchKernelGGL( - HIP_KERNEL_NAME(inclusive_scan_array_kernel), - dim3(grid_size), dim3(block_size), 0, 0, - device_output - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(inclusive_scan_array_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_output.get()); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + output = device_output.load(); // Validating results ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); - HIP_CHECK(hipFree(device_output)); } } @@ -602,65 +591,32 @@ auto test_block_scan_input_arrays() } // Writing to device memory - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - T* device_output_reductions; - HIP_CHECK( - test_common_utils::hipMallocHelper( - &device_output_reductions, - output_reductions.size() * sizeof(typename decltype(output_reductions)::value_type) - ) - ); - - HIP_CHECK( - hipMemcpy( - device_output, output.data(), - output.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - - HIP_CHECK( - hipMemcpy( - device_output_reductions, output_reductions.data(), - output_reductions.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_output(output); + test_utils::device_ptr device_output_reductions(output_reductions); // Launching kernel - hipLaunchKernelGGL( - HIP_KERNEL_NAME(inclusive_scan_reduce_array_kernel), - dim3(grid_size), dim3(block_size), 0, 0, - device_output, device_output_reductions - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(inclusive_scan_reduce_array_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_output.get(), + device_output_reductions.get()); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); - - HIP_CHECK( - hipMemcpy( - output_reductions.data(), device_output_reductions, - output_reductions.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + output = device_output.load(); + output_reductions = device_output_reductions.load(); // Validating results ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output_reductions, expected_reductions)); - - HIP_CHECK(hipFree(device_output)); - HIP_CHECK(hipFree(device_output_reductions)); } } @@ -721,67 +677,35 @@ auto test_block_scan_input_arrays() } // Writing to device memory - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - T* device_output_bp; - HIP_CHECK( - test_common_utils::hipMallocHelper( - &device_output_bp, - output_block_prefixes.size() * sizeof(typename decltype(output_block_prefixes)::value_type) - ) - ); - - HIP_CHECK( - hipMemcpy( - device_output, output.data(), - output.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); - - HIP_CHECK( - hipMemcpy( - device_output_bp, output_block_prefixes.data(), - output_block_prefixes.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_output(output); + test_utils::device_ptr device_output_bp(output_block_prefixes); // Launching kernel hipLaunchKernelGGL( - HIP_KERNEL_NAME( - inclusive_scan_array_prefix_callback_kernel - ), - dim3(grid_size), dim3(block_size), 0, 0, - device_output, device_output_bp, block_prefix - ); + HIP_KERNEL_NAME(inclusive_scan_array_prefix_callback_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_output.get(), + device_output_bp.get(), + block_prefix); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); - - HIP_CHECK( - hipMemcpy( - output_block_prefixes.data(), device_output_bp, - output_block_prefixes.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + output = device_output.load(); + output_block_prefixes = device_output_bp.load(); // Validating results ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output_block_prefixes, expected_block_prefixes)); - - HIP_CHECK(hipFree(device_output)); - HIP_CHECK(hipFree(device_output_bp)); + ASSERT_NO_FATAL_FAILURE( + test_utils::assert_eq(output_block_prefixes, expected_block_prefixes)); } } @@ -839,40 +763,29 @@ auto test_block_scan_input_arrays() } // Writing to device memory - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - - HIP_CHECK( - hipMemcpy( - device_output, output.data(), - output.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_output(output); // Launching kernel - hipLaunchKernelGGL( - HIP_KERNEL_NAME(exclusive_scan_array_kernel), - dim3(grid_size), dim3(block_size), 0, 0, - device_output, init - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(exclusive_scan_array_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_output.get(), + init); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + output = device_output.load(); // Validating results ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); - - HIP_CHECK(hipFree(device_output)); } } @@ -939,58 +852,33 @@ auto test_block_scan_input_arrays() } // Writing to device memory - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - T* device_output_reductions; - HIP_CHECK( - test_common_utils::hipMallocHelper( - &device_output_reductions, - output_reductions.size() * sizeof(typename decltype(output_reductions)::value_type) - ) - ); - - HIP_CHECK( - hipMemcpy( - device_output, output.data(), - output.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_output(output); + test_utils::device_ptr device_output_reductions(output_reductions.size()); // Launching kernel - hipLaunchKernelGGL( - HIP_KERNEL_NAME( - exclusive_scan_reduce_array_kernel - ), - dim3(grid_size), dim3(block_size), 0, 0, - device_output, device_output_reductions, init - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(exclusive_scan_reduce_array_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_output.get(), + device_output_reductions.get(), + init); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); - - HIP_CHECK( - hipMemcpy( - output_reductions.data(), device_output_reductions, - output_reductions.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + output = device_output.load(); + output_reductions = device_output_reductions.load(); // Validating results ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output_reductions, expected_reductions)); - HIP_CHECK(hipFree(device_output)); - HIP_CHECK(hipFree(device_output_reductions)); } } @@ -1056,59 +944,35 @@ auto test_block_scan_input_arrays() } // Writing to device memory - T* device_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_output, output.size() * sizeof(typename decltype(output)::value_type))); - T* device_output_bp; - HIP_CHECK( - test_common_utils::hipMallocHelper( - &device_output_bp, - output_block_prefixes.size() * sizeof(typename decltype(output_block_prefixes)::value_type) - ) - ); - - HIP_CHECK( - hipMemcpy( - device_output, output.data(), - output.size() * sizeof(T), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_output(output); + test_utils::device_ptr device_output_bp(output_block_prefixes.size()); // Launching kernel hipLaunchKernelGGL( - HIP_KERNEL_NAME( - exclusive_scan_prefix_callback_array_kernel - ), - dim3(grid_size), dim3(block_size), 0, 0, - device_output, device_output_bp, block_prefix - ); + HIP_KERNEL_NAME(exclusive_scan_prefix_callback_array_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_output.get(), + device_output_bp.get(), + block_prefix); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); // Read from device memory - HIP_CHECK( - hipMemcpy( - output.data(), device_output, - output.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); - - HIP_CHECK( - hipMemcpy( - output_block_prefixes.data(), device_output_bp, - output_block_prefixes.size() * sizeof(T), - hipMemcpyDeviceToHost - ) - ); + output = device_output.load(); + output_block_prefixes = device_output_bp.load(); // Validating results ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output_block_prefixes, expected_block_prefixes)); - - HIP_CHECK(hipFree(device_output)); - HIP_CHECK(hipFree(device_output_bp)); + ASSERT_NO_FATAL_FAILURE( + test_utils::assert_eq(output_block_prefixes, expected_block_prefixes)); } } diff --git a/test/rocprim/test_block_shuffle.cpp b/test/rocprim/test_block_shuffle.cpp index aa1092bd3..b12551d28 100644 --- a/test/rocprim/test_block_shuffle.cpp +++ b/test/rocprim/test_block_shuffle.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2021 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -29,6 +29,7 @@ #include // required test headers +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" // kernel definitions diff --git a/test/rocprim/test_block_shuffle.hpp b/test/rocprim/test_block_shuffle.hpp index a01ecc67c..bfed55c30 100644 --- a/test/rocprim/test_block_shuffle.hpp +++ b/test/rocprim/test_block_shuffle.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -41,40 +41,26 @@ typed_test_def(suite_name, name_suffix, BlockOffset) SCOPED_TRACE(testing::Message() << "with seed = " << seed_value << ", distance = " << distance); // Generate data - std::vector input_data = test_utils::get_random_data(size, -100, 100, seed_value); - std::vector output_data(input_data); + std::vector input_data + = test_utils::get_random_data(size, -100, 100, seed_value); // Preparing device - type * device_input; - type * device_output; - - HIP_CHECK(hipMalloc(reinterpret_cast(&device_input), input_data.size() * sizeof(type))); - HIP_CHECK(hipMalloc(reinterpret_cast(&device_output), input_data.size() * sizeof(type))); - - HIP_CHECK( - hipMemcpy( - device_input, input_data.data(), - input_data.size() * sizeof(type), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input_data); + test_utils::device_ptr device_output(input_data.size()); // Running kernel - hipLaunchKernelGGL( - HIP_KERNEL_NAME(shuffle_offset_kernel), - dim3(grid_size), dim3(block_size), 0, 0, - device_input, device_output, distance - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(shuffle_offset_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_input.get(), + device_output.get(), + distance); HIP_CHECK(hipGetLastError()); // Reading results back - HIP_CHECK( - hipMemcpy( - output_data.data(), device_output, - output_data.size() * sizeof(type), - hipMemcpyDeviceToHost - ) - ); + const auto output_data = device_output.load(); // Calculate expected results on host for(size_t block_index = 0; block_index < grid_size; block_index++) @@ -88,10 +74,6 @@ typed_test_def(suite_name, name_suffix, BlockOffset) } } } - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); - } } @@ -113,40 +95,26 @@ typed_test_def(suite_name, name_suffix, BlockRotate) SCOPED_TRACE(testing::Message() << "with seed = " << seed_value << ", distance = " << distance); // Generate data - std::vector input_data = test_utils::get_random_data(size, -100, 100, seed_value); - std::vector output_data(input_data); + std::vector input_data + = test_utils::get_random_data(size, -100, 100, seed_value); // Preparing device - type * device_input; - type * device_output; - - HIP_CHECK(hipMalloc(reinterpret_cast(&device_input), input_data.size() * sizeof(type))); - HIP_CHECK(hipMalloc(reinterpret_cast(&device_output), input_data.size() * sizeof(type))); - - HIP_CHECK( - hipMemcpy( - device_input, input_data.data(), - input_data.size() * sizeof(type), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr device_input(input_data); + test_utils::device_ptr device_output(input_data.size()); // Running kernel - hipLaunchKernelGGL( - HIP_KERNEL_NAME(shuffle_rotate_kernel), - dim3(grid_size), dim3(block_size), 0, 0, - device_input, device_output, distance - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(shuffle_rotate_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_input.get(), + device_output.get(), + distance); HIP_CHECK(hipGetLastError()); // Reading results back - HIP_CHECK( - hipMemcpy( - output_data.data(), device_output, - output_data.size() * sizeof(type), - hipMemcpyDeviceToHost - ) - ); + const auto output_data = device_output.load(); // Calculate expected results on host for(size_t block_index = 0; block_index < grid_size; block_index++) @@ -159,10 +127,6 @@ typed_test_def(suite_name, name_suffix, BlockRotate) test_utils::assert_eq(input_data[block_index*block_size + offset],output_data[block_index*block_size + thread_index]); } } - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); - } } @@ -183,47 +147,28 @@ typed_test_def(suite_name, name_suffix, BlockUp) unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); // Generate data - std::vector input_data = test_utils::get_random_data(ItemsPerThread * size, -100, 100, seed_value); - std::vector output_data(input_data); + std::vector input_data + = test_utils::get_random_data(ItemsPerThread * size, -100, 100, seed_value); std::vector arr_input(size); std::vector arr_output(size); // Preparing device - type * device_input; - type * device_output; - - - HIP_CHECK(hipMalloc(reinterpret_cast(&device_input), input_data.size() * sizeof(type))); - HIP_CHECK(hipMalloc(reinterpret_cast(&device_output), input_data.size() * sizeof(type))); - - - - HIP_CHECK( - hipMemcpy( - device_input, input_data.data(), - input_data.size() * sizeof(type), - hipMemcpyHostToDevice - ) - ); - + test_utils::device_ptr device_input(input_data); + test_utils::device_ptr device_output(input_data.size()); // Running kernel - hipLaunchKernelGGL( - HIP_KERNEL_NAME(shuffle_up_kernel), - dim3(grid_size), dim3(block_size), 0, 0, - device_input, device_output - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(shuffle_up_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_input.get(), + device_output.get()); HIP_CHECK(hipGetLastError()); // Reading results back - HIP_CHECK( - hipMemcpy( - output_data.data(), device_output, - output_data.size() * sizeof(type), - hipMemcpyDeviceToHost - ) - ); + const auto output_data = device_output.load(); // Calculate expected results on host for(size_t block_index = 0; block_index < grid_size; block_index++) @@ -240,10 +185,6 @@ typed_test_def(suite_name, name_suffix, BlockUp) } } } - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); - } } @@ -265,47 +206,28 @@ typed_test_def(suite_name, name_suffix, BlockDown) SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); // Generate data - std::vector input_data = test_utils::get_random_data(ItemsPerThread * size, -100, 100, seed_value); - std::vector output_data(input_data); + std::vector input_data + = test_utils::get_random_data(ItemsPerThread * size, -100, 100, seed_value); std::vector arr_input(size); std::vector arr_output(size); // Preparing device - type * device_input; - type * device_output; - - - HIP_CHECK(hipMalloc(reinterpret_cast(&device_input), input_data.size() * sizeof(type))); - HIP_CHECK(hipMalloc(reinterpret_cast(&device_output), input_data.size() * sizeof(type))); - - - - HIP_CHECK( - hipMemcpy( - device_input, input_data.data(), - input_data.size() * sizeof(type), - hipMemcpyHostToDevice - ) - ); - + test_utils::device_ptr device_input(input_data); + test_utils::device_ptr device_output(input_data.size()); // Running kernel - hipLaunchKernelGGL( - HIP_KERNEL_NAME(shuffle_down_kernel), - dim3(grid_size), dim3(block_size), 0, 0, - device_input, device_output - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(shuffle_down_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_input.get(), + device_output.get()); HIP_CHECK(hipGetLastError()); // Reading results back - HIP_CHECK( - hipMemcpy( - output_data.data(), device_output, - output_data.size() * sizeof(type), - hipMemcpyDeviceToHost - ) - ); + const auto output_data = device_output.load(); // Calculate expected results on host for(size_t block_index = 0; block_index < grid_size; block_index++) @@ -322,10 +244,6 @@ typed_test_def(suite_name, name_suffix, BlockDown) } } } - - HIP_CHECK(hipFree(device_input)); - HIP_CHECK(hipFree(device_output)); - } } diff --git a/test/rocprim/test_block_sort.hpp b/test/rocprim/test_block_sort.hpp index 7757434be..a18bd9f88 100644 --- a/test/rocprim/test_block_sort.hpp +++ b/test/rocprim/test_block_sort.hpp @@ -87,22 +87,8 @@ void TestSortKeyValue() } // Preparing device - key_type* device_key_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_key_output, - output_key.size() * sizeof(key_type))); - value_type* device_value_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_value_output, - output_value.size() * sizeof(value_type))); - - HIP_CHECK(hipMemcpy(device_key_output, - output_key.data(), - output_key.size() * sizeof(key_type), - hipMemcpyHostToDevice)); - - HIP_CHECK(hipMemcpy(device_value_output, - output_value.data(), - output_value.size() * sizeof(value_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr device_key_output(output_key); + test_utils::device_ptr device_value_output(output_value); // Running kernel, ignored if invalid size if(size > 0) @@ -117,22 +103,15 @@ void TestSortKeyValue() dim3(block_size), 0, stream, - device_key_output, - device_value_output, + device_key_output.get(), + device_value_output.get(), size); HIP_CHECK(hipGetLastError()); } // Reading results back - HIP_CHECK(hipMemcpy(output_key.data(), - device_key_output, - output_key.size() * sizeof(key_type), - hipMemcpyDeviceToHost)); - - HIP_CHECK(hipMemcpy(output_value.data(), - device_value_output, - output_value.size() * sizeof(value_type), - hipMemcpyDeviceToHost)); + output_key = device_key_output.load(); + output_value = device_value_output.load(); std::vector expected_key(expected.size()); std::vector expected_value(expected.size()); @@ -160,9 +139,6 @@ void TestSortKeyValue() test_utils::assert_eq(output_key, expected_key); test_utils::assert_eq(output_value, expected_value); - - HIP_CHECK(hipFree(device_value_output)); - HIP_CHECK(hipFree(device_key_output)); } } @@ -215,14 +191,7 @@ void TestSortKey(std::vector sizes) } // Preparing device - key_type* device_key_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_key_output, - output.size() * sizeof(key_type))); - - HIP_CHECK(hipMemcpy(device_key_output, - output.data(), - output.size() * sizeof(key_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr device_key_output(output); const unsigned int grid_size = rocprim::detail::ceiling_div(size, items_per_block); // Running kernel, ignored if invalid size @@ -237,19 +206,14 @@ void TestSortKey(std::vector sizes) dim3(block_size), 0, stream, - device_key_output, + device_key_output.get(), size); } // Reading results back - HIP_CHECK(hipMemcpy(output.data(), - device_key_output, - output.size() * sizeof(key_type), - hipMemcpyDeviceToHost)); + output = device_key_output.load(); test_utils::assert_eq(output, expected); - - HIP_CHECK(hipFree(device_key_output)); } } } @@ -314,14 +278,7 @@ void TestSortStableKey(std::vector sizes) } // Preparing device - tuple_type* device_tuples_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_tuples_output, - tuples.size() * sizeof(tuple_type))); - - HIP_CHECK(hipMemcpy(device_tuples_output, - tuples.data(), - tuples.size() * sizeof(tuple_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr device_tuples_output(tuples); const unsigned int grid_size = rocprim::detail::ceiling_div(size, items_per_block); @@ -335,14 +292,11 @@ void TestSortStableKey(std::vector sizes) dim3(block_size), 0, stream, - device_tuples_output, + device_tuples_output.get(), size); // Reading results back - HIP_CHECK(hipMemcpy(tuples.data(), - device_tuples_output, - tuples.size() * sizeof(tuple_type), - hipMemcpyDeviceToHost)); + tuples = device_tuples_output.load(); // Calculate expected results on host binary_op_type binary_op; @@ -355,8 +309,6 @@ void TestSortStableKey(std::vector sizes) } test_utils::assert_eq(tuples, expected); - - HIP_CHECK(hipFree(device_tuples_output)); } } } diff --git a/test/rocprim/test_block_sort_bitonic.cpp b/test/rocprim/test_block_sort_bitonic.cpp index 75f70e6d8..04859cefd 100644 --- a/test/rocprim/test_block_sort_bitonic.cpp +++ b/test/rocprim/test_block_sort_bitonic.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2022 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -28,6 +28,7 @@ #include // required test headers +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" // kernel definitions diff --git a/test/rocprim/test_block_sort_merge.cpp b/test/rocprim/test_block_sort_merge.cpp index 935871da9..8c08e0103 100644 --- a/test/rocprim/test_block_sort_merge.cpp +++ b/test/rocprim/test_block_sort_merge.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -28,6 +28,7 @@ #include // required test headers +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" // kernel definitions diff --git a/test/rocprim/test_block_sort_merge_stable.cpp b/test/rocprim/test_block_sort_merge_stable.cpp index 1439d58bc..e142f8d61 100644 --- a/test/rocprim/test_block_sort_merge_stable.cpp +++ b/test/rocprim/test_block_sort_merge_stable.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2023-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -28,6 +28,7 @@ #include // required test headers +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" // kernel definitions diff --git a/test/rocprim/test_device_adjacent_difference.cpp b/test/rocprim/test_device_adjacent_difference.cpp index f0df7d644..95b93f754 100644 --- a/test/rocprim/test_device_adjacent_difference.cpp +++ b/test/rocprim/test_device_adjacent_difference.cpp @@ -23,6 +23,7 @@ #include "../common_test_header.hpp" #include "indirect_iterator.hpp" +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" #include @@ -332,31 +333,26 @@ TYPED_TEST(RocprimDeviceAdjacentDifferenceTests, AdjacentDifference) // Default stream does not support hipGraph stream capture, so create one HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - + SCOPED_TRACE(testing::Message() << "with size = " << size); // Generate data std::vector input = test_utils::get_random_data(size, 1, 100, seed_value); - T* d_input; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(input[0]))); - HIP_CHECK(hipMemcpy( - d_input, input.data(), input.size() * sizeof(input[0]), hipMemcpyHostToDevice)); + test_utils::device_ptr d_input(input); static constexpr auto left_tag = rocprim::detail::bool_constant{}; static constexpr auto alias_tag = std::integral_constant{}; auto input_it = test_utils::wrap_in_indirect_iterator( - d_input); + d_input.get()); // Allocate temporary storage std::size_t temp_storage_size; - void* d_temp_storage = nullptr; HIP_CHECK(dispatch_adjacent_difference(left_tag, alias_tag, - d_temp_storage, + nullptr, temp_storage_size, input_it, (output_type*){nullptr}, @@ -367,7 +363,7 @@ TYPED_TEST(RocprimDeviceAdjacentDifferenceTests, AdjacentDifference) ASSERT_GT(temp_storage_size, 0); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size)); + test_utils::device_ptr d_temp_storage(temp_storage_size); test_utils::GraphHelper gHelper; @@ -385,7 +381,7 @@ TYPED_TEST(RocprimDeviceAdjacentDifferenceTests, AdjacentDifference) // Run HIP_CHECK(dispatch_adjacent_difference(left_tag, alias_tag, - d_temp_storage, + d_temp_storage.get(), temp_storage_size, input_it, output_it, @@ -431,30 +427,24 @@ TYPED_TEST(RocprimDeviceAdjacentDifferenceTests, AdjacentDifference) // if api_variant is not in_place we should check the non aliased function call if(aliasing != api_variant::in_place) { - output_type* d_output = nullptr; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, size * sizeof(*d_output))); + test_utils::device_ptr d_output(size); const auto output_it - = test_utils::wrap_in_identity_iterator(d_output); + = test_utils::wrap_in_identity_iterator(d_output.get()); - ASSERT_NO_FATAL_FAILURE(run_and_verify(output_it, d_output)); - - HIP_CHECK(hipFree(d_output)); + ASSERT_NO_FATAL_FAILURE(run_and_verify(output_it, d_output.get())); } // if api_variant is not no_alias we should check the inplace function call if(aliasing != api_variant::no_alias) { - ASSERT_NO_FATAL_FAILURE(run_and_verify(input_it, d_input)); + ASSERT_NO_FATAL_FAILURE(run_and_verify(input_it, d_input.get())); } if(TestFixture::use_graphs) { HIP_CHECK(hipStreamDestroy(stream)); } - - HIP_CHECK(hipFree(d_temp_storage)); - HIP_CHECK(hipFree(d_input)); } } } @@ -633,18 +623,20 @@ TYPED_TEST(RocprimDeviceAdjacentDifferenceLargeTests, LargeIndices) { SCOPED_TRACE(testing::Message() << "with size = " << size); - flag_type* d_incorrect_flag; - size_t* d_counter; + test_utils::device_ptr d_incorrect_flag(1); + HIP_CHECK(hipMemset(d_incorrect_flag.get(), + 0, + sizeof(typename decltype(d_incorrect_flag)::value_type))); + + test_utils::device_ptr d_counter(1); HIP_CHECK( - test_common_utils::hipMallocHelper(&d_incorrect_flag, sizeof(*d_incorrect_flag))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_counter, sizeof(*d_counter))); - HIP_CHECK(hipMemset(d_incorrect_flag, 0, sizeof(*d_incorrect_flag))); - HIP_CHECK(hipMemset(d_counter, 0, sizeof(*d_counter))); + hipMemset(d_counter.get(), 0, sizeof(typename decltype(d_counter)::value_type))); + // Note: hipMemset runs asynchronously unless the pointer it's passed refers to pinned memory. // Wait for it to complete here before we use the device variables we just set. HIP_CHECK(hipDeviceSynchronize()); - OutputIterator output(d_incorrect_flag, d_counter); + OutputIterator output(d_incorrect_flag.get(), d_counter.get()); const auto input = rocprim::make_counting_iterator(T{0}); @@ -659,10 +651,9 @@ TYPED_TEST(RocprimDeviceAdjacentDifferenceLargeTests, LargeIndices) // Allocate temporary storage std::size_t temp_storage_size; - void* d_temp_storage = nullptr; HIP_CHECK(dispatch_adjacent_difference(left_tag, aliasing_tag, - d_temp_storage, + nullptr, temp_storage_size, input, output, @@ -673,7 +664,7 @@ TYPED_TEST(RocprimDeviceAdjacentDifferenceLargeTests, LargeIndices) ASSERT_GT(temp_storage_size, 0); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size)); + test_utils::device_ptr d_temp_storage(temp_storage_size); test_utils::GraphHelper gHelper;; if(TestFixture::use_graphs) @@ -682,13 +673,19 @@ TYPED_TEST(RocprimDeviceAdjacentDifferenceLargeTests, LargeIndices) } // Capture the memset in the graph so that relaunching will have expected result - HIP_CHECK(hipMemsetAsync(d_incorrect_flag, 0, sizeof(*d_incorrect_flag), stream)); - HIP_CHECK(hipMemsetAsync(d_counter, 0, sizeof(*d_counter), stream)); + HIP_CHECK(hipMemsetAsync(d_incorrect_flag.get(), + 0, + sizeof(typename decltype(d_incorrect_flag)::value_type), + stream)); + HIP_CHECK(hipMemsetAsync(d_counter.get(), + 0, + sizeof(typename decltype(d_counter)::value_type), + stream)); // Run HIP_CHECK(dispatch_adjacent_difference(left_tag, aliasing_tag, - d_temp_storage, + d_temp_storage.get(), temp_storage_size, input, output, @@ -703,21 +700,12 @@ TYPED_TEST(RocprimDeviceAdjacentDifferenceLargeTests, LargeIndices) } // Copy output to host - flag_type incorrect_flag; - size_t counter; - HIP_CHECK(hipMemcpy(&incorrect_flag, - d_incorrect_flag, - sizeof(incorrect_flag), - hipMemcpyDeviceToHost)); - HIP_CHECK(hipMemcpy(&counter, d_counter, sizeof(counter), hipMemcpyDeviceToHost)); + const auto incorrect_flag = d_incorrect_flag.load()[0]; + const auto counter = d_counter.load()[0]; ASSERT_EQ(incorrect_flag, 0); ASSERT_EQ(counter, rocprim::detail::ceiling_div(size, sampling_rate)); - HIP_CHECK(hipFree(d_temp_storage)); - HIP_CHECK(hipFree(d_incorrect_flag)); - HIP_CHECK(hipFree(d_counter)); - if(TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); diff --git a/test/rocprim/test_device_batch_memcpy.cpp b/test/rocprim/test_device_batch_memcpy.cpp index 0736e648a..d97db5c0c 100644 --- a/test/rocprim/test_device_batch_memcpy.cpp +++ b/test/rocprim/test_device_batch_memcpy.cpp @@ -25,6 +25,7 @@ #include "test_utils_assertions.hpp" #include "test_utils_custom_test_types.hpp" #include "test_utils_data_generation.hpp" +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" #include "rocprim/detail/various.hpp" @@ -397,40 +398,28 @@ TYPED_TEST(RocprimDeviceBatchMemcpyTests, SizeAndTypeVariation) // Shuffle the sizes so that size classes aren't clustered std::shuffle(h_buffer_num_elements.begin(), h_buffer_num_elements.end(), rng); - // And the total byte size const byte_offset_type total_num_bytes = total_num_elements * sizeof(value_type); - // Device pointers - value_type* d_input = nullptr; - value_type* d_output = nullptr; - value_type** d_buffer_srcs = nullptr; - value_type** d_buffer_dsts = nullptr; - buffer_size_type* d_buffer_sizes = nullptr; + // Allocate memory. + test_utils::device_ptr d_input(total_num_elements); + test_utils::device_ptr d_output(total_num_elements); - // Calculate temporary storage + test_utils::device_ptr d_buffer_srcs(num_buffers); + test_utils::device_ptr d_buffer_dsts(num_buffers); + test_utils::device_ptr d_buffer_sizes(num_buffers); + // Calculate temporary storage size_t temp_storage_bytes = 0; - batch_copy(nullptr, temp_storage_bytes, - d_buffer_srcs, - d_buffer_dsts, - d_buffer_sizes, + d_buffer_srcs.get(), + d_buffer_dsts.get(), + d_buffer_sizes.get(), num_buffers, hipStreamDefault, debug_synchronous); - void* d_temp_storage = nullptr; - - // Allocate memory. - HIP_CHECK(hipMalloc(&d_input, total_num_bytes)); - HIP_CHECK(hipMalloc(&d_output, total_num_bytes)); - - HIP_CHECK(hipMalloc(&d_buffer_srcs, num_buffers * sizeof(*d_buffer_srcs))); - HIP_CHECK(hipMalloc(&d_buffer_dsts, num_buffers * sizeof(*d_buffer_dsts))); - HIP_CHECK(hipMalloc(&d_buffer_sizes, num_buffers * sizeof(*d_buffer_sizes))); - - HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_bytes); // Generate data. std::vector h_input_for_memcpy; @@ -474,54 +463,48 @@ TYPED_TEST(RocprimDeviceBatchMemcpyTests, SizeAndTypeVariation) for(int i = 0; i < num_buffers; ++i) { - h_buffer_srcs[i] = d_input + src_offsets[i]; - h_buffer_dsts[i] = d_output + dst_offsets[i]; + h_buffer_srcs[i] = d_input.get() + src_offsets[i]; + h_buffer_dsts[i] = d_output.get() + dst_offsets[i]; } // Prepare the batch memcpy. if(isMemCpy) { - HIP_CHECK(hipMemcpy(d_input, + HIP_CHECK(hipMemcpy(d_input.get(), h_input_for_memcpy.data(), total_num_bytes, hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_buffer_sizes, + HIP_CHECK(hipMemcpy(d_buffer_sizes.get(), h_buffer_num_bytes.data(), - h_buffer_num_bytes.size() * sizeof(*d_buffer_sizes), + h_buffer_num_bytes.size() * sizeof(buffer_size_type), hipMemcpyHostToDevice)); } else { - HIP_CHECK(hipMemcpy(d_input, + HIP_CHECK(hipMemcpy(d_input.get(), h_input_for_copy.data(), total_num_bytes, hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_buffer_sizes, + HIP_CHECK(hipMemcpy(d_buffer_sizes.get(), h_buffer_num_elements.data(), - h_buffer_num_elements.size() * sizeof(*d_buffer_sizes), + h_buffer_num_elements.size() * sizeof(buffer_size_type), hipMemcpyHostToDevice)); } - HIP_CHECK(hipMemcpy(d_buffer_srcs, - h_buffer_srcs.data(), - h_buffer_srcs.size() * sizeof(*d_buffer_srcs), - hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_buffer_dsts, - h_buffer_dsts.data(), - h_buffer_dsts.size() * sizeof(*d_buffer_dsts), - hipMemcpyHostToDevice)); + d_buffer_srcs.store(h_buffer_srcs); + d_buffer_dsts.store(h_buffer_dsts); const auto input_src_it - = test_utils::wrap_in_indirect_iterator(d_buffer_srcs); + = test_utils::wrap_in_indirect_iterator(d_buffer_srcs.get()); const auto output_src_it - = test_utils::wrap_in_indirect_iterator(d_buffer_dsts); + = test_utils::wrap_in_indirect_iterator(d_buffer_dsts.get()); // Run batched memcpy. - batch_copy(d_temp_storage, + batch_copy(d_temp_storage.get(), temp_storage_bytes, input_src_it, output_src_it, - d_buffer_sizes, + d_buffer_sizes.get(), num_buffers, hipStreamDefault, debug_synchronous); @@ -529,20 +512,13 @@ TYPED_TEST(RocprimDeviceBatchMemcpyTests, SizeAndTypeVariation) // Verify results. check_result(h_input_for_memcpy, h_input_for_copy, - d_output, + d_output.get(), total_num_bytes, total_num_elements, num_buffers, src_offsets, dst_offsets, h_buffer_num_bytes); - - HIP_CHECK(hipFree(d_temp_storage)); - HIP_CHECK(hipFree(d_buffer_sizes)); - HIP_CHECK(hipFree(d_buffer_dsts)); - HIP_CHECK(hipFree(d_buffer_srcs)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_input)); } } diff --git a/test/rocprim/test_device_binary_search.cpp b/test/rocprim/test_device_binary_search.cpp index 1a5f8b362..1f3adb2d8 100644 --- a/test/rocprim/test_device_binary_search.cpp +++ b/test/rocprim/test_device_binary_search.cpp @@ -27,6 +27,7 @@ #include // required test headers +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" template d_haystack(haystack); + test_utils::device_ptr d_needles(needles); + test_utils::device_ptr d_output(needles_size); // Calculate expected results on host std::vector expected(needles_size); @@ -157,13 +141,12 @@ TYPED_TEST(RocprimDeviceBinarySearch, LowerBound) haystack.begin(); } - void * d_temporary_storage = nullptr; size_t temporary_storage_bytes; - HIP_CHECK(rocprim::lower_bound(d_temporary_storage, + HIP_CHECK(rocprim::lower_bound(nullptr, temporary_storage_bytes, - d_haystack, - d_needles, - d_output, + d_haystack.get(), + d_needles.get(), + d_output.get(), haystack_size, needles_size, compare_op, @@ -172,7 +155,7 @@ TYPED_TEST(RocprimDeviceBinarySearch, LowerBound) ASSERT_GT(temporary_storage_bytes, 0); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::params::use_graphs) @@ -180,11 +163,11 @@ TYPED_TEST(RocprimDeviceBinarySearch, LowerBound) gHelper.startStreamCapture(stream); } - HIP_CHECK(rocprim::lower_bound(d_temporary_storage, + HIP_CHECK(rocprim::lower_bound(d_temporary_storage.get(), temporary_storage_bytes, - d_haystack, - d_needles, - d_output, + d_haystack.get(), + d_needles.get(), + d_output.get(), haystack_size, needles_size, compare_op, @@ -196,19 +179,7 @@ TYPED_TEST(RocprimDeviceBinarySearch, LowerBound) gHelper.createAndLaunchGraph(stream); } - std::vector output(needles_size); - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - needles_size * sizeof(output_type), - hipMemcpyDeviceToHost - ) - ); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_haystack)); - HIP_CHECK(hipFree(d_needles)); - HIP_CHECK(hipFree(d_output)); + const auto output = d_output.load(); if(TestFixture::params::use_graphs) { @@ -272,26 +243,9 @@ TYPED_TEST(RocprimDeviceBinarySearch, UpperBound) needles_size, d, haystack_size + d, seed_value ); - haystack_type * d_haystack; - needle_type * d_needles; - output_type * d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_haystack, haystack_size * sizeof(haystack_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_needles, needles_size * sizeof(needle_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, needles_size * sizeof(output_type))); - HIP_CHECK( - hipMemcpy( - d_haystack, haystack.data(), - haystack_size * sizeof(haystack_type), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_needles, needles.data(), - needles_size * sizeof(needle_type), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr d_haystack(haystack); + test_utils::device_ptr d_needles(needles); + test_utils::device_ptr d_output(needles_size); // Calculate expected results on host std::vector expected(needles_size); @@ -302,13 +256,12 @@ TYPED_TEST(RocprimDeviceBinarySearch, UpperBound) haystack.begin(); } - void * d_temporary_storage = nullptr; size_t temporary_storage_bytes; - HIP_CHECK(rocprim::upper_bound(d_temporary_storage, + HIP_CHECK(rocprim::upper_bound(nullptr, temporary_storage_bytes, - d_haystack, - d_needles, - d_output, + d_haystack.get(), + d_needles.get(), + d_output.get(), haystack_size, needles_size, compare_op, @@ -317,7 +270,7 @@ TYPED_TEST(RocprimDeviceBinarySearch, UpperBound) ASSERT_GT(temporary_storage_bytes, 0); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::params::use_graphs) @@ -325,11 +278,11 @@ TYPED_TEST(RocprimDeviceBinarySearch, UpperBound) gHelper.startStreamCapture(stream); } - HIP_CHECK(rocprim::upper_bound(d_temporary_storage, + HIP_CHECK(rocprim::upper_bound(d_temporary_storage.get(), temporary_storage_bytes, - d_haystack, - d_needles, - d_output, + d_haystack.get(), + d_needles.get(), + d_output.get(), haystack_size, needles_size, compare_op, @@ -341,19 +294,7 @@ TYPED_TEST(RocprimDeviceBinarySearch, UpperBound) gHelper.createAndLaunchGraph(stream); } - std::vector output(needles_size); - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - needles_size * sizeof(output_type), - hipMemcpyDeviceToHost - ) - ); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_haystack)); - HIP_CHECK(hipFree(d_needles)); - HIP_CHECK(hipFree(d_output)); + const auto output = d_output.load(); if(TestFixture::params::use_graphs) { @@ -420,26 +361,9 @@ TYPED_TEST(RocprimDeviceBinarySearch, BinarySearch) needles_size, d, haystack_size + d, seed_value ); - haystack_type * d_haystack; - needle_type * d_needles; - output_type * d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_haystack, haystack_size * sizeof(haystack_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_needles, needles_size * sizeof(needle_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, needles_size * sizeof(output_type))); - HIP_CHECK( - hipMemcpy( - d_haystack, haystack.data(), - haystack_size * sizeof(haystack_type), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_needles, needles.data(), - needles_size * sizeof(needle_type), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr d_haystack(haystack); + test_utils::device_ptr d_needles(needles); + test_utils::device_ptr d_output(needles_size); // Calculate expected results on host std::vector expected(needles_size); @@ -448,13 +372,12 @@ TYPED_TEST(RocprimDeviceBinarySearch, BinarySearch) expected[i] = std::binary_search(haystack.begin(), haystack.end(), needles[i], compare_op); } - void * d_temporary_storage = nullptr; size_t temporary_storage_bytes; - HIP_CHECK(rocprim::binary_search(d_temporary_storage, + HIP_CHECK(rocprim::binary_search(nullptr, temporary_storage_bytes, - d_haystack, - d_needles, - d_output, + d_haystack.get(), + d_needles.get(), + d_output.get(), haystack_size, needles_size, compare_op, @@ -463,7 +386,7 @@ TYPED_TEST(RocprimDeviceBinarySearch, BinarySearch) ASSERT_GT(temporary_storage_bytes, 0); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::params::use_graphs) @@ -471,11 +394,11 @@ TYPED_TEST(RocprimDeviceBinarySearch, BinarySearch) gHelper.startStreamCapture(stream); } - HIP_CHECK(rocprim::binary_search(d_temporary_storage, + HIP_CHECK(rocprim::binary_search(d_temporary_storage.get(), temporary_storage_bytes, - d_haystack, - d_needles, - d_output, + d_haystack.get(), + d_needles.get(), + d_output.get(), haystack_size, needles_size, compare_op, @@ -487,19 +410,7 @@ TYPED_TEST(RocprimDeviceBinarySearch, BinarySearch) gHelper.createAndLaunchGraph(stream); } - std::vector output(needles_size); - HIP_CHECK( - hipMemcpy( - output.data(), d_output, - needles_size * sizeof(output_type), - hipMemcpyDeviceToHost - ) - ); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_haystack)); - HIP_CHECK(hipFree(d_needles)); - HIP_CHECK(hipFree(d_output)); + const auto output = d_output.load(); if(TestFixture::params::use_graphs) { diff --git a/test/rocprim/test_device_find_end.cpp b/test/rocprim/test_device_find_end.cpp index 64da6f00e..9a3039bea 100644 --- a/test/rocprim/test_device_find_end.cpp +++ b/test/rocprim/test_device_find_end.cpp @@ -26,6 +26,7 @@ #include "test_utils_custom_float_type.hpp" #include "test_utils_custom_test_types.hpp" #include "test_utils_data_generation.hpp" +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" #include "../common_test_header.hpp" @@ -180,41 +181,22 @@ TYPED_TEST(RocprimDeviceFindEndTests, FindEnd) keys.assign(input.begin() + pattern, input.end()); } - value_type* d_input; - key_type* d_keys; - index_type* d_output; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(*d_input))); - - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_keys, keys.size() * sizeof(*d_keys))); - - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, sizeof(*d_output))); - - HIP_CHECK(hipMemcpy(d_input, - input.data(), - input.size() * sizeof(*d_input), - hipMemcpyHostToDevice)); - - HIP_CHECK(hipMemcpy(d_keys, - keys.data(), - keys.size() * sizeof(*d_keys), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_keys(keys); + test_utils::device_ptr d_output(1); const auto input_it - = test_utils::wrap_in_indirect_iterator(d_input); + = test_utils::wrap_in_indirect_iterator(d_input.get()); const auto keys_it - = test_utils::wrap_in_indirect_iterator(d_keys); + = test_utils::wrap_in_indirect_iterator(d_keys.get()); const auto output_keys - = test_utils::wrap_in_indirect_iterator(d_output); + = test_utils::wrap_in_indirect_iterator(d_output.get()); // compare function compare_function compare_op; - // temp storage - size_t temp_storage_size_bytes; - void* d_temp_storage = nullptr; // Get size of d_temp_storage + size_t temp_storage_size_bytes; HIP_CHECK(rocprim::find_end(nullptr, temp_storage_size_bytes, input_it, @@ -230,8 +212,7 @@ TYPED_TEST(RocprimDeviceFindEndTests, FindEnd) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) @@ -240,7 +221,7 @@ TYPED_TEST(RocprimDeviceFindEndTests, FindEnd) } // Run - HIP_CHECK(rocprim::find_end(d_temp_storage, + HIP_CHECK(rocprim::find_end(d_temp_storage.get(), temp_storage_size_bytes, input_it, keys_it, @@ -259,9 +240,8 @@ TYPED_TEST(RocprimDeviceFindEndTests, FindEnd) HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); - index_type output; // Copy output to host - HIP_CHECK(hipMemcpy(&output, d_output, sizeof(*d_output), hipMemcpyDeviceToHost)); + const auto output = d_output.load()[0]; index_type expected = std::find_end(input.begin(), input.end(), @@ -272,11 +252,6 @@ TYPED_TEST(RocprimDeviceFindEndTests, FindEnd) ASSERT_EQ(output, expected); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_keys)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if(TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -347,41 +322,23 @@ TYPED_TEST(RocprimDeviceFindEndTests, FindEndRepetition) std::copy(keys.begin(), keys.end(), input.begin() + i * key_size); } - value_type* d_input; - key_type* d_keys; - index_type* d_output; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(*d_input))); - - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys, keys.size() * sizeof(*d_keys))); - - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, sizeof(*d_output))); - - HIP_CHECK(hipMemcpy(d_input, - input.data(), - input.size() * sizeof(*d_input), - hipMemcpyHostToDevice)); - - HIP_CHECK(hipMemcpy(d_keys, - keys.data(), - keys.size() * sizeof(*d_keys), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_keys(keys); + test_utils::device_ptr d_output(1); const auto input_it - = test_utils::wrap_in_indirect_iterator(d_input); + = test_utils::wrap_in_indirect_iterator(d_input.get()); // compare function compare_function compare_op; - // temp storage - size_t temp_storage_size_bytes; - void* d_temp_storage = nullptr; // Get size of d_temp_storage + size_t temp_storage_size_bytes; HIP_CHECK(rocprim::find_end(nullptr, temp_storage_size_bytes, input_it, - d_keys, - d_output, + d_keys.get(), + d_output.get(), input.size(), keys.size(), compare_op, @@ -392,7 +349,7 @@ TYPED_TEST(RocprimDeviceFindEndTests, FindEndRepetition) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) @@ -401,11 +358,11 @@ TYPED_TEST(RocprimDeviceFindEndTests, FindEndRepetition) } // Run - HIP_CHECK(rocprim::find_end(d_temp_storage, + HIP_CHECK(rocprim::find_end(d_temp_storage.get(), temp_storage_size_bytes, input_it, - d_keys, - d_output, + d_keys.get(), + d_output.get(), input.size(), keys.size(), compare_op, @@ -420,9 +377,8 @@ TYPED_TEST(RocprimDeviceFindEndTests, FindEndRepetition) HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); - index_type output; // Copy output to host - HIP_CHECK(hipMemcpy(&output, d_output, sizeof(*d_output), hipMemcpyDeviceToHost)); + const auto output = d_output.load()[0]; index_type expected = std::find_end(input.begin(), input.end(), keys.begin(), keys.end(), compare_op) @@ -430,11 +386,6 @@ TYPED_TEST(RocprimDeviceFindEndTests, FindEndRepetition) ASSERT_EQ(output, expected); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_keys)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if(TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); diff --git a/test/rocprim/test_device_find_first_of.cpp b/test/rocprim/test_device_find_first_of.cpp index 98314cab6..e1544567c 100644 --- a/test/rocprim/test_device_find_first_of.cpp +++ b/test/rocprim/test_device_find_first_of.cpp @@ -26,6 +26,7 @@ #include "test_utils_custom_float_type.hpp" #include "test_utils_custom_test_types.hpp" #include "test_utils_data_generation.hpp" +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" #include "../common_test_header.hpp" @@ -205,43 +206,27 @@ TYPED_TEST(RocprimDeviceFindFirstOfTests, FindFirstOf) } } - type* d_input; - key_type* d_keys; - output_type* d_output; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(*d_input))); - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_keys, keys.size() * sizeof(*d_keys))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, sizeof(*d_output))); - - HIP_CHECK(hipMemcpy(d_input, - input.data(), - input.size() * sizeof(*d_input), - hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_keys, - keys.data(), - keys.size() * sizeof(*d_keys), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_keys(keys); + test_utils::device_ptr d_output(1); const auto input_it = test_utils::wrap_in_indirect_iterator( - d_input); + d_input.get()); const auto keys_it = test_utils::wrap_in_indirect_iterator( - d_keys); + d_keys.get()); // compare function compare_function compare_op; - // temp storage - size_t temp_storage_size_bytes; - void* d_temp_storage = nullptr; // Get size of d_temp_storage - HIP_CHECK(rocprim::find_first_of(d_temp_storage, + size_t temp_storage_size_bytes; + HIP_CHECK(rocprim::find_first_of(nullptr, temp_storage_size_bytes, input_it, keys_it, - d_output, + d_output.get(), input.size(), keys.size(), compare_op, @@ -252,8 +237,7 @@ TYPED_TEST(RocprimDeviceFindFirstOfTests, FindFirstOf) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); test_utils::GraphHelper gHelper; if(TestFixture::use_graphs) @@ -262,11 +246,11 @@ TYPED_TEST(RocprimDeviceFindFirstOfTests, FindFirstOf) } // Run - HIP_CHECK(rocprim::find_first_of(d_temp_storage, + HIP_CHECK(rocprim::find_first_of(d_temp_storage.get(), temp_storage_size_bytes, input_it, keys_it, - d_output, + d_output.get(), input.size(), keys.size(), compare_op, @@ -281,10 +265,8 @@ TYPED_TEST(RocprimDeviceFindFirstOfTests, FindFirstOf) HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); - output_type output; - // Copy output to host - HIP_CHECK(hipMemcpy(&output, d_output, sizeof(*d_output), hipMemcpyDeviceToHost)); + const auto output = d_output.load()[0]; // Check auto expected = std::find_first_of(input.begin(), @@ -296,11 +278,6 @@ TYPED_TEST(RocprimDeviceFindFirstOfTests, FindFirstOf) ASSERT_EQ(output, expected); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_keys)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_temp_storage)); - if(TestFixture::use_graphs) { gHelper.cleanupGraphHelper(); @@ -334,8 +311,7 @@ TEST(RocprimDeviceFindFirstOfTests, LargeIndices) hipStream_t stream = 0; // default - output_type* d_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, sizeof(*d_output))); + test_utils::device_ptr d_output(1); const output_type expected = std::min(size, static_cast(starting_point * size)); @@ -345,15 +321,13 @@ TEST(RocprimDeviceFindFirstOfTests, LargeIndices) rocprim::equal_to compare_op; - // temp storage - size_t temp_storage_size_bytes; - void* d_temp_storage = nullptr; // Get size of d_temp_storage - HIP_CHECK(rocprim::find_first_of(d_temp_storage, + size_t temp_storage_size_bytes; + HIP_CHECK(rocprim::find_first_of(nullptr, temp_storage_size_bytes, input_it, keys_it, - d_output, + d_output.get(), size, keys_size, compare_op, @@ -364,14 +338,14 @@ TEST(RocprimDeviceFindFirstOfTests, LargeIndices) ASSERT_GT(temp_storage_size_bytes, 0); // allocate temporary storage - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temp_storage, temp_storage_size_bytes)); + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); // Run - HIP_CHECK(rocprim::find_first_of(d_temp_storage, + HIP_CHECK(rocprim::find_first_of(d_temp_storage.get(), temp_storage_size_bytes, input_it, keys_it, - d_output, + d_output.get(), size, keys_size, compare_op, @@ -382,12 +356,8 @@ TEST(RocprimDeviceFindFirstOfTests, LargeIndices) HIP_CHECK(hipDeviceSynchronize()); // Copy output to host and check - output_type output; - HIP_CHECK(hipMemcpy(&output, d_output, sizeof(*d_output), hipMemcpyDeviceToHost)); + const auto output = d_output.load()[0]; ASSERT_EQ(output, expected); - - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_temp_storage)); } } } diff --git a/test/rocprim/test_device_histogram.cpp b/test/rocprim/test_device_histogram.cpp index 926353a63..94a836ce2 100644 --- a/test/rocprim/test_device_histogram.cpp +++ b/test/rocprim/test_device_histogram.cpp @@ -27,6 +27,7 @@ #include // required test headers +#include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" // rows, columns, (row_stride - columns * Channels) @@ -242,17 +243,8 @@ TYPED_TEST(RocprimDeviceHistogramEven, Even) // Generate data std::vector input = get_random_samples(size, lower_level, upper_level, seed_value); - sample_type * d_input; - counter_type * d_histogram; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, size * sizeof(sample_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_histogram, bins * sizeof(counter_type))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - size * sizeof(sample_type), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_histogram(bins); // Calculate expected results on host // native host types - used for low-precision floating-point types @@ -282,33 +274,36 @@ TYPED_TEST(RocprimDeviceHistogramEven, Even) size_t temporary_storage_bytes = 0; if(rows == 1) { - HIP_CHECK( - rocprim::histogram_even( - nullptr, temporary_storage_bytes, - d_input, static_cast(columns), - d_histogram, - bins + 1, lower_level, upper_level, - stream, debug_synchronous - ) - ); + HIP_CHECK(rocprim::histogram_even(nullptr, + temporary_storage_bytes, + d_input.get(), + static_cast(columns), + d_histogram.get(), + bins + 1, + lower_level, + upper_level, + stream, + debug_synchronous)); } else { - HIP_CHECK( - rocprim::histogram_even( - nullptr, temporary_storage_bytes, - d_input, columns, rows, row_stride_bytes, - d_histogram, - bins + 1, lower_level, upper_level, - stream, debug_synchronous - ) - ); + HIP_CHECK(rocprim::histogram_even(nullptr, + temporary_storage_bytes, + d_input.get(), + columns, + rows, + row_stride_bytes, + d_histogram.get(), + bins + 1, + lower_level, + upper_level, + stream, + debug_synchronous)); } ASSERT_GT(temporary_storage_bytes, 0U); - void * d_temporary_storage; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::params::use_graphs) @@ -318,27 +313,31 @@ TYPED_TEST(RocprimDeviceHistogramEven, Even) if(rows == 1) { - HIP_CHECK( - rocprim::histogram_even( - d_temporary_storage, temporary_storage_bytes, - d_input, columns, - d_histogram, - bins + 1, lower_level, upper_level, - stream, debug_synchronous - ) - ); + HIP_CHECK(rocprim::histogram_even(d_temporary_storage.get(), + temporary_storage_bytes, + d_input.get(), + columns, + d_histogram.get(), + bins + 1, + lower_level, + upper_level, + stream, + debug_synchronous)); } else { - HIP_CHECK( - rocprim::histogram_even( - d_temporary_storage, temporary_storage_bytes, - d_input, columns, rows, row_stride_bytes, - d_histogram, - bins + 1, lower_level, upper_level, - stream, debug_synchronous - ) - ); + HIP_CHECK(rocprim::histogram_even(d_temporary_storage.get(), + temporary_storage_bytes, + d_input.get(), + columns, + rows, + row_stride_bytes, + d_histogram.get(), + bins + 1, + lower_level, + upper_level, + stream, + debug_synchronous)); } if(TestFixture::params::use_graphs) @@ -346,18 +345,7 @@ TYPED_TEST(RocprimDeviceHistogramEven, Even) gHelper.createAndLaunchGraph(stream); } - std::vector histogram(bins); - HIP_CHECK( - hipMemcpy( - histogram.data(), d_histogram, - bins * sizeof(counter_type), - hipMemcpyDeviceToHost - ) - ); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_histogram)); + const auto histogram = d_histogram.load(); for(size_t i = 0; i < bins; i++) { @@ -510,26 +498,9 @@ TYPED_TEST(RocprimDeviceHistogramRange, Range) std::vector input = get_random_samples(size, levels[0], levels[bins], seed_value); - sample_type * d_input; - level_type * d_levels; - counter_type * d_histogram; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, size * sizeof(sample_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_levels, (bins + 1) * sizeof(level_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_histogram, bins * sizeof(counter_type))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - size * sizeof(sample_type), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_levels, levels.data(), - (bins + 1) * sizeof(level_type), - hipMemcpyHostToDevice - ) - ); + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_levels(levels); + test_utils::device_ptr d_histogram(bins); // Calculate expected results on host std::vector histogram_expected(bins, 0); @@ -547,10 +518,8 @@ TYPED_TEST(RocprimDeviceHistogramRange, Range) } } - rocprim::transform_iterator, sample_type> d_input2( - d_input, - transform_op() - ); + rocprim::transform_iterator, sample_type> + d_input2(d_input.get(), transform_op()); using config = typename TestFixture::params::config; @@ -561,9 +530,9 @@ TYPED_TEST(RocprimDeviceHistogramRange, Range) temporary_storage_bytes, d_input2, columns, - d_histogram, + d_histogram.get(), bins + 1, - d_levels, + d_levels.get(), stream, debug_synchronous)); } @@ -575,17 +544,16 @@ TYPED_TEST(RocprimDeviceHistogramRange, Range) columns, rows, row_stride_bytes, - d_histogram, + d_histogram.get(), bins + 1, - d_levels, + d_levels.get(), stream, debug_synchronous)); } ASSERT_GT(temporary_storage_bytes, 0U); - void * d_temporary_storage; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::params::use_graphs) @@ -595,27 +563,27 @@ TYPED_TEST(RocprimDeviceHistogramRange, Range) if(rows == 1) { - HIP_CHECK(rocprim::histogram_range(d_temporary_storage, + HIP_CHECK(rocprim::histogram_range(d_temporary_storage.get(), temporary_storage_bytes, d_input2, columns, - d_histogram, + d_histogram.get(), bins + 1, - d_levels, + d_levels.get(), stream, debug_synchronous)); } else { - HIP_CHECK(rocprim::histogram_range(d_temporary_storage, + HIP_CHECK(rocprim::histogram_range(d_temporary_storage.get(), temporary_storage_bytes, d_input2, columns, rows, row_stride_bytes, - d_histogram, + d_histogram.get(), bins + 1, - d_levels, + d_levels.get(), stream, debug_synchronous)); } @@ -625,19 +593,7 @@ TYPED_TEST(RocprimDeviceHistogramRange, Range) gHelper.createAndLaunchGraph(stream); } - std::vector histogram(bins); - HIP_CHECK( - hipMemcpy( - histogram.data(), d_histogram, - bins * sizeof(counter_type), - hipMemcpyDeviceToHost - ) - ); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_levels)); - HIP_CHECK(hipFree(d_histogram)); + const auto histogram = d_histogram.load(); for(size_t i = 0; i < bins; i++) { @@ -793,20 +749,12 @@ TYPED_TEST(RocprimDeviceHistogramMultiEven, MultiEven) } } - sample_type * d_input; - counter_type * d_histogram[active_channels]; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, size * sizeof(sample_type))); + test_utils::device_ptr d_input(input); + counter_type* d_histogram[active_channels]; for(unsigned int channel = 0; channel < active_channels; channel++) { HIP_CHECK(test_common_utils::hipMallocHelper(&d_histogram[channel], bins[channel] * sizeof(counter_type))); } - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - size * sizeof(sample_type), - hipMemcpyHostToDevice - ) - ); // Calculate expected results on host std::vector histogram_expected[active_channels]; @@ -830,10 +778,8 @@ TYPED_TEST(RocprimDeviceHistogramMultiEven, MultiEven) } } - rocprim::transform_iterator, sample_type> d_input2( - d_input, - transform_op() - ); + rocprim::transform_iterator, sample_type> + d_input2(d_input.get(), transform_op()); using config = typename TestFixture::params::config; @@ -871,8 +817,7 @@ TYPED_TEST(RocprimDeviceHistogramMultiEven, MultiEven) ASSERT_GT(temporary_storage_bytes, 0U); - void * d_temporary_storage; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::params::use_graphs) @@ -883,7 +828,7 @@ TYPED_TEST(RocprimDeviceHistogramMultiEven, MultiEven) if(rows == 1) { HIP_CHECK((rocprim::multi_histogram_even( - d_temporary_storage, + d_temporary_storage.get(), temporary_storage_bytes, d_input2, columns, @@ -897,7 +842,7 @@ TYPED_TEST(RocprimDeviceHistogramMultiEven, MultiEven) else { HIP_CHECK((rocprim::multi_histogram_even( - d_temporary_storage, + d_temporary_storage.get(), temporary_storage_bytes, d_input2, columns, @@ -930,9 +875,6 @@ TYPED_TEST(RocprimDeviceHistogramMultiEven, MultiEven) HIP_CHECK(hipFree(d_histogram[channel])); } - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_input)); - for(unsigned int channel = 0; channel < active_channels; channel++) { SCOPED_TRACE(testing::Message() << "with channel = " << channel); @@ -1107,22 +1049,14 @@ TYPED_TEST(RocprimDeviceHistogramMultiRange, MultiRange) } } - sample_type * d_input; + test_utils::device_ptr d_input(input); level_type * d_levels[active_channels]; - counter_type * d_histogram[active_channels]; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, size * sizeof(sample_type))); + counter_type* d_histogram[active_channels]; for(unsigned int channel = 0; channel < active_channels; channel++) { HIP_CHECK(test_common_utils::hipMallocHelper(&d_levels[channel], num_levels[channel] * sizeof(level_type))); HIP_CHECK(test_common_utils::hipMallocHelper(&d_histogram[channel], bins[channel] * sizeof(counter_type))); } - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - size * sizeof(sample_type), - hipMemcpyHostToDevice - ) - ); for(unsigned int channel = 0; channel < active_channels; channel++) { HIP_CHECK( @@ -1161,33 +1095,36 @@ TYPED_TEST(RocprimDeviceHistogramMultiRange, MultiRange) size_t temporary_storage_bytes = 0; if(rows == 1) { - HIP_CHECK(( - rocprim::multi_histogram_range( - nullptr, temporary_storage_bytes, - d_input, columns, - d_histogram, - num_levels, d_levels, - stream, debug_synchronous - ) - )); + HIP_CHECK((rocprim::multi_histogram_range( + nullptr, + temporary_storage_bytes, + d_input.get(), + columns, + d_histogram, + num_levels, + d_levels, + stream, + debug_synchronous))); } else { - HIP_CHECK(( - rocprim::multi_histogram_range( - nullptr, temporary_storage_bytes, - d_input, columns, rows, row_stride_bytes, - d_histogram, - num_levels, d_levels, - stream, debug_synchronous - ) - )); + HIP_CHECK((rocprim::multi_histogram_range( + nullptr, + temporary_storage_bytes, + d_input.get(), + columns, + rows, + row_stride_bytes, + d_histogram, + num_levels, + d_levels, + stream, + debug_synchronous))); } ASSERT_GT(temporary_storage_bytes, 0U); - void * d_temporary_storage; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::params::use_graphs) @@ -1197,27 +1134,31 @@ TYPED_TEST(RocprimDeviceHistogramMultiRange, MultiRange) if(rows == 1) { - HIP_CHECK(( - rocprim::multi_histogram_range( - d_temporary_storage, temporary_storage_bytes, - d_input, columns, - d_histogram, - num_levels, d_levels, - stream, debug_synchronous - ) - )); + HIP_CHECK((rocprim::multi_histogram_range( + d_temporary_storage.get(), + temporary_storage_bytes, + d_input.get(), + columns, + d_histogram, + num_levels, + d_levels, + stream, + debug_synchronous))); } else { - HIP_CHECK(( - rocprim::multi_histogram_range( - d_temporary_storage, temporary_storage_bytes, - d_input, columns, rows, row_stride_bytes, - d_histogram, - num_levels, d_levels, - stream, debug_synchronous - ) - )); + HIP_CHECK((rocprim::multi_histogram_range( + d_temporary_storage.get(), + temporary_storage_bytes, + d_input.get(), + columns, + rows, + row_stride_bytes, + d_histogram, + num_levels, + d_levels, + stream, + debug_synchronous))); } if(TestFixture::params::use_graphs) @@ -1240,9 +1181,6 @@ TYPED_TEST(RocprimDeviceHistogramMultiRange, MultiRange) HIP_CHECK(hipFree(d_histogram[channel])); } - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_input)); - if(TestFixture::params::use_graphs) { gHelper.cleanupGraphHelper(); From 8b7a36e1fa6f33e95b9c43939bec77d21581d1fc Mon Sep 17 00:00:00 2001 From: Nick Breed Date: Tue, 17 Dec 2024 13:11:24 +0000 Subject: [PATCH 46/84] Fix rules overwrite for other then MR pipelines --- .gitlab-ci.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a9321eea4..3342b4381 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -351,7 +351,6 @@ autotune:build: - $GPU extends: - .cmake-minimum - - .rules:test needs: - job: build:cmake-minimum parallel: @@ -381,10 +380,7 @@ test:any-gpu: extends: - .test:common rules: - - if: $CI_MERGE_REQUEST_TITLE =~ /Draft:/ && - $CI_MERGE_REQUEST_LABELS !~ /Arch::/ - when: always - - when: never + - if: $CI_MERGE_REQUEST_TITLE =~ /Draft:/ && $CI_MERGE_REQUEST_LABELS !~ /Arch::/ test:label-arch: extends: @@ -393,13 +389,12 @@ test:label-arch: - .rules:arch-labels test:all-gpus: + variables: + SHOULD_BE_UNDRAFTED: "true" extends: - .gpus:rocm - .test:common - rules: - - if: '$CI_MERGE_REQUEST_TITLE !~ /Draft:/' - when: always - - when: never + - .rules:test .test-windows-base: stage: test From 8d73701249868a75fe41c922e48bf69405e5b12d Mon Sep 17 00:00:00 2001 From: Cenxuan Tian Date: Tue, 17 Dec 2024 15:09:26 +0000 Subject: [PATCH 47/84] Resolve "Add stream, memset, array, and unique_ptr support to device_ptr" --- test/rocprim/detail/get_rocprim_version.cpp | 2 +- .../rocprim/test_block_radix_sort.kernels.hpp | 14 +- test/rocprim/test_config_dispatch.cpp | 2 +- test/rocprim/test_device_adjacent_find.cpp | 2 +- test/rocprim/test_device_partition.cpp | 8 +- test/rocprim/test_device_radix_sort.hpp | 355 +++++--------- test/rocprim/test_device_search_n.cpp | 26 +- test/rocprim/test_utils_device_ptr.hpp | 432 ++++++++++++++++-- test/rocprim/test_zip_iterator.cpp | 2 +- 9 files changed, 539 insertions(+), 304 deletions(-) diff --git a/test/rocprim/detail/get_rocprim_version.cpp b/test/rocprim/detail/get_rocprim_version.cpp index fec53dd71..a1f52d876 100644 --- a/test/rocprim/detail/get_rocprim_version.cpp +++ b/test/rocprim/detail/get_rocprim_version.cpp @@ -40,5 +40,5 @@ unsigned int get_rocprim_version_on_device() HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); - return d_version.load()[0]; + return d_version.load_value_at(0); } diff --git a/test/rocprim/test_block_radix_sort.kernels.hpp b/test/rocprim/test_block_radix_sort.kernels.hpp index ab808e22d..cef967a6b 100644 --- a/test/rocprim/test_block_radix_sort.kernels.hpp +++ b/test/rocprim/test_block_radix_sort.kernels.hpp @@ -216,7 +216,7 @@ auto test_block_radix_sort() -> typename std::enable_if::type } // Preparing device - test_utils::device_ptr device_keys_output(keys_output.get(), size); + test_utils::device_ptr device_keys_output(keys_output, size); sort_key_kernel <<>>(device_keys_output.get(), @@ -227,10 +227,7 @@ auto test_block_radix_sort() -> typename std::enable_if::type HIP_CHECK(hipGetLastError()); // Getting results to host - HIP_CHECK(hipMemcpy(keys_output.get(), - device_keys_output.get(), - size * sizeof(keys_output[0]), - hipMemcpyDeviceToHost)); + keys_output = device_keys_output.load_to_unique_ptr(); // Verifying results ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(keys_output.get(), @@ -324,7 +321,7 @@ auto test_block_radix_sort() -> typename std::enable_if::type values_expected[i] = expected[i].second; } - test_utils::device_ptr device_keys_output(keys_output.get(), size); + test_utils::device_ptr device_keys_output(keys_output, size); test_utils::device_ptr device_values_output(values_output); // Running kernel @@ -342,10 +339,7 @@ auto test_block_radix_sort() -> typename std::enable_if::type HIP_CHECK(hipGetLastError()); // Getting results to host - HIP_CHECK(hipMemcpy(keys_output.get(), - device_keys_output.get(), - size * sizeof(keys_output[0]), - hipMemcpyDeviceToHost)); + keys_output = device_keys_output.load_to_unique_ptr(); values_output = device_values_output.load(); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(keys_output.get(), diff --git a/test/rocprim/test_config_dispatch.cpp b/test/rocprim/test_config_dispatch.cpp index 7f5f47a3e..630665bda 100644 --- a/test/rocprim/test_config_dispatch.cpp +++ b/test/rocprim/test_config_dispatch.cpp @@ -78,7 +78,7 @@ TEST(RocprimConfigDispatchTests, HostMatchesDevice) hipLaunchKernelGGL(write_target_arch, dim3(1), dim3(1), 0, stream, device_arch_ptr.get()); HIP_CHECK(hipGetLastError()); - target_arch device_arch = device_arch_ptr.load()[0]; + const auto device_arch = device_arch_ptr.load_value_at(0); ASSERT_NE(host_arch, target_arch::invalid); ASSERT_EQ(host_arch, device_arch); diff --git a/test/rocprim/test_device_adjacent_find.cpp b/test/rocprim/test_device_adjacent_find.cpp index 614dba654..459e662b6 100644 --- a/test/rocprim/test_device_adjacent_find.cpp +++ b/test/rocprim/test_device_adjacent_find.cpp @@ -204,7 +204,7 @@ TYPED_TEST(RocprimDeviceAdjacentFindTests, AdjacentFind) HIP_CHECK(hipDeviceSynchronize()); // Allocate memory for output and copy to host side - auto output = d_output.load()[0]; + auto output = d_output.load_value_at(0); // Calculate expected results on host const auto expected diff --git a/test/rocprim/test_device_partition.cpp b/test/rocprim/test_device_partition.cpp index 8268acce7..09af28b88 100644 --- a/test/rocprim/test_device_partition.cpp +++ b/test/rocprim/test_device_partition.cpp @@ -621,9 +621,6 @@ TYPED_TEST(RocprimDevicePartitionTests, PredicateThreeWay) // Generate data const auto input = test_utils::get_random_data(size, 1, 100, seed_value); - // Output - auto selected_counts = std::array{}; - test_utils::device_ptr d_input(input); test_utils::device_ptr d_first_output(input.size()); test_utils::device_ptr d_second_output(input.size()); @@ -709,10 +706,7 @@ TYPED_TEST(RocprimDevicePartitionTests, PredicateThreeWay) HIP_CHECK(hipDeviceSynchronize()); // Check if number of selected value is as expected_selected - HIP_CHECK(hipMemcpy(selected_counts.data(), - d_selected_counts.get(), - sizeof(selected_counts), - hipMemcpyDeviceToHost)); + const auto selected_counts = d_selected_counts.load_to_array<2>(); ASSERT_EQ(selected_counts, expected_counts); // Check if output values are as expected_selected diff --git a/test/rocprim/test_device_radix_sort.hpp b/test/rocprim/test_device_radix_sort.hpp index e41308b10..dfa15ee56 100644 --- a/test/rocprim/test_device_radix_sort.hpp +++ b/test/rocprim/test_device_radix_sort.hpp @@ -33,6 +33,7 @@ #include "test_utils_custom_float_traits_type.hpp" #include "test_utils_custom_float_type.hpp" #include "test_utils_custom_test_types.hpp" +#include "test_utils_device_ptr.hpp" #include "test_utils_sort_comparator.hpp" #include "test_utils_types.hpp" @@ -273,22 +274,11 @@ void sort_keys() auto keys_input = std::make_unique(size); generate_key_input(keys_input.get(), size, rng_engine); - key_type* d_keys_input; - key_type* d_keys_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_input, size * sizeof(key_type))); - if(in_place) - { - d_keys_output = d_keys_input; - } - else - { - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_keys_output, size * sizeof(key_type))); - } - HIP_CHECK(hipMemcpy(d_keys_input, - keys_input.get(), - size * sizeof(key_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_keys_input(keys_input, size); + test_utils::device_ptr d_keys_output_alloc; + test_utils::device_ptr& d_keys_output + = in_place ? d_keys_input : d_keys_output_alloc; + d_keys_output_alloc.resize(in_place ? 0 : size); // Calculate expected results on host std::vector expected(keys_input.get(), keys_input.get() + size); @@ -306,18 +296,15 @@ void sort_keys() size_t temporary_storage_bytes; HIP_CHECK((invoke_sort_keys(nullptr, temporary_storage_bytes, - d_keys_input, - d_keys_output, + d_keys_input.get(), + d_keys_output.get(), size, start_bit, end_bit, stream, debug_synchronous))); ASSERT_GT(temporary_storage_bytes, 0); - - void* d_temporary_storage; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::params::use_graphs) @@ -325,34 +312,22 @@ void sort_keys() gHelper.startStreamCapture(stream); } - HIP_CHECK((invoke_sort_keys(d_temporary_storage, + HIP_CHECK((invoke_sort_keys(d_temporary_storage.get(), temporary_storage_bytes, - d_keys_input, - d_keys_output, + d_keys_input.get(), + d_keys_output.get(), size, start_bit, end_bit, stream, debug_synchronous))); - if(TestFixture::params::use_graphs) { gHelper.createAndLaunchGraph(stream); } - auto keys_output = std::make_unique(size); - HIP_CHECK(hipMemcpy(keys_output.get(), - d_keys_output, - size * sizeof(key_type), - hipMemcpyDeviceToHost)); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_keys_input)); - if(!in_place) - { - HIP_CHECK(hipFree(d_keys_output)); - } + const auto keys_output = d_keys_output.load_to_unique_ptr(); if(TestFixture::params::use_graphs) { @@ -570,40 +545,17 @@ void sort_pairs() std::vector values_input(size); test_utils::iota(values_input.begin(), values_input.end(), 0); - key_type* d_keys_input; - key_type* d_keys_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_input, size * sizeof(key_type))); - if(in_place) - { - d_keys_output = d_keys_input; - } - else - { - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_keys_output, size * sizeof(key_type))); - } - HIP_CHECK(hipMemcpy(d_keys_input, - keys_input.get(), - size * sizeof(key_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_keys_input(keys_input, size); + test_utils::device_ptr d_keys_output_alloc; + test_utils::device_ptr& d_keys_output + = in_place ? d_keys_input : d_keys_output_alloc; + d_keys_output_alloc.resize(in_place ? 0 : size); - value_type* d_values_input; - value_type* d_values_output; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_values_input, size * sizeof(value_type))); - if(in_place) - { - d_values_output = d_values_input; - } - else - { - HIP_CHECK(test_common_utils::hipMallocHelper(&d_values_output, - size * sizeof(value_type))); - } - HIP_CHECK(hipMemcpy(d_values_input, - values_input.data(), - size * sizeof(value_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_values_input(values_input); + test_utils::device_ptr d_values_output_alloc; + test_utils::device_ptr& d_values_output + = in_place ? d_values_input : d_values_output_alloc; + d_values_output_alloc.resize(in_place ? 0 : size); using key_value = std::pair; @@ -642,14 +594,13 @@ void sort_pairs() gHelper.startStreamCapture(stream); } - void* d_temporary_storage = nullptr; size_t temporary_storage_bytes; - HIP_CHECK((invoke_sort_pairs(d_temporary_storage, + HIP_CHECK((invoke_sort_pairs(nullptr, temporary_storage_bytes, - d_keys_input, - d_keys_output, - d_values_input, - d_values_output, + d_keys_input.get(), + d_keys_output.get(), + d_values_input.get(), + d_values_output.get(), size, start_bit, end_bit, @@ -662,21 +613,19 @@ void sort_pairs() } ASSERT_GT(temporary_storage_bytes, 0); - - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); if(TestFixture::params::use_graphs) { gHelper.resetGraphHelper(stream); } - HIP_CHECK((invoke_sort_pairs(d_temporary_storage, + HIP_CHECK((invoke_sort_pairs(d_temporary_storage.get(), temporary_storage_bytes, - d_keys_input, - d_keys_output, - d_values_input, - d_values_output, + d_keys_input.get(), + d_keys_output.get(), + d_values_input.get(), + d_values_output.get(), size, start_bit, end_bit, @@ -688,26 +637,8 @@ void sort_pairs() gHelper.createAndLaunchGraph(stream); } - auto keys_output = std::make_unique(size); - HIP_CHECK(hipMemcpy(keys_output.get(), - d_keys_output, - size * sizeof(key_type), - hipMemcpyDeviceToHost)); - - std::vector values_output(size); - HIP_CHECK(hipMemcpy(values_output.data(), - d_values_output, - size * sizeof(value_type), - hipMemcpyDeviceToHost)); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_values_input)); - if(!in_place) - { - HIP_CHECK(hipFree(d_keys_output)); - HIP_CHECK(hipFree(d_values_output)); - } + const auto keys_output = d_keys_output.load_to_unique_ptr(); + const auto values_output = d_values_output.load(); if(TestFixture::params::use_graphs) { @@ -891,14 +822,8 @@ void sort_keys_double_buffer() auto keys_input = std::make_unique(size); generate_key_input(keys_input.get(), size, rng_engine); - key_type* d_keys_input; - key_type* d_keys_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_input, size * sizeof(key_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_output, size * sizeof(key_type))); - HIP_CHECK(hipMemcpy(d_keys_input, - keys_input.get(), - size * sizeof(key_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_keys_input(keys_input, size); + test_utils::device_ptr d_keys_output(size); // Calculate expected results on host std::vector expected(keys_input.get(), keys_input.get() + size); @@ -907,7 +832,7 @@ void sort_keys_double_buffer() expected.end(), test_utils::key_comparator()); - rocprim::double_buffer d_keys(d_keys_input, d_keys_output); + rocprim::double_buffer d_keys(d_keys_input.get(), d_keys_output.get()); size_t temporary_storage_bytes; HIP_CHECK( @@ -922,9 +847,7 @@ void sort_keys_double_buffer() ASSERT_GT(temporary_storage_bytes, 0); - void* d_temporary_storage; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::params::use_graphs) @@ -933,7 +856,7 @@ void sort_keys_double_buffer() } HIP_CHECK( - (invoke_sort_keys(d_temporary_storage, + (invoke_sort_keys(d_temporary_storage.get(), temporary_storage_bytes, d_keys, size, @@ -942,23 +865,17 @@ void sort_keys_double_buffer() stream, debug_synchronous))); - if(TestFixture::params::use_graphs) { gHelper.createAndLaunchGraph(stream); } - HIP_CHECK(hipFree(d_temporary_storage)); - auto keys_output = std::make_unique(size); HIP_CHECK(hipMemcpy(keys_output.get(), d_keys.current(), size * sizeof(key_type), hipMemcpyDeviceToHost)); - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_keys_output)); - if(TestFixture::params::use_graphs) { gHelper.cleanupGraphHelper(); @@ -1152,25 +1069,11 @@ void sort_pairs_double_buffer() std::vector values_input(size); test_utils::iota(values_input.begin(), values_input.end(), 0); - key_type* d_keys_input; - key_type* d_keys_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_input, size * sizeof(key_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_output, size * sizeof(key_type))); - HIP_CHECK(hipMemcpy(d_keys_input, - keys_input.get(), - size * sizeof(key_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_keys_input(keys_input, size); + test_utils::device_ptr d_keys_output(size); - value_type* d_values_input; - value_type* d_values_output; - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_values_input, size * sizeof(value_type))); - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_values_output, size * sizeof(value_type))); - HIP_CHECK(hipMemcpy(d_values_input, - values_input.data(), - size * sizeof(value_type), - hipMemcpyHostToDevice)); + test_utils::device_ptr d_values_input(values_input); + test_utils::device_ptr d_values_output(size); using key_value = std::pair; @@ -1193,13 +1096,13 @@ void sort_pairs_double_buffer() values_expected[i] = expected[i].second; } - rocprim::double_buffer d_keys(d_keys_input, d_keys_output); - rocprim::double_buffer d_values(d_values_input, d_values_output); + rocprim::double_buffer d_keys(d_keys_input.get(), d_keys_output.get()); + rocprim::double_buffer d_values(d_values_input.get(), + d_values_output.get()); - void* d_temporary_storage = nullptr; size_t temporary_storage_bytes; HIP_CHECK( - (invoke_sort_pairs(d_temporary_storage, + (invoke_sort_pairs(nullptr, temporary_storage_bytes, d_keys, d_values, @@ -1210,9 +1113,7 @@ void sort_pairs_double_buffer() debug_synchronous))); ASSERT_GT(temporary_storage_bytes, 0); - - HIP_CHECK( - test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); test_utils::GraphHelper gHelper;; if(TestFixture::params::use_graphs) @@ -1221,7 +1122,7 @@ void sort_pairs_double_buffer() } HIP_CHECK( - (invoke_sort_pairs(d_temporary_storage, + (invoke_sort_pairs(d_temporary_storage.get(), temporary_storage_bytes, d_keys, d_values, @@ -1231,14 +1132,11 @@ void sort_pairs_double_buffer() stream, debug_synchronous))); - if(TestFixture::params::use_graphs) { gHelper.createAndLaunchGraph(stream); } - HIP_CHECK(hipFree(d_temporary_storage)); - auto keys_output = std::make_unique(size); HIP_CHECK(hipMemcpy(keys_output.get(), d_keys.current(), @@ -1251,11 +1149,6 @@ void sort_pairs_double_buffer() size * sizeof(value_type), hipMemcpyDeviceToHost)); - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_keys_output)); - HIP_CHECK(hipFree(d_values_input)); - HIP_CHECK(hipFree(d_values_output)); - if(TestFixture::params::use_graphs) { gHelper.cleanupGraphHelper(); @@ -1311,20 +1204,14 @@ void sort_keys_over_4g() //generate histogram of the randomly generated values std::for_each(keys_input.begin(), keys_input.end(), [&](const key_type& a) { histogram[a]++; }); - key_type* d_keys_input_output{}; + test_utils::device_ptr d_keys_input_output(keys_input); size_t key_type_storage_bytes = size * sizeof(key_type); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_input_output, key_type_storage_bytes)); - HIP_CHECK(hipMemcpy(d_keys_input_output, - keys_input.data(), - key_type_storage_bytes, - hipMemcpyHostToDevice)); - size_t temporary_storage_bytes; HIP_CHECK(rocprim::radix_sort_keys(nullptr, temporary_storage_bytes, - d_keys_input_output, - d_keys_input_output, + d_keys_input_output.get(), + d_keys_input_output.get(), size, start_bit, end_bit, @@ -1337,57 +1224,50 @@ void sort_keys_over_4g() HIP_CHECK(hipGetDeviceProperties(&prop, device_id)); size_t total_storage_bytes = key_type_storage_bytes + temporary_storage_bytes; - if (total_storage_bytes > (static_cast(prop.totalGlobalMem * 0.90))) { - HIP_CHECK(hipFree(d_keys_input_output)); - GTEST_SKIP() << "Test case device memory requirement (" << total_storage_bytes << " bytes) exceeds available memory on current device (" - << prop.totalGlobalMem << " bytes). Skipping test"; - } - - void* d_temporary_storage; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); - - test_utils::GraphHelper gHelper;; - if(UseGraphs) - { - gHelper.startStreamCapture(stream); - } - - HIP_CHECK(rocprim::radix_sort_keys(d_temporary_storage, - temporary_storage_bytes, - d_keys_input_output, - d_keys_input_output, - size, - start_bit, - end_bit, - stream, - debug_synchronous)); - - - if(UseGraphs) - { - gHelper.createAndLaunchGraph(stream); - } - - std::vector output(keys_input.size()); - HIP_CHECK(hipMemcpy(output.data(), - d_keys_input_output, - size * sizeof(key_type), - hipMemcpyDeviceToHost)); - - size_t counter = 0; - for(size_t i = 0; i <= test_utils::numeric_limits::max(); ++i) - { - for(size_t j = 0; j < histogram[i]; ++j) - { - ASSERT_EQ(static_cast(output[counter]), i); - ++counter; - } - } + if(total_storage_bytes > (static_cast(prop.totalGlobalMem * 0.90))) + { + GTEST_SKIP() << "Test case device memory requirement (" << total_storage_bytes + << " bytes) exceeds available memory on current device (" << prop.totalGlobalMem + << " bytes). Skipping test"; + } + + test_utils::device_ptr d_temporary_storage(temporary_storage_bytes); + + test_utils::GraphHelper gHelper; + ; + if(UseGraphs) + { + gHelper.startStreamCapture(stream); + } + + HIP_CHECK(rocprim::radix_sort_keys(d_temporary_storage.get(), + temporary_storage_bytes, + d_keys_input_output.get(), + d_keys_input_output.get(), + size, + start_bit, + end_bit, + stream, + debug_synchronous)); + + if(UseGraphs) + { + gHelper.createAndLaunchGraph(stream); + } + + const auto output = d_keys_input_output.load(); + + size_t counter = 0; + for(size_t i = 0; i <= test_utils::numeric_limits::max(); ++i) + { + for(size_t j = 0; j < histogram[i]; ++j) + { + ASSERT_EQ(static_cast(output[counter]), i); + ++counter; + } + } ASSERT_EQ(counter, size); - HIP_CHECK(hipFree(d_keys_input_output)); - HIP_CHECK(hipFree(d_temporary_storage)); - if (UseGraphs) { gHelper.cleanupGraphHelper(); @@ -1414,47 +1294,46 @@ inline void sort_keys_large_sizes() { SCOPED_TRACE(testing::Message() << "with size = " << size); + test_utils::device_ptr d_keys; + if(!d_keys.resize_with_memory_check(size)) + { + std::cout << "Out of memory. Skipping size = " << size << std::endl; + break; + } + // Generate data std::vector keys_input(size); std::iota(keys_input.begin(), keys_input.end(), 0); + d_keys.store(keys_input); - key_type* d_keys; - HIP_CHECK_MEMORY(test_common_utils::hipMallocHelper(&d_keys, size * sizeof(key_type))); - HIP_CHECK( - hipMemcpy(d_keys, keys_input.data(), size * sizeof(key_type), hipMemcpyHostToDevice)); - - void* d_temporary_storage = nullptr; size_t temporary_storage_bytes = 0; - HIP_CHECK(rocprim::radix_sort_keys(d_temporary_storage, + HIP_CHECK(rocprim::radix_sort_keys(nullptr, temporary_storage_bytes, - d_keys, - d_keys, + d_keys.get(), + d_keys.get(), size, start_bit, end_bit, stream)); ASSERT_GT(temporary_storage_bytes, 0U); + test_utils::device_ptr d_temporary_storage; + if(!d_temporary_storage.resize_with_memory_check(temporary_storage_bytes)) + { + std::cout << "Out of memory. Skipping size = " << size << std::endl; + break; + } - HIP_CHECK_MEMORY( - test_common_utils::hipMallocHelper(&d_temporary_storage, temporary_storage_bytes)); - - HIP_CHECK(rocprim::radix_sort_keys(d_temporary_storage, + HIP_CHECK(rocprim::radix_sort_keys(d_temporary_storage.get(), temporary_storage_bytes, - d_keys, - d_keys, + d_keys.get(), + d_keys.get(), size, start_bit, end_bit, stream)); - HIP_CHECK(hipFree(d_temporary_storage)); - - std::vector keys_output(size); - HIP_CHECK( - hipMemcpy(keys_output.data(), d_keys, size * sizeof(key_type), hipMemcpyDeviceToHost)); - - HIP_CHECK(hipFree(d_keys)); + const auto keys_output = d_keys.load(); // Check if output values are as expected const size_t unique_keys = size_t(rocprim::numeric_limits::max()) + 1; diff --git a/test/rocprim/test_device_search_n.cpp b/test/rocprim/test_device_search_n.cpp index 1764e6493..a7557e51b 100644 --- a/test/rocprim/test_device_search_n.cpp +++ b/test/rocprim/test_device_search_n.cpp @@ -138,7 +138,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, RandomTest) output_type h_output; test_utils::device_ptr d_input(h_input); - test_utils::device_ptr d_value(&h_value, 1); + test_utils::device_ptr d_value(std::vector({h_value})); test_utils::device_ptr d_output(1); SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); @@ -231,7 +231,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, MaxCount) std::vector h_input(size, h_value); output_type h_output; test_utils::device_ptr d_input(h_input); - test_utils::device_ptr d_value(&h_value, 1); + test_utils::device_ptr d_value(std::vector({h_value})); test_utils::device_ptr d_output(1); SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); @@ -326,7 +326,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, MinCount) std::vector h_input(size, h_value); output_type h_output; test_utils::device_ptr d_input(h_input); - test_utils::device_ptr d_value(&h_value, 1); + test_utils::device_ptr d_value(std::vector({h_value})); test_utils::device_ptr d_output(1); SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); @@ -418,7 +418,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, SmallCount) std::vector h_input(size, h_noise); output_type h_output; test_utils::device_ptr d_input(h_input); - test_utils::device_ptr d_value(&h_value, 1); + test_utils::device_ptr d_value(std::vector({h_value})); test_utils::device_ptr d_output(1); if(size > 0 && size - 1 > 0) @@ -518,7 +518,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, StartFromBegin) std::fill(h_input.begin() + count, h_input.end(), 0); output_type h_output; test_utils::device_ptr d_input(h_input); - test_utils::device_ptr d_value(&h_value, 1); + test_utils::device_ptr d_value(std::vector({h_value})); test_utils::device_ptr d_output(1); SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); @@ -612,7 +612,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, StartFromMiddle) std::fill(h_input.begin() + count, h_input.end(), h_value); output_type h_output; test_utils::device_ptr d_input(h_input); - test_utils::device_ptr d_value(&h_value, 1); + test_utils::device_ptr d_value(std::vector({h_value})); test_utils::device_ptr d_output(1); SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); @@ -706,7 +706,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, StartFromEnd) std::fill(h_input.begin() + (size - count), h_input.end(), h_value); output_type h_output; test_utils::device_ptr d_input(h_input); - test_utils::device_ptr d_value(&h_value, 1); + test_utils::device_ptr d_value(std::vector({h_value})); test_utils::device_ptr d_output(1); SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); @@ -804,7 +804,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, StartFromEndButFail) } output_type h_output; test_utils::device_ptr d_input(h_input); - test_utils::device_ptr d_value(&h_value, 1); + test_utils::device_ptr d_value(std::vector({h_value})); test_utils::device_ptr d_output(1); SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); @@ -924,7 +924,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, NoiseTest_1block) } test_utils::device_ptr d_input(h_input); - test_utils::device_ptr d_value(&h_value, 1); + test_utils::device_ptr d_value(std::vector({h_value})); test_utils::device_ptr d_output(1); SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); @@ -1044,7 +1044,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, NoiseTest_2block) } test_utils::device_ptr d_input(h_input); - test_utils::device_ptr d_value(&h_value, 1); + test_utils::device_ptr d_value(std::vector({h_value})); test_utils::device_ptr d_output(1); SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); @@ -1164,7 +1164,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, NoiseTest_3block) } test_utils::device_ptr d_input(h_input); - test_utils::device_ptr d_value(&h_value, 1); + test_utils::device_ptr d_value(std::vector({h_value})); test_utils::device_ptr d_output(1); SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); @@ -1286,7 +1286,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, MultiResult1) } test_utils::device_ptr d_input(h_input); - test_utils::device_ptr d_value(&h_value, 1); + test_utils::device_ptr d_value(std::vector({h_value})); test_utils::device_ptr d_output(1); SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); @@ -1402,7 +1402,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, MultiResult2) } test_utils::device_ptr d_input(h_input); - test_utils::device_ptr d_value(&h_value, 1); + test_utils::device_ptr d_value(std::vector({h_value})); test_utils::device_ptr d_output(1); SCOPED_TRACE(testing::Message() << "with size = " << h_input.size()); diff --git a/test/rocprim/test_utils_device_ptr.hpp b/test/rocprim/test_utils_device_ptr.hpp index 6f78ec8c4..4ec4de6a2 100644 --- a/test/rocprim/test_utils_device_ptr.hpp +++ b/test/rocprim/test_utils_device_ptr.hpp @@ -21,7 +21,10 @@ #ifndef ROCPRIM_TEST_UTILS_DEVICE_PTR_HPP #define ROCPRIM_TEST_UTILS_DEVICE_PTR_HPP +#include #include +#include +#include #include #include "../common_test_header.hpp" @@ -60,29 +63,91 @@ class device_ptr }; /// \brief Construct by host vectors with the same sized value_type - template - explicit device_ptr(std::vector const& data) + template + explicit device_ptr(std::vector const& data) : device_raw_ptr_(nullptr), number_of_ele_(data.size()) { - static_assert( - sizeof(InVecValueType) == sizeof(value_type), - "value_type of input vector must have the same size with device_ptr::value_type"); + static_assert(sizeof(InValueType) == sizeof(value_type), + "value_type of input must have the same size with device_ptr::value_type"); size_type storage_size = number_of_ele_ * sizeof(value_type); HIP_CHECK(test_common_utils::hipMallocHelper(&device_raw_ptr_, storage_size)); HIP_CHECK(hipMemcpy(device_raw_ptr_, data.data(), storage_size, hipMemcpyHostToDevice)); } - /// \brief Construct with a copy of this `host_buffer` - /// - /// \param _number_of_ele be aware, this is NOT the sizeof `host_buffer`, this is the `number of elements` in the `host_buffer` - device_ptr(const void* host_buffer, size_type _number_of_ele) - : device_raw_ptr_(nullptr), number_of_ele_(_number_of_ele) + template + explicit device_ptr(std::vector const& data, hipStream_t stream) + : device_raw_ptr_(nullptr), number_of_ele_(data.size()) { + static_assert(sizeof(InValueType) == sizeof(value_type), + "value_type of input must have the same size with device_ptr::value_type"); + size_type storage_size = number_of_ele_ * sizeof(value_type); HIP_CHECK(test_common_utils::hipMallocHelper(&device_raw_ptr_, storage_size)); - HIP_CHECK(hipMemcpy(device_raw_ptr_, host_buffer, storage_size, hipMemcpyHostToDevice)); - }; + HIP_CHECK(hipMemcpyAsync(device_raw_ptr_, + data.data(), + storage_size, + hipMemcpyHostToDevice, + stream)); + } + + template + explicit device_ptr(std::array const& data) + : device_raw_ptr_(nullptr), number_of_ele_(Size) + { + static_assert(sizeof(InValueType) == sizeof(value_type), + "value_type of input must have the same size with device_ptr::value_type"); + + size_type storage_size = Size * sizeof(value_type); + HIP_CHECK(test_common_utils::hipMallocHelper(&device_raw_ptr_, storage_size)); + HIP_CHECK(hipMemcpy(device_raw_ptr_, data.data(), storage_size, hipMemcpyHostToDevice)); + } + + template + explicit device_ptr(std::array const& data, hipStream_t stream) + : device_raw_ptr_(nullptr), number_of_ele_(Size) + { + static_assert(sizeof(InValueType) == sizeof(value_type), + "value_type of input must have the same size with device_ptr::value_type"); + + size_type storage_size = Size * sizeof(value_type); + HIP_CHECK(test_common_utils::hipMallocHelper(&device_raw_ptr_, storage_size)); + HIP_CHECK(hipMemcpyAsync(device_raw_ptr_, + data.data(), + storage_size, + hipMemcpyHostToDevice, + stream)); + } + + template + explicit device_ptr(std::unique_ptr const& uptr, size_type size) + : device_raw_ptr_(nullptr), number_of_ele_(size) + { + static_assert(sizeof(InValueType) == sizeof(value_type), + "value_type of input must have the same size with device_ptr::value_type"); + + size_type storage_size = size * sizeof(value_type); + HIP_CHECK(test_common_utils::hipMallocHelper(&device_raw_ptr_, storage_size)); + HIP_CHECK(hipMemcpy(device_raw_ptr_, uptr.get(), storage_size, hipMemcpyHostToDevice)); + } + + template + explicit device_ptr(std::unique_ptr const& uptr, + size_type size, + hipStream_t stream) + : device_raw_ptr_(nullptr), number_of_ele_(size) + { + static_assert(sizeof(InValueType) == sizeof(value_type), + "value_type of input must have the same size with device_ptr::value_type"); + + size_type storage_size = size * sizeof(value_type); + HIP_CHECK(test_common_utils::hipMallocHelper(&device_raw_ptr_, storage_size)); + HIP_CHECK(hipMemcpyAsync(device_raw_ptr_, + uptr.get(), + storage_size, + hipMemcpyHostToDevice, + stream)); + } ~device_ptr() { @@ -113,6 +178,20 @@ class device_ptr return ret; } + device_ptr duplicate_async(hipStream_t stream) const + { + device_ptr ret; + ret.number_of_ele_ = number_of_ele_; + size_type storage_size = number_of_ele_ * sizeof(value_type); + HIP_CHECK(test_common_utils::hipMallocHelper(&ret.device_raw_ptr_, storage_size)); + HIP_CHECK(hipMemcpyAsync(ret.device_raw_ptr_, + device_raw_ptr_, + storage_size, + hipMemcpyDeviceToDevice, + stream)); + return ret; + } + /// \brief Do type cast and move the ownership to the new `device_ptr`. /// /// \return A new `device_ptr` rvalue. @@ -151,9 +230,9 @@ class device_ptr leak(); } - void resize(size_type _new_number_of_ele) + void resize(size_type new_number_of_ele) { - if(_new_number_of_ele == 0) + if(new_number_of_ele == 0) { free_manually(); } @@ -161,17 +240,99 @@ class device_ptr { value_type* device_temp_ptr = nullptr; HIP_CHECK(test_common_utils::hipMallocHelper(&device_temp_ptr, - _new_number_of_ele * sizeof(value_type))); + new_number_of_ele * sizeof(value_type))); HIP_CHECK(hipMemcpy(device_temp_ptr, device_raw_ptr_, - std::min(_new_number_of_ele, number_of_ele_) * sizeof(value_type), + std::min(new_number_of_ele, number_of_ele_) * sizeof(value_type), hipMemcpyDeviceToDevice)); free_manually(); device_raw_ptr_ = device_temp_ptr; - number_of_ele_ = _new_number_of_ele; + number_of_ele_ = new_number_of_ele; } } + void resize_async(size_type new_number_of_ele, hipStream_t stream) + { + if(new_number_of_ele == 0) + { + free_manually(); + } + else + { + value_type* device_temp_ptr = nullptr; + HIP_CHECK(test_common_utils::hipMallocHelper(&device_temp_ptr, + new_number_of_ele * sizeof(value_type))); + HIP_CHECK( + hipMemcpyAsync(device_temp_ptr, + device_raw_ptr_, + std::min(new_number_of_ele, number_of_ele_) * sizeof(value_type), + hipMemcpyDeviceToDevice, + stream)); + free_manually(); + device_raw_ptr_ = device_temp_ptr; + number_of_ele_ = new_number_of_ele; + } + } + + // if got error hipErrorOutOfMemory` return false, else return `true` + bool resize_with_memory_check(size_type new_number_of_ele) + { + if(new_number_of_ele == 0) + { + free_manually(); + } + else + { + value_type* device_temp_ptr = nullptr; + const auto err + = test_common_utils::hipMallocHelper(&device_temp_ptr, + new_number_of_ele * sizeof(value_type)); + if(err == hipErrorOutOfMemory) + { + return false; + } + HIP_CHECK(err); + HIP_CHECK(hipMemcpy(device_temp_ptr, + device_raw_ptr_, + std::min(new_number_of_ele, number_of_ele_) * sizeof(value_type), + hipMemcpyDeviceToDevice)); + free_manually(); + device_raw_ptr_ = device_temp_ptr; + number_of_ele_ = new_number_of_ele; + } + return true; + } + + bool resize_with_memory_check_async(size_type new_number_of_ele, hipStream_t stream) + { + if(new_number_of_ele == 0) + { + free_manually(); + } + else + { + value_type* device_temp_ptr = nullptr; + const auto err + = test_common_utils::hipMallocHelper(&device_temp_ptr, + new_number_of_ele * sizeof(value_type)); + if(err == hipErrorOutOfMemory) + { + return false; + } + HIP_CHECK(err); + HIP_CHECK( + hipMemcpyAsync(device_temp_ptr, + device_raw_ptr_, + std::min(new_number_of_ele, number_of_ele_) * sizeof(value_type), + hipMemcpyDeviceToDevice, + stream)); + free_manually(); + device_raw_ptr_ = device_temp_ptr; + number_of_ele_ = new_number_of_ele; + } + return true; + } + /// \brief Get the size of this memory space size_type msize() const noexcept { @@ -185,12 +346,11 @@ class device_ptr } /// \brief Copy from host to device - template - void store(std::vector const& host_vec, size_type offset = 0) + template + void store(std::vector const& host_vec, size_type offset = 0) { - static_assert( - sizeof(InVecValueType) == sizeof(value_type), - "value_type of input vector must have the same size with device_ptr::value_type"); + static_assert(sizeof(InValueType) == sizeof(value_type), + "value_type of input must have the same size with device_ptr::value_type"); if(host_vec.size() + offset > number_of_ele_) { @@ -203,26 +363,170 @@ class device_ptr hipMemcpyHostToDevice)); } + template + void store(std::array const& host_arr) + { + static_assert(sizeof(InValueType) == sizeof(value_type), + "value_type of input must have the same size with device_ptr::value_type"); + + if(Size > number_of_ele_) + { + resize(Size); + } + + HIP_CHECK(hipMemcpy(device_raw_ptr_, + host_arr.data(), + Size * sizeof(value_type), + hipMemcpyHostToDevice)); + } + + template + void + store(std::unique_ptr const& uptr, size_type offset, size_type number_of_ele) + { + static_assert( + sizeof(InValueType) == sizeof(value_type), + "value_type of input unique_ptr must have the same size with device_ptr::value_type"); + + if(offset + number_of_ele > number_of_ele_) + { + resize(offset + number_of_ele); + } + HIP_CHECK(hipMemcpy(device_raw_ptr_ + offset, + uptr.get(), + number_of_ele * sizeof(value_type), + hipMemcpyHostToDevice)); + } + + template + void store_async(std::vector const& host_vec, hipStream_t stream) + { + static_assert( + sizeof(InValueType) == sizeof(value_type), + "value_type of input vector must have the same size with device_ptr::value_type"); + + if(host_vec.size() > number_of_ele_) + { + resize(host_vec.size()); + } + + HIP_CHECK(hipMemcpyAsync(device_raw_ptr_, + host_vec.data(), + host_vec.size() * sizeof(value_type), + hipMemcpyHostToDevice, + stream)); + } + + template + void store_async(std::array const& host_arr, hipStream_t stream) + { + static_assert(sizeof(InValueType) == sizeof(value_type), + "value_type of input must have the same size with device_ptr::value_type"); + + if(Size > number_of_ele_) + { + resize(Size); + } + + HIP_CHECK(hipMemcpyAsync(device_raw_ptr_, + host_arr.data(), + Size * sizeof(value_type), + hipMemcpyHostToDevice, + stream)); + } + + template + void store_async(std::unique_ptr const& uptr, + size_type offset, + size_type number_of_ele, + hipStream_t stream) + { + static_assert( + sizeof(InValueType) == sizeof(value_type), + "value_type of input unique_ptr must have the same size with device_ptr::value_type"); + + if(offset + number_of_ele > number_of_ele_) + { + resize(offset + number_of_ele); + } + HIP_CHECK(hipMemcpyAsync(device_raw_ptr_ + offset, + uptr.get(), + number_of_ele * sizeof(value_type), + hipMemcpyHostToDevice, + stream)); + } + + // will not check the boundary + void store_value_at(size_type pos, value_type const& value) + { + HIP_CHECK( + hipMemcpy(device_raw_ptr_ + pos, &value, sizeof(value_type), hipMemcpyHostToDevice)); + } + + // will not check the boundary + void store_value_at_async(size_type pos, value_type const& value, hipStream_t stream) + { + HIP_CHECK(hipMemcpy(device_raw_ptr_ + pos, + &value, + sizeof(value_type), + hipMemcpyHostToDevice, + stream)); + } + /// \brief Copy from device to device template - void replace(device_ptr const& device_ptr, size_type offset = 0) + void replace(device_ptr const& device_ptr) { static_assert(sizeof(InPtrValueType) == sizeof(value_type), "sizeof(InPtrValueType) must equal to sizeof(value_type)"); - if(device_ptr.number_of_ele_ + offset > number_of_ele_) + if(device_ptr.number_of_ele_ > number_of_ele_) { - resize(device_ptr.number_of_ele_ + offset); + resize(device_ptr.number_of_ele_); } - HIP_CHECK(hipMemcpy(device_raw_ptr_ + offset, + HIP_CHECK(hipMemcpy(device_raw_ptr_, device_ptr.device_raw_ptr_, device_ptr.number_of_ele_ * sizeof(value_type), hipMemcpyDeviceToDevice)); } + template + void replace_async(device_ptr const& device_ptr, hipStream_t stream) + { + static_assert(sizeof(InPtrValueType) == sizeof(value_type), + "sizeof(InPtrValueType) must equal to sizeof(value_type)"); + + if(device_ptr.number_of_ele_ > number_of_ele_) + { + resize(device_ptr.number_of_ele_); + } + + HIP_CHECK(hipMemcpyAsync(device_raw_ptr_, + device_ptr.device_raw_ptr_, + device_ptr.number_of_ele_ * sizeof(value_type), + hipMemcpyDeviceToDevice, + stream)); + } + + void memset(size_type offset, int value, size_type size_bytes) + { + HIP_CHECK(hipMemset(reinterpret_cast(device_raw_ptr_) + offset, + value, + static_cast(size_bytes))); + } + + void memset_async(size_type offset, int value, size_type size_bytes, hipStream_t stream) + { + HIP_CHECK(hipMemsetAsync(reinterpret_cast(device_raw_ptr_) + offset, + value, + static_cast(size_bytes), + stream)); + } + /// \brief Copy from device to host - std::vector load() const + /// This function will store loaded values into std::vector + auto load() const { std::vector ret(number_of_ele_); HIP_CHECK(hipMemcpy(ret.data(), @@ -231,17 +535,81 @@ class device_ptr hipMemcpyDeviceToHost)); return ret; } - /// \brief Copy from device to host - std::vector load(size_type target_size, size_type offset = 0) const + + auto load_async(hipStream_t stream) const { - std::vector ret(target_size); + std::vector ret(number_of_ele_); + HIP_CHECK(hipMemcpyAsync(ret.data(), + device_raw_ptr_, + number_of_ele_ * sizeof(value_type), + hipMemcpyDeviceToHost, + stream)); + return ret; + } + + template + auto load_to_array() const + { + std::array ret; HIP_CHECK(hipMemcpy(ret.data(), - device_raw_ptr_ + offset, - target_size * sizeof(value_type), + device_raw_ptr_, + std::min(number_of_ele_, Size) * sizeof(value_type), hipMemcpyDeviceToHost)); return ret; } + template + auto load_to_array_async(hipStream_t stream) const + { + std::array ret; + HIP_CHECK(hipMemcpyAsync(ret.data(), + device_raw_ptr_, + std::min(number_of_ele_, Size) * sizeof(value_type), + hipMemcpyDeviceToHost, + stream)); + return ret; + } + + auto load_to_unique_ptr() const + { + std::unique_ptr ret(new value_type[number_of_ele_]); + HIP_CHECK(hipMemcpy(ret.get(), + device_raw_ptr_, + number_of_ele_ * sizeof(value_type), + hipMemcpyDeviceToHost)); + return ret; + } + + auto load_to_unique_ptr_async(hipStream_t stream) const + { + std::unique_ptr ret(new value_type[number_of_ele_]); + HIP_CHECK(hipMemcpyAsync(ret.get(), + device_raw_ptr_, + number_of_ele_ * sizeof(value_type), + hipMemcpyDeviceToHost, + stream)); + return ret; + } + + auto load_value_at(size_type pos) const + { + value_type ret; + HIP_CHECK( + hipMemcpy(&ret, device_raw_ptr_ + pos, sizeof(value_type), hipMemcpyDeviceToHost)); + return ret; + } + + auto load_value_at_async(size_type pos, hipStream_t stream) const + { + value_type ret; + HIP_CHECK(hipMemcpyAsync(&ret, + device_raw_ptr_ + pos, + sizeof(value_type), + hipMemcpyDeviceToHost, + stream)); + return ret; + } + private: value_type* device_raw_ptr_; size_type number_of_ele_; diff --git a/test/rocprim/test_zip_iterator.cpp b/test/rocprim/test_zip_iterator.cpp index 9ab54d429..b93847f8a 100644 --- a/test/rocprim/test_zip_iterator.cpp +++ b/test/rocprim/test_zip_iterator.cpp @@ -206,7 +206,7 @@ TEST(RocprimZipIteratorTests, Transform) HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - output = d_output.load(output.size()); + output = d_output.load(); // Check if output values are as expected // precision of tuple3_transform_op is precision * 2 From b3b6497b9a922c50de67c274bb50baa2c07dc741 Mon Sep 17 00:00:00 2001 From: Nick Breed Date: Wed, 18 Dec 2024 07:50:19 +0000 Subject: [PATCH 48/84] Resolve "Reduce amount of test runs based on environment variable." --- .gitlab-ci.yml | 6 +++++ test/rocprim/test_arg_index_iterator.cpp | 4 +-- ...test_block_adjacent_difference.kernels.hpp | 14 +++++----- .../test_block_discontinuity.kernels.hpp | 6 ++--- test/rocprim/test_block_histogram.kernels.hpp | 2 +- test/rocprim/test_block_load_store.hpp | 6 ++--- test/rocprim/test_block_radix_rank.hpp | 2 +- .../rocprim/test_block_radix_sort.kernels.hpp | 4 +-- test/rocprim/test_block_reduce.hpp | 8 +++--- test/rocprim/test_block_reduce.kernels.hpp | 2 +- test/rocprim/test_block_run_length_decode.cpp | 2 +- test/rocprim/test_block_scan.hpp | 12 ++++----- test/rocprim/test_block_scan.kernels.hpp | 12 ++++----- test/rocprim/test_block_shuffle.hpp | 8 +++--- test/rocprim/test_block_sort.hpp | 6 ++--- test/rocprim/test_constant_iterator.cpp | 2 +- test/rocprim/test_counting_iterator.cpp | 2 +- .../test_device_adjacent_difference.cpp | 4 +-- test/rocprim/test_device_adjacent_find.cpp | 2 +- test/rocprim/test_device_batch_memcpy.cpp | 2 +- test/rocprim/test_device_binary_search.cpp | 6 ++--- test/rocprim/test_device_find_end.cpp | 4 +-- test/rocprim/test_device_find_first_of.cpp | 2 +- test/rocprim/test_device_histogram.cpp | 9 +++---- test/rocprim/test_device_merge.cpp | 4 +-- test/rocprim/test_device_merge_sort.cpp | 4 +-- test/rocprim/test_device_nth_element.cpp | 2 +- test/rocprim/test_device_partial_sort.cpp | 4 +-- test/rocprim/test_device_partition.cpp | 10 +++---- test/rocprim/test_device_radix_sort.hpp | 8 +++--- test/rocprim/test_device_reduce.cpp | 8 +++--- test/rocprim/test_device_reduce_by_key.cpp | 4 +-- .../rocprim/test_device_run_length_encode.cpp | 4 +-- test/rocprim/test_device_scan.cpp | 14 +++++----- test/rocprim/test_device_search.cpp | 4 +-- test/rocprim/test_device_search_n.cpp | 26 +++++++++---------- .../test_device_segmented_radix_sort.hpp | 14 +++++----- test/rocprim/test_device_segmented_reduce.cpp | 2 +- test/rocprim/test_device_segmented_scan.cpp | 8 +++--- test/rocprim/test_device_select.cpp | 14 +++++----- test/rocprim/test_device_transform.cpp | 6 ++--- test/rocprim/test_discard_iterator.cpp | 4 +-- test/rocprim/test_intrinsics.cpp | 16 ++++++------ .../rocprim/test_lookback_reproducibility.cpp | 6 ++--- test/rocprim/test_radix_key_codec.cpp | 2 +- test/rocprim/test_seed.hpp | 21 ++++++++++++--- test/rocprim/test_texture_cache_iterator.cpp | 2 +- test/rocprim/test_thread.cpp | 4 +-- test/rocprim/test_thread_algos.cpp | 10 +++---- test/rocprim/test_transform_iterator.cpp | 2 +- test/rocprim/test_warp_reduce.hpp | 14 +++++----- test/rocprim/test_warp_scan.hpp | 18 ++++++------- test/rocprim/test_warp_sort.hpp | 4 +-- test/rocprim/test_zip_iterator.cpp | 4 +-- 54 files changed, 194 insertions(+), 176 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3342b4381..e2843dd37 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,6 +44,12 @@ stages: workflow: rules: + - if: $CI_MERGE_REQUEST_TITLE !~ /Draft:/ + variables: + ROCPRIM_TEST_RUNS: 3 + - if: $CI_MERGE_REQUEST_TITLE =~ /Draft:/ + variables: + ROCPRIM_TEST_RUNS: 1 - if: $CI_MERGE_REQUEST_LABELS !~ /CI Skip/ variables: diff --git a/test/rocprim/test_arg_index_iterator.cpp b/test/rocprim/test_arg_index_iterator.cpp index d9f134b07..a14c60d8b 100644 --- a/test/rocprim/test_arg_index_iterator.cpp +++ b/test/rocprim/test_arg_index_iterator.cpp @@ -64,7 +64,7 @@ TYPED_TEST(RocprimArgIndexIteratorTests, Equal) using T = typename TestFixture::input_type; using Iterator = typename rocprim::arg_index_iterator; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -120,7 +120,7 @@ TYPED_TEST(RocprimArgIndexIteratorTests, ReduceArgMinimum) hipStream_t stream = 0; // default - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_block_adjacent_difference.kernels.hpp b/test/rocprim/test_block_adjacent_difference.kernels.hpp index bee409992..bd24c3efa 100644 --- a/test/rocprim/test_block_adjacent_difference.kernels.hpp +++ b/test/rocprim/test_block_adjacent_difference.kernels.hpp @@ -397,7 +397,7 @@ auto test_block_adjacent_difference() -> typename std::enable_if::t return; } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -496,7 +496,7 @@ auto test_block_adjacent_difference() -> typename std::enable_if::t return; } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -597,7 +597,7 @@ auto test_block_adjacent_difference() -> typename std::enable_if::t return; } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -710,7 +710,7 @@ auto test_block_adjacent_difference() -> typename std::enable_if::t return; } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { const unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -796,7 +796,7 @@ auto test_block_adjacent_difference() -> typename std::enable_if::t return; } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { const unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -882,7 +882,7 @@ auto test_block_adjacent_difference() -> typename std::enable_if::t return; } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { const unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -980,7 +980,7 @@ auto test_block_adjacent_difference() -> typename std::enable_if::t return; } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { const unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_block_discontinuity.kernels.hpp b/test/rocprim/test_block_discontinuity.kernels.hpp index 81556fec0..fb043e169 100644 --- a/test/rocprim/test_block_discontinuity.kernels.hpp +++ b/test/rocprim/test_block_discontinuity.kernels.hpp @@ -231,7 +231,7 @@ auto test_block_discontinuity() return; } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -328,7 +328,7 @@ auto test_block_discontinuity() return; } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -425,7 +425,7 @@ auto test_block_discontinuity() return; } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_block_histogram.kernels.hpp b/test/rocprim/test_block_histogram.kernels.hpp index 62fb8d97d..be6177a76 100644 --- a/test/rocprim/test_block_histogram.kernels.hpp +++ b/test/rocprim/test_block_histogram.kernels.hpp @@ -127,7 +127,7 @@ void test_block_histogram_input_arrays() GTEST_SKIP() << "Temporary skipped test"; } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_block_load_store.hpp b/test/rocprim/test_block_load_store.hpp index 1a5a7f1f1..f306bb509 100644 --- a/test/rocprim/test_block_load_store.hpp +++ b/test/rocprim/test_block_load_store.hpp @@ -60,7 +60,7 @@ typed_test_def(suite_name, name_suffix, LoadStoreClass) return; } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -138,7 +138,7 @@ typed_test_def(suite_name, name_suffix, LoadStoreClassValid) const size_t valid = items_per_block - 32; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -225,7 +225,7 @@ typed_test_def(suite_name, name_suffix, LoadStoreClassDefault) const size_t valid = items_per_thread + 1; Type _default = (Type)-1; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_block_radix_rank.hpp b/test/rocprim/test_block_radix_rank.hpp index 4f35e6474..60aab0a5a 100644 --- a/test/rocprim/test_block_radix_rank.hpp +++ b/test/rocprim/test_block_radix_rank.hpp @@ -132,7 +132,7 @@ void test_block_radix_rank() SCOPED_TRACE(testing::Message() << "with grid_size = " << size); SCOPED_TRACE(testing::Message() << "with size = " << size); - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; ++seed_index) + for(size_t seed_index = 0; seed_index < number_of_runs; ++seed_index) { seed_type seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_block_radix_sort.kernels.hpp b/test/rocprim/test_block_radix_sort.kernels.hpp index cef967a6b..0aac1bd1b 100644 --- a/test/rocprim/test_block_radix_sort.kernels.hpp +++ b/test/rocprim/test_block_radix_sort.kernels.hpp @@ -189,7 +189,7 @@ auto test_block_radix_sort() -> typename std::enable_if::type SCOPED_TRACE(testing::Message() << "with size = " << size); SCOPED_TRACE(testing::Message() << "with grid_size = " << grid_size); - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -277,7 +277,7 @@ auto test_block_radix_sort() -> typename std::enable_if::type SCOPED_TRACE(testing::Message() << "with size = " << size); SCOPED_TRACE(testing::Message() << "with grid_size = " << grid_size); - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { seed_type seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_block_reduce.hpp b/test/rocprim/test_block_reduce.hpp index 105514d7f..d740174bf 100644 --- a/test/rocprim/test_block_reduce.hpp +++ b/test/rocprim/test_block_reduce.hpp @@ -50,7 +50,7 @@ typed_test_def(suite_name_single, name_suffix, Reduce) const size_t size = block_size * 58; const size_t grid_size = size / block_size; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -128,7 +128,7 @@ typed_test_def(suite_name_single, name_suffix, ReduceMultiplies) const size_t size = block_size * 58; const size_t grid_size = size / block_size; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -208,7 +208,7 @@ typed_test_def(suite_name_single, name_suffix, ReduceMultipliesExact) const size_t size = block_size * 58; const size_t grid_size = size / block_size; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -289,7 +289,7 @@ typed_test_def(suite_name_single, name_suffix, ReduceValid) constexpr size_t block_size = TestFixture::block_size; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_block_reduce.kernels.hpp b/test/rocprim/test_block_reduce.kernels.hpp index 874776fc0..79e844eb9 100644 --- a/test/rocprim/test_block_reduce.kernels.hpp +++ b/test/rocprim/test_block_reduce.kernels.hpp @@ -241,7 +241,7 @@ void test_block_reduce_input_arrays() SCOPED_TRACE(testing::Message() << "with size = " << size); SCOPED_TRACE(testing::Message() << "with grid_size = " << grid_size); - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_block_run_length_decode.cpp b/test/rocprim/test_block_run_length_decode.cpp index 5f60754b2..8c98e15fe 100644 --- a/test/rocprim/test_block_run_length_decode.cpp +++ b/test/rocprim/test_block_run_length_decode.cpp @@ -160,7 +160,7 @@ TYPED_TEST(HipcubBlockRunLengthDecodeTest, TestDecode) constexpr unsigned runs_per_thread = TestFixture::params::runs_per_thread; constexpr unsigned decoded_items_per_thread = TestFixture::params::decoded_items_per_thread; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_block_scan.hpp b/test/rocprim/test_block_scan.hpp index 82d123740..643ea2ce6 100644 --- a/test/rocprim/test_block_scan.hpp +++ b/test/rocprim/test_block_scan.hpp @@ -49,7 +49,7 @@ typed_test_def(suite_name_single, name_suffix, InclusiveScan) const size_t size = block_size * 58; const size_t grid_size = size / block_size; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -120,7 +120,7 @@ typed_test_def(suite_name_single, name_suffix, InclusiveScanReduce) const size_t size = block_size * 58; const size_t grid_size = size / block_size; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -195,7 +195,7 @@ typed_test_def(suite_name_single, name_suffix, InclusiveScanPrefixCallback) const size_t size = block_size * 58; const size_t grid_size = size / block_size; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -271,7 +271,7 @@ typed_test_def(suite_name_single, name_suffix, ExclusiveScan) const size_t size = block_size * 58; const size_t grid_size = size / block_size; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -343,7 +343,7 @@ typed_test_def(suite_name_single, name_suffix, ExclusiveScanReduce) const size_t size = block_size * 58; const size_t grid_size = size / block_size; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -429,7 +429,7 @@ typed_test_def(suite_name_single, name_suffix, ExclusiveScanPrefixCallback) const size_t size = block_size * 58; const size_t grid_size = size / block_size; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_block_scan.kernels.hpp b/test/rocprim/test_block_scan.kernels.hpp index baea8f02e..bcb2f34fc 100644 --- a/test/rocprim/test_block_scan.kernels.hpp +++ b/test/rocprim/test_block_scan.kernels.hpp @@ -488,7 +488,7 @@ auto test_block_scan_input_arrays() SCOPED_TRACE(testing::Message() << "with size = " << size); SCOPED_TRACE(testing::Message() << "with grid_size = " << grid_size); - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -565,7 +565,7 @@ auto test_block_scan_input_arrays() SCOPED_TRACE(testing::Message() << "with size = " << size); SCOPED_TRACE(testing::Message() << "with grid_size = " << grid_size); - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -651,7 +651,7 @@ auto test_block_scan_input_arrays() SCOPED_TRACE(testing::Message() << "with size = " << size); SCOPED_TRACE(testing::Message() << "with grid_size = " << grid_size); - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -740,7 +740,7 @@ auto test_block_scan_input_arrays() SCOPED_TRACE(testing::Message() << "with size = " << size); SCOPED_TRACE(testing::Message() << "with grid_size = " << grid_size); - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -820,7 +820,7 @@ auto test_block_scan_input_arrays() SCOPED_TRACE(testing::Message() << "with size = " << size); SCOPED_TRACE(testing::Message() << "with grid_size = " << grid_size); - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -913,7 +913,7 @@ auto test_block_scan_input_arrays() SCOPED_TRACE(testing::Message() << "with size = " << size); SCOPED_TRACE(testing::Message() << "with grid_size = " << grid_size); - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_block_shuffle.hpp b/test/rocprim/test_block_shuffle.hpp index bfed55c30..586fb536e 100644 --- a/test/rocprim/test_block_shuffle.hpp +++ b/test/rocprim/test_block_shuffle.hpp @@ -34,7 +34,7 @@ typed_test_def(suite_name, name_suffix, BlockOffset) static constexpr size_t block_size = TestFixture::block_size; static constexpr size_t size = block_size * 11; static constexpr size_t grid_size = size / block_size; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; int distance = (rand()%std::min(10,block_size/2))-std::min(10,block_size/2); @@ -88,7 +88,7 @@ typed_test_def(suite_name, name_suffix, BlockRotate) static constexpr size_t block_size = TestFixture::block_size; static constexpr size_t size = block_size * 11; static constexpr size_t grid_size = size / block_size; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; int distance = (rand()%std::min(5,block_size/2)); @@ -142,7 +142,7 @@ typed_test_def(suite_name, name_suffix, BlockUp) static constexpr size_t size = block_size * 11; static constexpr size_t grid_size = size / block_size; static constexpr unsigned int ItemsPerThread = 128; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -200,7 +200,7 @@ typed_test_def(suite_name, name_suffix, BlockDown) static constexpr size_t size = block_size * 11; static constexpr size_t grid_size = size / block_size; static constexpr unsigned int ItemsPerThread = 128; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_block_sort.hpp b/test/rocprim/test_block_sort.hpp index a18bd9f88..22749cff8 100644 --- a/test/rocprim/test_block_sort.hpp +++ b/test/rocprim/test_block_sort.hpp @@ -53,7 +53,7 @@ void TestSortKeyValue() GTEST_SKIP(); } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -162,7 +162,7 @@ void TestSortKey(std::vector sizes) GTEST_SKIP(); } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -247,7 +247,7 @@ void TestSortStableKey(std::vector sizes) GTEST_SKIP(); } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_constant_iterator.cpp b/test/rocprim/test_constant_iterator.cpp index 2224ce360..6d27b0d67 100644 --- a/test/rocprim/test_constant_iterator.cpp +++ b/test/rocprim/test_constant_iterator.cpp @@ -78,7 +78,7 @@ TYPED_TEST(RocprimConstantIteratorTests, Transform) hipStream_t stream = 0; // default - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_counting_iterator.cpp b/test/rocprim/test_counting_iterator.cpp index 555acd114..e7ab3e7b8 100644 --- a/test/rocprim/test_counting_iterator.cpp +++ b/test/rocprim/test_counting_iterator.cpp @@ -78,7 +78,7 @@ TYPED_TEST(RocprimCountingIteratorTests, Transform) hipStream_t stream = 0; // default - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_device_adjacent_difference.cpp b/test/rocprim/test_device_adjacent_difference.cpp index 95b93f754..00895599b 100644 --- a/test/rocprim/test_device_adjacent_difference.cpp +++ b/test/rocprim/test_device_adjacent_difference.cpp @@ -319,7 +319,7 @@ TYPED_TEST(RocprimDeviceAdjacentDifferenceTests, AdjacentDifference) SCOPED_TRACE(testing::Message() << "left = " << left << ", api_variant = " << to_string(aliasing)); - for(std::size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(std::size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { const unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -613,7 +613,7 @@ TYPED_TEST(RocprimDeviceAdjacentDifferenceLargeTests, LargeIndices) HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for(std::size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(std::size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_device_adjacent_find.cpp b/test/rocprim/test_device_adjacent_find.cpp index 459e662b6..1f5453ee4 100644 --- a/test/rocprim/test_device_adjacent_find.cpp +++ b/test/rocprim/test_device_adjacent_find.cpp @@ -125,7 +125,7 @@ TYPED_TEST(RocprimDeviceAdjacentFindTests, AdjacentFind) op_type op{}; - for(std::size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(std::size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { const unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_device_batch_memcpy.cpp b/test/rocprim/test_device_batch_memcpy.cpp index d97db5c0c..8e83411dd 100644 --- a/test/rocprim/test_device_batch_memcpy.cpp +++ b/test/rocprim/test_device_batch_memcpy.cpp @@ -365,7 +365,7 @@ TYPED_TEST(RocprimDeviceBatchMemcpyTests, SizeAndTypeVariation) constexpr int num_tlev = num_buffers - num_blev - num_wlev; // Get random buffer sizes - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; ++seed_index) + for(size_t seed_index = 0; seed_index < number_of_runs; ++seed_index) { seed_type seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_device_binary_search.cpp b/test/rocprim/test_device_binary_search.cpp index 1f3adb2d8..2ec949645 100644 --- a/test/rocprim/test_device_binary_search.cpp +++ b/test/rocprim/test_device_binary_search.cpp @@ -104,7 +104,7 @@ TYPED_TEST(RocprimDeviceBinarySearch, LowerBound) compare_op_type compare_op; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -220,7 +220,7 @@ TYPED_TEST(RocprimDeviceBinarySearch, UpperBound) compare_op_type compare_op; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { seed_type seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -337,7 +337,7 @@ TYPED_TEST(RocprimDeviceBinarySearch, BinarySearch) compare_op_type compare_op; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_device_find_end.cpp b/test/rocprim/test_device_find_end.cpp index 9a3039bea..d6074524b 100644 --- a/test/rocprim/test_device_find_end.cpp +++ b/test/rocprim/test_device_find_end.cpp @@ -126,7 +126,7 @@ TYPED_TEST(RocprimDeviceFindEndTests, FindEnd) std::vector key_sizes = {0, 1, 10, 1000, 10000}; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -278,7 +278,7 @@ TYPED_TEST(RocprimDeviceFindEndTests, FindEndRepetition) size_t key_size = 10; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_device_find_first_of.cpp b/test/rocprim/test_device_find_first_of.cpp index e1544567c..e73a11b4d 100644 --- a/test/rocprim/test_device_find_first_of.cpp +++ b/test/rocprim/test_device_find_first_of.cpp @@ -148,7 +148,7 @@ TYPED_TEST(RocprimDeviceFindFirstOfTests, FindFirstOf) constexpr bool debug_synchronous = TestFixture::debug_synchronous; constexpr bool use_indirect_iterator = TestFixture::use_indirect_iterator; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_device_histogram.cpp b/test/rocprim/test_device_histogram.cpp index 94a836ce2..855201c4b 100644 --- a/test/rocprim/test_device_histogram.cpp +++ b/test/rocprim/test_device_histogram.cpp @@ -234,7 +234,7 @@ TYPED_TEST(RocprimDeviceHistogramEven, Even) const size_t row_stride_bytes = row_stride * sizeof(sample_type); const size_t size = std::max(1, rows * row_stride); - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -490,7 +490,7 @@ TYPED_TEST(RocprimDeviceHistogramRange, Range) } levels.push_back(level); - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -712,8 +712,7 @@ TYPED_TEST(RocprimDeviceHistogramMultiEven, MultiEven) const size_t row_stride_bytes = row_stride * sizeof(sample_type); const size_t size = std::max(1, rows * row_stride); - - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -998,7 +997,7 @@ TYPED_TEST(RocprimDeviceHistogramMultiRange, MultiRange) const size_t row_stride_bytes = row_stride * sizeof(sample_type); const size_t size = std::max(1, rows * row_stride); - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_device_merge.cpp b/test/rocprim/test_device_merge.cpp index ad534a7e4..46a49fc89 100644 --- a/test/rocprim/test_device_merge.cpp +++ b/test/rocprim/test_device_merge.cpp @@ -146,7 +146,7 @@ TYPED_TEST(RocprimDeviceMergeTests, MergeKey) // compare function compare_op_type compare_op; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -280,7 +280,7 @@ TYPED_TEST(RocprimDeviceMergeTests, MergeKeyValue) // compare function compare_op_type compare_op; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_device_merge_sort.cpp b/test/rocprim/test_device_merge_sort.cpp index 0ccc803db..152d0496f 100644 --- a/test/rocprim/test_device_merge_sort.cpp +++ b/test/rocprim/test_device_merge_sort.cpp @@ -100,7 +100,7 @@ TYPED_TEST(RocprimDeviceSortTests, SortKey) bool in_place = false; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -207,7 +207,7 @@ TYPED_TEST(RocprimDeviceSortTests, SortKeyValue) bool in_place = false; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_device_nth_element.cpp b/test/rocprim/test_device_nth_element.cpp index 4d4199227..7fda72873 100644 --- a/test/rocprim/test_device_nth_element.cpp +++ b/test/rocprim/test_device_nth_element.cpp @@ -192,7 +192,7 @@ TYPED_TEST(RocprimDeviceNthelementTests, NthelementKey) // The size loop alternates between in place and not in place bool in_place = false; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_device_partial_sort.cpp b/test/rocprim/test_device_partial_sort.cpp index 0a7a4295d..2746924c4 100644 --- a/test/rocprim/test_device_partial_sort.cpp +++ b/test/rocprim/test_device_partial_sort.cpp @@ -195,7 +195,7 @@ TYPED_TEST(RocprimDevicePartialSortTests, PartialSort) const bool debug_synchronous = TestFixture::debug_synchronous; constexpr bool use_indirect_iterator = TestFixture::use_indirect_iterator; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; ++seed_index) + for(size_t seed_index = 0; seed_index < number_of_runs; ++seed_index) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -382,7 +382,7 @@ TYPED_TEST(RocprimDevicePartialSortTests, PartialSortCopy) constexpr bool input_is_const = std::is_const_v; constexpr bool use_indirect_iterator = TestFixture::use_indirect_iterator; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; ++seed_index) + for(size_t seed_index = 0; seed_index < number_of_runs; ++seed_index) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_device_partition.cpp b/test/rocprim/test_device_partition.cpp index 09af28b88..d7d34f2d3 100644 --- a/test/rocprim/test_device_partition.cpp +++ b/test/rocprim/test_device_partition.cpp @@ -105,7 +105,7 @@ TYPED_TEST(RocprimDevicePartitionTests, Flagged) HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -330,7 +330,7 @@ TYPED_TEST(RocprimDevicePartitionTests, Predicate) auto select_op = select_op_t{}; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -459,7 +459,7 @@ TYPED_TEST(RocprimDevicePartitionTests, PredicateTwoWay) auto select_op = select_op_t{}; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -605,7 +605,7 @@ TYPED_TEST(RocprimDevicePartitionTests, PredicateThreeWay) { static_cast(30), static_cast(101) } // unselected is empty }; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { const unsigned int seed_value = seed_index < random_seeds_count ? static_cast(rand()) : seeds[seed_index - random_seeds_count]; @@ -1395,7 +1395,7 @@ TEST(RocprimDevicePartitionBlockSizeTests, BlockSize) // Use some power of two and off-by-one-from-power-of-two data sizes. const std::vector sizes = {256, 257, 511, 512, 1024, 1025}; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_device_radix_sort.hpp b/test/rocprim/test_device_radix_sort.hpp index dfa15ee56..a299efe6f 100644 --- a/test/rocprim/test_device_radix_sort.hpp +++ b/test/rocprim/test_device_radix_sort.hpp @@ -249,7 +249,7 @@ void sort_keys() bool in_place = false; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -517,7 +517,7 @@ void sort_pairs() bool in_place = false; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -798,7 +798,7 @@ void sort_keys_double_buffer() const bool debug_synchronous = false; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -1042,7 +1042,7 @@ void sort_pairs_double_buffer() const bool debug_synchronous = false; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_device_reduce.cpp b/test/rocprim/test_device_reduce.cpp index 9b63c813c..29753865e 100644 --- a/test/rocprim/test_device_reduce.cpp +++ b/test/rocprim/test_device_reduce.cpp @@ -234,7 +234,7 @@ TYPED_TEST(RocprimDeviceReduceTests, ReduceSum) static constexpr bool use_identity_iterator = TestFixture::use_identity_iterator; using Config = size_limit_config_t; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -363,7 +363,7 @@ TYPED_TEST(RocprimDeviceReduceTests, ReduceArgMinimum) static constexpr bool use_identity_iterator = TestFixture::use_identity_iterator; using Config = size_limit_config_t; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -479,7 +479,7 @@ void testLargeIndices() HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -687,7 +687,7 @@ TYPED_TEST(RocprimDeviceReduceTests, ReduceMinimum) static constexpr bool use_identity_iterator = TestFixture::use_identity_iterator; using Config = size_limit_config_t; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_device_reduce_by_key.cpp b/test/rocprim/test_device_reduce_by_key.cpp index 278fe186c..657dd9e29 100644 --- a/test/rocprim/test_device_reduce_by_key.cpp +++ b/test/rocprim/test_device_reduce_by_key.cpp @@ -172,7 +172,7 @@ TYPED_TEST(RocprimDeviceReduceByKey, ReduceByKey) const unsigned int seed = 123; std::default_random_engine gen(seed); - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -586,7 +586,7 @@ TEST(RocprimDeviceReduceByKey, ReduceByNonEqualKeys) ::rocprim::plus reduce_op; auto key_compare_op = [](const auto&, const auto&) { return false; }; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_device_run_length_encode.cpp b/test/rocprim/test_device_run_length_encode.cpp index 22d758fef..d40624214 100644 --- a/test/rocprim/test_device_run_length_encode.cpp +++ b/test/rocprim/test_device_run_length_encode.cpp @@ -133,7 +133,7 @@ TYPED_TEST(RocprimDeviceRunLengthEncode, Encode) std::default_random_engine gen(seed); std::vector random_keys = test_utils::get_random_data(64, -100, 100, seed); - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -263,7 +263,7 @@ TYPED_TEST(RocprimDeviceRunLengthEncode, NonTrivialRuns) std::default_random_engine gen(seed); std::vector random_keys = test_utils::get_random_data(64, -100, 100, seed); - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_device_scan.cpp b/test/rocprim/test_device_scan.cpp index 56deecc84..a3b128068 100644 --- a/test/rocprim/test_device_scan.cpp +++ b/test/rocprim/test_device_scan.cpp @@ -347,7 +347,7 @@ TYPED_TEST(RocprimDeviceScanTests, InclusiveScan) SCOPED_TRACE(testing::Message() << "with device_id = " << device_id); HIP_CHECK(hipSetDevice(device_id)); - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -480,7 +480,7 @@ TYPED_TEST(RocprimDeviceScanTests, ExclusiveScan) SCOPED_TRACE(testing::Message() << "with device_id = " << device_id); HIP_CHECK(hipSetDevice(device_id)); - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -610,7 +610,7 @@ TYPED_TEST(RocprimDeviceScanTests, InclusiveScanByKey) SCOPED_TRACE(testing::Message() << "with device_id = " << device_id); HIP_CHECK(hipSetDevice(device_id)); - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -756,7 +756,7 @@ TYPED_TEST(RocprimDeviceScanTests, ExclusiveScanByKey) SCOPED_TRACE(testing::Message() << "with device_id = " << device_id); HIP_CHECK(hipSetDevice(device_id)); - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -967,7 +967,7 @@ void testLargeIndicesInclusiveScan() HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -1081,7 +1081,7 @@ void testLargeIndicesExclusiveScan() HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -1517,7 +1517,7 @@ TYPED_TEST(RocprimDeviceScanFutureTests, ExclusiveScan) SCOPED_TRACE(testing::Message() << "with device_id = " << device_id); HIP_CHECK(hipSetDevice(device_id)); - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { const unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_device_search.cpp b/test/rocprim/test_device_search.cpp index 52cafc901..00ec55d9b 100644 --- a/test/rocprim/test_device_search.cpp +++ b/test/rocprim/test_device_search.cpp @@ -126,7 +126,7 @@ TYPED_TEST(RocprimDeviceSearchTests, Search) std::vector key_sizes = {0, 1, 10, 1000, 10000}; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -276,7 +276,7 @@ TYPED_TEST(RocprimDeviceSearchTests, SearchRepetition) size_t key_size = 10; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_device_search_n.cpp b/test/rocprim/test_device_search_n.cpp index a7557e51b..26e1aed99 100644 --- a/test/rocprim/test_device_search_n.cpp +++ b/test/rocprim/test_device_search_n.cpp @@ -20,6 +20,7 @@ #include "../common_test_header.hpp" +#include "test_seed.hpp" #include "test_utils_custom_test_types.hpp" #include "test_utils_device_ptr.hpp" #include "test_utils_types.hpp" @@ -110,7 +111,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, RandomTest) constexpr bool debug_synchronous = TestFixture::debug_synchronous; op_type op{}; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -214,7 +215,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, MaxCount) constexpr bool debug_synchronous = TestFixture::debug_synchronous; op_type op{}; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -309,7 +310,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, MinCount) constexpr bool debug_synchronous = TestFixture::debug_synchronous; op_type op{}; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -502,7 +503,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, StartFromBegin) constexpr bool debug_synchronous = TestFixture::debug_synchronous; op_type op{}; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -596,7 +597,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, StartFromMiddle) constexpr bool debug_synchronous = TestFixture::debug_synchronous; op_type op{}; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -690,7 +691,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, StartFromEnd) constexpr bool debug_synchronous = TestFixture::debug_synchronous; op_type op{}; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -784,7 +785,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, StartFromEndButFail) constexpr bool debug_synchronous = TestFixture::debug_synchronous; op_type op{}; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -882,7 +883,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, NoiseTest_1block) constexpr bool debug_synchronous = TestFixture::debug_synchronous; op_type op{}; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -1002,7 +1003,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, NoiseTest_2block) constexpr bool debug_synchronous = TestFixture::debug_synchronous; op_type op{}; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -1122,7 +1123,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, NoiseTest_3block) constexpr bool debug_synchronous = TestFixture::debug_synchronous; op_type op{}; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -1242,7 +1243,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, MultiResult1) constexpr bool debug_synchronous = TestFixture::debug_synchronous; op_type op{}; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -1364,11 +1365,10 @@ TYPED_TEST(RocprimDeviceSearchNTests, MultiResult2) constexpr bool debug_synchronous = TestFixture::debug_synchronous; op_type op{}; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; - for(const auto size : test_utils::get_sizes(seed_value)) { using wrapped_config = rocprim::detail::wrapped_search_n_config; diff --git a/test/rocprim/test_device_segmented_radix_sort.hpp b/test/rocprim/test_device_segmented_radix_sort.hpp index d26f88970..020d45cc5 100644 --- a/test/rocprim/test_device_segmented_radix_sort.hpp +++ b/test/rocprim/test_device_segmented_radix_sort.hpp @@ -129,7 +129,7 @@ inline void sort_keys() TestFixture::params::min_segment_length, TestFixture::params::max_segment_length); - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -343,7 +343,7 @@ inline void sort_keys_large_segments() size_t size = 1 << 20; size_t segments_count = 2; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -451,7 +451,7 @@ inline void sort_keys_unspecified_ranges() TestFixture::params::min_segment_length, TestFixture::params::max_segment_length); - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -591,7 +591,7 @@ inline void sort_pairs() TestFixture::params::min_segment_length, TestFixture::params::max_segment_length); - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -745,7 +745,7 @@ inline void sort_pairs_unspecified_ranges() TestFixture::params::min_segment_length, TestFixture::params::max_segment_length); - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -911,7 +911,7 @@ inline void sort_keys_double_buffer() TestFixture::params::min_segment_length, TestFixture::params::max_segment_length); - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -1039,7 +1039,7 @@ inline void sort_pairs_double_buffer() TestFixture::params::min_segment_length, TestFixture::params::max_segment_length); - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_device_segmented_reduce.cpp b/test/rocprim/test_device_segmented_reduce.cpp index 6fd35160c..675d1dae8 100644 --- a/test/rocprim/test_device_segmented_reduce.cpp +++ b/test/rocprim/test_device_segmented_reduce.cpp @@ -175,7 +175,7 @@ TYPED_TEST(RocprimDeviceSegmentedReduce, Reduce) TestFixture::params::min_segment_length, TestFixture::params::max_segment_length); - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_device_segmented_scan.cpp b/test/rocprim/test_device_segmented_scan.cpp index e4492b98a..985f934e6 100644 --- a/test/rocprim/test_device_segmented_scan.cpp +++ b/test/rocprim/test_device_segmented_scan.cpp @@ -117,7 +117,7 @@ TYPED_TEST(RocprimDeviceSegmentedScan, InclusiveScan) HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -270,7 +270,7 @@ TYPED_TEST(RocprimDeviceSegmentedScan, ExclusiveScan) HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -418,7 +418,7 @@ TYPED_TEST(RocprimDeviceSegmentedScan, InclusiveScanUsingHeadFlags) HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -578,7 +578,7 @@ TYPED_TEST(RocprimDeviceSegmentedScan, ExclusiveScanUsingHeadFlags) HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_device_select.cpp b/test/rocprim/test_device_select.cpp index c5f4c7b9c..15ab2eae0 100644 --- a/test/rocprim/test_device_select.cpp +++ b/test/rocprim/test_device_select.cpp @@ -95,7 +95,7 @@ TYPED_TEST(RocprimDeviceSelectTests, Flagged) HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -222,7 +222,7 @@ TYPED_TEST(RocprimDeviceSelectTests, SelectOp) HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -336,7 +336,7 @@ TYPED_TEST(RocprimDeviceSelectTests, SelectFlagged) HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -464,7 +464,7 @@ TYPED_TEST(RocprimDeviceSelectTests, Unique) HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -618,7 +618,7 @@ void testUniqueGuardedOperator() HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -816,7 +816,7 @@ TYPED_TEST(RocprimDeviceUniqueByKeyTests, UniqueByKey) HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -976,7 +976,7 @@ TYPED_TEST(RocprimDeviceUniqueByKeyTests, UniqueByKeyAlias) HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_device_transform.cpp b/test/rocprim/test_device_transform.cpp index 6afa8327e..7ad6c6cbd 100644 --- a/test/rocprim/test_device_transform.cpp +++ b/test/rocprim/test_device_transform.cpp @@ -123,7 +123,7 @@ TYPED_TEST(RocprimDeviceTransformTests, Transform) static constexpr bool use_identity_iterator = TestFixture::use_identity_iterator; using Config = size_limit_config_t; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -213,7 +213,7 @@ TYPED_TEST(RocprimDeviceTransformTests, BinaryTransform) const bool debug_synchronous = TestFixture::debug_synchronous; using Config = size_limit_config_t; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -327,7 +327,7 @@ void testLargeIndices() HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_discard_iterator.cpp b/test/rocprim/test_discard_iterator.cpp index 7b949e642..1002a6842 100644 --- a/test/rocprim/test_discard_iterator.cpp +++ b/test/rocprim/test_discard_iterator.cpp @@ -40,7 +40,7 @@ TEST(RocprimDiscardIteratorTests, Equal) using Iterator = typename rocprim::discard_iterator; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -70,7 +70,7 @@ TEST(RocprimDiscardIteratorTests, Less) using Iterator = typename rocprim::discard_iterator; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_intrinsics.cpp b/test/rocprim/test_intrinsics.cpp index 25f85182a..c5171fdc9 100644 --- a/test/rocprim/test_intrinsics.cpp +++ b/test/rocprim/test_intrinsics.cpp @@ -316,7 +316,7 @@ void test_shuffle() test_utils::device_ptr d_data(size); - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -410,7 +410,7 @@ TYPED_TEST(RocprimIntrinsicsTests, ShuffleIndex) HIP_CHECK(::rocprim::host_warp_size(device_id, hardware_warp_size)); const size_t size = hardware_warp_size; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -562,7 +562,7 @@ TEST(RocprimIntrinsicsTests, MaskedBitCount) std::vector expected(out_size); - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -677,7 +677,7 @@ void warp_any_all_test() test_utils::device_ptr d_output(out_size); std::vector expected(out_size); - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -823,7 +823,7 @@ TYPED_TEST(RocprimIntrinsicsTests, WarpPermute) std::vector expected(size); std::vector indices(size); - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -920,7 +920,7 @@ TEST(RocprimIntrinsicsTests, MatchAny) std::vector expected(size); - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -1029,7 +1029,7 @@ TEST(RocprimIntrinsicsTests, Ballot) std::vector expected(size); - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -1119,7 +1119,7 @@ TEST(RocprimIntrinsicsTests, GroupElect) std::vector output; output.reserve(number_of_warps); - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_lookback_reproducibility.cpp b/test/rocprim/test_lookback_reproducibility.cpp index 60c29eb94..4a0b190c1 100644 --- a/test/rocprim/test_lookback_reproducibility.cpp +++ b/test/rocprim/test_lookback_reproducibility.cpp @@ -146,7 +146,7 @@ TYPED_TEST(RocprimLookbackReproducibilityTests, Scan) const hipStream_t stream = 0; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -216,7 +216,7 @@ TYPED_TEST(RocprimLookbackReproducibilityTests, ScanByKey) const hipStream_t stream = 0; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -296,7 +296,7 @@ TYPED_TEST(RocprimLookbackReproducibilityTests, ReduceByKey) const hipStream_t stream = 0; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_radix_key_codec.cpp b/test/rocprim/test_radix_key_codec.cpp index 8189ccd61..03c2e6a27 100644 --- a/test/rocprim/test_radix_key_codec.cpp +++ b/test/rocprim/test_radix_key_codec.cpp @@ -418,7 +418,7 @@ TYPED_TEST(TypedRadixKeyCodecTest, EncodeDecodeExtract) CustomDecomposer custom_decomposer{}; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_seed.hpp b/test/rocprim/test_seed.hpp index c276f44ec..814b5eb78 100644 --- a/test/rocprim/test_seed.hpp +++ b/test/rocprim/test_seed.hpp @@ -24,10 +24,23 @@ #include using engine_type = std::default_random_engine; -using seed_type = typename engine_type::result_type; +using seed_type = typename engine_type::result_type; -static constexpr size_t random_seeds_count = 1; -static constexpr seed_type seeds [] = {0, 1997132004}; -static constexpr size_t seed_size = sizeof(seeds) / sizeof(seeds[0]); +static const char* env_p = std::getenv("ROCPRIM_TEST_RUNS"); +// "env_var" determines the number of iterations. +// If undefined or incorrectly defined, it defaults to 0. +static const size_t env_var = (env_p == nullptr) ? 0ul : std::atoi(env_p); + +// Predefined seeds. +static constexpr seed_type seeds[] = {0, 1997132004}; +static constexpr size_t seed_size = sizeof(seeds) / sizeof(seeds[0]); + +// Calculate the number of random seeds. +// Always at least 1, or (env_var - seed_size) if env_var exceeds seed_size. +static const size_t random_seeds_count = (env_var > seed_size) ? (env_var - seed_size) : 1; + +// Calculate the total number of runs. +// Default to random_seeds_count + seed_size if env_var is 0, otherwise use env_var. +static const size_t number_of_runs = (env_var == 0) ? (random_seeds_count + seed_size) : env_var; #endif // TEST_SEED_HPP_ diff --git a/test/rocprim/test_texture_cache_iterator.cpp b/test/rocprim/test_texture_cache_iterator.cpp index 0bf60e0a6..05bea707c 100644 --- a/test/rocprim/test_texture_cache_iterator.cpp +++ b/test/rocprim/test_texture_cache_iterator.cpp @@ -92,7 +92,7 @@ TYPED_TEST(RocprimTextureCacheIteratorTests, Transform) std::vector input(size); - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_thread.cpp b/test/rocprim/test_thread.cpp index 705f86cc8..ab9465594 100644 --- a/test/rocprim/test_thread.cpp +++ b/test/rocprim/test_thread.cpp @@ -97,7 +97,7 @@ TYPED_TEST(RocprimThreadTests, FlatBlockThreadID) return; } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -164,7 +164,7 @@ TYPED_TEST(RocprimThreadTests, FlatBlockID) return; } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_thread_algos.cpp b/test/rocprim/test_thread_algos.cpp index de8bb381e..ed98f7412 100644 --- a/test/rocprim/test_thread_algos.cpp +++ b/test/rocprim/test_thread_algos.cpp @@ -85,7 +85,7 @@ TYPED_TEST(RocprimThreadOperationTests, Load) static constexpr uint32_t grid_size = 128; static constexpr uint32_t size = block_size * grid_size; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -129,7 +129,7 @@ TYPED_TEST(RocprimThreadOperationTests, Store) static constexpr uint32_t grid_size = 128; static constexpr uint32_t size = block_size * grid_size; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -184,7 +184,7 @@ TYPED_TEST(RocprimThreadOperationTests, Reduction) static constexpr uint32_t size = block_size * grid_size * length; sum_op operation; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -250,7 +250,7 @@ TYPED_TEST(RocprimThreadOperationTests, Scan) static constexpr uint32_t size = block_size * grid_size * length; sum_op operation; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -361,7 +361,7 @@ void merge_path_search_test() BinaryFunction bin_op; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_transform_iterator.cpp b/test/rocprim/test_transform_iterator.cpp index b4549856a..496c1e3e4 100644 --- a/test/rocprim/test_transform_iterator.cpp +++ b/test/rocprim/test_transform_iterator.cpp @@ -100,7 +100,7 @@ TYPED_TEST(RocprimTransformIteratorTests, TransformReduce) const size_t size = 1024; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_warp_reduce.hpp b/test/rocprim/test_warp_reduce.hpp index 8fad511c1..83626a12a 100644 --- a/test/rocprim/test_warp_reduce.hpp +++ b/test/rocprim/test_warp_reduce.hpp @@ -71,7 +71,7 @@ typed_test_def(RocprimWarpReduceTests, name_suffix, ReduceSum) GTEST_SKIP(); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -177,7 +177,7 @@ typed_test_def(RocprimWarpReduceTests, name_suffix, AllReduceSum) GTEST_SKIP(); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -288,7 +288,7 @@ typed_test_def(RocprimWarpReduceTests, name_suffix, ReduceSumValid) GTEST_SKIP(); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -398,7 +398,7 @@ typed_test_def(RocprimWarpReduceTests, name_suffix, AllReduceSumValid) GTEST_SKIP(); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -509,7 +509,7 @@ typed_test_def(RocprimWarpReduceTests, name_suffix, ReduceCustomStruct) GTEST_SKIP(); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -628,7 +628,7 @@ typed_test_def(RocprimWarpReduceTests, name_suffix, HeadSegmentedReduceSum) GTEST_SKIP(); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -768,7 +768,7 @@ typed_test_def(RocprimWarpReduceTests, name_suffix, TailSegmentedReduceSum) GTEST_SKIP(); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_warp_scan.hpp b/test/rocprim/test_warp_scan.hpp index 8c0e15101..1330103ee 100644 --- a/test/rocprim/test_warp_scan.hpp +++ b/test/rocprim/test_warp_scan.hpp @@ -71,7 +71,7 @@ typed_test_def(RocprimWarpScanTests, name_suffix, InclusiveScan) GTEST_SKIP(); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -180,7 +180,7 @@ typed_test_def(RocprimWarpScanTests, name_suffix, InclusiveScanReduce) GTEST_SKIP(); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -301,7 +301,7 @@ typed_test_def(RocprimWarpScanTests, name_suffix, ExclusiveScan) GTEST_SKIP(); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -418,7 +418,7 @@ typed_test_def(RocprimWarpScanTests, name_suffix, ExclusiveScanWoInit) GTEST_SKIP(); } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -542,7 +542,7 @@ typed_test_def(RocprimWarpScanTests, name_suffix, ExclusiveReduceScan) GTEST_SKIP(); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -677,7 +677,7 @@ typed_test_def(RocprimWarpScanTests, name_suffix, ExclusiveReduceScanWoInit) GTEST_SKIP(); } - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -818,7 +818,7 @@ typed_test_def(RocprimWarpScanTests, name_suffix, Scan) GTEST_SKIP(); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -947,7 +947,7 @@ typed_test_def(RocprimWarpScanTests, name_suffix, ScanReduce) GTEST_SKIP(); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -1085,7 +1085,7 @@ typed_test_def(RocprimWarpScanTests, name_suffix, InclusiveScanCustomType) GTEST_SKIP(); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_warp_sort.hpp b/test/rocprim/test_warp_sort.hpp index 19e8b88f7..836d62895 100644 --- a/test/rocprim/test_warp_sort.hpp +++ b/test/rocprim/test_warp_sort.hpp @@ -60,7 +60,7 @@ typed_test_def(RocprimWarpSortShuffleBasedTests, name_suffix, Sort) GTEST_SKIP(); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -140,7 +140,7 @@ typed_test_def(RocprimWarpSortShuffleBasedTests, name_suffix, SortKeyInt) GTEST_SKIP(); } - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); diff --git a/test/rocprim/test_zip_iterator.cpp b/test/rocprim/test_zip_iterator.cpp index b93847f8a..684b4709b 100644 --- a/test/rocprim/test_zip_iterator.cpp +++ b/test/rocprim/test_zip_iterator.cpp @@ -163,7 +163,7 @@ TEST(RocprimZipIteratorTests, Transform) // using default stream hipStream_t stream = 0; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); @@ -261,7 +261,7 @@ TEST(RocprimZipIteratorTests, TransformReduce) // using default stream hipStream_t stream = 0; - for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); From 57e268a4d949c6c120479a1aef0094a71e7d9d59 Mon Sep 17 00:00:00 2001 From: Saiyang Zhang Date: Wed, 18 Dec 2024 08:25:15 +0000 Subject: [PATCH 49/84] Resolve "Get clarity on skipping of adjacent difference tests" --- test/rocprim/test_device_adjacent_difference.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/rocprim/test_device_adjacent_difference.cpp b/test/rocprim/test_device_adjacent_difference.cpp index 00895599b..16fb23e97 100644 --- a/test/rocprim/test_device_adjacent_difference.cpp +++ b/test/rocprim/test_device_adjacent_difference.cpp @@ -587,9 +587,6 @@ TYPED_TEST_SUITE(RocprimDeviceAdjacentDifferenceLargeTests, TYPED_TEST(RocprimDeviceAdjacentDifferenceLargeTests, LargeIndices) { - if (TestFixture::use_graphs) - GTEST_SKIP() << "Temporarily skipping test within hipGraphs. Will re-enable when issues with atomics inside graphs are resolved."; - const int device_id = test_common_utils::obtain_device_from_ctest(); SCOPED_TRACE(testing::Message() << "with device_id = " << device_id); From f29ab0d20ab65797206ab235297811d7c0206b18 Mon Sep 17 00:00:00 2001 From: Nick Breed Date: Tue, 17 Dec 2024 13:43:25 +0000 Subject: [PATCH 50/84] Remove windows CI check --- .gitlab-ci.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e2843dd37..2a47b5524 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -294,14 +294,6 @@ build:windows: - $CI_PROJECT_DIR/build/.ninja_log - $CI_PROJECT_DIR/build/CTestTestfile.cmake expire_in: 2 weeks - rules: - - if: $CI_MERGE_REQUEST_TITLE !~ /Draft:/ || $CI_MERGE_REQUEST_LABELS =~ /windows/ - when: always - - if: $CI_MERGE_REQUEST_TITLE =~ /Draft:/ && - $CI_MERGE_REQUEST_LABELS !~ /windows/ && - $BUILD_TYPE == "Release" - when: always - - when: never autotune:build: stage: autotune From fe76c50493ec4d72783fb9f9842166db71cbc3a2 Mon Sep 17 00:00:00 2001 From: Nick Breed Date: Wed, 18 Dec 2024 10:54:01 +0000 Subject: [PATCH 51/84] Resolve "Add UnrolledThreadLoad and UnrolledThreadCopy functions" --- CHANGELOG.md | 1 + .../include/rocprim/thread/thread_load.hpp | 28 ++++++++++++ test/rocprim/test_device_search_n.cpp | 2 +- test/rocprim/test_thread_algos.cpp | 45 +++++++++++++++++++ 4 files changed, 75 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f22b6665b..efd631558 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Full documentation for rocPRIM is available at [https://rocm.docs.amd.com/projec * Added gfx950 support. * Added missing operator `rocprim::key_value_pair::operator==`. +* Added `rocprim::unrolled_copy` thread function to copy multiple items inside the thread. ### Resolved issues diff --git a/rocprim/include/rocprim/thread/thread_load.hpp b/rocprim/include/rocprim/thread/thread_load.hpp index f9876ac1a..b3d577763 100644 --- a/rocprim/include/rocprim/thread/thread_load.hpp +++ b/rocprim/include/rocprim/thread/thread_load.hpp @@ -33,6 +33,8 @@ #include "../config.hpp" #include "../detail/various.hpp" +#include + BEGIN_ROCPRIM_NAMESPACE /// \defgroup thread_load Thread Load Functions @@ -160,6 +162,32 @@ template return detail::AsmThreadLoad(ptr); } +namespace detail +{ + +template +ROCPRIM_DEVICE ROCPRIM_INLINE +void unrolled_copy_impl(InputIteratorT src, T* dst, std::integer_sequence) +{ + int dummy[] = {(dst[Is] = src[Is], 0)...}; + (void)dummy; +} + +} // namespace detail + +/// \brief Copy Count number of items from src to dst. +/// \tparam Count number of items to copy +/// \tparam InputIteratorT the input iterator type +/// \tparam T Type of Data to be copied to +/// \param src [in] - Input iterator for data that will be copied +/// \param dst [out] - The pointer the data will be copied to. +template +ROCPRIM_DEVICE ROCPRIM_INLINE +void unrolled_copy(InputIteratorT src, T* dst) +{ + detail::unrolled_copy_impl(src, dst, std::make_integer_sequence{}); +} + /// @} // end of group thread_load diff --git a/test/rocprim/test_device_search_n.cpp b/test/rocprim/test_device_search_n.cpp index 26e1aed99..657fda243 100644 --- a/test/rocprim/test_device_search_n.cpp +++ b/test/rocprim/test_device_search_n.cpp @@ -404,7 +404,7 @@ TYPED_TEST(RocprimDeviceSearchNTests, SmallCount) constexpr bool debug_synchronous = TestFixture::debug_synchronous; op_type op{}; - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; diff --git a/test/rocprim/test_thread_algos.cpp b/test/rocprim/test_thread_algos.cpp index ed98f7412..415f3121a 100644 --- a/test/rocprim/test_thread_algos.cpp +++ b/test/rocprim/test_thread_algos.cpp @@ -112,6 +112,51 @@ TYPED_TEST(RocprimThreadOperationTests, Load) } } +template +__global__ +void thread_copy_unroll_kernel(Type* device_input, Type* device_output) +{ + size_t index = (blockIdx.x * blockDim.x + threadIdx.x) * ItemsPerThread; + rocprim::unrolled_copy(device_input + index, device_output + index); +} + +TYPED_TEST(RocprimThreadOperationTests, CopyUnroll) +{ + using T = typename TestFixture::type; + static constexpr uint32_t block_size = 256; + static constexpr uint32_t grid_size = 128; + static constexpr uint32_t ItemsPerThread = 4; + static constexpr uint32_t size = block_size * grid_size * ItemsPerThread; + + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) + { + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); + + // Generate data + std::vector input = test_utils::get_random_data(size, 2, 200, seed_value); + std::vector output(size); + + // Calculate expected results on host + std::vector expected = input; + + // Preparing device + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(input.size()); + + thread_copy_unroll_kernel + <<>>(device_input.get(), device_output.get()); + HIP_CHECK(hipGetLastError()); + + // Reading results back + output = device_output.load(); + + // Verifying results + ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); + } +} + template __global__ void thread_store_kernel(Type* const device_input, Type* device_output) From eba0af4132816ae1dc37aa5c395853049ed5439e Mon Sep 17 00:00:00 2001 From: Nick Breed Date: Fri, 20 Dec 2024 11:35:24 +0000 Subject: [PATCH 52/84] Resolve "General changes CCCL 2.6" --- .../benchmark_device_search_n.parallel.hpp | 19 +- .../include/rocprim/block/block_load_func.hpp | 2 +- .../rocprim/block/block_store_func.hpp | 4 +- rocprim/include/rocprim/detail/various.hpp | 2 +- .../iterator/texture_cache_iterator.hpp | 13 +- rocprim/include/rocprim/type_traits.hpp | 23 +- rocprim/include/rocprim/warp/warp_reduce.hpp | 4 +- rocprim/include/rocprim/warp/warp_scan.hpp | 4 +- rocprim/include/rocprim/warp/warp_sort.hpp | 4 +- test/rocprim/test_arg_index_iterator.cpp | 11 +- test/rocprim/test_block_load_store.cpp | 8 +- .../rocprim/test_block_load_store.kernels.hpp | 116 +++++----- test/rocprim/test_block_radix_sort.cpp.in | 4 +- test/rocprim/test_block_scan.hpp | 8 +- test/rocprim/test_constant_iterator.cpp | 11 +- test/rocprim/test_counting_iterator.cpp | 11 +- test/rocprim/test_device_batch_memcpy.cpp | 5 +- test/rocprim/test_device_binary_search.cpp | 5 +- test/rocprim/test_device_histogram.cpp | 49 ++-- test/rocprim/test_device_merge.cpp | 5 +- test/rocprim/test_device_partition.cpp | 5 +- test/rocprim/test_device_reduce.cpp | 17 +- test/rocprim/test_device_reduce_by_key.cpp | 4 +- .../rocprim/test_device_run_length_encode.cpp | 5 +- test/rocprim/test_device_scan.cpp | 5 +- test/rocprim/test_device_segmented_reduce.cpp | 5 +- test/rocprim/test_device_segmented_scan.cpp | 5 +- test/rocprim/test_device_select.cpp | 50 ++-- test/rocprim/test_device_transform.cpp | 34 +-- test/rocprim/test_intrinsics.cpp | 13 +- test/rocprim/test_invoke_result.cpp | 10 +- test/rocprim/test_texture_cache_iterator.cpp | 17 +- test/rocprim/test_thread.cpp | 42 ++-- test/rocprim/test_thread_algos.cpp | 23 +- test/rocprim/test_transform_iterator.cpp | 11 +- test/rocprim/test_utils.hpp | 26 +-- test/rocprim/test_utils_types.hpp | 219 +++++++++--------- 37 files changed, 377 insertions(+), 422 deletions(-) diff --git a/benchmark/benchmark_device_search_n.parallel.hpp b/benchmark/benchmark_device_search_n.parallel.hpp index eeb8b1857..fc2e0b7b6 100644 --- a/benchmark/benchmark_device_search_n.parallel.hpp +++ b/benchmark/benchmark_device_search_n.parallel.hpp @@ -488,16 +488,15 @@ inline void add_benchmark_search_n(std::vector& add_one_benchmark_search_n(benchmarks, _seed, _stream, _size_byte); } -typedef type_arr - benchmark_search_n_types; +using benchmark_search_n_types = type_arr; template struct device_search_n_benchmark_generator diff --git a/rocprim/include/rocprim/block/block_load_func.hpp b/rocprim/include/rocprim/block/block_load_func.hpp index 0df327030..4e578a16f 100644 --- a/rocprim/include/rocprim/block/block_load_func.hpp +++ b/rocprim/include/rocprim/block/block_load_func.hpp @@ -186,7 +186,7 @@ block_load_direct_blocked_vectorized(unsigned int flat_id, T* block_input, U (&items)[ItemsPerThread]) -> typename std::enable_if::value>::type { - typedef typename detail::match_vector_type::type vector_type; + using vector_type = typename detail::match_vector_type::type; constexpr unsigned int vectors_per_thread = (sizeof(T) * ItemsPerThread) / sizeof(vector_type); vector_type vector_items[vectors_per_thread]; diff --git a/rocprim/include/rocprim/block/block_store_func.hpp b/rocprim/include/rocprim/block/block_store_func.hpp index a0b22647b..ba6abc495 100644 --- a/rocprim/include/rocprim/block/block_store_func.hpp +++ b/rocprim/include/rocprim/block/block_store_func.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -155,7 +155,7 @@ block_store_direct_blocked_vectorized(unsigned int flat_id, static_assert(std::is_convertible::value, "The type U must be such that it can be implicitly converted to T."); - typedef typename detail::match_vector_type::type vector_type; + using vector_type = typename detail::match_vector_type::type; constexpr unsigned int vectors_per_thread = (sizeof(T) * ItemsPerThread) / sizeof(vector_type); vector_type *vectors_ptr = reinterpret_cast(const_cast(block_output)); diff --git a/rocprim/include/rocprim/detail/various.hpp b/rocprim/include/rocprim/detail/various.hpp index a8dbfbffd..7e20f44f3 100644 --- a/rocprim/include/rocprim/detail/various.hpp +++ b/rocprim/include/rocprim/detail/various.hpp @@ -193,7 +193,7 @@ struct [[deprecated("To store non default-constructible types in local memory, u "rocprim::uninitialized_array instead")]] raw_storage { // Biggest memory-access word that T is a whole multiple of and is not larger than the alignment of T - typedef typename detail::match_fundamental_type::type device_word; + using device_word = typename detail::match_fundamental_type::type; // Backing storage alignas(device_word) unsigned char storage[sizeof(T)]; diff --git a/rocprim/include/rocprim/iterator/texture_cache_iterator.hpp b/rocprim/include/rocprim/iterator/texture_cache_iterator.hpp index 4cd3799eb..63e0cfbc6 100644 --- a/rocprim/include/rocprim/iterator/texture_cache_iterator.hpp +++ b/rocprim/include/rocprim/iterator/texture_cache_iterator.hpp @@ -148,14 +148,11 @@ class texture_cache_iterator using self_type = texture_cache_iterator; #endif - ROCPRIM_HOST_DEVICE inline - ~texture_cache_iterator() = default; + ROCPRIM_HOST_DEVICE ROCPRIM_INLINE ~texture_cache_iterator() = default; - ROCPRIM_HOST_DEVICE inline - texture_cache_iterator() - : ptr(NULL), texture_offset(0), texture_object(0) - { - } + ROCPRIM_HOST_DEVICE ROCPRIM_INLINE texture_cache_iterator() + : ptr(nullptr), texture_offset(0), texture_object(0) + {} /// \brief Creates a \p hipTextureObject_t and binds this iterator to it. /// @@ -185,7 +182,7 @@ class texture_cache_iterator resourse_desc.res.linear.sizeInBytes = bytes; texture_desc.readMode = hipReadModeElementType; - return hipCreateTextureObject(&texture_object, &resourse_desc, &texture_desc, NULL); + return hipCreateTextureObject(&texture_object, &resourse_desc, &texture_desc, nullptr); } /// \brief Destroys the texture object that this iterator points at. diff --git a/rocprim/include/rocprim/type_traits.hpp b/rocprim/include/rocprim/type_traits.hpp index 2bed27199..40beba513 100644 --- a/rocprim/include/rocprim/type_traits.hpp +++ b/rocprim/include/rocprim/type_traits.hpp @@ -116,40 +116,39 @@ struct numeric_limits : std::numeric_limits template struct get_unsigned_bits_type { - typedef typename get_unsigned_bits_type::unsigned_type unsigned_type; ///< Typedefed to the unsigned type. + using unsigned_type = typename get_unsigned_bits_type:: + unsigned_type; ///< Typedefed to the unsigned type. }; #ifndef DOXYGEN_SHOULD_SKIP_THIS // skip specialized versions template -struct get_unsigned_bits_type +struct get_unsigned_bits_type { - typedef uint8_t unsigned_type; + using unsigned_type = uint8_t; }; - template -struct get_unsigned_bits_type +struct get_unsigned_bits_type { - typedef uint16_t unsigned_type; + using unsigned_type = uint16_t; }; - template -struct get_unsigned_bits_type +struct get_unsigned_bits_type { - typedef uint32_t unsigned_type; + using unsigned_type = uint32_t; }; template -struct get_unsigned_bits_type +struct get_unsigned_bits_type { - typedef uint64_t unsigned_type; + using unsigned_type = uint64_t; }; template struct get_unsigned_bits_type { - typedef ::rocprim::uint128_t unsigned_type; + using unsigned_type = ::rocprim::uint128_t; }; #endif // DOXYGEN_SHOULD_SKIP_THIS diff --git a/rocprim/include/rocprim/warp/warp_reduce.hpp b/rocprim/include/rocprim/warp/warp_reduce.hpp index 34b7df901..197d166a2 100644 --- a/rocprim/include/rocprim/warp/warp_reduce.hpp +++ b/rocprim/include/rocprim/warp/warp_reduce.hpp @@ -45,12 +45,12 @@ namespace detail template struct select_warp_reduce_impl { - typedef typename std::conditional< + using type = typename std::conditional< // can we use crosslane (DPP or shuffle-based) implementation? detail::is_warpsize_shuffleable::value, detail::warp_reduce_crosslane, // yes detail::warp_reduce_shared_mem // no - >::type type; + >::type; }; } // end namespace detail diff --git a/rocprim/include/rocprim/warp/warp_scan.hpp b/rocprim/include/rocprim/warp/warp_scan.hpp index 22daed51f..73ae72af2 100644 --- a/rocprim/include/rocprim/warp/warp_scan.hpp +++ b/rocprim/include/rocprim/warp/warp_scan.hpp @@ -45,12 +45,12 @@ namespace detail template struct select_warp_scan_impl { - typedef typename std::conditional< + using type = typename std::conditional< // can we use crosslane (DPP or shuffle-based) implementation? detail::is_warpsize_shuffleable::value, detail::warp_scan_crosslane, // yes detail::warp_scan_shared_mem // no - >::type type; + >::type; }; } // end namespace detail diff --git a/rocprim/include/rocprim/warp/warp_sort.hpp b/rocprim/include/rocprim/warp/warp_sort.hpp index 940fec91e..3ed2736a0 100644 --- a/rocprim/include/rocprim/warp/warp_sort.hpp +++ b/rocprim/include/rocprim/warp/warp_sort.hpp @@ -107,7 +107,7 @@ template< > class warp_sort : detail::warp_sort_shuffle { - typedef typename detail::warp_sort_shuffle base_type; + using base_type = typename detail::warp_sort_shuffle; // Check if WarpSize is valid for the targets static_assert(WarpSize <= ROCPRIM_MAX_WARP_SIZE, @@ -122,7 +122,7 @@ class warp_sort : detail::warp_sort_shuffle /// using keywords \p __shared__. It can be aliased to /// an externally allocated memory, or be a part of a union with other storage types /// to increase shared memory reusability. - typedef typename base_type::storage_type storage_type; + using storage_type = typename base_type::storage_type; /// \brief Warp sort for any data type. /// diff --git a/test/rocprim/test_arg_index_iterator.cpp b/test/rocprim/test_arg_index_iterator.cpp index a14c60d8b..7b914a129 100644 --- a/test/rocprim/test_arg_index_iterator.cpp +++ b/test/rocprim/test_arg_index_iterator.cpp @@ -46,12 +46,11 @@ class RocprimArgIndexIteratorTests : public ::testing::Test const bool debug_synchronous = false; }; -typedef ::testing::Types< - RocprimArgIndexIteratorParams, - RocprimArgIndexIteratorParams, - RocprimArgIndexIteratorParams, - RocprimArgIndexIteratorParams -> RocprimArgIndexIteratorTestsParams; +using RocprimArgIndexIteratorTestsParams + = ::testing::Types, + RocprimArgIndexIteratorParams, + RocprimArgIndexIteratorParams, + RocprimArgIndexIteratorParams>; TYPED_TEST_SUITE(RocprimArgIndexIteratorTests, RocprimArgIndexIteratorTestsParams); diff --git a/test/rocprim/test_block_load_store.cpp b/test/rocprim/test_block_load_store.cpp index d9b533b97..1d73a1185 100644 --- a/test/rocprim/test_block_load_store.cpp +++ b/test/rocprim/test_block_load_store.cpp @@ -53,11 +53,11 @@ TYPED_TEST(RocprimVectorizationTests, IsVectorizable) TYPED_TEST(RocprimVectorizationTests, MatchVectorType) { - using T = typename TestFixture::params::type; - using U = typename TestFixture::params::vector_type; + using T = typename TestFixture::params::type; + using U = typename TestFixture::params::vector_type; constexpr size_t items_per_thread = TestFixture::params::items_per_thread; - typedef typename rocprim::detail::match_vector_type::type Vector; - bool input = std::is_same::value; + using Vector = typename rocprim::detail::match_vector_type::type; + bool input = std::is_same::value; ASSERT_TRUE(input); } diff --git a/test/rocprim/test_block_load_store.kernels.hpp b/test/rocprim/test_block_load_store.kernels.hpp index 9b5bf2bee..fab4f6f34 100644 --- a/test/rocprim/test_block_load_store.kernels.hpp +++ b/test/rocprim/test_block_load_store.kernels.hpp @@ -34,7 +34,7 @@ #include #include -typedef ::testing::Types< +using ClassParamsFirstPart = ::testing::Types< // block_load_direct class_params - - > - ClassParamsFirstPart; + 4>>; -typedef ::testing::Types< +using ClassParamsSecondPart = ::testing::Types< class_params - > - ClassParamsSecondPart; + >; -typedef ::testing::Types< +using ClassParamsThirdPart = ::testing::Types< // block_load_striped class_params - > - ClassParamsThirdPart; - -typedef ::testing::Types, - vector_params, - vector_params, - vector_params, - vector_params, - vector_params, - - vector_params, - vector_params, - vector_params, - vector_params, - vector_params, - vector_params, - - vector_params, - vector_params, - vector_params, - vector_params, - vector_params, - vector_params, - - vector_params, - vector_params, - vector_params, - vector_params, - vector_params, - vector_params, - - vector_params, - vector_params, - vector_params, - vector_params, - vector_params, - vector_params, - - vector_params, - vector_params, - vector_params, - vector_params, - vector_params, - vector_params, - - vector_params, - vector_params, - vector_params, - vector_params, - vector_params, - vector_params> - VectorParams; + >; + +using VectorParams = ::testing::Types, + vector_params, + vector_params, + vector_params, + vector_params, + vector_params, + + vector_params, + vector_params, + vector_params, + vector_params, + vector_params, + vector_params, + + vector_params, + vector_params, + vector_params, + vector_params, + vector_params, + vector_params, + + vector_params, + vector_params, + vector_params, + vector_params, + vector_params, + vector_params, + + vector_params, + vector_params, + vector_params, + vector_params, + vector_params, + vector_params, + + vector_params, + vector_params, + vector_params, + vector_params, + vector_params, + vector_params, + + vector_params, + vector_params, + vector_params, + vector_params, + vector_params, + vector_params>; template, int) , block_param_type(test_utils::custom_test_type, int8_t) , block_param_type(test_utils::custom_test_type, uint16_t) #if ROCPRIM_HAS_INT128_SUPPORT , block_param_type(test_utils::custom_test_type, double) #endif -> BlockParamsCustom; +>; struct Custom; #define suite_name RocprimBlockRadixSort diff --git a/test/rocprim/test_block_scan.hpp b/test/rocprim/test_block_scan.hpp index 643ea2ce6..b6eefd886 100644 --- a/test/rocprim/test_block_scan.hpp +++ b/test/rocprim/test_block_scan.hpp @@ -80,7 +80,7 @@ typed_test_def(suite_name_single, name_suffix, InclusiveScan) expected, expected, device_output.get(), - NULL, + nullptr, T(0), grid_size); @@ -90,7 +90,7 @@ typed_test_def(suite_name_single, name_suffix, InclusiveScan) expected, expected, device_output.get(), - NULL, + nullptr, T(0), grid_size); } @@ -304,7 +304,7 @@ typed_test_def(suite_name_single, name_suffix, ExclusiveScan) expected, expected, device_output.get(), - NULL, + nullptr, init, grid_size); @@ -314,7 +314,7 @@ typed_test_def(suite_name_single, name_suffix, ExclusiveScan) expected, expected, device_output.get(), - NULL, + nullptr, init, grid_size); } diff --git a/test/rocprim/test_constant_iterator.cpp b/test/rocprim/test_constant_iterator.cpp index 6d27b0d67..c9561a3d0 100644 --- a/test/rocprim/test_constant_iterator.cpp +++ b/test/rocprim/test_constant_iterator.cpp @@ -45,12 +45,11 @@ class RocprimConstantIteratorTests : public ::testing::Test const bool debug_synchronous = false; }; -typedef ::testing::Types< - RocprimConstantIteratorParams, - RocprimConstantIteratorParams, - RocprimConstantIteratorParams, - RocprimConstantIteratorParams -> RocprimConstantIteratorTestsParams; +using RocprimConstantIteratorTestsParams + = ::testing::Types, + RocprimConstantIteratorParams, + RocprimConstantIteratorParams, + RocprimConstantIteratorParams>; TYPED_TEST_SUITE(RocprimConstantIteratorTests, RocprimConstantIteratorTestsParams); diff --git a/test/rocprim/test_counting_iterator.cpp b/test/rocprim/test_counting_iterator.cpp index e7ab3e7b8..56a09514e 100644 --- a/test/rocprim/test_counting_iterator.cpp +++ b/test/rocprim/test_counting_iterator.cpp @@ -45,12 +45,11 @@ class RocprimCountingIteratorTests : public ::testing::Test const bool debug_synchronous = false; }; -typedef ::testing::Types< - RocprimCountingIteratorParams, - RocprimCountingIteratorParams, - RocprimCountingIteratorParams, - RocprimCountingIteratorParams -> RocprimCountingIteratorTestsParams; +using RocprimCountingIteratorTestsParams + = ::testing::Types, + RocprimCountingIteratorParams, + RocprimCountingIteratorParams, + RocprimCountingIteratorParams>; TYPED_TEST_SUITE(RocprimCountingIteratorTests, RocprimCountingIteratorTestsParams); diff --git a/test/rocprim/test_device_batch_memcpy.cpp b/test/rocprim/test_device_batch_memcpy.cpp index 8e83411dd..cbb4046b8 100644 --- a/test/rocprim/test_device_batch_memcpy.cpp +++ b/test/rocprim/test_device_batch_memcpy.cpp @@ -78,7 +78,7 @@ struct RocprimDeviceBatchMemcpyTests : public ::testing::Test static constexpr bool use_indirect_iterator = Params::use_indirect_iterator; }; -typedef ::testing::Types< +using RocprimDeviceBatchMemcpyTestsParams = ::testing::Types< // Ignore copy/move DeviceBatchMemcpyParams, unsigned int, @@ -127,8 +127,7 @@ typedef ::testing::Types< DeviceBatchMemcpyParams, // Test iterator input for BatchCopy - DeviceBatchMemcpyParams> - RocprimDeviceBatchMemcpyTestsParams; + DeviceBatchMemcpyParams>; TYPED_TEST_SUITE(RocprimDeviceBatchMemcpyTests, RocprimDeviceBatchMemcpyTestsParams); diff --git a/test/rocprim/test_device_binary_search.cpp b/test/rocprim/test_device_binary_search.cpp index 2ec949645..d10ab771f 100644 --- a/test/rocprim/test_device_binary_search.cpp +++ b/test/rocprim/test_device_binary_search.cpp @@ -58,7 +58,7 @@ using custom_double2 = test_utils::custom_test_type; struct use_custom_config {}; -typedef ::testing::Types< +using Params = ::testing::Types< params, params>, params>, @@ -73,8 +73,7 @@ typedef ::testing::Types< use_custom_config>, params, params>, - params, rocprim::default_config, true>> - Params; + params, rocprim::default_config, true>>; TYPED_TEST_SUITE(RocprimDeviceBinarySearch, Params); diff --git a/test/rocprim/test_device_histogram.cpp b/test/rocprim/test_device_histogram.cpp index 855201c4b..42f1c16d0 100644 --- a/test/rocprim/test_device_histogram.cpp +++ b/test/rocprim/test_device_histogram.cpp @@ -131,23 +131,23 @@ class RocprimDeviceHistogramEven : public ::testing::Test { using custom_config1 = rocprim::histogram_config>; -typedef ::testing::Types, - params1, - params1, - params1, - params1, - params1, - params1, - params1, - params1, - params1, - params1, - params1, - params1, - params1, - params1, - params1> - Params1; +using Params1 + = ::testing::Types, + params1, + params1, + params1, + params1, + params1, + params1, + params1, + params1, + params1, + params1, + params1, + params1, + params1, + params1, + params1>; TYPED_TEST_SUITE(RocprimDeviceHistogramEven, Params1); @@ -395,7 +395,7 @@ class RocprimDeviceHistogramRange : public ::testing::Test { using custom_config2 = rocprim::histogram_config>; -typedef ::testing::Types< +using Params2 = ::testing::Types< params2, params2, params2, @@ -404,8 +404,7 @@ typedef ::testing::Types< params2, params2, - params2> - Params2; + params2>; TYPED_TEST_SUITE(RocprimDeviceHistogramRange, Params2); @@ -644,7 +643,7 @@ class RocprimDeviceHistogramMultiEven : public ::testing::Test { using custom_config3 = rocprim::histogram_config>; -typedef ::testing::Types< +using Params3 = ::testing::Types< params3, params3, params3, @@ -656,8 +655,7 @@ typedef ::testing::Types< params3, params3, params3, - params3> - Params3; + params3>; TYPED_TEST_SUITE(RocprimDeviceHistogramMultiEven, Params3); @@ -931,7 +929,7 @@ class RocprimDeviceHistogramMultiRange : public ::testing::Test { using custom_config4 = rocprim::histogram_config>; -typedef ::testing::Types< +using Params4 = ::testing::Types< params4, params4, params4, @@ -940,8 +938,7 @@ typedef ::testing::Types< params4, params4, - params4> - Params4; + params4>; TYPED_TEST_SUITE(RocprimDeviceHistogramMultiRange, Params4); diff --git a/test/rocprim/test_device_merge.cpp b/test/rocprim/test_device_merge.cpp index 46a49fc89..7a3cf1c07 100644 --- a/test/rocprim/test_device_merge.cpp +++ b/test/rocprim/test_device_merge.cpp @@ -72,7 +72,7 @@ class RocprimDeviceMergeTests : public ::testing::Test using custom_int2 = test_utils::custom_test_type; using custom_double2 = test_utils::custom_test_type; -typedef ::testing::Types< +using RocprimDeviceMergeTestsParams = ::testing::Types< DeviceMergeParams, DeviceMergeParams>, DeviceMergeParams, @@ -84,8 +84,7 @@ typedef ::testing::Types< DeviceMergeParams>, DeviceMergeParams>, DeviceMergeParams, - DeviceMergeParams, true>> - RocprimDeviceMergeTestsParams; + DeviceMergeParams, true>>; // size1, size2 std::vector> get_sizes() diff --git a/test/rocprim/test_device_partition.cpp b/test/rocprim/test_device_partition.cpp index d7d34f2d3..54467e39e 100644 --- a/test/rocprim/test_device_partition.cpp +++ b/test/rocprim/test_device_partition.cpp @@ -69,7 +69,7 @@ using config = rocprim::select_config<512, ::rocprim::block_load_method::block_load_transpose, ::rocprim::block_scan_algorithm::using_warp_scan>; -typedef ::testing::Types< +using RocprimDevicePartitionTestsParams = ::testing::Types< DevicePartitionParams, DevicePartitionParams, DevicePartitionParams, @@ -80,8 +80,7 @@ typedef ::testing::Types< DevicePartitionParams, DevicePartitionParams, DevicePartitionParams>, - DevicePartitionParams> - RocprimDevicePartitionTestsParams; + DevicePartitionParams>; TYPED_TEST_SUITE(RocprimDevicePartitionTests, RocprimDevicePartitionTestsParams); diff --git a/test/rocprim/test_device_reduce.cpp b/test/rocprim/test_device_reduce.cpp index 29753865e..ce62ab58d 100644 --- a/test/rocprim/test_device_reduce.cpp +++ b/test/rocprim/test_device_reduce.cpp @@ -93,7 +93,7 @@ class RocprimDeviceReduceTests : public ::testing::Test template class RocprimDeviceReducePrecisionTests : public RocprimDeviceReduceTests{}; -typedef ::testing::Types< +using RocprimDeviceReduceTestsParams = ::testing::Types< DeviceReduceParams, DeviceReduceParams, DeviceReduceParams, @@ -138,14 +138,13 @@ typedef ::testing::Types< bra::default_algorithm, false, true>, - DeviceReduceParams> - RocprimDeviceReduceTestsParams; - -typedef ::testing::Types, - DeviceReduceParamsList(float, float, false, 2048), - DeviceReduceParams, - DeviceReduceParams> - RocprimDeviceReducePrecisionTestsParams; + DeviceReduceParams>; + +using RocprimDeviceReducePrecisionTestsParams + = ::testing::Types, + DeviceReduceParamsList(float, float, false, 2048), + DeviceReduceParams, + DeviceReduceParams>; TYPED_TEST_SUITE(RocprimDeviceReduceTests, RocprimDeviceReduceTestsParams); TYPED_TEST_SUITE(RocprimDeviceReducePrecisionTests, RocprimDeviceReducePrecisionTestsParams); diff --git a/test/rocprim/test_device_reduce_by_key.cpp b/test/rocprim/test_device_reduce_by_key.cpp index 657dd9e29..ec59d2214 100644 --- a/test/rocprim/test_device_reduce_by_key.cpp +++ b/test/rocprim/test_device_reduce_by_key.cpp @@ -92,7 +92,7 @@ using custom_int2 = test_utils::custom_test_type; using custom_double2 = test_utils::custom_test_type; // clang-format off -typedef ::testing::Types< +using Params = ::testing::Types< params, 1, 1, int, rocprim::equal_to, true>, params, 3, 5, long long, custom_key_compare_op1, false, false, true>, params, 1, 10000>, @@ -120,7 +120,7 @@ typedef ::testing::Types< params, 100000, 100000>, params, unsigned long, rocprim::plus<>, 69, 420>, params, 1, 10, int, ::rocprim::equal_to, false, true> -> Params; +>; // clang-format on template diff --git a/test/rocprim/test_device_run_length_encode.cpp b/test/rocprim/test_device_run_length_encode.cpp index d40624214..21c30dccd 100644 --- a/test/rocprim/test_device_run_length_encode.cpp +++ b/test/rocprim/test_device_run_length_encode.cpp @@ -61,7 +61,7 @@ class RocprimDeviceRunLengthEncode : public ::testing::Test { using custom_int2 = test_utils::custom_test_type; using custom_double2 = test_utils::custom_test_type; -typedef ::testing::Types< +using Params = ::testing::Types< // Tests with default configuration params, params, @@ -98,8 +98,7 @@ typedef ::testing::Types< rocprim::run_length_encode_config, rocprim::select_config<256, 13>>>, // Tests for when output's value_type is void - params> - Params; + params>; TYPED_TEST_SUITE(RocprimDeviceRunLengthEncode, Params); diff --git a/test/rocprim/test_device_scan.cpp b/test/rocprim/test_device_scan.cpp index a3b128068..c98827f18 100644 --- a/test/rocprim/test_device_scan.cpp +++ b/test/rocprim/test_device_scan.cpp @@ -155,7 +155,7 @@ class RocprimDeviceScanTests : public ::testing::Test static constexpr bool deterministic = Params::deterministic; }; -typedef ::testing::Types< +using RocprimDeviceScanTestsParams = ::testing::Types< // Small DeviceScanParams, DeviceScanParams, @@ -222,8 +222,7 @@ typedef ::testing::Types< DeviceScanParams>, DeviceScanParams>, // With graphs - DeviceScanParams, false, default_config_helper, true>> - RocprimDeviceScanTestsParams; + DeviceScanParams, false, default_config_helper, true>>; // use float for accumulation of bfloat16 and half inputs if operator is plus template struct accum_type { diff --git a/test/rocprim/test_device_segmented_reduce.cpp b/test/rocprim/test_device_segmented_reduce.cpp index 675d1dae8..74e3186de 100644 --- a/test/rocprim/test_device_segmented_reduce.cpp +++ b/test/rocprim/test_device_segmented_reduce.cpp @@ -102,7 +102,7 @@ using bfloat16 = rocprim::bfloat16; #define maximum rocprim::maximum #define minimum rocprim::minimum -typedef ::testing::Types< +using Params = ::testing::Types< // Integer types SegmentedReduceParamsList(int, int, plus, -100, 0, 10000, false), SegmentedReduceParamsList(int8_t, int8_t, maximum, 0, 0, 2000, false), @@ -138,8 +138,7 @@ typedef ::testing::Types< false, bra::default_algorithm, false, - true>> - Params; + true>>; #undef plus #undef maximum diff --git a/test/rocprim/test_device_segmented_scan.cpp b/test/rocprim/test_device_segmented_scan.cpp index 985f934e6..3341fab45 100644 --- a/test/rocprim/test_device_segmented_scan.cpp +++ b/test/rocprim/test_device_segmented_scan.cpp @@ -68,7 +68,7 @@ using custom_double2 = test_utils::custom_test_type; using half = rocprim::half; using bfloat16 = rocprim::bfloat16; -typedef ::testing::Types< +using Params = ::testing::Types< params>, params, -100, 0, 10000>, params, -100, 0, 10000>, @@ -81,8 +81,7 @@ typedef ::testing::Types< params, 0, 10, 200, true>, params, 0, 1000, 30000>, params, 10, 3000, 4000>, - params, 0, 0, 1000, false, true>> - Params; + params, 0, 0, 1000, false, true>>; TYPED_TEST_SUITE(RocprimDeviceSegmentedScan, Params); diff --git a/test/rocprim/test_device_select.cpp b/test/rocprim/test_device_select.cpp index 15ab2eae0..4f68ee30b 100644 --- a/test/rocprim/test_device_select.cpp +++ b/test/rocprim/test_device_select.cpp @@ -60,20 +60,20 @@ class RocprimDeviceSelectTests : public ::testing::Test static constexpr bool use_graphs = Params::use_graphs; }; -typedef ::testing::Types, - DeviceSelectParams, - DeviceSelectParams, - DeviceSelectParams, - DeviceSelectParams, - DeviceSelectParams, - DeviceSelectParams, - DeviceSelectParams, - DeviceSelectParams, - test_utils::custom_test_type, - int, - true>, - DeviceSelectParams> - RocprimDeviceSelectTestsParams; +using RocprimDeviceSelectTestsParams + = ::testing::Types, + DeviceSelectParams, + DeviceSelectParams, + DeviceSelectParams, + DeviceSelectParams, + DeviceSelectParams, + DeviceSelectParams, + DeviceSelectParams, + DeviceSelectParams, + test_utils::custom_test_type, + int, + true>, + DeviceSelectParams>; TYPED_TEST_SUITE(RocprimDeviceSelectTests, RocprimDeviceSelectTestsParams); @@ -778,17 +778,17 @@ class RocprimDeviceUniqueByKeyTests : public ::testing::Test const bool use_graphs = Params::use_graphs; }; -typedef ::testing::Types, - DeviceUniqueByKeyParams, - DeviceUniqueByKeyParams, - DeviceUniqueByKeyParams, - DeviceUniqueByKeyParams, - DeviceUniqueByKeyParams, - DeviceUniqueByKeyParams, - DeviceUniqueByKeyParams, - test_utils::custom_test_type>, - DeviceUniqueByKeyParams> - RocprimDeviceUniqueByKeyTestParams; +using RocprimDeviceUniqueByKeyTestParams + = ::testing::Types, + DeviceUniqueByKeyParams, + DeviceUniqueByKeyParams, + DeviceUniqueByKeyParams, + DeviceUniqueByKeyParams, + DeviceUniqueByKeyParams, + DeviceUniqueByKeyParams, + DeviceUniqueByKeyParams, + test_utils::custom_test_type>, + DeviceUniqueByKeyParams>; TYPED_TEST_SUITE(RocprimDeviceUniqueByKeyTests, RocprimDeviceUniqueByKeyTestParams); diff --git a/test/rocprim/test_device_transform.cpp b/test/rocprim/test_device_transform.cpp index 7ad6c6cbd..e448ed182 100644 --- a/test/rocprim/test_device_transform.cpp +++ b/test/rocprim/test_device_transform.cpp @@ -66,23 +66,23 @@ using custom_short2 = test_utils::custom_test_type; using custom_int2 = test_utils::custom_test_type; using custom_double2 = test_utils::custom_test_type; -typedef ::testing::Types, - DeviceTransformParams, - DeviceTransformParams, - DeviceTransformParams, - DeviceTransformParams, - DeviceTransformParams, - DeviceTransformParams, - DeviceTransformParams, - DeviceTransformParams, - DeviceTransformParams, - DeviceTransformParams, - DeviceTransformParams, - DeviceTransformParams, - DeviceTransformParams, - DeviceTransformParams, - DeviceTransformParams> - RocprimDeviceTransformTestsParams; +using RocprimDeviceTransformTestsParams + = ::testing::Types, + DeviceTransformParams, + DeviceTransformParams, + DeviceTransformParams, + DeviceTransformParams, + DeviceTransformParams, + DeviceTransformParams, + DeviceTransformParams, + DeviceTransformParams, + DeviceTransformParams, + DeviceTransformParams, + DeviceTransformParams, + DeviceTransformParams, + DeviceTransformParams, + DeviceTransformParams, + DeviceTransformParams>; template struct size_limit_config diff --git a/test/rocprim/test_intrinsics.cpp b/test/rocprim/test_intrinsics.cpp index c5171fdc9..f5cded8d1 100644 --- a/test/rocprim/test_intrinsics.cpp +++ b/test/rocprim/test_intrinsics.cpp @@ -195,13 +195,12 @@ class RocprimIntrinsicsTests : public ::testing::Test using type = typename Params::type; }; -typedef ::testing::Types, - params, - params, - params, - params, - params> - IntrinsicsTestParams; +using IntrinsicsTestParams = ::testing::Types, + params, + params, + params, + params, + params>; TYPED_TEST_SUITE(RocprimIntrinsicsTests, IntrinsicsTestParams); diff --git a/test/rocprim/test_invoke_result.cpp b/test/rocprim/test_invoke_result.cpp index c956426c8..e6f163cc5 100644 --- a/test/rocprim/test_invoke_result.cpp +++ b/test/rocprim/test_invoke_result.cpp @@ -55,7 +55,7 @@ class RocprimInvokeResultBinOpTests : public ::testing::Test using expected_type = typename Params::expected_type; }; -typedef ::testing::Types< +using RocprimInvokeResultBinOpTestsParams = ::testing::Types< RocprimTypeInvokeResultParams>, RocprimTypeInvokeResultParams>, RocprimTypeInvokeResultParams>, @@ -64,8 +64,7 @@ typedef ::testing::Types< RocprimTypeInvokeResultParams>, RocprimTypeInvokeResultParams, bool>, RocprimTypeInvokeResultParams, bool>, - RocprimTypeInvokeResultParams, bool>> - RocprimInvokeResultBinOpTestsParams; + RocprimTypeInvokeResultParams, bool>>; TYPED_TEST_SUITE(RocprimInvokeResultBinOpTests, RocprimInvokeResultBinOpTestsParams); @@ -100,13 +99,12 @@ class RocprimInvokeResultUnOpTests : public ::testing::Test using expected_type = typename Params::expected_type; }; -typedef ::testing::Types< +using RocprimInvokeResultUnOpTestsParams = ::testing::Types< RocprimTypeInvokeResultParams, float>, RocprimTypeInvokeResultParams, rocprim::bfloat16>, - RocprimTypeInvokeResultParams>> - RocprimInvokeResultUnOpTestsParams; + RocprimTypeInvokeResultParams>>; TYPED_TEST_SUITE(RocprimInvokeResultUnOpTests, RocprimInvokeResultUnOpTestsParams); diff --git a/test/rocprim/test_texture_cache_iterator.cpp b/test/rocprim/test_texture_cache_iterator.cpp index 05bea707c..aeb5b6a49 100644 --- a/test/rocprim/test_texture_cache_iterator.cpp +++ b/test/rocprim/test_texture_cache_iterator.cpp @@ -45,15 +45,14 @@ class RocprimTextureCacheIteratorTests : public ::testing::Test const bool debug_synchronous = false; }; -typedef ::testing::Types< - RocprimTextureCacheIteratorParams, - RocprimTextureCacheIteratorParams, - RocprimTextureCacheIteratorParams, - RocprimTextureCacheIteratorParams, - RocprimTextureCacheIteratorParams, - RocprimTextureCacheIteratorParams>, - RocprimTextureCacheIteratorParams> -> RocprimTextureCacheIteratorTestsParams; +using RocprimTextureCacheIteratorTestsParams + = ::testing::Types, + RocprimTextureCacheIteratorParams, + RocprimTextureCacheIteratorParams, + RocprimTextureCacheIteratorParams, + RocprimTextureCacheIteratorParams, + RocprimTextureCacheIteratorParams>, + RocprimTextureCacheIteratorParams>>; TYPED_TEST_SUITE(RocprimTextureCacheIteratorTests, RocprimTextureCacheIteratorTestsParams); diff --git a/test/rocprim/test_thread.cpp b/test/rocprim/test_thread.cpp index ab9465594..3a34d15d1 100644 --- a/test/rocprim/test_thread.cpp +++ b/test/rocprim/test_thread.cpp @@ -46,28 +46,26 @@ class RocprimThreadTests : public ::testing::Test { using params = Params; }; -typedef ::testing::Types< - params<32, 1, 1>, - params<64, 1, 1>, - params<128, 1, 1>, - params<256, 1, 1>, - params<512, 1, 1>, - params<1024, 1, 1>, - - params<16, 2, 1>, - params<32, 2, 1>, - params<64, 2, 1>, - params<128, 2, 1>, - params<256, 2, 1>, - params<512, 2, 1>, - - params<8, 2, 2>, - params<16, 2, 2>, - params<32, 2, 2>, - params<64, 2, 2>, - params<128, 2, 2>, - params<256, 2, 2> -> Params; +using Params = ::testing::Types, + params<64, 1, 1>, + params<128, 1, 1>, + params<256, 1, 1>, + params<512, 1, 1>, + params<1024, 1, 1>, + + params<16, 2, 1>, + params<32, 2, 1>, + params<64, 2, 1>, + params<128, 2, 1>, + params<256, 2, 1>, + params<512, 2, 1>, + + params<8, 2, 2>, + params<16, 2, 2>, + params<32, 2, 2>, + params<64, 2, 2>, + params<128, 2, 2>, + params<256, 2, 2>>; TYPED_TEST_SUITE(RocprimThreadTests, Params); diff --git a/test/rocprim/test_thread_algos.cpp b/test/rocprim/test_thread_algos.cpp index 415f3121a..1da0d3882 100644 --- a/test/rocprim/test_thread_algos.cpp +++ b/test/rocprim/test_thread_algos.cpp @@ -53,18 +53,17 @@ class RocprimThreadOperationTests : public ::testing::Test using type = typename Params::type; }; -typedef ::testing::Types, - params, - params, - params, - params, - params, - params, - params, - params, - params>, - params>> - ThreadOperationTestParams; +using ThreadOperationTestParams = ::testing::Types, + params, + params, + params, + params, + params, + params, + params, + params, + params>, + params>>; TYPED_TEST_SUITE(RocprimThreadOperationTests, ThreadOperationTestParams); diff --git a/test/rocprim/test_transform_iterator.cpp b/test/rocprim/test_transform_iterator.cpp index 496c1e3e4..c60cc818a 100644 --- a/test/rocprim/test_transform_iterator.cpp +++ b/test/rocprim/test_transform_iterator.cpp @@ -74,12 +74,11 @@ class RocprimTransformIteratorTests : public ::testing::Test const bool debug_synchronous = false; }; -typedef ::testing::Types< - RocprimTransformIteratorParams>, - RocprimTransformIteratorParams, - RocprimTransformIteratorParams, - RocprimTransformIteratorParams, double> -> RocprimTransformIteratorTestsParams; +using RocprimTransformIteratorTestsParams + = ::testing::Types>, + RocprimTransformIteratorParams, + RocprimTransformIteratorParams, + RocprimTransformIteratorParams, double>>; TYPED_TEST_SUITE(RocprimTransformIteratorTests, RocprimTransformIteratorTestsParams); diff --git a/test/rocprim/test_utils.hpp b/test/rocprim/test_utils.hpp index 4d1a0c4be..894756cda 100644 --- a/test/rocprim/test_utils.hpp +++ b/test/rocprim/test_utils.hpp @@ -78,43 +78,43 @@ static constexpr float precision> = precision; template struct is_plus_operator : std::false_type { - typedef uint8_t value_type; + using value_type = uint8_t; }; template struct is_plus_operator> : std::true_type { - typedef T value_type; + using value_type = T; }; template struct is_add_operator : std::false_type { - typedef uint8_t value_type; + using value_type = uint8_t; }; template struct is_add_operator> : std::true_type { - typedef T value_type; + using value_type = T; }; template struct is_add_operator> : std::true_type { - typedef T value_type; + using value_type = T; }; template struct is_multiply_operator : std::false_type { - typedef uint8_t value_type; + using value_type = uint8_t; }; template struct is_multiply_operator> : std::true_type { - typedef T value_type; + using value_type = T; }; /* Plus to operator selector for host-side @@ -126,22 +126,22 @@ struct is_multiply_operator> : std::true_type template struct select_plus_operator_host { - typedef ::rocprim::plus type; - typedef T acc_type; + using type = ::rocprim::plus; + using acc_type = T; }; template<> struct select_plus_operator_host<::rocprim::half> { - typedef ::rocprim::plus type; - typedef double acc_type; + using type = ::rocprim::plus; + using acc_type = double; }; template<> struct select_plus_operator_host<::rocprim::bfloat16> { - typedef ::rocprim::plus type; - typedef double acc_type; + using type = ::rocprim::plus; + using acc_type = double; }; template // clang-format on -typedef ::testing::Types< - warp_param_type(int), - warp_param_type(int8_t), - warp_param_type(uint8_t) -> WarpParamsIntegral; - -typedef ::testing::Types - WarpParamsFloating; +using WarpParamsIntegral + = ::testing::Types; + +using WarpParamsFloating = ::testing::Types; // Separate sort params (only power of two warp sizes) #define warp_sort_param_type(type, items_per_thread) \ @@ -128,112 +124,105 @@ typedef ::testing::Types, \ warp_params -typedef ::testing::Types< - warp_sort_param_type(int, 1), - warp_sort_param_type(test_utils::custom_test_type, 1), - warp_sort_param_type(uint8_t, 1), - warp_sort_param_type(int8_t, 1) -> WarpSortParamsIntegral; - -typedef ::testing::Types - WarpSortParamsFloating; - -typedef ::testing::Types< - warp_sort_param_type(int, 2), - warp_sort_param_type(test_utils::custom_test_type, 2), - warp_sort_param_type(uint8_t, 2), - warp_sort_param_type(int8_t, 2), - warp_sort_param_type(int, 4), - warp_sort_param_type(test_utils::custom_test_type, 4), - warp_sort_param_type(uint8_t, 4), - warp_sort_param_type(int8_t, 4) -> WarpSortParamsIntegralMultiThread; - -typedef ::testing::Types< - block_param_type(int, test_utils::custom_test_type), - block_param_type(uint8_t, short), - block_param_type(int8_t, float) -> BlockParamsIntegral; - -typedef ::testing::Types), - block_param_type(uint8_t, short), - block_param_type(int8_t, float), - block_param_type(bool, rocprim::half) +using WarpSortParamsIntegral + = ::testing::Types, 1), + warp_sort_param_type(uint8_t, 1), + warp_sort_param_type(int8_t, 1)>; + +using WarpSortParamsFloating = ::testing::Types; + +using WarpSortParamsIntegralMultiThread + = ::testing::Types, 2), + warp_sort_param_type(uint8_t, 2), + warp_sort_param_type(int8_t, 2), + warp_sort_param_type(int, 4), + warp_sort_param_type(test_utils::custom_test_type, 4), + warp_sort_param_type(uint8_t, 4), + warp_sort_param_type(int8_t, 4)>; + +using BlockParamsIntegral + = ::testing::Types), + block_param_type(uint8_t, short), + block_param_type(int8_t, float)>; + +using BlockParamsIntegralExtended + = ::testing::Types), + block_param_type(uint8_t, short), + block_param_type(int8_t, float), + block_param_type(bool, rocprim::half) #if ROCPRIM_HAS_INT128_SUPPORT - , - block_param_type(rocprim::uint128_t, short), - block_param_type(rocprim::int128_t, float) + , + block_param_type(rocprim::uint128_t, short), + block_param_type(rocprim::int128_t, float) #endif - > - BlockParamsIntegralExtended; - -typedef ::testing::Types), - block_param_type(rocprim::half, int), - block_param_type(rocprim::half, rocprim::half), - block_param_type(rocprim::bfloat16, int), - block_param_type(rocprim::bfloat16, rocprim::bfloat16)> - BlockParamsFloating; - -typedef ::testing::Types< - block_param_type(test_utils::custom_test_type, int), - block_param_type(uint8_t, bool), - block_param_type(int8_t, bool) -> BlockDiscParamsIntegral; - -typedef ::testing::Types - BlockDiscParamsFloating; - -typedef ::testing::Types - BlockDiscParamsFloatingHalf; - -typedef ::testing::Types - BlockHistAtomicParamsIntegral; - -typedef ::testing::Types), - block_param_type(double, int8_t), - block_param_type(rocprim::half, rocprim::half), - block_param_type(rocprim::half, int16_t), - block_param_type(rocprim::bfloat16, rocprim::bfloat16)> - BlockExchParamsFloating; - -typedef ::testing::Types - BlockHistAtomicParamsFloating; - -typedef ::testing::Types - BlockHistSortParamsIntegral; - -typedef ::testing::Types - BlockHistSortParamsFloating; + >; + +using BlockParamsFloating + = ::testing::Types), + block_param_type(rocprim::half, int), + block_param_type(rocprim::half, rocprim::half), + block_param_type(rocprim::bfloat16, int), + block_param_type(rocprim::bfloat16, rocprim::bfloat16)>; + +using BlockDiscParamsIntegral + = ::testing::Types, int), + block_param_type(uint8_t, bool), + block_param_type(int8_t, bool)>; + +using BlockDiscParamsFloating + = ::testing::Types; + +using BlockDiscParamsFloatingHalf + = ::testing::Types; + +using BlockHistAtomicParamsIntegral = ::testing::Types; + +using BlockExchParamsFloating + = ::testing::Types), + block_param_type(double, int8_t), + block_param_type(rocprim::half, rocprim::half), + block_param_type(rocprim::half, int16_t), + block_param_type(rocprim::bfloat16, rocprim::bfloat16)>; + +using BlockHistAtomicParamsFloating + = ::testing::Types; + +using BlockHistSortParamsIntegral = ::testing::Types; + +using BlockHistSortParamsFloating + = ::testing::Types; static constexpr size_t n_items = 7; static constexpr unsigned int items[n_items] = { From 55f4df59166600aec50f873eb0286f084878a4e9 Mon Sep 17 00:00:00 2001 From: Robin Voetter Date: Fri, 20 Dec 2024 13:02:39 +0000 Subject: [PATCH 53/84] Reduce CI artifact retention time --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2a47b5524..8ba5f934f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -207,7 +207,7 @@ build:cmake-minimum-apt: - $BUILD_DIR/test/rocprim/CTestTestfile.cmake - $BUILD_DIR/test/rocprim/test_* - $BUILD_DIR/test/test_* - expire_in: 2 weeks + expire_in: 1 day build:cmake-latest: stage: build @@ -257,7 +257,7 @@ build:package: paths: - $PACKAGE_DIR/rocprim*.deb - $PACKAGE_DIR/rocprim*.zip - expire_in: 2 weeks + expire_in: 1 day build:windows: stage: build @@ -293,7 +293,7 @@ build:windows: - $CI_PROJECT_DIR/build/CMakeCache.txt - $CI_PROJECT_DIR/build/.ninja_log - $CI_PROJECT_DIR/build/CTestTestfile.cmake - expire_in: 2 weeks + expire_in: 1 day autotune:build: stage: autotune From 7414e034dee8d65e79003f3aabfbcb62e6845c68 Mon Sep 17 00:00:00 2001 From: Nick Breed Date: Fri, 20 Dec 2024 15:41:51 +0000 Subject: [PATCH 54/84] Resolve "Add thread_load_volatile_pointer to rocprim" --- CHANGELOG.md | 13 +++- .../include/rocprim/thread/thread_load.hpp | 41 +++++++++++++ rocprim/include/rocprim/type_traits.hpp | 60 +++++++++++++++++++ test/rocprim/test_thread_algos.cpp | 51 +++++++++++++++- 4 files changed, 163 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index efd631558..46bedbdee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,22 @@ Full documentation for rocPRIM is available at [https://rocm.docs.amd.com/projec * Added gfx950 support. * Added missing operator `rocprim::key_value_pair::operator==`. * Added `rocprim::unrolled_copy` thread function to copy multiple items inside the thread. +* Added `rocprim::unrolled_thread_load` thread function to load multiple items inside the thread using `rocprim::thread_load`. +* Added `rocprim::int128_t` and `rocprim::uint128_t` to benchmarks for improved performance evaluation on 128-bit integers. +* Added `rocprim::int128_t` to autotuning supported types to improve performance for 128-bit integers. +* Added `rocprim::merge_inplace` function that allows merging in-place. +* Added initial value support for warp- and block-level inclusive scan. + +### Changed + +* Undeprecated `rocprim::thread_load` uses dereference as default and updated other cache load modifiers. +* Undeprecated `rocprim::thread_store` uses dereference as default and updated other cache store modifiers. +* Changed symbol visibility: all kernels now have hidden visibility, inline namespace with the library version is added, e.g. `rocprim::ROCPRIM_300400_NS::...`. This allows to link multiple libraries built with different versions of rocPRIM. ### Resolved issues * Fixed device radix sort not returning the correct required temporary storage when a double buffer contains `nullptr`. -* Fixed constness of equality operators (`==` and `!=`) in `rocprim::key_value_pair`. +* Fixed constness of equality operators (`==` and `!=`) in `rocprim::key_value_pair`. ## rocPRIM 3.4.0 for ROCm 6.4.0 diff --git a/rocprim/include/rocprim/thread/thread_load.hpp b/rocprim/include/rocprim/thread/thread_load.hpp index b3d577763..0dd9e8a50 100644 --- a/rocprim/include/rocprim/thread/thread_load.hpp +++ b/rocprim/include/rocprim/thread/thread_load.hpp @@ -32,6 +32,7 @@ #include "../config.hpp" #include "../detail/various.hpp" +#include "../type_traits.hpp" #include @@ -188,6 +189,46 @@ void unrolled_copy(InputIteratorT src, T* dst) detail::unrolled_copy_impl(src, dst, std::make_integer_sequence{}); } +/// \brief Thread loads with volatile_pointer for fundamental types +/// \tparam T Type of Data to be copied +/// \param ptr [in] - Input pointer for data that will be copied +/// \return returns loaded value +template +ROCPRIM_DEVICE ROCPRIM_INLINE +T thread_load_volatile(const T* ptr, Int2Type /*is_fundamental*/) +{ + return *reinterpret_cast(ptr); +} + +/// \brief Thread loads with volatile_pointer for non-fundamental types +/// \tparam T Type of Data to be copied +/// \param ptr [in] - Input pointer for data that will be copied +/// \return returns loaded value +template +ROCPRIM_DEVICE ROCPRIM_INLINE +T thread_load_volatile(const T* ptr, Int2Type /*is_fundamental*/) +{ + using device_word = typename detail::word_type::type; + constexpr int volatile_multiple = sizeof(T) / sizeof(device_word); + + T retval; + device_word* words = reinterpret_cast(&retval); + unrolled_copy(reinterpret_cast(ptr), words); + return retval; +} + +/// \brief Thread loads with volatile_pointer +/// \tparam T Type of Data to be copied +/// \param ptr [in] - Input pointer for data that will be copied +/// \return returns loaded value +template +ROCPRIM_DEVICE ROCPRIM_INLINE +T thread_load_volatile(const T* ptr) +{ + static constexpr bool is_constructible = std::is_constructible::value; + return thread_load_volatile(ptr, Int2Type()); +} + /// @} // end of group thread_load diff --git a/rocprim/include/rocprim/type_traits.hpp b/rocprim/include/rocprim/type_traits.hpp index 40beba513..517afd98f 100644 --- a/rocprim/include/rocprim/type_traits.hpp +++ b/rocprim/include/rocprim/type_traits.hpp @@ -497,6 +497,66 @@ struct is_binary_functional> } // namespace detail +/// \brief Helper struct it has the Type and the number of aligned bytes. +/// +/// \tparam T is the Type used to get the number of aligned bytes. +template +struct align_bytes +{ + /// Number of aligned bytes for type T + static constexpr unsigned value = alignof(T); + /// Type defined by T + using Type = T; +}; + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // skip specialized versions +template +struct align_bytes : align_bytes +{}; +template +struct align_bytes : align_bytes +{}; +template +struct align_bytes : align_bytes +{}; +#endif + +namespace detail +{ + +template +struct word_type +{ + static constexpr auto align_bytes_value = align_bytes::value; + + template + struct IsMultiple + { + static constexpr auto unit_align_bytes = align_bytes::value; + static constexpr bool is_multiple + = (sizeof(T) % sizeof(Unit) == 0) + && (int(align_bytes_value) % int(unit_align_bytes) == 0); + }; + + using type = typename std::conditional::is_multiple, + unsigned int, + typename std::conditional::is_multiple, + unsigned short, + unsigned char>::type>::type; +}; + +template +struct word_type : word_type +{}; +template +struct word_type : word_type +{}; +template +struct word_type : word_type +{}; + +} // namespace detail + END_ROCPRIM_NAMESPACE /// @} diff --git a/test/rocprim/test_thread_algos.cpp b/test/rocprim/test_thread_algos.cpp index 1da0d3882..a245179c3 100644 --- a/test/rocprim/test_thread_algos.cpp +++ b/test/rocprim/test_thread_algos.cpp @@ -63,7 +63,12 @@ using ThreadOperationTestParams = ::testing::Types, params, params, params>, - params>>; + params> +#if ROCPRIM_HAS_INT128_SUPPORT + , + params +#endif + >; TYPED_TEST_SUITE(RocprimThreadOperationTests, ThreadOperationTestParams); @@ -111,6 +116,50 @@ TYPED_TEST(RocprimThreadOperationTests, Load) } } +template +__global__ +void thread_volatile_load_kernel(Type* volatile const device_input, Type* device_output) +{ + size_t index = blockIdx.x * blockDim.x + threadIdx.x; + device_output[index] = rocprim::thread_load_volatile(device_input + index); +} + +TYPED_TEST(RocprimThreadOperationTests, VolatileLoad) +{ + using T = typename TestFixture::type; + static constexpr uint32_t block_size = 256; + static constexpr uint32_t grid_size = 128; + static constexpr uint32_t size = block_size * grid_size; + + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) + { + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); + + // Generate data + std::vector input = test_utils::get_random_data(size, 2, 200, seed_value); + std::vector output(size); + + // Calculate expected results on host + std::vector expected = input; + + // Preparing device + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(input.size()); + + thread_volatile_load_kernel + <<>>(device_input.get(), device_output.get()); + HIP_CHECK(hipGetLastError()); + + // Reading results back + output = device_output.load(); + + // Verifying results + ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); + } +} + template __global__ void thread_copy_unroll_kernel(Type* device_input, Type* device_output) From a8cbe5cd2b7dd3a8c3dbb3428da7230443b5bc48 Mon Sep 17 00:00:00 2001 From: Anton Gorenko Date: Mon, 30 Dec 2024 11:27:28 +0000 Subject: [PATCH 55/84] Fix test:install on rocm/dev-ubuntu --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8ba5f934f..2dff23450 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -459,7 +459,8 @@ test:install: -D CMAKE_CXX_STANDARD=14 -B build -S $CI_PROJECT_DIR - - $SUDO_CMD cmake --build build --target install + # Preserve $PATH when sudoing + - $SUDO_CMD env PATH="$PATH" cmake --build build --target install - !reference [.test-package, script] test:deb: From 337797c1393ceaa276e3e10917d745a64438016e Mon Sep 17 00:00:00 2001 From: Saiyang Zhang Date: Fri, 3 Jan 2025 08:39:38 +0000 Subject: [PATCH 56/84] Resolve "Nth element - Investigate smaller index sizes where possible" --- .../device/detail/device_nth_element.hpp | 163 +++++++++--------- .../rocprim/device/device_nth_element.hpp | 38 ++-- test/rocprim/test_device_nth_element.cpp | 21 +-- 3 files changed, 109 insertions(+), 113 deletions(-) diff --git a/rocprim/include/rocprim/device/detail/device_nth_element.hpp b/rocprim/include/rocprim/device/detail/device_nth_element.hpp index dbda6b86b..f4ab53e61 100644 --- a/rocprim/include/rocprim/device/detail/device_nth_element.hpp +++ b/rocprim/include/rocprim/device/detail/device_nth_element.hpp @@ -27,8 +27,8 @@ #include "../../block/block_sort.hpp" #include "../../block/block_store.hpp" -#include "../../config.hpp" #include "../../common.hpp" +#include "../../config.hpp" #include "../../intrinsics.hpp" #include "../../type_traits.hpp" @@ -69,34 +69,36 @@ struct nth_element_onesweep_lookback_state underlying_type state; - ROCPRIM_DEVICE ROCPRIM_INLINE explicit nth_element_onesweep_lookback_state( - underlying_type state) + ROCPRIM_DEVICE ROCPRIM_INLINE explicit nth_element_onesweep_lookback_state(underlying_type state) : state(state) {} ROCPRIM_DEVICE ROCPRIM_INLINE nth_element_onesweep_lookback_state(prefix_flag status, - underlying_type value) + underlying_type value) : state(static_cast(status) | value) {} - ROCPRIM_DEVICE ROCPRIM_INLINE underlying_type value() const + ROCPRIM_DEVICE ROCPRIM_INLINE + underlying_type value() const { return this->state & value_mask; } - ROCPRIM_DEVICE ROCPRIM_INLINE prefix_flag status() const + ROCPRIM_DEVICE ROCPRIM_INLINE + prefix_flag status() const { return static_cast(this->state & status_mask); } - ROCPRIM_DEVICE ROCPRIM_INLINE static nth_element_onesweep_lookback_state - load(nth_element_onesweep_lookback_state* ptr) + ROCPRIM_DEVICE ROCPRIM_INLINE + static nth_element_onesweep_lookback_state load(nth_element_onesweep_lookback_state* ptr) { underlying_type state = atomic_load(&ptr->state); return nth_element_onesweep_lookback_state(state); } - ROCPRIM_DEVICE ROCPRIM_INLINE void store(nth_element_onesweep_lookback_state* ptr) const + ROCPRIM_DEVICE ROCPRIM_INLINE + void store(nth_element_onesweep_lookback_state* ptr) const { atomic_store(&ptr->state, this->state); } @@ -112,15 +114,15 @@ struct nth_element_onesweep_lookback_state struct n_th_element_iteration_data { - size_t bucket_idx; - size_t offset; - size_t size; - bool equality_bucket; + unsigned int bucket_idx; + unsigned int offset; + unsigned int size; + bool equality_bucket; }; template ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void - kernel_block_sort_impl(KeysIterator keys, const size_t size, BinaryFunction compare_function) + kernel_block_sort_impl(KeysIterator keys, const unsigned int size, BinaryFunction compare_function) { constexpr nth_element_config_params params = device_params(); @@ -154,7 +156,7 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void template ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().stop_recursion_size) void - kernel_block_sort(KeysIterator keys, const size_t size, BinaryFunction compare_function) + kernel_block_sort(KeysIterator keys, const unsigned int size, BinaryFunction compare_function) { kernel_block_sort_impl(keys, size, compare_function); } @@ -164,7 +166,7 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void kernel_find_splitters_impl(KeysIterator keys, typename std::iterator_traits::value_type* tree, bool* equality_buckets, - const size_t size, + const unsigned int size, BinaryFunction compare_function) { constexpr nth_element_config_params params = device_params(); @@ -176,7 +178,7 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void ROCPRIM_SHARED_MEMORY typename block_sort_key::storage_type storage; - const size_t stride = size / num_splitters; + const unsigned int stride = size / num_splitters; const unsigned int idx = threadIdx.x; // Find values to split data in buckets @@ -202,13 +204,11 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void } template -ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS( - device_params().number_of_buckets - - 1) void +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().number_of_buckets - 1) void kernel_find_splitters(KeysIterator keys, typename std::iterator_traits::value_type* tree, bool* equality_buckets, - const size_t size, + const unsigned int size, BinaryFunction compare_function) { kernel_find_splitters_impl(keys, tree, equality_buckets, size, compare_function); @@ -218,8 +218,8 @@ template ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void kernel_count_bucket_sizes_impl(KeysIterator keys, typename std::iterator_traits::value_type* tree, - const size_t size, - size_t* buckets, + const unsigned int size, + unsigned int* buckets, bool* equality_buckets, BinaryFunction compare_function) { @@ -249,8 +249,8 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void ROCPRIM_SHARED_MEMORY struct { uninitialized_array buffer; - unsigned int shared_buckets[num_buckets]; - bool shared_equality_buckets[num_buckets]; + unsigned int shared_buckets[num_buckets]; + bool shared_equality_buckets[num_buckets]; } storage; if(threadIdx.x < num_buckets) @@ -265,9 +265,9 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void } const key_type* search_tree = storage.buffer.get_unsafe_array(); - key_type elements[num_items_per_thread]; - const size_t offset = blockIdx.x * num_items_per_block; - const bool is_complete_block = offset + num_items_per_block <= size; + key_type elements[num_items_per_thread]; + const unsigned int offset = blockIdx.x * num_items_per_block; + const bool is_complete_block = offset + num_items_per_block <= size; if(is_complete_block) { @@ -280,9 +280,9 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void syncthreads(); - for(size_t item = 0; item < num_items_per_thread; item++) + for(unsigned int item = 0; item < num_items_per_thread; item++) { - size_t idx = offset + threadIdx.x * num_items_per_thread + item; + unsigned int idx = offset + threadIdx.x * num_items_per_thread + item; if(idx < size) { const key_type element = elements[item]; @@ -290,7 +290,7 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void unsigned int right = num_splitters; unsigned int bucket; // Binary search through splitters to put in bucket - for(unsigned int i = 0; i < Log2::VALUE; i++) + for(unsigned int i = 0; i < Log2(num_buckets)>::VALUE; i++) { const unsigned int mid = (left + right) >> 1; const bool comp = compare_function(element, search_tree[mid]); @@ -319,14 +319,11 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void } template -ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS( - device_params() - .kernel_config - .block_size) void +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().kernel_config.block_size) void kernel_count_bucket_sizes(KeysIterator keys, typename std::iterator_traits::value_type* tree, - const size_t size, - size_t* buckets, + const unsigned int size, + unsigned int* buckets, bool* equality_buckets, BinaryFunction compare_function) { @@ -340,26 +337,26 @@ ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS( template ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void - kernel_find_nth_element_bucket_impl(size_t* buckets, + kernel_find_nth_element_bucket_impl(unsigned int* buckets, n_th_element_iteration_data* nth_element_data, bool* equality_buckets, - const size_t rank) + const unsigned int rank) { constexpr nth_element_config_params params = device_params(); constexpr unsigned int num_buckets = params.number_of_buckets; - using block_scan_buckets = block_scan; + using block_scan_buckets = block_scan; ROCPRIM_SHARED_MEMORY struct { typename block_scan_buckets::storage_type scan; - size_t bucket_offsets[num_buckets]; + unsigned int bucket_offsets[num_buckets]; } storage; - size_t bucket_size = buckets[threadIdx.x]; - size_t bucket_offset; + unsigned int bucket_size = buckets[threadIdx.x]; + unsigned int bucket_offset; // Calculate the global offset of the buckets based on bucket sizes block_scan_buckets().exclusive_scan(bucket_size, bucket_offset, 0, storage.scan); @@ -367,7 +364,7 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void syncthreads(); - size_t num_buckets_before; + unsigned int num_buckets_before; // Find in which bucket the nth element sits bool in_nth = storage.bucket_offsets[threadIdx.x] <= rank; @@ -376,7 +373,7 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void if(threadIdx.x == (num_buckets - 1)) { // Store nth_element data - size_t nth_element = num_buckets_before - 1; + unsigned int nth_element = num_buckets_before - 1; nth_element_data->offset = storage.bucket_offsets[nth_element]; nth_element_data->size = buckets[nth_element]; nth_element_data->equality_bucket = equality_buckets[nth_element]; @@ -385,13 +382,11 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void } template -ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS( - device_params() - .number_of_buckets) void - kernel_find_nth_element_bucket(size_t* buckets, +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().number_of_buckets) void + kernel_find_nth_element_bucket(unsigned int* buckets, n_th_element_iteration_data* nth_element_data, bool* equality_buckets, - const size_t rank) + const unsigned int rank) { kernel_find_nth_element_bucket_impl(buckets, nth_element_data, equality_buckets, rank); @@ -401,7 +396,7 @@ template::value_type* tree, - const size_t size, + const unsigned int size, nth_element_onesweep_lookback_state* lookback_states, n_th_element_iteration_data* nth_element_data, typename std::iterator_traits::value_type* keys_buffer, @@ -421,7 +416,7 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void constexpr unsigned int num_partitions = NumPartitions; using block_rank = rocprim::block_radix_rank::VALUE, + Log2(num_partitions + 1)>::VALUE, params.radix_rank_algorithm>; using block_load_element = block_load; @@ -437,14 +432,14 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void uint8_t buckets[num_items_per_thread]; - const size_t nth_element = nth_element_data->bucket_idx; - const key_type lower_bound = nth_element > 0 ? tree[nth_element - 1] : key_type(0); - const key_type upper_bound = nth_element < num_splitters ? tree[nth_element] : key_type(0); - const bool equality_bucket = nth_element_data->equality_bucket; - const bool equality_bucket_before = nth_element > 0 && equality_buckets[nth_element - 1]; + const unsigned int nth_element = nth_element_data->bucket_idx; + const key_type lower_bound = nth_element > 0 ? tree[nth_element - 1] : key_type(0); + const key_type upper_bound = nth_element < num_splitters ? tree[nth_element] : key_type(0); + const bool equality_bucket = nth_element_data->equality_bucket; + const bool equality_bucket_before = nth_element > 0 && equality_buckets[nth_element - 1]; - const size_t offset = blockIdx.x * num_items_per_block; - const bool is_complete_block = offset + num_items_per_block <= size; + const unsigned int offset = blockIdx.x * num_items_per_block; + const bool is_complete_block = offset + num_items_per_block <= size; key_type elements[num_items_per_thread]; if(is_complete_block) @@ -456,12 +451,12 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void block_load_element().load(keys + offset, elements, size - offset, 0, storage.load_element); } - const size_t thread_id = (threadIdx.x * num_items_per_thread) + offset; + const unsigned int thread_id = (threadIdx.x * num_items_per_thread) + offset; if(is_complete_block) { ROCPRIM_UNROLL - for(size_t item = 0; item < num_items_per_thread; item++) + for(unsigned int item = 0; item < num_items_per_thread; item++) { const key_type element = elements[item]; // Assign items to the offset of buckets: @@ -480,10 +475,10 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void else { ROCPRIM_UNROLL - for(size_t item = 0; item < num_items_per_thread; item++) + for(unsigned int item = 0; item < num_items_per_thread; item++) { - const key_type element = elements[item]; - const size_t idx = item + thread_id; + const key_type element = elements[item]; + const unsigned int idx = item + thread_id; if(idx < size) { buckets[item] @@ -549,18 +544,18 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void // of all not finished yet is progressing. state(state::COMPLETE, inclusive_digit_prefix).store(block_state); - size_t global_offset = 0; + unsigned int global_offset = 0; // The global offsets based on the nth element bucket. if(partition > 0) { - const size_t nth_bucket_offset = nth_element_data->offset; + const unsigned int nth_bucket_offset = nth_element_data->offset; global_offset += nth_bucket_offset; } if((partition > 0 && nth_element == 0) || partition > 1) { - const size_t nth_bucket_size = nth_element_data->size; + const unsigned int nth_bucket_size = nth_element_data->size; global_offset += nth_bucket_size; } @@ -574,24 +569,23 @@ ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void // Scatter the keys based on their placement in the buckets ROCPRIM_UNROLL - for(size_t item = 0; item < num_items_per_thread; item++) + for(unsigned int item = 0; item < num_items_per_thread; item++) { - const size_t idx = item + thread_id; + const unsigned int idx = item + thread_id; if(idx < size) { - const uint8_t bucket = buckets[item]; - const size_t index = storage.buckets_block_offsets_shared[bucket] + ranks[item]; - keys_buffer[index] = elements[item]; + const uint8_t bucket = buckets[item]; + const unsigned int index = storage.buckets_block_offsets_shared[bucket] + ranks[item]; + keys_buffer[index] = elements[item]; } } } template -ROCPRIM_KERNEL - ROCPRIM_LAUNCH_BOUNDS(device_params().kernel_config.block_size) void +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().kernel_config.block_size) void kernel_copy_buckets(KeysIterator keys, typename std::iterator_traits::value_type* tree, - const size_t size, + const unsigned int size, nth_element_onesweep_lookback_state* lookback_states, n_th_element_iteration_data* nth_element_data, typename std::iterator_traits::value_type* keys_buffer, @@ -609,13 +603,14 @@ ROCPRIM_KERNEL } template -ROCPRIM_INLINE hipError_t +ROCPRIM_INLINE +hipError_t nth_element_keys_impl(KeysIterator keys, typename std::iterator_traits::value_type* keys_buffer, typename std::iterator_traits::value_type* tree, - size_t rank, - size_t size, - size_t* buckets, + unsigned int rank, + unsigned int size, + unsigned int* buckets, bool* equality_buckets, nth_element_onesweep_lookback_state* lookback_states, const unsigned int num_buckets, @@ -632,7 +627,7 @@ ROCPRIM_INLINE hipError_t constexpr unsigned int num_partitions = NumPartitions; const unsigned int num_splitters = num_buckets - 1; const unsigned int num_items_per_block = num_threads_per_block * num_items_per_thread; - size_t iteration = 0; + unsigned int iteration = 0; // Start point for time measurements std::chrono::steady_clock::time_point start; @@ -717,9 +712,9 @@ ROCPRIM_INLINE hipError_t ROCPRIM_RETURN_ON_ERROR(hipStreamSynchronize(stream)); - size_t offset = h_nth_element_data.offset; - size_t bucket_size = h_nth_element_data.size; - bool equality_bucket = h_nth_element_data.equality_bucket; + unsigned int offset = h_nth_element_data.offset; + unsigned int bucket_size = h_nth_element_data.size; + bool equality_bucket = h_nth_element_data.equality_bucket; // If all values are the same it is already sorted if(equality_bucket) @@ -739,8 +734,6 @@ ROCPRIM_INLINE hipError_t return hipSuccess; } - - } // namespace detail END_ROCPRIM_NAMESPACE diff --git a/rocprim/include/rocprim/device/device_nth_element.hpp b/rocprim/include/rocprim/device/device_nth_element.hpp index b1e38aa50..3f07dc22b 100644 --- a/rocprim/include/rocprim/device/device_nth_element.hpp +++ b/rocprim/include/rocprim/device/device_nth_element.hpp @@ -77,7 +77,7 @@ hipError_t const unsigned int num_blocks = ceiling_div(size, num_items_per_block); key_type* tree = nullptr; - size_t* buckets = nullptr; + unsigned int* buckets = nullptr; n_th_element_iteration_data* nth_element_data = nullptr; bool* equality_buckets = nullptr; nth_element_onesweep_lookback_state* lookback_states = nullptr; @@ -248,14 +248,15 @@ template::value_type>> -ROCPRIM_INLINE hipError_t nth_element(void* temporary_storage, - size_t& storage_size, - KeysIterator keys, - size_t nth, - size_t size, - BinaryFunction compare_function = BinaryFunction(), - hipStream_t stream = 0, - bool debug_synchronous = false) +ROCPRIM_INLINE +hipError_t nth_element(void* temporary_storage, + size_t& storage_size, + KeysIterator keys, + size_t nth, + size_t size, + BinaryFunction compare_function = BinaryFunction(), + hipStream_t stream = 0, + bool debug_synchronous = false) { return detail::nth_element_impl(temporary_storage, storage_size, @@ -358,15 +359,16 @@ template::value_type>> -ROCPRIM_INLINE hipError_t nth_element(void* temporary_storage, - size_t& storage_size, - KeysInputIterator keys_input, - KeysOutputIterator keys_output, - size_t nth, - size_t size, - BinaryFunction compare_function = BinaryFunction(), - hipStream_t stream = 0, - bool debug_synchronous = false) +ROCPRIM_INLINE +hipError_t nth_element(void* temporary_storage, + size_t& storage_size, + KeysInputIterator keys_input, + KeysOutputIterator keys_output, + size_t nth, + size_t size, + BinaryFunction compare_function = BinaryFunction(), + hipStream_t stream = 0, + bool debug_synchronous = false) { using key_type = typename std::iterator_traits::value_type; static_assert( diff --git a/test/rocprim/test_device_nth_element.cpp b/test/rocprim/test_device_nth_element.cpp index 7fda72873..a0b41d50f 100644 --- a/test/rocprim/test_device_nth_element.cpp +++ b/test/rocprim/test_device_nth_element.cpp @@ -63,7 +63,7 @@ struct DeviceNthelementParams template void inline compare_cpp_14(InputVector input, OutputVector output, - size_t nth_element, + unsigned int nth_element, CompareFunction compare_op) { using key_type = typename InputVector::value_type; @@ -89,7 +89,7 @@ void inline compare_cpp_14(InputVector input, template void inline compare_cpp_17(InputVector input, OutputVector output, - size_t nth_element, + unsigned int nth_element, CompareFunction compare_op) { using key_type = typename InputVector::value_type; @@ -122,7 +122,7 @@ void inline compare_cpp_17(InputVector input, template void inline compare(InputVector input, OutputVector output, - size_t nth_element, + unsigned int nth_element, CompareFunction compare_op) { compare_cpp_14(input, output, nth_element, compare_op); @@ -192,13 +192,13 @@ TYPED_TEST(RocprimDeviceNthelementTests, NthelementKey) // The size loop alternates between in place and not in place bool in_place = false; - for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) + for(unsigned int seed_index = 0; seed_index < number_of_runs; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); - for(size_t size : test_utils::get_sizes(seed_value)) + for(unsigned int size : test_utils::get_sizes(seed_value)) { hipStream_t stream = 0; // default if(TestFixture::use_graphs) @@ -209,11 +209,11 @@ TYPED_TEST(RocprimDeviceNthelementTests, NthelementKey) SCOPED_TRACE(testing::Message() << "with size = " << size); - in_place = !in_place; - size_t nth_element = 0; + in_place = !in_place; + unsigned int nth_element = 0; if(size > 0) { - nth_element = test_utils::get_random_value(0, size - 1, seed_value); + nth_element = test_utils::get_random_value(0, size - 1, seed_value); } SCOPED_TRACE(testing::Message() << "with nth_element = " << nth_element); @@ -268,7 +268,8 @@ TYPED_TEST(RocprimDeviceNthelementTests, NthelementKey) // allocate temporary storage test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); - test_utils::GraphHelper gHelper;; + test_utils::GraphHelper gHelper; + if(TestFixture::use_graphs) { gHelper.startStreamCapture(stream); @@ -338,7 +339,7 @@ TEST(RocprimNthelementKeySameTests, NthelementKeySame) const bool debug_synchronous = false; unsigned int seed_value = rand(); - for(size_t size : test_utils::get_sizes(seed_value)) + for(unsigned int size : test_utils::get_sizes(seed_value)) { hipStream_t stream = 0; // default SCOPED_TRACE(testing::Message() << "with size = " << size); From b57e16305514dcc15d109e35a537f1f93849daf0 Mon Sep 17 00:00:00 2001 From: Anton Gorenko Date: Fri, 3 Jan 2025 10:31:32 +0000 Subject: [PATCH 57/84] Enable int128 tests on Windows --- test/common_test_header.hpp | 24 +++++++++--------------- test/rocprim/CMakeLists.txt | 7 +++++++ test/rocprim/test_seed.hpp | 6 ++++-- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/test/common_test_header.hpp b/test/common_test_header.hpp index b6e2a1ac1..5a8bf282b 100755 --- a/test/common_test_header.hpp +++ b/test/common_test_header.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,8 +20,8 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#ifndef ROCPRIM_COMMON_TEST_HEADER -#define ROCPRIM_COMMON_TEST_HEADER +#ifndef ROCPRIM_COMMON_TEST_HEADER_HPP_ +#define ROCPRIM_COMMON_TEST_HEADER_HPP_ #include #include @@ -75,10 +75,8 @@ } \ } -#if(defined(__GNUC__) || defined(__clang__)) && (defined(__GLIBCXX__) || defined(_LIBCPP_VERSION)) +#ifndef ROCPRIM_HAS_INT128_SUPPORT #define ROCPRIM_HAS_INT128_SUPPORT 1 -#else - #define ROCPRIM_HAS_INT128_SUPPORT 0 #endif #define INSTANTIATE_TYPED_TEST_EXPANDED_1(line, test_suite_name, ...) \ @@ -111,8 +109,7 @@ inline char* __get_env(const char* name) { char* env; #ifdef _MSC_VER - size_t len; - errno_t err = _dupenv_s(&env, &len, name); + errno_t err = _dupenv_s(&env, nullptr, name); if(err) { return nullptr; @@ -123,15 +120,12 @@ inline char* __get_env(const char* name) return env; } -inline void clean_env(char* name) +inline void clean_env(char* env) { #ifdef _MSC_VER - if(name != nullptr) - { - free(name); - } + free(env); #endif - (void)name; + (void)env; } inline int obtain_device_from_ctest() @@ -184,4 +178,4 @@ hipError_t hipMallocHelper(T** devPtr, size_t size) } -#endif +#endif // ROCPRIM_COMMON_TEST_HEADER_HPP_ diff --git a/test/rocprim/CMakeLists.txt b/test/rocprim/CMakeLists.txt index bfc447cc4..07d81d3fe 100644 --- a/test/rocprim/CMakeLists.txt +++ b/test/rocprim/CMakeLists.txt @@ -74,6 +74,13 @@ function(add_rocprim_test_internal TEST_NAME TEST_SOURCES TEST_TARGET) $<$:$<$:/bigobj>> ) + if(WIN32) + # Usage of 128-bit integral types (__int128_t and __uint128_t, whose alignment is 16 bytes) + # requires an extended alignment support. Otherwise, a static assert will be triggered in + # functions like std::stable_sort. _ENABLE_EXTENDED_ALIGNED_STORAGE turns it on: + target_compile_definitions(${TEST_TARGET} PRIVATE _ENABLE_EXTENDED_ALIGNED_STORAGE) + endif() + set_target_properties(${TEST_TARGET} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/test/rocprim" diff --git a/test/rocprim/test_seed.hpp b/test/rocprim/test_seed.hpp index 814b5eb78..134aff545 100644 --- a/test/rocprim/test_seed.hpp +++ b/test/rocprim/test_seed.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -23,10 +23,12 @@ #include +#include "../common_test_header.hpp" + using engine_type = std::default_random_engine; using seed_type = typename engine_type::result_type; -static const char* env_p = std::getenv("ROCPRIM_TEST_RUNS"); +static const char* env_p = test_common_utils::__get_env("ROCPRIM_TEST_RUNS"); // "env_var" determines the number of iterations. // If undefined or incorrectly defined, it defaults to 0. static const size_t env_var = (env_p == nullptr) ? 0ul : std::atoi(env_p); From 9911f17b6b5af10a7830b828b1962482180873b6 Mon Sep 17 00:00:00 2001 From: Yung-Sheng Tu Date: Fri, 3 Jan 2025 15:05:24 +0000 Subject: [PATCH 58/84] Resolve "Benchmark and tuning for 128-bit integers" --- CHANGELOG.md | 9 - benchmark/ConfigAutotuneSettings.cmake | 10 +- .../benchmark_block_adjacent_difference.cpp | 265 +- benchmark/benchmark_block_discontinuity.cpp | 224 +- benchmark/benchmark_block_exchange.cpp | 255 +- benchmark/benchmark_block_histogram.cpp | 163 +- benchmark/benchmark_block_radix_rank.cpp | 86 +- benchmark/benchmark_block_radix_sort.cpp | 230 +- benchmark/benchmark_block_reduce.cpp | 178 +- .../benchmark_block_run_length_decode.cpp | 61 +- benchmark/benchmark_block_scan.cpp | 244 +- benchmark/benchmark_block_sort.cpp | 19 +- benchmark/benchmark_block_sort.parallel.hpp | 54 +- benchmark/benchmark_config_dispatch.cpp | 35 +- .../benchmark_device_adjacent_difference.cpp | 15 +- ...device_adjacent_difference.parallel.cpp.in | 9 +- ...rk_device_adjacent_difference.parallel.hpp | 13 +- benchmark/benchmark_device_adjacent_find.cpp | 10 + ...hmark_device_adjacent_find.parallel.cpp.in | 5 +- ...enchmark_device_adjacent_find.parallel.hpp | 11 +- benchmark/benchmark_device_batch_memcpy.cpp | 107 +- benchmark/benchmark_device_binary_search.cpp | 78 +- ...hmark_device_binary_search.parallel.cpp.in | 7 +- ...enchmark_device_binary_search.parallel.hpp | 23 +- benchmark/benchmark_device_find_end.cpp | 13 +- benchmark/benchmark_device_find_end.hpp | 12 +- benchmark/benchmark_device_find_first_of.cpp | 10 + ...hmark_device_find_first_of.parallel.cpp.in | 4 + ...enchmark_device_find_first_of.parallel.hpp | 9 +- benchmark/benchmark_device_histogram.cpp | 367 +- ...benchmark_device_histogram.parallel.cpp.in | 8 +- .../benchmark_device_histogram.parallel.hpp | 93 +- benchmark/benchmark_device_memory.cpp | 3004 ++++++++++++++--- benchmark/benchmark_device_merge.cpp | 25 +- .../benchmark_device_merge.parallel.cpp.in | 6 +- benchmark/benchmark_device_merge.parallel.hpp | 163 +- benchmark/benchmark_device_merge_sort.cpp | 20 +- benchmark/benchmark_device_merge_sort.hpp | 29 +- ...enchmark_device_merge_sort_block_merge.cpp | 17 +- ...ice_merge_sort_block_merge.parallel.cpp.in | 8 +- ...device_merge_sort_block_merge.parallel.hpp | 161 +- ...benchmark_device_merge_sort_block_sort.cpp | 17 +- ...vice_merge_sort_block_sort.parallel.cpp.in | 8 +- ..._device_merge_sort_block_sort.parallel.hpp | 101 +- benchmark/benchmark_device_nth_element.cpp | 21 +- benchmark/benchmark_device_nth_element.hpp | 13 +- benchmark/benchmark_device_partial_sort.cpp | 21 +- benchmark/benchmark_device_partial_sort.hpp | 13 +- .../benchmark_device_partial_sort_copy.cpp | 10 +- .../benchmark_device_partial_sort_copy.hpp | 13 +- benchmark/benchmark_device_partition.cpp | 38 +- ...benchmark_device_partition.parallel.cpp.in | 6 +- .../benchmark_device_partition.parallel.hpp | 53 +- benchmark/benchmark_device_radix_sort.cpp | 10 +- benchmark/benchmark_device_radix_sort.hpp | 30 +- ...benchmark_device_radix_sort_block_sort.cpp | 17 +- ...vice_radix_sort_block_sort.parallel.cpp.in | 8 +- ..._device_radix_sort_block_sort.parallel.hpp | 118 +- .../benchmark_device_radix_sort_onesweep.cpp | 6 +- ...device_radix_sort_onesweep.parallel.cpp.in | 8 +- ...rk_device_radix_sort_onesweep.parallel.hpp | 236 +- benchmark/benchmark_device_reduce.cpp | 29 +- .../benchmark_device_reduce.parallel.cpp.in | 11 +- .../benchmark_device_reduce.parallel.hpp | 75 +- benchmark/benchmark_device_reduce_by_key.cpp | 37 +- ...hmark_device_reduce_by_key.parallel.cpp.in | 6 +- ...enchmark_device_reduce_by_key.parallel.hpp | 22 +- ...ark_device_reduce_by_key_deterministic.cpp | 36 +- .../benchmark_device_run_length_encode.cpp | 7 +- ...k_device_run_length_encode.parallel.cpp.in | 6 +- ...mark_device_run_length_encode.parallel.hpp | 12 +- ...ice_run_length_encode_non_trivial_runs.cpp | 7 +- ...th_encode_non_trivial_runs.parallel.cpp.in | 7 +- ...ength_encode_non_trivial_runs.parallel.hpp | 17 +- benchmark/benchmark_device_scan.cpp | 20 +- .../benchmark_device_scan.parallel.cpp.in | 9 +- benchmark/benchmark_device_scan.parallel.hpp | 35 +- benchmark/benchmark_device_scan_by_key.cpp | 32 +- ...nchmark_device_scan_by_key.parallel.cpp.in | 9 +- .../benchmark_device_scan_by_key.parallel.hpp | 36 +- ...hmark_device_scan_by_key_deterministic.cpp | 30 +- .../benchmark_device_scan_deterministic.cpp | 16 +- benchmark/benchmark_device_search.cpp | 13 +- benchmark/benchmark_device_search.hpp | 12 +- benchmark/benchmark_device_search_n.cpp | 13 + .../benchmark_device_search_n.parallel.cpp.in | 8 +- .../benchmark_device_search_n.parallel.hpp | 27 +- ...hmark_device_segmented_radix_sort_keys.cpp | 111 +- ..._segmented_radix_sort_keys.parallel.cpp.in | 6 +- ...ice_segmented_radix_sort_keys.parallel.hpp | 30 +- ...mark_device_segmented_radix_sort_pairs.cpp | 133 +- ...segmented_radix_sort_pairs.parallel.cpp.in | 6 +- ...ce_segmented_radix_sort_pairs.parallel.hpp | 34 +- .../benchmark_device_segmented_reduce.cpp | 174 +- benchmark/benchmark_device_select.cpp | 35 +- .../benchmark_device_select.parallel.cpp.in | 6 +- .../benchmark_device_select.parallel.hpp | 58 +- benchmark/benchmark_device_transform.cpp | 28 +- ...benchmark_device_transform.parallel.cpp.in | 6 +- .../benchmark_device_transform.parallel.hpp | 19 +- benchmark/benchmark_predicate_iterator.cpp | 60 +- benchmark/benchmark_utils.hpp | 406 ++- benchmark/benchmark_warp_exchange.cpp | 237 +- benchmark/benchmark_warp_reduce.cpp | 104 +- benchmark/benchmark_warp_scan.cpp | 104 +- benchmark/benchmark_warp_sort.cpp | 138 +- scripts/autotune/fallback_config.json | 3 +- scripts/autotune/templates/config_template | 4 +- .../autotune/templates/merge_config_template | 2 +- 109 files changed, 6029 insertions(+), 3014 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46bedbdee..885dc1e5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,17 +9,8 @@ Full documentation for rocPRIM is available at [https://rocm.docs.amd.com/projec * Added gfx950 support. * Added missing operator `rocprim::key_value_pair::operator==`. * Added `rocprim::unrolled_copy` thread function to copy multiple items inside the thread. -* Added `rocprim::unrolled_thread_load` thread function to load multiple items inside the thread using `rocprim::thread_load`. * Added `rocprim::int128_t` and `rocprim::uint128_t` to benchmarks for improved performance evaluation on 128-bit integers. * Added `rocprim::int128_t` to autotuning supported types to improve performance for 128-bit integers. -* Added `rocprim::merge_inplace` function that allows merging in-place. -* Added initial value support for warp- and block-level inclusive scan. - -### Changed - -* Undeprecated `rocprim::thread_load` uses dereference as default and updated other cache load modifiers. -* Undeprecated `rocprim::thread_store` uses dereference as default and updated other cache store modifiers. -* Changed symbol visibility: all kernels now have hidden visibility, inline namespace with the library version is added, e.g. `rocprim::ROCPRIM_300400_NS::...`. This allows to link multiple libraries built with different versions of rocPRIM. ### Resolved issues diff --git a/benchmark/ConfigAutotuneSettings.cmake b/benchmark/ConfigAutotuneSettings.cmake index 37f3a97bc..af1cb9798 100644 --- a/benchmark/ConfigAutotuneSettings.cmake +++ b/benchmark/ConfigAutotuneSettings.cmake @@ -21,11 +21,11 @@ # SOFTWARE. # All default fallback types as listed in scripts/autotune/fallback_config.json -set(TUNING_TYPES "int64_t int short int8_t double float rocprim::half") +set(TUNING_TYPES "rocprim::int128_t int64_t int short int8_t double float rocprim::half") # If config selection happens based on two types, the second type has limited fallbacks. The selection is based # on the size and it is ignored whether the type is floating-point or integral. The autotuning script uses the # benchmarks for the integral types as fallback, hence tuning for the floating-point types is not needed. -set(LIMITED_TUNING_TYPES "int64_t int short int8_t") +set(LIMITED_TUNING_TYPES "rocprim::int128_t int64_t int short int8_t") function(read_config_autotune_settings file list_across_names list_across output_pattern_suffix) if(file STREQUAL "benchmark_device_adjacent_difference") @@ -44,13 +44,13 @@ true;false true;32 64 128 256 512 1024" PARENT_SCOPE) elseif(file STREQUAL "benchmark_device_merge_sort_block_merge") set(list_across_names "KeyType;ValueType;BlockSize;UseMergePath" PARENT_SCOPE) set(list_across "\ -${TUNING_TYPES};rocprim::empty_type ${LIMITED_TUNING_TYPES} custom_type;\ +${TUNING_TYPES};rocprim::empty_type ${LIMITED_TUNING_TYPES};\ 128 256 512 1024;true" PARENT_SCOPE) set(output_pattern_suffix "@KeyType@_@ValueType@_@BlockSize@_@UseMergePath@" PARENT_SCOPE) elseif(file STREQUAL "benchmark_device_merge_sort_block_sort") set(list_across_names "KeyType;ValueType;BlockSize;BlockSortMethod" PARENT_SCOPE) set(list_across "\ -${TUNING_TYPES};rocprim::empty_type ${LIMITED_TUNING_TYPES} custom_type;\ +${TUNING_TYPES};rocprim::empty_type ${LIMITED_TUNING_TYPES};\ 256 512 1024;rocprim::block_sort_algorithm::stable_merge_sort" PARENT_SCOPE) set(output_pattern_suffix "@KeyType@_@ValueType@_@BlockSize@_@BlockSortMethod@" PARENT_SCOPE) elseif(file STREQUAL "benchmark_device_radix_sort_block_sort") @@ -94,7 +94,7 @@ KeyType;LongBits;BlockSize;ItemsPerThread;WarpSmallLWS;WarpSmallIPT;WarpSmallBS; elseif(file STREQUAL "benchmark_device_segmented_radix_sort_pairs") set(list_across_names "\ KeyType;ValueType;LongBits;BlockSize;ItemsPerThread;WarpSmallLWS;WarpSmallIPT;WarpSmallBS;WarpPartition;WarpMediumLWS;WarpMediumIPT;WarpMediumBS" PARENT_SCOPE) - set(list_across "${TUNING_TYPES};int8_t;8;256;4 8 16;8;4;256;64;16;8;256" PARENT_SCOPE) + set(list_across "${TUNING_TYPES};${LIMITED_TUNING_TYPES};8;256;4 8 16;8;4;256;64;16;8;256" PARENT_SCOPE) set(output_pattern_suffix "\ @KeyType@_@ValueType@_@LongBits@_@BlockSize@_@ItemsPerThread@_@WarpSmallLWS@_@WarpSmallIPT@_@WarpSmallBS@_@WarpPartition@_@WarpMediumLWS@_@WarpMediumIPT@_@WarpMediumBS@" PARENT_SCOPE) elseif(file STREQUAL "benchmark_device_transform") diff --git a/benchmark/benchmark_block_adjacent_difference.cpp b/benchmark/benchmark_block_adjacent_difference.cpp index 8d237f763..d9f7728b4 100644 --- a/benchmark/benchmark_block_adjacent_difference.cpp +++ b/benchmark/benchmark_block_adjacent_difference.cpp @@ -34,83 +34,85 @@ #include #include #include +#include +#include +#include +#include -#include -#include -#include +#include +#include #include +#include #include -#include -#include - #ifndef DEFAULT_N const size_t DEFAULT_BYTES = 1024 * 1024 * 128 * 4; #endif -namespace rp = rocprim; - -template -__global__ __launch_bounds__(BlockSize) void kernel(Args ...args) +template +__global__ __launch_bounds__(BlockSize) +void kernel(Args... args) { Benchmark::template run(args...); } struct subtract_left { - template - __device__ static void run(const T* d_input, T* d_output, unsigned int trials) + template + __device__ + static void run(const T* d_input, T* d_output, unsigned int trials) { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; T input[ItemsPerThread]; - rp::block_load_direct_striped(lid, d_input + block_offset, input); + rocprim::block_load_direct_striped(lid, d_input + block_offset, input); - using adjacent_diff_t = rp::block_adjacent_difference; + using adjacent_diff_t = rocprim::block_adjacent_difference; __shared__ typename adjacent_diff_t::storage_type storage; ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < trials; trial++) + for(unsigned int trial = 0; trial < trials; ++trial) { T output[ItemsPerThread]; if(WithTile) { - adjacent_diff_t().subtract_left(input, output, rp::minus<>{}, T(123), storage); + adjacent_diff_t().subtract_left(input, output, rocprim::minus<>{}, T(123), storage); } else { - adjacent_diff_t().subtract_left(input, output, rp::minus<>{}, storage); + adjacent_diff_t().subtract_left(input, output, rocprim::minus<>{}, storage); } for(unsigned int i = 0; i < ItemsPerThread; ++i) { input[i] += output[i]; } - rp::syncthreads(); + rocprim::syncthreads(); } - rp::block_store_direct_striped(lid, d_output + block_offset, input); + rocprim::block_store_direct_striped(lid, d_output + block_offset, input); } }; struct subtract_left_partial { - template - __device__ static void + template + __device__ + static void run(const T* d_input, const unsigned int* tile_sizes, T* d_output, unsigned int trials) { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; T input[ItemsPerThread]; - rp::block_load_direct_striped(lid, d_input + block_offset, input); + rocprim::block_load_direct_striped(lid, d_input + block_offset, input); - using adjacent_diff_t = rp::block_adjacent_difference; + using adjacent_diff_t = rocprim::block_adjacent_difference; __shared__ typename adjacent_diff_t::storage_type storage; unsigned int tile_size = tile_sizes[blockIdx.x]; @@ -119,16 +121,25 @@ struct subtract_left_partial const auto tile_size_diff = (BlockSize * ItemsPerThread) / trials + 1; ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < trials; trial++) + for(unsigned int trial = 0; trial < trials; ++trial) { T output[ItemsPerThread]; if(WithTile) { - adjacent_diff_t().subtract_left_partial(input, output, rp::minus<>{}, T(123), tile_size, storage); + adjacent_diff_t().subtract_left_partial(input, + output, + rocprim::minus<>{}, + T(123), + tile_size, + storage); } else { - adjacent_diff_t().subtract_left_partial(input, output, rp::minus<>{}, tile_size, storage); + adjacent_diff_t().subtract_left_partial(input, + output, + rocprim::minus<>{}, + tile_size, + storage); } for(unsigned int i = 0; i < ItemsPerThread; ++i) @@ -138,66 +149,69 @@ struct subtract_left_partial // Change the tile_size to even out the distribution tile_size = (tile_size + tile_size_diff) % (BlockSize * ItemsPerThread); - rp::syncthreads(); + rocprim::syncthreads(); } - rp::block_store_direct_striped(lid, d_output + block_offset, input); + rocprim::block_store_direct_striped(lid, d_output + block_offset, input); } }; struct subtract_right { - template - __device__ static void run(const T* d_input, T* d_output, unsigned int trials) + template + __device__ + static void run(const T* d_input, T* d_output, unsigned int trials) { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; T input[ItemsPerThread]; - rp::block_load_direct_striped(lid, d_input + block_offset, input); + rocprim::block_load_direct_striped(lid, d_input + block_offset, input); - using adjacent_diff_t = rp::block_adjacent_difference; + using adjacent_diff_t = rocprim::block_adjacent_difference; __shared__ typename adjacent_diff_t::storage_type storage; ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < trials; trial++) + for(unsigned int trial = 0; trial < trials; ++trial) { T output[ItemsPerThread]; if(WithTile) { - adjacent_diff_t().subtract_right(input, output, rp::minus<>{}, T(123), storage); + adjacent_diff_t().subtract_right(input, + output, + rocprim::minus<>{}, + T(123), + storage); } else { - adjacent_diff_t().subtract_right(input, output, rp::minus<>{}, storage); + adjacent_diff_t().subtract_right(input, output, rocprim::minus<>{}, storage); } for(unsigned int i = 0; i < ItemsPerThread; ++i) { input[i] += output[i]; } - rp::syncthreads(); + rocprim::syncthreads(); } - rp::block_store_direct_striped(lid, d_output + block_offset, input); + rocprim::block_store_direct_striped(lid, d_output + block_offset, input); } }; struct subtract_right_partial { - template - __device__ static void + template + __device__ + static void run(const T* d_input, const unsigned int* tile_sizes, T* d_output, unsigned int trials) { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; T input[ItemsPerThread]; - rp::block_load_direct_striped(lid, d_input + block_offset, input); + rocprim::block_load_direct_striped(lid, d_input + block_offset, input); - using adjacent_diff_t = rp::block_adjacent_difference; + using adjacent_diff_t = rocprim::block_adjacent_difference; __shared__ typename adjacent_diff_t::storage_type storage; unsigned int tile_size = tile_sizes[blockIdx.x]; @@ -205,10 +219,14 @@ struct subtract_right_partial const auto tile_size_diff = (BlockSize * ItemsPerThread) / trials + 1; ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < trials; trial++) + for(unsigned int trial = 0; trial < trials; ++trial) { T output[ItemsPerThread]; - adjacent_diff_t().subtract_right_partial(input, output, rp::minus<>{}, tile_size, storage); + adjacent_diff_t().subtract_right_partial(input, + output, + rocprim::minus<>{}, + tile_size, + storage); for(unsigned int i = 0; i < ItemsPerThread; ++i) { @@ -216,27 +234,30 @@ struct subtract_right_partial } // Change the tile_size to even out the distribution tile_size = (tile_size + tile_size_diff) % (BlockSize * ItemsPerThread); - rp::syncthreads(); + rocprim::syncthreads(); } - rp::block_store_direct_striped(lid, d_output + block_offset, input); + rocprim::block_store_direct_striped(lid, d_output + block_offset, input); } }; -template -auto run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& seed, hipStream_t stream) +auto run_benchmark(benchmark::State& state, + size_t bytes, + const managed_seed& seed, + hipStream_t stream) -> std::enable_if_t::value && !std::is_same::value> { // Calculate the number of elements N size_t N = bytes / sizeof(T); - + constexpr auto items_per_block = BlockSize * ItemsPerThread; - const auto num_blocks = (N + items_per_block - 1) / items_per_block; + const auto num_blocks = (N + items_per_block - 1) / items_per_block; // Round up size to the next multiple of items_per_block const auto size = num_blocks * items_per_block; @@ -249,12 +270,7 @@ auto run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& se HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(input[0]))); HIP_CHECK(hipMalloc(&d_output, input.size() * sizeof(T))); HIP_CHECK( - hipMemcpy( - d_input, input.data(), - input.size() * sizeof(input[0]), - hipMemcpyHostToDevice - ) - ); + hipMemcpy(d_input, input.data(), input.size() * sizeof(input[0]), hipMemcpyHostToDevice)); // HIP events creation hipEvent_t start, stop; @@ -266,11 +282,14 @@ auto run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& se // Record start event HIP_CHECK(hipEventRecord(start, stream)); - hipLaunchKernelGGL( - HIP_KERNEL_NAME(kernel), - dim3(num_blocks), dim3(BlockSize), 0, stream, - d_input, d_output, Trials - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel), + dim3(num_blocks), + dim3(BlockSize), + 0, + stream, + d_input, + d_output, + Trials); HIP_CHECK(hipGetLastError()); // Record stop event and wait until it completes @@ -293,13 +312,16 @@ auto run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& se HIP_CHECK(hipFree(d_output)); } -template -auto run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& seed, hipStream_t stream) +auto run_benchmark(benchmark::State& state, + size_t bytes, + const managed_seed& seed, + hipStream_t stream) -> std::enable_if_t::value || std::is_same::value> { @@ -307,7 +329,7 @@ auto run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& se size_t N = bytes / sizeof(T); static constexpr auto items_per_block = BlockSize * ItemsPerThread; - const auto num_blocks = (N + items_per_block - 1) / items_per_block; + const auto num_blocks = (N + items_per_block - 1) / items_per_block; // Round up size to the next multiple of items_per_block const auto size = num_blocks * items_per_block; @@ -330,19 +352,11 @@ auto run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& se HIP_CHECK(hipMalloc(&d_tile_sizes, tile_sizes.size() * sizeof(tile_sizes[0]))); HIP_CHECK(hipMalloc(&d_output, input.size() * sizeof(input[0]))); HIP_CHECK( - hipMemcpy( - d_input, input.data(), - input.size() * sizeof(input[0]), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_tile_sizes, tile_sizes.data(), - tile_sizes.size() * sizeof(tile_sizes[0]), - hipMemcpyHostToDevice - ) - ); + hipMemcpy(d_input, input.data(), input.size() * sizeof(input[0]), hipMemcpyHostToDevice)); + HIP_CHECK(hipMemcpy(d_tile_sizes, + tile_sizes.data(), + tile_sizes.size() * sizeof(tile_sizes[0]), + hipMemcpyHostToDevice)); // HIP events creation hipEvent_t start, stop; @@ -354,11 +368,15 @@ auto run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& se // Record start event HIP_CHECK(hipEventRecord(start, stream)); - hipLaunchKernelGGL( - HIP_KERNEL_NAME(kernel), - dim3(num_blocks), dim3(BlockSize), 0, stream, - d_input, d_tile_sizes, d_output, Trials - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel), + dim3(num_blocks), + dim3(BlockSize), + 0, + stream, + d_input, + d_tile_sizes, + d_output, + Trials); HIP_CHECK(hipGetLastError()); // Record stop event and wait until it completes @@ -389,50 +407,49 @@ auto run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& se ",with_tile:" #WITH_TILE "}}") \ .c_str(), \ run_benchmark, \ - bytes, \ + bytes, \ seed, \ stream) -#define BENCHMARK_TYPE(type, block, with_tile) \ - CREATE_BENCHMARK(type, block, 1, with_tile), \ - CREATE_BENCHMARK(type, block, 3, with_tile), \ - CREATE_BENCHMARK(type, block, 4, with_tile), \ - CREATE_BENCHMARK(type, block, 8, with_tile), \ - CREATE_BENCHMARK(type, block, 16, with_tile), \ - CREATE_BENCHMARK(type, block, 32, with_tile) +#define BENCHMARK_TYPE(type, block, with_tile) \ + CREATE_BENCHMARK(type, block, 1, with_tile), CREATE_BENCHMARK(type, block, 3, with_tile), \ + CREATE_BENCHMARK(type, block, 4, with_tile), CREATE_BENCHMARK(type, block, 8, with_tile), \ + CREATE_BENCHMARK(type, block, 16, with_tile), CREATE_BENCHMARK(type, block, 32, with_tile) -template +template void add_benchmarks(const std::string& name, std::vector& benchmarks, size_t bytes, const managed_seed& seed, hipStream_t stream) { - std::vector bs = + std::vector bs + = {BENCHMARK_TYPE(int, 256, false), + BENCHMARK_TYPE(float, 256, false), + BENCHMARK_TYPE(int8_t, 256, false), + BENCHMARK_TYPE(rocprim::half, 256, false), + BENCHMARK_TYPE(long long, 256, false), + BENCHMARK_TYPE(double, 256, false), + BENCHMARK_TYPE(rocprim::int128_t, 256, false), + BENCHMARK_TYPE(rocprim::uint128_t, 256, false)}; + + if(!std::is_same::value) { - BENCHMARK_TYPE(int, 256, false), - BENCHMARK_TYPE(float, 256, false), - BENCHMARK_TYPE(int8_t, 256, false), - BENCHMARK_TYPE(rocprim::half, 256, false), - BENCHMARK_TYPE(long long, 256, false), - BENCHMARK_TYPE(double, 256, false) - }; - - if(!std::is_same::value) { - bs.insert(bs.end(), { - BENCHMARK_TYPE(int, 256, true), - BENCHMARK_TYPE(float, 256, true), - BENCHMARK_TYPE(int8_t, 256, true), - BENCHMARK_TYPE(rocprim::half, 256, true), - BENCHMARK_TYPE(long long, 256, true), - BENCHMARK_TYPE(double, 256, true) - }); + bs.insert(bs.end(), + {BENCHMARK_TYPE(int, 256, true), + BENCHMARK_TYPE(float, 256, true), + BENCHMARK_TYPE(int8_t, 256, true), + BENCHMARK_TYPE(rocprim::half, 256, true), + BENCHMARK_TYPE(long long, 256, true), + BENCHMARK_TYPE(double, 256, true), + BENCHMARK_TYPE(rocprim::int128_t, 256, true), + BENCHMARK_TYPE(rocprim::uint128_t, 256, true)}); } benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); } -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { cli::Parser parser(argc, argv); parser.set_optional("size", "size", DEFAULT_BYTES, "number of bytes"); @@ -446,8 +463,8 @@ int main(int argc, char *argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); - const int trials = parser.get("trials"); + const size_t bytes = parser.get("size"); + const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); const managed_seed seed(seed_type); diff --git a/benchmark/benchmark_block_discontinuity.cpp b/benchmark/benchmark_block_discontinuity.cpp index 1d07cdb9b..010def843 100644 --- a/benchmark/benchmark_block_discontinuity.cpp +++ b/benchmark/benchmark_block_discontinuity.cpp @@ -34,195 +34,190 @@ #include #include #include +#include +#include +#include +#include -#include -#include +#include +#include #include #include -#include -#include - #ifndef DEFAULT_N const size_t DEFAULT_BYTES = 1024 * 1024 * 128 * 4; #endif -namespace rp = rocprim; - -template< - class Runner, - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - bool WithTile, - unsigned int Trials -> -__global__ -__launch_bounds__(BlockSize) -void kernel(const T * d_input, T * d_output) +template +__global__ __launch_bounds__(BlockSize) +void kernel(const T* d_input, T* d_output) { Runner::template run(d_input, d_output); } struct flag_heads { - template< - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - bool WithTile, - unsigned int Trials - > + template __device__ - static void run(const T * d_input, T * d_output) + static void run(const T* d_input, T* d_output) { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; T input[ItemsPerThread]; - rp::block_load_direct_striped(lid, d_input + block_offset, input); + rocprim::block_load_direct_striped(lid, d_input + block_offset, input); ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < Trials; trial++) + for(unsigned int trial = 0; trial < Trials; ++trial) { - rp::block_discontinuity bdiscontinuity; - bool head_flags[ItemsPerThread]; + rocprim::block_discontinuity bdiscontinuity; + bool head_flags[ItemsPerThread]; if(WithTile) { - bdiscontinuity.flag_heads(head_flags, T(123), input, rp::equal_to()); + bdiscontinuity.flag_heads(head_flags, T(123), input, rocprim::equal_to()); } else { - bdiscontinuity.flag_heads(head_flags, input, rp::equal_to()); + bdiscontinuity.flag_heads(head_flags, input, rocprim::equal_to()); } - for(unsigned int i = 0; i < ItemsPerThread; i++) + for(unsigned int i = 0; i < ItemsPerThread; ++i) { input[i] += head_flags[i]; } - rp::syncthreads(); + rocprim::syncthreads(); } - rp::block_store_direct_striped(lid, d_output + block_offset, input); + rocprim::block_store_direct_striped(lid, d_output + block_offset, input); } }; struct flag_tails { - template< - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - bool WithTile, - unsigned int Trials - > + template __device__ - static void run(const T * d_input, T * d_output) + static void run(const T* d_input, T* d_output) { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; T input[ItemsPerThread]; - rp::block_load_direct_striped(lid, d_input + block_offset, input); + rocprim::block_load_direct_striped(lid, d_input + block_offset, input); ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < Trials; trial++) + for(unsigned int trial = 0; trial < Trials; ++trial) { - rp::block_discontinuity bdiscontinuity; - bool tail_flags[ItemsPerThread]; + rocprim::block_discontinuity bdiscontinuity; + bool tail_flags[ItemsPerThread]; if(WithTile) { - bdiscontinuity.flag_tails(tail_flags, T(123), input, rp::equal_to()); + bdiscontinuity.flag_tails(tail_flags, T(123), input, rocprim::equal_to()); } else { - bdiscontinuity.flag_tails(tail_flags, input, rp::equal_to()); + bdiscontinuity.flag_tails(tail_flags, input, rocprim::equal_to()); } - for(unsigned int i = 0; i < ItemsPerThread; i++) + for(unsigned int i = 0; i < ItemsPerThread; ++i) { input[i] += tail_flags[i]; } - rp::syncthreads(); + rocprim::syncthreads(); } - rp::block_store_direct_striped(lid, d_output + block_offset, input); + rocprim::block_store_direct_striped(lid, d_output + block_offset, input); } }; struct flag_heads_and_tails { - template< - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - bool WithTile, - unsigned int Trials - > + template __device__ - static void run(const T * d_input, T * d_output) + static void run(const T* d_input, T* d_output) { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; T input[ItemsPerThread]; - rp::block_load_direct_striped(lid, d_input + block_offset, input); + rocprim::block_load_direct_striped(lid, d_input + block_offset, input); ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < Trials; trial++) + for(unsigned int trial = 0; trial < Trials; ++trial) { - rp::block_discontinuity bdiscontinuity; - bool head_flags[ItemsPerThread]; - bool tail_flags[ItemsPerThread]; + rocprim::block_discontinuity bdiscontinuity; + bool head_flags[ItemsPerThread]; + bool tail_flags[ItemsPerThread]; if(WithTile) { - bdiscontinuity.flag_heads_and_tails(head_flags, T(123), tail_flags, T(234), input, rp::equal_to()); + bdiscontinuity.flag_heads_and_tails(head_flags, + T(123), + tail_flags, + T(234), + input, + rocprim::equal_to()); } else { - bdiscontinuity.flag_heads_and_tails(head_flags, tail_flags, input, rp::equal_to()); + bdiscontinuity.flag_heads_and_tails(head_flags, + tail_flags, + input, + rocprim::equal_to()); } - for(unsigned int i = 0; i < ItemsPerThread; i++) + for(unsigned int i = 0; i < ItemsPerThread; ++i) { input[i] += head_flags[i]; input[i] += tail_flags[i]; } - rp::syncthreads(); + rocprim::syncthreads(); } - rp::block_store_direct_striped(lid, d_output + block_offset, input); + rocprim::block_store_direct_striped(lid, d_output + block_offset, input); } }; -template -void run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& seed, hipStream_t stream) +void run_benchmark(benchmark::State& state, + size_t bytes, + const managed_seed& seed, + hipStream_t stream) { // Calculate the number of elements N size_t N = bytes / sizeof(T); constexpr auto items_per_block = BlockSize * ItemsPerThread; - const auto size = items_per_block * ((N + items_per_block - 1)/items_per_block); + const auto size = items_per_block * ((N + items_per_block - 1) / items_per_block); const auto random_range = limit_random_range(0, 10); std::vector input = get_random_data(size, random_range.first, random_range.second, seed.get_0()); - T * d_input; - T * d_output; + T* d_input; + T* d_output; HIP_CHECK(hipMalloc(reinterpret_cast(&d_input), size * sizeof(T))); HIP_CHECK(hipMalloc(reinterpret_cast(&d_output), size * sizeof(T))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - size * sizeof(T), - hipMemcpyHostToDevice - ) - ); + HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); HIP_CHECK(hipDeviceSynchronize()); // HIP events creation @@ -237,9 +232,12 @@ void run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& se hipLaunchKernelGGL( HIP_KERNEL_NAME(kernel), - dim3(size/items_per_block), dim3(BlockSize), 0, stream, - d_input, d_output - ); + dim3(size / items_per_block), + dim3(BlockSize), + 0, + stream, + d_input, + d_output); HIP_CHECK(hipGetLastError()); // Record stop event and wait until it completes @@ -269,42 +267,42 @@ void run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& se ",with_tile:" #WITH_TILE "}}") \ .c_str(), \ run_benchmark, \ - bytes, \ + bytes, \ seed, \ stream) -#define BENCHMARK_TYPE(type, block, bool) \ - CREATE_BENCHMARK(type, block, 1, bool), \ - CREATE_BENCHMARK(type, block, 2, bool), \ - CREATE_BENCHMARK(type, block, 3, bool), \ - CREATE_BENCHMARK(type, block, 4, bool), \ - CREATE_BENCHMARK(type, block, 8, bool) +#define BENCHMARK_TYPE(type, block, bool) \ + CREATE_BENCHMARK(type, block, 1, bool), CREATE_BENCHMARK(type, block, 2, bool), \ + CREATE_BENCHMARK(type, block, 3, bool), CREATE_BENCHMARK(type, block, 4, bool), \ + CREATE_BENCHMARK(type, block, 8, bool) -template +template void add_benchmarks(const std::string& name, std::vector& benchmarks, size_t bytes, const managed_seed& seed, hipStream_t stream) { - std::vector bs = - { - BENCHMARK_TYPE(int, 256, false), - BENCHMARK_TYPE(int, 256, true), - BENCHMARK_TYPE(int8_t, 256, false), - BENCHMARK_TYPE(int8_t, 256, true), - BENCHMARK_TYPE(uint8_t, 256, false), - BENCHMARK_TYPE(uint8_t, 256, true), - BENCHMARK_TYPE(rocprim::half, 256, false), - BENCHMARK_TYPE(rocprim::half, 256, true), - BENCHMARK_TYPE(long long, 256, false), - BENCHMARK_TYPE(long long, 256, true), - }; + std::vector bs + = {BENCHMARK_TYPE(int, 256, false), + BENCHMARK_TYPE(int, 256, true), + BENCHMARK_TYPE(int8_t, 256, false), + BENCHMARK_TYPE(int8_t, 256, true), + BENCHMARK_TYPE(uint8_t, 256, false), + BENCHMARK_TYPE(uint8_t, 256, true), + BENCHMARK_TYPE(rocprim::half, 256, false), + BENCHMARK_TYPE(rocprim::half, 256, true), + BENCHMARK_TYPE(long long, 256, false), + BENCHMARK_TYPE(long long, 256, true), + BENCHMARK_TYPE(rocprim::int128_t, 256, false), + BENCHMARK_TYPE(rocprim::int128_t, 256, true), + BENCHMARK_TYPE(rocprim::uint128_t, 256, false), + BENCHMARK_TYPE(rocprim::uint128_t, 256, true)}; benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); } -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { cli::Parser parser(argc, argv); parser.set_optional("size", "size", DEFAULT_BYTES, "number of bytes"); @@ -318,8 +316,8 @@ int main(int argc, char *argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); - const int trials = parser.get("trials"); + const size_t bytes = parser.get("size"); + const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); const managed_seed seed(seed_type); diff --git a/benchmark/benchmark_block_exchange.cpp b/benchmark/benchmark_block_exchange.cpp index ec3c95f48..87832664a 100644 --- a/benchmark/benchmark_block_exchange.cpp +++ b/benchmark/benchmark_block_exchange.cpp @@ -21,8 +21,8 @@ // SOFTWARE. // CmdParser -#include "cmdparser.hpp" #include "benchmark_utils.hpp" +#include "cmdparser.hpp" // Google Benchmark #include @@ -34,261 +34,219 @@ #include #include #include - -#include -#include +#include +#include +#include + +#include +#include +#include +#include #include #include -#include -#include - #ifndef DEFAULT_N const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif -namespace rp = rocprim; - -template< - class Runner, - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - unsigned int Trials -> -__global__ -__launch_bounds__(BlockSize) -void kernel(const T * d_input, const unsigned int * d_ranks, T * d_output) +template +__global__ __launch_bounds__(BlockSize) +void kernel(const T* d_input, const unsigned int* d_ranks, T* d_output) { Runner::template run(d_input, d_ranks, d_output); } struct blocked_to_striped { - template< - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - unsigned int Trials - > + template __device__ - static void run(const T * d_input, const unsigned int *, T * d_output) + static void run(const T* d_input, const unsigned int*, T* d_output) { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; T input[ItemsPerThread]; - rp::block_load_direct_striped(lid, d_input + block_offset, input); + rocprim::block_load_direct_striped(lid, d_input + block_offset, input); ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < Trials; trial++) + for(unsigned int trial = 0; trial < Trials; ++trial) { - rp::block_exchange exchange; + rocprim::block_exchange exchange; exchange.blocked_to_striped(input, input); ::rocprim::syncthreads(); } - rp::block_store_direct_striped(lid, d_output + block_offset, input); + rocprim::block_store_direct_striped(lid, d_output + block_offset, input); } }; struct striped_to_blocked { - template< - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - unsigned int Trials - > + template __device__ - static void run(const T * d_input, const unsigned int *, T * d_output) + static void run(const T* d_input, const unsigned int*, T* d_output) { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; T input[ItemsPerThread]; - rp::block_load_direct_striped(lid, d_input + block_offset, input); + rocprim::block_load_direct_striped(lid, d_input + block_offset, input); ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < Trials; trial++) + for(unsigned int trial = 0; trial < Trials; ++trial) { - rp::block_exchange exchange; + rocprim::block_exchange exchange; exchange.striped_to_blocked(input, input); ::rocprim::syncthreads(); } - rp::block_store_direct_striped(lid, d_output + block_offset, input); + rocprim::block_store_direct_striped(lid, d_output + block_offset, input); } }; struct blocked_to_warp_striped { - template< - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - unsigned int Trials - > + template __device__ - static void run(const T * d_input, const unsigned int *, T * d_output) + static void run(const T* d_input, const unsigned int*, T* d_output) { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; T input[ItemsPerThread]; - rp::block_load_direct_striped(lid, d_input + block_offset, input); + rocprim::block_load_direct_striped(lid, d_input + block_offset, input); ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < Trials; trial++) + for(unsigned int trial = 0; trial < Trials; ++trial) { - rp::block_exchange exchange; + rocprim::block_exchange exchange; exchange.blocked_to_warp_striped(input, input); ::rocprim::syncthreads(); } - rp::block_store_direct_striped(lid, d_output + block_offset, input); + rocprim::block_store_direct_striped(lid, d_output + block_offset, input); } }; struct warp_striped_to_blocked { - template< - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - unsigned int Trials - > + template __device__ - static void run(const T * d_input, const unsigned int *, T * d_output) + static void run(const T* d_input, const unsigned int*, T* d_output) { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; T input[ItemsPerThread]; - rp::block_load_direct_striped(lid, d_input + block_offset, input); + rocprim::block_load_direct_striped(lid, d_input + block_offset, input); ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < Trials; trial++) + for(unsigned int trial = 0; trial < Trials; ++trial) { - rp::block_exchange exchange; + rocprim::block_exchange exchange; exchange.warp_striped_to_blocked(input, input); ::rocprim::syncthreads(); } - rp::block_store_direct_striped(lid, d_output + block_offset, input); + rocprim::block_store_direct_striped(lid, d_output + block_offset, input); } }; struct scatter_to_blocked { - template< - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - unsigned int Trials - > + template __device__ - static void run(const T * d_input, const unsigned int * d_ranks, T * d_output) + static void run(const T* d_input, const unsigned int* d_ranks, T* d_output) { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; - T input[ItemsPerThread]; + T input[ItemsPerThread]; unsigned int ranks[ItemsPerThread]; - rp::block_load_direct_striped(lid, d_input + block_offset, input); - rp::block_load_direct_striped(lid, d_ranks + block_offset, ranks); + rocprim::block_load_direct_striped(lid, d_input + block_offset, input); + rocprim::block_load_direct_striped(lid, d_ranks + block_offset, ranks); ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < Trials; trial++) + for(unsigned int trial = 0; trial < Trials; ++trial) { - rp::block_exchange exchange; + rocprim::block_exchange exchange; exchange.scatter_to_blocked(input, input, ranks); ::rocprim::syncthreads(); } - rp::block_store_direct_striped(lid, d_output + block_offset, input); + rocprim::block_store_direct_striped(lid, d_output + block_offset, input); } }; struct scatter_to_striped { - template< - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - unsigned int Trials - > + template __device__ - static void run(const T * d_input, const unsigned int * d_ranks, T * d_output) + static void run(const T* d_input, const unsigned int* d_ranks, T* d_output) { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; - T input[ItemsPerThread]; + T input[ItemsPerThread]; unsigned int ranks[ItemsPerThread]; - rp::block_load_direct_striped(lid, d_input + block_offset, input); - rp::block_load_direct_striped(lid, d_ranks + block_offset, ranks); + rocprim::block_load_direct_striped(lid, d_input + block_offset, input); + rocprim::block_load_direct_striped(lid, d_ranks + block_offset, ranks); ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < Trials; trial++) + for(unsigned int trial = 0; trial < Trials; ++trial) { - rp::block_exchange exchange; + rocprim::block_exchange exchange; exchange.scatter_to_striped(input, input, ranks); ::rocprim::syncthreads(); } - rp::block_store_direct_striped(lid, d_output + block_offset, input); + rocprim::block_store_direct_striped(lid, d_output + block_offset, input); } }; -template -void run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& seed, hipStream_t stream) +void run_benchmark(benchmark::State& state, + size_t bytes, + const managed_seed& seed, + hipStream_t stream) { // Calculate the number of elements N size_t N = bytes / sizeof(T); - + constexpr auto items_per_block = BlockSize * ItemsPerThread; - const auto size = items_per_block * ((N + items_per_block - 1)/items_per_block); + const auto size = items_per_block * ((N + items_per_block - 1) / items_per_block); std::vector input(size); // Fill input - for(size_t i = 0; i < size; i++) + for(size_t i = 0; i < size; ++i) { input[i] = T(i); } std::vector ranks(size); // Fill ranks (for scatter operations) engine_type gen(seed.get_0()); - for(size_t bi = 0; bi < size / items_per_block; bi++) + for(size_t bi = 0; bi < size / items_per_block; ++bi) { auto block_ranks = ranks.begin() + bi * items_per_block; std::iota(block_ranks, block_ranks + items_per_block, 0); std::shuffle(block_ranks, block_ranks + items_per_block, gen); } - T * d_input; - unsigned int * d_ranks; - T * d_output; + T* d_input; + unsigned int* d_ranks; + T* d_output; HIP_CHECK(hipMalloc(reinterpret_cast(&d_input), size * sizeof(T))); HIP_CHECK(hipMalloc(reinterpret_cast(&d_ranks), size * sizeof(unsigned int))); HIP_CHECK(hipMalloc(reinterpret_cast(&d_output), size * sizeof(T))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - size * sizeof(T), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_ranks, ranks.data(), - size * sizeof(unsigned int), - hipMemcpyHostToDevice - ) - ); + HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); + HIP_CHECK(hipMemcpy(d_ranks, ranks.data(), size * sizeof(unsigned int), hipMemcpyHostToDevice)); HIP_CHECK(hipDeviceSynchronize()); // HIP events creation @@ -301,11 +259,14 @@ void run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& se // Record start event HIP_CHECK(hipEventRecord(start, stream)); - hipLaunchKernelGGL( - HIP_KERNEL_NAME(kernel), - dim3(size/items_per_block), dim3(BlockSize), 0, stream, - d_input, d_ranks, d_output - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel), + dim3(size / items_per_block), + dim3(BlockSize), + 0, + stream, + d_input, + d_ranks, + d_output); HIP_CHECK(hipGetLastError()); // Record stop event and wait until it completes @@ -335,45 +296,41 @@ void run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& se + ",key_type:" #T ",cfg:{bs:" #BS ",ipt:" #IPT "}}") \ .c_str(), \ run_benchmark, \ - bytes, \ + bytes, \ seed, \ stream) -#define BENCHMARK_TYPE(type, block) \ - CREATE_BENCHMARK(type, block, 1), \ - CREATE_BENCHMARK(type, block, 2), \ - CREATE_BENCHMARK(type, block, 3), \ - CREATE_BENCHMARK(type, block, 4), \ - CREATE_BENCHMARK(type, block, 7), \ - CREATE_BENCHMARK(type, block, 8) +#define BENCHMARK_TYPE(type, block) \ + CREATE_BENCHMARK(type, block, 1), CREATE_BENCHMARK(type, block, 2), \ + CREATE_BENCHMARK(type, block, 3), CREATE_BENCHMARK(type, block, 4), \ + CREATE_BENCHMARK(type, block, 7), CREATE_BENCHMARK(type, block, 8) -template +template void add_benchmarks(const std::string& name, std::vector& benchmarks, size_t bytes, const managed_seed& seed, hipStream_t stream) { - using custom_float2 = custom_type; + using custom_float2 = custom_type; using custom_double2 = custom_type; - std::vector bs = - { - BENCHMARK_TYPE(int, 256), - BENCHMARK_TYPE(int8_t, 256), - BENCHMARK_TYPE(rocprim::half, 256), - BENCHMARK_TYPE(long long, 256), - BENCHMARK_TYPE(custom_float2, 256), - BENCHMARK_TYPE(float2, 256), - BENCHMARK_TYPE(custom_double2, 256), - BENCHMARK_TYPE(double2, 256), - BENCHMARK_TYPE(float4, 256), - }; + std::vector bs = {BENCHMARK_TYPE(int, 256), + BENCHMARK_TYPE(int8_t, 256), + BENCHMARK_TYPE(rocprim::half, 256), + BENCHMARK_TYPE(long long, 256), + BENCHMARK_TYPE(custom_float2, 256), + BENCHMARK_TYPE(float2, 256), + BENCHMARK_TYPE(custom_double2, 256), + BENCHMARK_TYPE(double2, 256), + BENCHMARK_TYPE(float4, 256), + BENCHMARK_TYPE(rocprim::int128_t, 256), + BENCHMARK_TYPE(rocprim::uint128_t, 256)}; benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); } -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { cli::Parser parser(argc, argv); parser.set_optional("bytes", "bytes", DEFAULT_BYTES, "number of values"); @@ -387,8 +344,8 @@ int main(int argc, char *argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("bytes"); - const int trials = parser.get("trials"); + const size_t bytes = parser.get("bytes"); + const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); const managed_seed seed(seed_type); diff --git a/benchmark/benchmark_block_histogram.cpp b/benchmark/benchmark_block_histogram.cpp index 7dc282749..cf454b60c 100644 --- a/benchmark/benchmark_block_histogram.cpp +++ b/benchmark/benchmark_block_histogram.cpp @@ -32,33 +32,24 @@ // rocPRIM #include -#include -#include +#include +#include -#include -#include +#include #include #include -#include -#include - #ifndef DEFAULT_N const size_t DEFAULT_BYTES = 1024 * 1024 * 128 * 4; #endif -namespace rp = rocprim; - -template< - class Runner, - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - unsigned int BinSize, - unsigned int Trials -> -__global__ -__launch_bounds__(BlockSize) +template +__global__ __launch_bounds__(BlockSize) void kernel(const T* input, T* output) { Runner::template run(input, output); @@ -67,42 +58,42 @@ void kernel(const T* input, T* output) template struct histogram { - template< - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - unsigned int BinSize, - unsigned int Trials - > + static constexpr auto algorithm_type = algorithm; + template __device__ static void run(const T* input, T* output) { // TODO: Move global_offset into final loop const unsigned int index = ((blockIdx.x * BlockSize) + threadIdx.x) * ItemsPerThread; - unsigned int global_offset = blockIdx.x * BinSize; + unsigned int global_offset = blockIdx.x * BinSize; T values[ItemsPerThread]; - for(unsigned int k = 0; k < ItemsPerThread; k++) + for(unsigned int k = 0; k < ItemsPerThread; ++k) { values[k] = input[index + k]; } - using bhistogram_t = rp::block_histogram; + using bhistogram_t + = rocprim::block_histogram; __shared__ T histogram[BinSize]; __shared__ typename bhistogram_t::storage_type storage; ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < Trials; trial++) + for(unsigned int trial = 0; trial < Trials; ++trial) { bhistogram_t().histogram(values, histogram, storage); - for(unsigned int k = 0; k < ItemsPerThread; k++) + for(unsigned int k = 0; k < ItemsPerThread; ++k) { values[k] = BinSize - 1 - values[k]; } } ROCPRIM_UNROLL - for (unsigned int offset = 0; offset < BinSize; offset += BlockSize) + for(unsigned int offset = 0; offset < BinSize; offset += BlockSize) { if(offset + threadIdx.x < BinSize) { @@ -113,35 +104,27 @@ struct histogram } }; -template< - class Benchmark, - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - unsigned int BinSize = BlockSize, - unsigned int Trials = 100 -> +template void run_benchmark(benchmark::State& state, hipStream_t stream, size_t bytes) { // Calculate the number of elements N size_t N = bytes / sizeof(T); // Make sure size is a multiple of BlockSize constexpr auto items_per_block = BlockSize * ItemsPerThread; - const auto size = items_per_block * ((N + items_per_block - 1)/items_per_block); - const auto bin_size = BinSize * ((N + items_per_block - 1)/items_per_block); + const auto size = items_per_block * ((N + items_per_block - 1) / items_per_block); + const auto bin_size = BinSize * ((N + items_per_block - 1) / items_per_block); // Allocate and fill memory std::vector input(size, 0.0f); - T * d_input; - T * d_output; + T* d_input; + T* d_output; HIP_CHECK(hipMalloc(reinterpret_cast(&d_input), size * sizeof(T))); HIP_CHECK(hipMalloc(reinterpret_cast(&d_output), bin_size * sizeof(T))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - size * sizeof(T), - hipMemcpyHostToDevice - ) - ); + HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); HIP_CHECK(hipDeviceSynchronize()); // HIP events creation @@ -149,16 +132,19 @@ void run_benchmark(benchmark::State& state, hipStream_t stream, size_t bytes) HIP_CHECK(hipEventCreate(&start)); HIP_CHECK(hipEventCreate(&stop)); - for (auto _ : state) + for(auto _ : state) { // Record start event HIP_CHECK(hipEventRecord(start, stream)); hipLaunchKernelGGL( HIP_KERNEL_NAME(kernel), - dim3(size/items_per_block), dim3(BlockSize), 0, stream, - d_input, d_output - ); + dim3(size / items_per_block), + dim3(BlockSize), + 0, + stream, + d_input, + d_output); HIP_CHECK(hipGetLastError()); // Record stop event and wait until it completes @@ -192,33 +178,62 @@ void run_benchmark(benchmark::State& state, hipStream_t stream, size_t bytes) stream, \ bytes) -#define BENCHMARK_TYPE(type, block) \ - CREATE_BENCHMARK(type, block, 1), \ - CREATE_BENCHMARK(type, block, 2), \ - CREATE_BENCHMARK(type, block, 3), \ - CREATE_BENCHMARK(type, block, 4), \ - CREATE_BENCHMARK(type, block, 8), \ - CREATE_BENCHMARK(type, block, 16) +#define BENCHMARK_TYPE(type, block) \ + CREATE_BENCHMARK(type, block, 1), CREATE_BENCHMARK(type, block, 2), \ + CREATE_BENCHMARK(type, block, 3), CREATE_BENCHMARK(type, block, 4), \ + CREATE_BENCHMARK(type, block, 8), CREATE_BENCHMARK(type, block, 16) + +#define BENCHMARK_TYPE_128(type, block) \ + CREATE_BENCHMARK(type, block, 1), CREATE_BENCHMARK(type, block, 2), \ + CREATE_BENCHMARK(type, block, 3), CREATE_BENCHMARK(type, block, 4), \ + CREATE_BENCHMARK(type, block, 8), CREATE_BENCHMARK(type, block, 12) -template +template< + typename Benchmark, + std::enable_if_t< + std::is_same>::value, + bool> + = true> void add_benchmarks(std::vector& benchmarks, const std::string& method_name, hipStream_t stream, size_t bytes) { - std::vector new_benchmarks = - { - BENCHMARK_TYPE(int, 256), - BENCHMARK_TYPE(int, 320), - BENCHMARK_TYPE(int, 512), + std::vector new_benchmarks + = {BENCHMARK_TYPE(int, 256), + BENCHMARK_TYPE(int, 320), + BENCHMARK_TYPE(int, 512), + + BENCHMARK_TYPE(unsigned long long, 256), + BENCHMARK_TYPE(unsigned long long, 320)}; + benchmarks.insert(benchmarks.end(), new_benchmarks.begin(), new_benchmarks.end()); +} + +template< + typename Benchmark, + std::enable_if_t< + std::is_same>::value, + bool> + = true> +void add_benchmarks(std::vector& benchmarks, + const std::string& method_name, + hipStream_t stream, + size_t bytes) +{ + std::vector new_benchmarks + = {BENCHMARK_TYPE(int, 256), + BENCHMARK_TYPE(int, 320), + BENCHMARK_TYPE(int, 512), + + BENCHMARK_TYPE(unsigned long long, 256), + BENCHMARK_TYPE(unsigned long long, 320), - BENCHMARK_TYPE(unsigned long long, 256), - BENCHMARK_TYPE(unsigned long long, 320) - }; + BENCHMARK_TYPE_128(rocprim::int128_t, 256), + BENCHMARK_TYPE_128(rocprim::uint128_t, 256)}; benchmarks.insert(benchmarks.end(), new_benchmarks.begin(), new_benchmarks.end()); } -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { cli::Parser parser(argc, argv); parser.set_optional("size", "size", DEFAULT_BYTES, "number of bytes"); @@ -231,8 +246,8 @@ int main(int argc, char *argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); - const int trials = parser.get("trials"); + const size_t bytes = parser.get("size"); + const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); // HIP diff --git a/benchmark/benchmark_block_radix_rank.cpp b/benchmark/benchmark_block_radix_rank.cpp index faee6d37c..a8d97a85b 100644 --- a/benchmark/benchmark_block_radix_rank.cpp +++ b/benchmark/benchmark_block_radix_rank.cpp @@ -34,34 +34,35 @@ #include #include #include +#include +#include #include -#include +#include #include +#include #ifndef DEFAULT_N const size_t DEFAULT_BYTES = 1024 * 1024 * 128 * 4; #endif -namespace rp = rocprim; - template -__global__ __launch_bounds__(BlockSize) void rank_kernel(const T* keys_input, - unsigned int* ranks_output) + unsigned int BlockSize, + unsigned int ItemsPerThread, + unsigned int RadixBits, + bool Descending, + rocprim::block_radix_rank_algorithm Algorithm, + unsigned int Trials> +__global__ __launch_bounds__(BlockSize) +void rank_kernel(const T* keys_input, unsigned int* ranks_output) { - using rank_type = rp::block_radix_rank; + using rank_type = rocprim::block_radix_rank; const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; T keys[ItemsPerThread]; - rp::block_load_direct_striped(lid, keys_input + block_offset, keys); + rocprim::block_load_direct_striped(lid, keys_input + block_offset, keys); unsigned int ranks[ItemsPerThread]; @@ -87,20 +88,23 @@ __global__ __launch_bounds__(BlockSize) void rank_kernel(const T* keys_inpu } } - rp::block_store_direct_striped(lid, ranks_output + block_offset, ranks); + rocprim::block_store_direct_striped(lid, ranks_output + block_offset, ranks); } template -void run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& seed, hipStream_t stream) + unsigned int BlockSize, + unsigned int ItemsPerThread, + rocprim::block_radix_rank_algorithm Algorithm, + unsigned int RadixBits = 4, + bool Descending = false, + unsigned int Trials = 10> +void run_benchmark(benchmark::State& state, + size_t bytes, + const managed_seed& seed, + hipStream_t stream) { // Calculate the number of elements N - size_t N = bytes / sizeof(T); + size_t N = bytes / sizeof(T); constexpr unsigned int items_per_block = BlockSize * ItemsPerThread; const unsigned int grid_size = ((N + items_per_block - 1) / items_per_block); const unsigned int size = items_per_block * grid_size; @@ -155,15 +159,15 @@ void run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& se ",ipt:" #IPT ",method:" #KIND "}}") \ .c_str(), \ run_benchmark, \ - bytes, \ + bytes, \ seed, \ stream) // clang-format off #define CREATE_BENCHMARK_KINDS(type, block, ipt) \ - CREATE_BENCHMARK(type, block, ipt, rp::block_radix_rank_algorithm::basic), \ - CREATE_BENCHMARK(type, block, ipt, rp::block_radix_rank_algorithm::basic_memoize), \ - CREATE_BENCHMARK(type, block, ipt, rp::block_radix_rank_algorithm::match) + CREATE_BENCHMARK(type, block, ipt, rocprim::block_radix_rank_algorithm::basic), \ + CREATE_BENCHMARK(type, block, ipt, rocprim::block_radix_rank_algorithm::basic_memoize), \ + CREATE_BENCHMARK(type, block, ipt, rocprim::block_radix_rank_algorithm::match) #define BENCHMARK_TYPE(type, block) \ CREATE_BENCHMARK_KINDS(type, block, 1), \ @@ -179,19 +183,25 @@ void add_benchmarks(std::vector& benchmarks, const managed_seed& seed, hipStream_t stream) { - std::vector bs = { - BENCHMARK_TYPE(int, 128), - BENCHMARK_TYPE(int, 256), - BENCHMARK_TYPE(int, 512), + std::vector bs = {BENCHMARK_TYPE(int, 128), + BENCHMARK_TYPE(int, 256), + BENCHMARK_TYPE(int, 512), + + BENCHMARK_TYPE(uint8_t, 128), + BENCHMARK_TYPE(uint8_t, 256), + BENCHMARK_TYPE(uint8_t, 512), + + BENCHMARK_TYPE(long long, 128), + BENCHMARK_TYPE(long long, 256), + BENCHMARK_TYPE(long long, 512), - BENCHMARK_TYPE(uint8_t, 128), - BENCHMARK_TYPE(uint8_t, 256), - BENCHMARK_TYPE(uint8_t, 512), + BENCHMARK_TYPE(rocprim::int128_t, 128), + BENCHMARK_TYPE(rocprim::int128_t, 256), + BENCHMARK_TYPE(rocprim::int128_t, 512), - BENCHMARK_TYPE(long long, 128), - BENCHMARK_TYPE(long long, 256), - BENCHMARK_TYPE(long long, 512), - }; + BENCHMARK_TYPE(rocprim::uint128_t, 128), + BENCHMARK_TYPE(rocprim::uint128_t, 256), + BENCHMARK_TYPE(rocprim::uint128_t, 512)}; benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); } @@ -210,7 +220,7 @@ int main(int argc, char* argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); + const size_t bytes = parser.get("size"); const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); diff --git a/benchmark/benchmark_block_radix_sort.cpp b/benchmark/benchmark_block_radix_sort.cpp index ff144732f..7f6829f67 100644 --- a/benchmark/benchmark_block_radix_sort.cpp +++ b/benchmark/benchmark_block_radix_sort.cpp @@ -34,16 +34,15 @@ #include #include #include +#include +#include -#include -#include +#include +#include #include #include #include -#include -#include - #ifndef DEFAULT_N const size_t DEFAULT_BYTES = 1024 * 1024 * 128 * 4; #endif @@ -54,75 +53,76 @@ enum class benchmark_kinds sort_pairs }; -namespace rp = rocprim; - -template -using select_decomposer_t = std:: - conditional_t::value, custom_type_decomposer, rp::identity_decomposer>; +template +using select_decomposer_t = std::conditional_t::value, + custom_type_decomposer, + rocprim::identity_decomposer>; -template -__global__ __launch_bounds__(BlockSize) void sort_keys_kernel(const T* input, T* output) +__global__ __launch_bounds__(BlockSize) +void sort_keys_kernel(const T* input, T* output) { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; T keys[ItemsPerThread]; - rp::block_load_direct_striped(lid, input + block_offset, keys); + rocprim::block_load_direct_striped(lid, input + block_offset, keys); ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < Trials; trial++) + for(unsigned int trial = 0; trial < Trials; ++trial) { - rp::block_radix_sort + rocprim::block_radix_sort sort; sort.sort(keys, 0, sizeof(T) * 8, select_decomposer_t{}); } - rp::block_store_direct_striped(lid, output + block_offset, keys); + rocprim::block_store_direct_striped(lid, output + block_offset, keys); } -template -__global__ __launch_bounds__(BlockSize) void sort_pairs_kernel(const T* input, T* output) +__global__ __launch_bounds__(BlockSize) +void sort_pairs_kernel(const T* input, T* output) { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; T keys[ItemsPerThread]; T values[ItemsPerThread]; - rp::block_load_direct_striped(lid, input + block_offset, keys); - for(unsigned int i = 0; i < ItemsPerThread; i++) + rocprim::block_load_direct_striped(lid, input + block_offset, keys); + for(unsigned int i = 0; i < ItemsPerThread; ++i) { values[i] = keys[i] + T(1); } ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < Trials; trial++) + for(unsigned int trial = 0; trial < Trials; ++trial) { - rp::block_radix_sort sort; + rocprim::block_radix_sort sort; sort.sort(keys, values, 0, sizeof(T) * 8, select_decomposer_t{}); } - for(unsigned int i = 0; i < ItemsPerThread; i++) + for(unsigned int i = 0; i < ItemsPerThread; ++i) { keys[i] += values[i]; } - rp::block_store_direct_striped(lid, output + block_offset, keys); + rocprim::block_store_direct_striped(lid, output + block_offset, keys); } -template input = get_random_data(size, generate_limits::min(), generate_limits::max(), seed.get_0()); - T* d_input; - T * d_output; + T* d_input; + T* d_output; HIP_CHECK(hipMalloc(reinterpret_cast(&d_input), size * sizeof(T))); HIP_CHECK(hipMalloc(reinterpret_cast(&d_output), size * sizeof(T))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - size * sizeof(T), - hipMemcpyHostToDevice - ) - ); + HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); HIP_CHECK(hipDeviceSynchronize()); // HIP events creation @@ -220,7 +214,7 @@ void run_benchmark(benchmark::State& state, .c_str(), \ run_benchmark, \ benchmark_kind, \ - bytes, \ + bytes, \ seed, \ stream) @@ -238,72 +232,90 @@ void add_benchmarks(benchmark_kinds benchmark_kind { using custom_int_type = custom_type; - std::vector bs = { - BENCHMARK_TYPE(int, 64, 3), - BENCHMARK_TYPE(int, 512, 3), - - BENCHMARK_TYPE(int, 64, 4), - BENCHMARK_TYPE(int, 128, 4), - BENCHMARK_TYPE(int, 192, 4), - BENCHMARK_TYPE(int, 256, 4), - BENCHMARK_TYPE(int, 320, 4), - BENCHMARK_TYPE(int, 512, 4), - - BENCHMARK_TYPE(int8_t, 64, 3), - BENCHMARK_TYPE(int8_t, 512, 3), - - BENCHMARK_TYPE(int8_t, 64, 4), - BENCHMARK_TYPE(int8_t, 128, 4), - BENCHMARK_TYPE(int8_t, 192, 4), - BENCHMARK_TYPE(int8_t, 256, 4), - BENCHMARK_TYPE(int8_t, 320, 4), - BENCHMARK_TYPE(int8_t, 512, 4), - - BENCHMARK_TYPE(uint8_t, 64, 3), - BENCHMARK_TYPE(uint8_t, 512, 3), - - BENCHMARK_TYPE(uint8_t, 64, 4), - BENCHMARK_TYPE(uint8_t, 128, 4), - BENCHMARK_TYPE(uint8_t, 192, 4), - BENCHMARK_TYPE(uint8_t, 256, 4), - BENCHMARK_TYPE(uint8_t, 320, 4), - BENCHMARK_TYPE(uint8_t, 512, 4), - - BENCHMARK_TYPE(rocprim::half, 64, 3), - BENCHMARK_TYPE(rocprim::half, 512, 3), - - BENCHMARK_TYPE(rocprim::half, 64, 4), - BENCHMARK_TYPE(rocprim::half, 128, 4), - BENCHMARK_TYPE(rocprim::half, 192, 4), - BENCHMARK_TYPE(rocprim::half, 256, 4), - BENCHMARK_TYPE(rocprim::half, 320, 4), - BENCHMARK_TYPE(rocprim::half, 512, 4), - - BENCHMARK_TYPE(long long, 64, 3), - BENCHMARK_TYPE(long long, 512, 3), - - BENCHMARK_TYPE(long long, 64, 4), - BENCHMARK_TYPE(long long, 128, 4), - BENCHMARK_TYPE(long long, 192, 4), - BENCHMARK_TYPE(long long, 256, 4), - BENCHMARK_TYPE(long long, 320, 4), - BENCHMARK_TYPE(long long, 512, 4), - - BENCHMARK_TYPE(custom_int_type, 64, 3), - BENCHMARK_TYPE(custom_int_type, 512, 3), - - BENCHMARK_TYPE(custom_int_type, 64, 4), - BENCHMARK_TYPE(custom_int_type, 128, 4), - BENCHMARK_TYPE(custom_int_type, 192, 4), - BENCHMARK_TYPE(custom_int_type, 256, 4), - BENCHMARK_TYPE(custom_int_type, 320, 4), - BENCHMARK_TYPE(custom_int_type, 512, 4), - }; + std::vector bs = {BENCHMARK_TYPE(int, 64, 3), + BENCHMARK_TYPE(int, 512, 3), + + BENCHMARK_TYPE(int, 64, 4), + BENCHMARK_TYPE(int, 128, 4), + BENCHMARK_TYPE(int, 192, 4), + BENCHMARK_TYPE(int, 256, 4), + BENCHMARK_TYPE(int, 320, 4), + BENCHMARK_TYPE(int, 512, 4), + + BENCHMARK_TYPE(int8_t, 64, 3), + BENCHMARK_TYPE(int8_t, 512, 3), + + BENCHMARK_TYPE(int8_t, 64, 4), + BENCHMARK_TYPE(int8_t, 128, 4), + BENCHMARK_TYPE(int8_t, 192, 4), + BENCHMARK_TYPE(int8_t, 256, 4), + BENCHMARK_TYPE(int8_t, 320, 4), + BENCHMARK_TYPE(int8_t, 512, 4), + + BENCHMARK_TYPE(uint8_t, 64, 3), + BENCHMARK_TYPE(uint8_t, 512, 3), + + BENCHMARK_TYPE(uint8_t, 64, 4), + BENCHMARK_TYPE(uint8_t, 128, 4), + BENCHMARK_TYPE(uint8_t, 192, 4), + BENCHMARK_TYPE(uint8_t, 256, 4), + BENCHMARK_TYPE(uint8_t, 320, 4), + BENCHMARK_TYPE(uint8_t, 512, 4), + + BENCHMARK_TYPE(rocprim::half, 64, 3), + BENCHMARK_TYPE(rocprim::half, 512, 3), + + BENCHMARK_TYPE(rocprim::half, 64, 4), + BENCHMARK_TYPE(rocprim::half, 128, 4), + BENCHMARK_TYPE(rocprim::half, 192, 4), + BENCHMARK_TYPE(rocprim::half, 256, 4), + BENCHMARK_TYPE(rocprim::half, 320, 4), + BENCHMARK_TYPE(rocprim::half, 512, 4), + + BENCHMARK_TYPE(long long, 64, 3), + BENCHMARK_TYPE(long long, 512, 3), + + BENCHMARK_TYPE(long long, 64, 4), + BENCHMARK_TYPE(long long, 128, 4), + BENCHMARK_TYPE(long long, 192, 4), + BENCHMARK_TYPE(long long, 256, 4), + BENCHMARK_TYPE(long long, 320, 4), + BENCHMARK_TYPE(long long, 512, 4), + + BENCHMARK_TYPE(custom_int_type, 64, 3), + BENCHMARK_TYPE(custom_int_type, 512, 3), + + BENCHMARK_TYPE(custom_int_type, 64, 4), + BENCHMARK_TYPE(custom_int_type, 128, 4), + BENCHMARK_TYPE(custom_int_type, 192, 4), + BENCHMARK_TYPE(custom_int_type, 256, 4), + BENCHMARK_TYPE(custom_int_type, 320, 4), + BENCHMARK_TYPE(custom_int_type, 512, 4), + + BENCHMARK_TYPE(rocprim::int128_t, 64, 3), + BENCHMARK_TYPE(rocprim::int128_t, 512, 3), + + BENCHMARK_TYPE(rocprim::int128_t, 64, 4), + BENCHMARK_TYPE(rocprim::int128_t, 128, 4), + BENCHMARK_TYPE(rocprim::int128_t, 192, 4), + BENCHMARK_TYPE(rocprim::int128_t, 256, 4), + BENCHMARK_TYPE(rocprim::int128_t, 320, 4), + BENCHMARK_TYPE(rocprim::int128_t, 512, 4), + + BENCHMARK_TYPE(rocprim::uint128_t, 64, 3), + BENCHMARK_TYPE(rocprim::uint128_t, 512, 3), + + BENCHMARK_TYPE(rocprim::uint128_t, 64, 4), + BENCHMARK_TYPE(rocprim::uint128_t, 128, 4), + BENCHMARK_TYPE(rocprim::uint128_t, 192, 4), + BENCHMARK_TYPE(rocprim::uint128_t, 256, 4), + BENCHMARK_TYPE(rocprim::uint128_t, 320, 4), + BENCHMARK_TYPE(rocprim::uint128_t, 512, 4)}; benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); } -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { cli::Parser parser(argc, argv); parser.set_optional("size", "size", DEFAULT_BYTES, "number of bytes"); @@ -317,8 +329,8 @@ int main(int argc, char *argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); - const int trials = parser.get("trials"); + const size_t bytes = parser.get("size"); + const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); const managed_seed seed(seed_type); diff --git a/benchmark/benchmark_block_reduce.cpp b/benchmark/benchmark_block_reduce.cpp index 257165459..58c3ba1e6 100644 --- a/benchmark/benchmark_block_reduce.cpp +++ b/benchmark/benchmark_block_reduce.cpp @@ -32,30 +32,24 @@ // rocPRIM #include +#include +#include -#include -#include +#include +#include #include #include -#include -#include - #ifndef DEFAULT_N const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif -namespace rp = rocprim; - -template< - class Runner, - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - unsigned int Trials -> -__global__ -__launch_bounds__(BlockSize) +template +__global__ __launch_bounds__(BlockSize) void kernel(const T* input, T* output) { Runner::template run(input, output); @@ -64,12 +58,7 @@ void kernel(const T* input, T* output) template struct reduce { - template< - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - unsigned int Trials - > + template __device__ static void run(const T* input, T* output) { @@ -77,16 +66,16 @@ struct reduce T values[ItemsPerThread]; T reduced_value; - for(unsigned int k = 0; k < ItemsPerThread; k++) + for(unsigned int k = 0; k < ItemsPerThread; ++k) { values[k] = input[i * ItemsPerThread + k]; } - using breduce_t = rp::block_reduce; + using breduce_t = rocprim::block_reduce; __shared__ typename breduce_t::storage_type storage; ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < Trials; trial++) + for(unsigned int trial = 0; trial < Trials; ++trial) { breduce_t().reduce(values, reduced_value, storage); values[0] = reduced_value; @@ -99,33 +88,25 @@ struct reduce } }; -template< - class Benchmark, - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - unsigned int Trials = 100 -> +template void run_benchmark(benchmark::State& state, hipStream_t stream, size_t bytes) { // Calculate the number of elements N size_t N = bytes / sizeof(T); // Make sure size is a multiple of BlockSize constexpr auto items_per_block = BlockSize * ItemsPerThread; - const auto size = items_per_block * ((N + items_per_block - 1)/items_per_block); + const auto size = items_per_block * ((N + items_per_block - 1) / items_per_block); // Allocate and fill memory std::vector input(size, T(1)); - T * d_input; - T * d_output; + T* d_input; + T* d_output; HIP_CHECK(hipMalloc(reinterpret_cast(&d_input), size * sizeof(T))); HIP_CHECK(hipMalloc(reinterpret_cast(&d_output), size * sizeof(T))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - size * sizeof(T), - hipMemcpyHostToDevice - ) - ); + HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); HIP_CHECK(hipDeviceSynchronize()); // HIP events creation @@ -133,16 +114,18 @@ void run_benchmark(benchmark::State& state, hipStream_t stream, size_t bytes) HIP_CHECK(hipEventCreate(&start)); HIP_CHECK(hipEventCreate(&stop)); - for (auto _ : state) + for(auto _ : state) { // Record start event HIP_CHECK(hipEventRecord(start, stream)); - hipLaunchKernelGGL( - HIP_KERNEL_NAME(kernel), - dim3(size/items_per_block), dim3(BlockSize), 0, stream, - d_input, d_output - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel), + dim3(size / items_per_block), + dim3(BlockSize), + 0, + stream, + d_input, + d_output); HIP_CHECK(hipGetLastError()); // Record stop event and wait until it completes @@ -175,65 +158,64 @@ void run_benchmark(benchmark::State& state, hipStream_t stream, size_t bytes) stream, \ bytes) -#define BENCHMARK_TYPE(type, block) \ - CREATE_BENCHMARK(type, block, 1), \ - CREATE_BENCHMARK(type, block, 2), \ - CREATE_BENCHMARK(type, block, 3), \ - CREATE_BENCHMARK(type, block, 4), \ - CREATE_BENCHMARK(type, block, 8), \ - CREATE_BENCHMARK(type, block, 11), \ - CREATE_BENCHMARK(type, block, 16) +#define BENCHMARK_TYPE(type, block) \ + CREATE_BENCHMARK(type, block, 1), CREATE_BENCHMARK(type, block, 2), \ + CREATE_BENCHMARK(type, block, 3), CREATE_BENCHMARK(type, block, 4), \ + CREATE_BENCHMARK(type, block, 8), CREATE_BENCHMARK(type, block, 11), \ + CREATE_BENCHMARK(type, block, 16) -template +template void add_benchmarks(std::vector& benchmarks, const std::string& method_name, hipStream_t stream, size_t bytes) { - using custom_float2 = custom_type; + using custom_float2 = custom_type; using custom_double2 = custom_type; - std::vector new_benchmarks = - { - // When block size is less than or equal to warp size - BENCHMARK_TYPE(int, 64), - BENCHMARK_TYPE(float, 64), - BENCHMARK_TYPE(double, 64), - BENCHMARK_TYPE(int8_t, 64), - BENCHMARK_TYPE(uint8_t, 64), - BENCHMARK_TYPE(rocprim::half, 64), - - BENCHMARK_TYPE(int, 256), - BENCHMARK_TYPE(float, 256), - BENCHMARK_TYPE(double, 256), - BENCHMARK_TYPE(int8_t, 256), - BENCHMARK_TYPE(uint8_t, 256), - BENCHMARK_TYPE(rocprim::half, 256), - - CREATE_BENCHMARK(custom_float2, 256, 1), - CREATE_BENCHMARK(custom_float2, 256, 4), - CREATE_BENCHMARK(custom_float2, 256, 8), - - CREATE_BENCHMARK(float2, 256, 1), - CREATE_BENCHMARK(float2, 256, 4), - CREATE_BENCHMARK(float2, 256, 8), - - CREATE_BENCHMARK(custom_double2, 256, 1), - CREATE_BENCHMARK(custom_double2, 256, 4), - CREATE_BENCHMARK(custom_double2, 256, 8), - - CREATE_BENCHMARK(double2, 256, 1), - CREATE_BENCHMARK(double2, 256, 4), - CREATE_BENCHMARK(double2, 256, 8), - - CREATE_BENCHMARK(float4, 256, 1), - CREATE_BENCHMARK(float4, 256, 4), - CREATE_BENCHMARK(float4, 256, 8), - }; + std::vector new_benchmarks + = {// When block size is less than or equal to warp size + BENCHMARK_TYPE(int, 64), + BENCHMARK_TYPE(float, 64), + BENCHMARK_TYPE(double, 64), + BENCHMARK_TYPE(int8_t, 64), + BENCHMARK_TYPE(uint8_t, 64), + BENCHMARK_TYPE(rocprim::half, 64), + BENCHMARK_TYPE(rocprim::int128_t, 64), + BENCHMARK_TYPE(rocprim::uint128_t, 64), + + BENCHMARK_TYPE(int, 256), + BENCHMARK_TYPE(float, 256), + BENCHMARK_TYPE(double, 256), + BENCHMARK_TYPE(int8_t, 256), + BENCHMARK_TYPE(uint8_t, 256), + BENCHMARK_TYPE(rocprim::half, 256), + BENCHMARK_TYPE(rocprim::int128_t, 256), + BENCHMARK_TYPE(rocprim::uint128_t, 256), + + CREATE_BENCHMARK(custom_float2, 256, 1), + CREATE_BENCHMARK(custom_float2, 256, 4), + CREATE_BENCHMARK(custom_float2, 256, 8), + + CREATE_BENCHMARK(float2, 256, 1), + CREATE_BENCHMARK(float2, 256, 4), + CREATE_BENCHMARK(float2, 256, 8), + + CREATE_BENCHMARK(custom_double2, 256, 1), + CREATE_BENCHMARK(custom_double2, 256, 4), + CREATE_BENCHMARK(custom_double2, 256, 8), + + CREATE_BENCHMARK(double2, 256, 1), + CREATE_BENCHMARK(double2, 256, 4), + CREATE_BENCHMARK(double2, 256, 8), + + CREATE_BENCHMARK(float4, 256, 1), + CREATE_BENCHMARK(float4, 256, 4), + CREATE_BENCHMARK(float4, 256, 8)}; benchmarks.insert(benchmarks.end(), new_benchmarks.begin(), new_benchmarks.end()); } -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { cli::Parser parser(argc, argv); parser.set_optional("size", "size", DEFAULT_BYTES, "number of bytes"); @@ -246,8 +228,8 @@ int main(int argc, char *argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); - const int trials = parser.get("trials"); + const size_t bytes = parser.get("size"); + const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); // HIP diff --git a/benchmark/benchmark_block_run_length_decode.cpp b/benchmark/benchmark_block_run_length_decode.cpp index 604042f3f..4af5743c3 100644 --- a/benchmark/benchmark_block_run_length_decode.cpp +++ b/benchmark/benchmark_block_run_length_decode.cpp @@ -28,26 +28,30 @@ #include #include #include +#include +#include #include -#include +#include +#include +#include #include #ifndef DEFAULT_N const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif -template -__global__ - __launch_bounds__(BlockSize) void block_run_length_decode_kernel(const ItemT* d_run_items, - const OffsetT* d_run_offsets, - ItemT* d_decoded_items, - bool enable_store = false) +__global__ __launch_bounds__(BlockSize) +void block_run_length_decode_kernel(const ItemT* d_run_items, + const OffsetT* d_run_offsets, + ItemT* d_decoded_items, + bool enable_store = false) { using BlockRunLengthDecodeT = rocprim::block_run_length_decode; @@ -86,18 +90,21 @@ __global__ } } -template -void run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& seed, hipStream_t stream) +void run_benchmark(benchmark::State& state, + size_t bytes, + const managed_seed& seed, + hipStream_t stream) { // Calculate the number of elements N - size_t N = bytes / sizeof(ItemT); + size_t N = bytes / sizeof(ItemT); constexpr auto runs_per_block = BlockSize * RunsPerThread; const auto target_num_runs = 2 * N / (MinRunLength + MaxRunLength); const auto num_runs @@ -107,11 +114,11 @@ void run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& se std::vector run_offsets(num_runs + 1); engine_type prng(seed.get_0()); - using ItemDistribution = std::conditional_t::value, - std::uniform_int_distribution, + using ItemDistribution = std::conditional_t::value, + uniform_int_distribution, std::uniform_real_distribution>; - ItemDistribution run_item_dist(0, 100); - std::uniform_int_distribution run_length_dist(MinRunLength, MaxRunLength); + ItemDistribution run_item_dist(0, 100); + uniform_int_distribution run_length_dist(MinRunLength, MaxRunLength); for(size_t i = 0; i < num_runs; ++i) { @@ -182,7 +189,7 @@ void run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& se ",run_per_thread:" #RPT ",decoded_items_per_thread:" #DIPT "}}") \ .c_str(), \ &run_benchmark, \ - bytes, \ + bytes, \ seed, \ stream) @@ -200,7 +207,7 @@ int main(int argc, char* argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); + const size_t bytes = parser.get("size"); const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); @@ -230,7 +237,23 @@ int main(int argc, char* argv[]) CREATE_BENCHMARK(double, long long, 1, 100, 128, 2, 4), CREATE_BENCHMARK(double, long long, 1, 500, 128, 2, 4), CREATE_BENCHMARK(double, long long, 1, 1000, 128, 2, 4), - CREATE_BENCHMARK(double, long long, 1, 5000, 128, 2, 4)}; + CREATE_BENCHMARK(double, long long, 1, 5000, 128, 2, 4), + + CREATE_BENCHMARK(rocprim::int128_t, rocprim::int128_t, 1, 5, 128, 2, 4), + CREATE_BENCHMARK(rocprim::int128_t, rocprim::int128_t, 1, 10, 128, 2, 4), + CREATE_BENCHMARK(rocprim::int128_t, rocprim::int128_t, 1, 50, 128, 2, 4), + CREATE_BENCHMARK(rocprim::int128_t, rocprim::int128_t, 1, 100, 128, 2, 4), + CREATE_BENCHMARK(rocprim::int128_t, rocprim::int128_t, 1, 500, 128, 2, 4), + CREATE_BENCHMARK(rocprim::int128_t, rocprim::int128_t, 1, 1000, 128, 2, 4), + CREATE_BENCHMARK(rocprim::int128_t, rocprim::int128_t, 1, 5000, 128, 2, 4), + + CREATE_BENCHMARK(rocprim::uint128_t, rocprim::uint128_t, 1, 5, 128, 2, 4), + CREATE_BENCHMARK(rocprim::uint128_t, rocprim::uint128_t, 1, 10, 128, 2, 4), + CREATE_BENCHMARK(rocprim::uint128_t, rocprim::uint128_t, 1, 50, 128, 2, 4), + CREATE_BENCHMARK(rocprim::uint128_t, rocprim::uint128_t, 1, 100, 128, 2, 4), + CREATE_BENCHMARK(rocprim::uint128_t, rocprim::uint128_t, 1, 500, 128, 2, 4), + CREATE_BENCHMARK(rocprim::uint128_t, rocprim::uint128_t, 1, 1000, 128, 2, 4), + CREATE_BENCHMARK(rocprim::uint128_t, rocprim::uint128_t, 1, 5000, 128, 2, 4)}; // Use manual timing for(auto& b : benchmarks) diff --git a/benchmark/benchmark_block_scan.cpp b/benchmark/benchmark_block_scan.cpp index 9c49c47d2..073c870df 100644 --- a/benchmark/benchmark_block_scan.cpp +++ b/benchmark/benchmark_block_scan.cpp @@ -32,30 +32,25 @@ // rocPRIM #include +#include +#include -#include -#include +#include +#include #include +#include #include -#include -#include - #ifndef DEFAULT_N const size_t DEFAULT_BYTES = 1024 * 1024 * 128 * 4; #endif -namespace rp = rocprim; - -template< - class Runner, - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - unsigned int Trials -> -__global__ -__launch_bounds__(BlockSize) +template +__global__ __launch_bounds__(BlockSize) void kernel(const T* input, T* output) { Runner::template run(input, output); @@ -64,107 +59,87 @@ void kernel(const T* input, T* output) template struct inclusive_scan { - template< - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - unsigned int Trials - > + template __device__ static void run(const T* input, T* output) { const unsigned int i = blockIdx.x * blockDim.x + threadIdx.x; T values[ItemsPerThread]; - for(unsigned int k = 0; k < ItemsPerThread; k++) + for(unsigned int k = 0; k < ItemsPerThread; ++k) { values[k] = input[i * ItemsPerThread + k]; } - using bscan_t = rp::block_scan; + using bscan_t = rocprim::block_scan; __shared__ typename bscan_t::storage_type storage; ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < Trials; trial++) + for(unsigned int trial = 0; trial < Trials; ++trial) { bscan_t().inclusive_scan(values, values, storage); } - for(unsigned int k = 0; k < ItemsPerThread; k++) + for(unsigned int k = 0; k < ItemsPerThread; ++k) { output[i * ItemsPerThread + k] = values[k]; } } - }; template struct exclusive_scan { - template< - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - unsigned int Trials - > + template __device__ static void run(const T* input, T* output) { const unsigned int i = blockIdx.x * blockDim.x + threadIdx.x; - using U = typename std::remove_reference::type; + using U = typename std::remove_reference::type; T values[ItemsPerThread]; U init = U(100); - for(unsigned int k = 0; k < ItemsPerThread; k++) + for(unsigned int k = 0; k < ItemsPerThread; ++k) { values[k] = input[i * ItemsPerThread + k]; } - using bscan_t = rp::block_scan; + using bscan_t = rocprim::block_scan; __shared__ typename bscan_t::storage_type storage; ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < Trials; trial++) + for(unsigned int trial = 0; trial < Trials; ++trial) { bscan_t().exclusive_scan(values, values, init, storage); } - for(unsigned int k = 0; k < ItemsPerThread; k++) + for(unsigned int k = 0; k < ItemsPerThread; ++k) { output[i * ItemsPerThread + k] = values[k]; } } - }; -template< - class Benchmark, - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - unsigned int Trials = 100 -> +template void run_benchmark(benchmark::State& state, hipStream_t stream, size_t bytes) { // Calculate the number of elements N size_t N = bytes / sizeof(T); // Make sure size is a multiple of BlockSize constexpr auto items_per_block = BlockSize * ItemsPerThread; - const auto size = items_per_block * ((N + items_per_block - 1)/items_per_block); + const auto size = items_per_block * ((N + items_per_block - 1) / items_per_block); // Allocate and fill memory std::vector input(size, T(1)); - T * d_input; - T * d_output; + T* d_input; + T* d_output; HIP_CHECK(hipMalloc(reinterpret_cast(&d_input), size * sizeof(T))); HIP_CHECK(hipMalloc(reinterpret_cast(&d_output), size * sizeof(T))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - size * sizeof(T), - hipMemcpyHostToDevice - ) - ); + HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); HIP_CHECK(hipDeviceSynchronize()); // HIP events creation @@ -172,16 +147,18 @@ void run_benchmark(benchmark::State& state, hipStream_t stream, size_t bytes) HIP_CHECK(hipEventCreate(&start)); HIP_CHECK(hipEventCreate(&stop)); - for (auto _ : state) + for(auto _ : state) { // Record start event HIP_CHECK(hipEventRecord(start, stream)); - hipLaunchKernelGGL( - HIP_KERNEL_NAME(kernel), - dim3(size/items_per_block), dim3(BlockSize), 0, stream, - d_input, d_output - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel), + dim3(size / items_per_block), + dim3(BlockSize), + 0, + stream, + d_input, + d_output); HIP_CHECK(hipGetLastError()); // Record stop event and wait until it completes @@ -215,66 +192,69 @@ void run_benchmark(benchmark::State& state, hipStream_t stream, size_t bytes) stream, \ bytes) -#define BENCHMARK_TYPE(type, block) \ - CREATE_BENCHMARK(type, block, 1), \ - CREATE_BENCHMARK(type, block, 2), \ - CREATE_BENCHMARK(type, block, 3), \ - CREATE_BENCHMARK(type, block, 4), \ - CREATE_BENCHMARK(type, block, 8), \ - CREATE_BENCHMARK(type, block, 11), \ - CREATE_BENCHMARK(type, block, 16) +#define BENCHMARK_TYPE(type, block) \ + CREATE_BENCHMARK(type, block, 1), CREATE_BENCHMARK(type, block, 2), \ + CREATE_BENCHMARK(type, block, 3), CREATE_BENCHMARK(type, block, 4), \ + CREATE_BENCHMARK(type, block, 8), CREATE_BENCHMARK(type, block, 11), \ + CREATE_BENCHMARK(type, block, 16) -template +template void add_benchmarks(std::vector& benchmarks, - const std::string& method_name, - const std::string& algorithm_name, - hipStream_t stream, - size_t bytes) + const std::string& method_name, + const std::string& algorithm_name, + hipStream_t stream, + size_t bytes) { - using custom_float2 = custom_type; + using custom_float2 = custom_type; using custom_double2 = custom_type; - std::vector new_benchmarks = - { - // When block size is less than or equal to warp size - BENCHMARK_TYPE(int, 64), - BENCHMARK_TYPE(float, 64), - BENCHMARK_TYPE(double, 64), - BENCHMARK_TYPE(int8_t, 64), - BENCHMARK_TYPE(uint8_t, 64), - BENCHMARK_TYPE(rocprim::half, 64), - - BENCHMARK_TYPE(int, 256), - BENCHMARK_TYPE(float, 256), - BENCHMARK_TYPE(double, 256), - BENCHMARK_TYPE(int8_t, 256), - BENCHMARK_TYPE(uint8_t, 256), - BENCHMARK_TYPE(rocprim::half, 256), - - CREATE_BENCHMARK(custom_float2, 256, 1), - CREATE_BENCHMARK(custom_float2, 256, 4), - CREATE_BENCHMARK(custom_float2, 256, 8), - - CREATE_BENCHMARK(float2, 256, 1), - CREATE_BENCHMARK(float2, 256, 4), - CREATE_BENCHMARK(float2, 256, 8), - - CREATE_BENCHMARK(custom_double2, 256, 1), - CREATE_BENCHMARK(custom_double2, 256, 4), - CREATE_BENCHMARK(custom_double2, 256, 8), - - CREATE_BENCHMARK(double2, 256, 1), - CREATE_BENCHMARK(double2, 256, 4), - CREATE_BENCHMARK(double2, 256, 8), - - CREATE_BENCHMARK(float4, 256, 1), - CREATE_BENCHMARK(float4, 256, 4), - CREATE_BENCHMARK(float4, 256, 8), - }; + std::vector new_benchmarks + = {// When block size is less than or equal to warp size + BENCHMARK_TYPE(int, 64), + BENCHMARK_TYPE(float, 64), + BENCHMARK_TYPE(double, 64), + BENCHMARK_TYPE(int8_t, 64), + BENCHMARK_TYPE(uint8_t, 64), + BENCHMARK_TYPE(rocprim::half, 64), + + BENCHMARK_TYPE(int, 256), + BENCHMARK_TYPE(float, 256), + BENCHMARK_TYPE(double, 256), + BENCHMARK_TYPE(int8_t, 256), + BENCHMARK_TYPE(uint8_t, 256), + BENCHMARK_TYPE(rocprim::half, 256), + + CREATE_BENCHMARK(custom_float2, 256, 1), + CREATE_BENCHMARK(custom_float2, 256, 4), + CREATE_BENCHMARK(custom_float2, 256, 8), + + CREATE_BENCHMARK(float2, 256, 1), + CREATE_BENCHMARK(float2, 256, 4), + CREATE_BENCHMARK(float2, 256, 8), + + CREATE_BENCHMARK(custom_double2, 256, 1), + CREATE_BENCHMARK(custom_double2, 256, 4), + CREATE_BENCHMARK(custom_double2, 256, 8), + + CREATE_BENCHMARK(double2, 256, 1), + CREATE_BENCHMARK(double2, 256, 4), + CREATE_BENCHMARK(double2, 256, 8), + + CREATE_BENCHMARK(float4, 256, 1), + CREATE_BENCHMARK(float4, 256, 4), + CREATE_BENCHMARK(float4, 256, 8), + + CREATE_BENCHMARK(rocprim::int128_t, 256, 1), + CREATE_BENCHMARK(rocprim::int128_t, 256, 4), + CREATE_BENCHMARK(rocprim::int128_t, 256, 8), + + CREATE_BENCHMARK(rocprim::uint128_t, 256, 1), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 4), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 8)}; benchmarks.insert(benchmarks.end(), new_benchmarks.begin(), new_benchmarks.end()); } -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { cli::Parser parser(argc, argv); parser.set_optional("size", "size", DEFAULT_BYTES, "number of bytes"); @@ -287,8 +267,8 @@ int main(int argc, char *argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); - const int trials = parser.get("trials"); + const size_t bytes = parser.get("size"); + const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); // HIP @@ -302,24 +282,32 @@ int main(int argc, char *argv[]) std::vector benchmarks; // inclusive_scan using_warp_scan using inclusive_scan_uws_t = inclusive_scan; - add_benchmarks( - benchmarks, "inclusive_scan", "using_warp_scan", stream, bytes - ); + add_benchmarks(benchmarks, + "inclusive_scan", + "using_warp_scan", + stream, + bytes); // exclusive_scan using_warp_scan using exclusive_scan_uws_t = exclusive_scan; - add_benchmarks( - benchmarks, "exclusive_scan", "using_warp_scan", stream, bytes - ); + add_benchmarks(benchmarks, + "exclusive_scan", + "using_warp_scan", + stream, + bytes); // inclusive_scan reduce then scan using inclusive_scan_rts_t = inclusive_scan; - add_benchmarks( - benchmarks, "inclusive_scan", "reduce_then_scan", stream, bytes - ); + add_benchmarks(benchmarks, + "inclusive_scan", + "reduce_then_scan", + stream, + bytes); // exclusive_scan reduce then scan using exclusive_scan_rts_t = exclusive_scan; - add_benchmarks( - benchmarks, "exclusive_scan", "reduce_then_scan", stream, bytes - ); + add_benchmarks(benchmarks, + "exclusive_scan", + "reduce_then_scan", + stream, + bytes); // Use manual timing for(auto& b : benchmarks) diff --git a/benchmark/benchmark_block_sort.cpp b/benchmark/benchmark_block_sort.cpp index 9556f5f0d..83c42c15f 100644 --- a/benchmark/benchmark_block_sort.cpp +++ b/benchmark/benchmark_block_sort.cpp @@ -33,12 +33,17 @@ #include // rocPRIM -#include - -#include -#include +#ifndef BENCHMARK_CONFIG_TUNING + #include + #include +#endif #include +#include +#include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif #ifndef DEFAULT_N const size_t DEFAULT_BYTES = 1024 * 1024 * 128 * 4; @@ -91,7 +96,7 @@ int main(int argc, char* argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); + const size_t bytes = parser.get("size"); const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); @@ -115,11 +120,15 @@ int main(int argc, char* argv[]) CREATE_BENCHMARK(int, rocprim::empty_type, 256) CREATE_BENCHMARK(int, rocprim::empty_type, 512) CREATE_BENCHMARK(double, rocprim::empty_type, 512) + CREATE_BENCHMARK(rocprim::int128_t, rocprim::empty_type, 256) + CREATE_BENCHMARK(rocprim::uint128_t, rocprim::empty_type, 256) CREATE_BENCHMARK(int, int, 512) CREATE_BENCHMARK(float, double, 512) CREATE_BENCHMARK(double, int64_t, 512) CREATE_BENCHMARK(rocprim::half, int16_t, 512) CREATE_BENCHMARK(uint8_t, uint32_t, 512) + CREATE_BENCHMARK(int64_t, rocprim::int128_t, 512) + CREATE_BENCHMARK(uint64_t, rocprim::uint128_t, 512) #endif std::vector benchmarks = {}; diff --git a/benchmark/benchmark_block_sort.parallel.hpp b/benchmark/benchmark_block_sort.parallel.hpp index d6e2fe15b..424dcf9f9 100644 --- a/benchmark/benchmark_block_sort.parallel.hpp +++ b/benchmark/benchmark_block_sort.parallel.hpp @@ -35,20 +35,25 @@ #include #include #include -#include +#include +#include +#include +#include +#include +#include #include +#include #include -#include - -template::value, bool> = true> -__global__ __launch_bounds__(BlockSize) void sort_kernel(const KeyType* input, KeyType* output) +__global__ __launch_bounds__(BlockSize) +void sort_kernel(const KeyType* input, KeyType* output) { const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; @@ -62,13 +67,14 @@ __global__ __launch_bounds__(BlockSize) void sort_kernel(const KeyType* input, K rocprim::block_store_direct_blocked(lid, output + block_offset, keys); } -template::value, bool> = true> -__global__ __launch_bounds__(BlockSize) void sort_kernel(const KeyType* input, KeyType* output) +__global__ __launch_bounds__(BlockSize) +void sort_kernel(const KeyType* input, KeyType* output) { const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; @@ -95,13 +101,13 @@ __global__ __launch_bounds__(BlockSize) void sort_kernel(const KeyType* input, K rocprim::block_store_direct_blocked(lid, output + block_offset, keys); } -template -__global__ __launch_bounds__(BlockSize) void stable_sort_kernel(const KeyType* input, - KeyType* output) +__global__ __launch_bounds__(BlockSize) +void stable_sort_kernel(const KeyType* input, KeyType* output) { const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; @@ -145,8 +151,8 @@ __global__ __launch_bounds__(BlockSize) void stable_sort_kernel(const KeyType* i rocprim::block_store_direct_blocked(lid, output + block_offset, keys); } -template -#include +#include +#include +#include #ifndef DEFAULT_N const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif - enum class stream_kind { default_stream, @@ -51,7 +52,9 @@ static void BM_host_target_arch(benchmark::State& state, const stream_kind strea } } -__global__ void empty_kernel() {} +__global__ +void empty_kernel() +{} // An empty kernel launch for baseline static void BM_kernel_launch(benchmark::State& state) @@ -66,17 +69,12 @@ static void BM_kernel_launch(benchmark::State& state) HIP_CHECK(hipStreamSynchronize(stream)); } -#define CREATE_BENCHMARK(ST, SK) \ - benchmark::RegisterBenchmark( \ - bench_naming::format_name( \ - "{lvl:na" \ - ",algo:" #ST \ - ",cfg:default_config}" \ - ).c_str(), \ - &BM_host_target_arch, \ - SK \ - ) \ - +#define CREATE_BENCHMARK(ST, SK) \ + benchmark::RegisterBenchmark(bench_naming::format_name("{lvl:na" \ + ",algo:" #ST ",cfg:default_config}") \ + .c_str(), \ + &BM_host_target_arch, \ + SK) int main(int argc, char** argv) { @@ -84,14 +82,14 @@ int main(int argc, char** argv) parser.set_optional("size", "size", DEFAULT_BYTES, "number of bytes"); parser.set_optional("trials", "trials", 100, "number of iterations"); parser.set_optional("name_format", - "name_format", - "human", - "either: json,human,txt"); + "name_format", + "human", + "either: json,human,txt"); parser.run_and_exit_if_error(); // Parse argv benchmark::Initialize(&argc, argv); - const int trials = parser.get("trials"); + const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); // HIP @@ -124,5 +122,4 @@ int main(int argc, char** argv) // Run benchmarks benchmark::RunSpecifiedBenchmarks(); return 0; - } diff --git a/benchmark/benchmark_device_adjacent_difference.cpp b/benchmark/benchmark_device_adjacent_difference.cpp index aa11754ae..e85b766c9 100644 --- a/benchmark/benchmark_device_adjacent_difference.cpp +++ b/benchmark/benchmark_device_adjacent_difference.cpp @@ -30,15 +30,19 @@ #include // rocPRIM -#include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif // CmdParser #include "cmdparser.hpp" -#include -#include - #include +#include +#include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif #ifndef DEFAULT_BYTES constexpr std::size_t DEFAULT_BYTES = 1024LL * 1024LL * 1024LL * 2LL; @@ -122,6 +126,9 @@ int main(int argc, char* argv[]) CREATE_BENCHMARKS(custom_float2) CREATE_BENCHMARKS(custom_double2) + + CREATE_BENCHMARKS(rocprim::int128_t) + CREATE_BENCHMARKS(rocprim::uint128_t) #endif // BENCHMARK_CONFIG_TUNING // Use manual timing diff --git a/benchmark/benchmark_device_adjacent_difference.parallel.cpp.in b/benchmark/benchmark_device_adjacent_difference.parallel.cpp.in index 03c316d5b..5fab2877d 100644 --- a/benchmark/benchmark_device_adjacent_difference.parallel.cpp.in +++ b/benchmark/benchmark_device_adjacent_difference.parallel.cpp.in @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,11 +20,13 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include - #include "benchmark_utils.hpp" #include "benchmark_device_adjacent_difference.parallel.hpp" +#include + +#include + namespace { auto benchmarks = config_autotune_register::create_bulk( device_adjacent_difference_benchmark_generator< @@ -32,5 +34,4 @@ namespace { @BlockSize@, @Left@, @InPlace@>::create); - } diff --git a/benchmark/benchmark_device_adjacent_difference.parallel.hpp b/benchmark/benchmark_device_adjacent_difference.parallel.hpp index 66b67ca06..13f5aee8d 100644 --- a/benchmark/benchmark_device_adjacent_difference.parallel.hpp +++ b/benchmark/benchmark_device_adjacent_difference.parallel.hpp @@ -32,18 +32,23 @@ #include // rocPRIM +#include +#include +#include #include -#include +#include #include #include +#include #include +#include +#include #include template std::string config_name() { - //const rocprim::adjacent_difference_config = Config(); auto config = Config(); return "{bs:" + std::to_string(config.block_size) + ",ipt:" + std::to_string(config.items_per_thread) + "}"; @@ -189,7 +194,7 @@ struct device_adjacent_difference_benchmark : public config_autotune_interface HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_size)); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { HIP_CHECK(launch()); } @@ -206,7 +211,7 @@ struct device_adjacent_difference_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { HIP_CHECK(launch()); } diff --git a/benchmark/benchmark_device_adjacent_find.cpp b/benchmark/benchmark_device_adjacent_find.cpp index 719061de1..24c0671aa 100644 --- a/benchmark/benchmark_device_adjacent_find.cpp +++ b/benchmark/benchmark_device_adjacent_find.cpp @@ -30,9 +30,17 @@ // HIP #include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif + // C++ Standard Library +#include #include #include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif #ifndef DEFAULT_N const size_t DEFAULT_BYTES = size_t{2} << 30; // 2 GiB @@ -115,6 +123,8 @@ int main(int argc, char* argv[]) CREATE_ADJACENT_FIND_BENCHMARKS(rocprim::half) CREATE_ADJACENT_FIND_BENCHMARKS(float) CREATE_ADJACENT_FIND_BENCHMARKS(double) + CREATE_ADJACENT_FIND_BENCHMARKS(rocprim::int128_t) + CREATE_ADJACENT_FIND_BENCHMARKS(rocprim::uint128_t) // Custom types CREATE_ADJACENT_FIND_BENCHMARKS(custom_float2) CREATE_ADJACENT_FIND_BENCHMARKS(custom_double2) diff --git a/benchmark/benchmark_device_adjacent_find.parallel.cpp.in b/benchmark/benchmark_device_adjacent_find.parallel.cpp.in index 95f53bdf3..6a5ac95a0 100644 --- a/benchmark/benchmark_device_adjacent_find.parallel.cpp.in +++ b/benchmark/benchmark_device_adjacent_find.parallel.cpp.in @@ -20,10 +20,13 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. - #include "benchmark_utils.hpp" #include "benchmark_device_adjacent_find.parallel.hpp" +#include + +#include + namespace { auto benchmarks = config_autotune_register::create_bulk( device_adjacent_find_benchmark_generator< diff --git a/benchmark/benchmark_device_adjacent_find.parallel.hpp b/benchmark/benchmark_device_adjacent_find.parallel.hpp index 016ee3d1e..30ce1450b 100644 --- a/benchmark/benchmark_device_adjacent_find.parallel.hpp +++ b/benchmark/benchmark_device_adjacent_find.parallel.hpp @@ -32,15 +32,18 @@ #include // rocPRIM -#include +#include #include #include +#include // C++ Standard Library -#include +#include +#include #include #include #include +#include #include #include @@ -157,7 +160,7 @@ struct device_adjacent_find_benchmark : public config_autotune_interface HIP_CHECK(hipMalloc(&d_tmp_storage, tmp_storage_size)); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { launch_adjacent_find(); } @@ -174,7 +177,7 @@ struct device_adjacent_find_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { launch_adjacent_find(); } diff --git a/benchmark/benchmark_device_batch_memcpy.cpp b/benchmark/benchmark_device_batch_memcpy.cpp index 27280b220..d115e7cf1 100644 --- a/benchmark/benchmark_device_batch_memcpy.cpp +++ b/benchmark/benchmark_device_batch_memcpy.cpp @@ -31,11 +31,19 @@ #include #include #include +#include +#ifdef BUILD_NAIVE_BENCHMARK + #include + #include +#endif -#include +#include +#include #include #include #include +#include +#include #include #include @@ -65,7 +73,7 @@ constexpr int32_t blev_min_size = rocprim::batch_memcpy_config<>::blev_size_thr // ┌───┬───┬───┬───┬───┬───┬───┬───┐ // │c0'│a0'│a1'│a2'│d0'│d1'│b0'│b1'│ buffer y contains buffers a', b', c', d' // └───┴───┴───┴───┴───┴───┴───┴───┘ -template +template std::vector shuffled_exclusive_scan(const std::vector& input, RandomGenerator& rng) { const auto n = input.size(); @@ -89,8 +97,8 @@ std::vector shuffled_exclusive_scan(const std::vector& input, RandomGenera using offset_type = size_t; template::type = 0> void init_input(ContainerMemCpy& h_input_for_memcpy, ContainerCopy& /*h_input_for_copy*/, @@ -111,9 +119,9 @@ void init_input(ContainerMemCpy& h_input_for_memcpy, } template::type = 0> void init_input(ContainerMemCpy& /*h_input_for_memcpy*/, ContainerCopy& h_input_for_copy, @@ -137,9 +145,9 @@ void init_input(ContainerMemCpy& /*h_input_for_memcpy*/, } template::type = 0> void batch_copy(void* temporary_storage, size_t& storage_size, @@ -159,9 +167,9 @@ void batch_copy(void* temporary_storage, } template::type = 0> void batch_copy(void* temporary_storage, size_t& storage_size, @@ -230,7 +238,7 @@ struct BatchMemcpyData } }; -template +template BatchMemcpyData prepare_data(const managed_seed& seed, const int32_t num_tlev_buffers = 1024, const int32_t num_wlev_buffers = 1024, @@ -355,7 +363,7 @@ BatchMemcpyData prepare_data(const managed_seed& seed return result; } -template +template void run_benchmark(benchmark::State& state, const managed_seed& seed, hipStream_t stream, @@ -384,7 +392,7 @@ void run_benchmark(benchmark::State& state, num_blev_buffers); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { batch_copy(d_temp_storage, temp_storage_bytes, @@ -436,7 +444,7 @@ void run_benchmark(benchmark::State& state, template __launch_bounds__(BlockSize) __global__ - void naive_kernel(void** in_ptr, void** out_ptr, const OffsetType* sizes) +void naive_kernel(void** in_ptr, void** out_ptr, const OffsetType* sizes) { using underlying_type = unsigned char; constexpr int32_t items_per_thread = 4; @@ -470,7 +478,7 @@ __launch_bounds__(BlockSize) __global__ } } -template +template void run_naive_benchmark(benchmark::State& state, const managed_seed& seed, hipStream_t stream, @@ -486,7 +494,7 @@ void run_naive_benchmark(benchmark::State& state, num_blev_buffers); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { naive_kernel <<>>((void**)data.d_buffer_srcs, @@ -572,21 +580,38 @@ void run_naive_benchmark(benchmark::State& state, }) #ifndef BUILD_NAIVE_BENCHMARK - #define BENCHMARK_TYPE(item_size, item_alignment) \ - CREATE_BENCHMARK(item_size, item_alignment, uint32_t, 100000, 0, 0), \ - CREATE_BENCHMARK(item_size, item_alignment, uint32_t, 0, 100000, 0), \ - CREATE_BENCHMARK(item_size, item_alignment, uint32_t, 0, 0, 1000), \ - CREATE_BENCHMARK(item_size, item_alignment, uint32_t, 1000, 1000, 1000) + #define BENCHMARK_TYPE(item_size, item_alignment) \ + CREATE_BENCHMARK(item_size, item_alignment, uint32_t, 100000, 0, 0), \ + CREATE_BENCHMARK(item_size, item_alignment, uint32_t, 0, 100000, 0), \ + CREATE_BENCHMARK(item_size, item_alignment, uint32_t, 0, 0, 1000), \ + CREATE_BENCHMARK(item_size, item_alignment, uint32_t, 1000, 1000, 1000), \ + CREATE_BENCHMARK(item_size, item_alignment, rocprim::uint128_t, 100000, 0, 0), \ + CREATE_BENCHMARK(item_size, item_alignment, rocprim::uint128_t, 0, 100000, 0), \ + CREATE_BENCHMARK(item_size, item_alignment, rocprim::uint128_t, 0, 0, 1000), \ + CREATE_BENCHMARK(item_size, item_alignment, rocprim::uint128_t, 1000, 1000, 1000) #else - #define BENCHMARK_TYPE(item_size, item_alignment) \ - CREATE_BENCHMARK(item_size, item_alignment, uint32_t, 100000, 0, 0), \ - CREATE_BENCHMARK(item_size, item_alignment, uint32_t, 0, 100000, 0), \ - CREATE_BENCHMARK(item_size, item_alignment, uint32_t, 0, 0, 1000), \ - CREATE_BENCHMARK(item_size, item_alignment, uint32_t, 1000, 1000, 1000), \ - CREATE_NAIVE_BENCHMARK(item_size, item_alignment, uint32_t, 100000, 0, 0), \ - CREATE_NAIVE_BENCHMARK(item_size, item_alignment, uint32_t, 0, 100000, 0), \ - CREATE_NAIVE_BENCHMARK(item_size, item_alignment, uint32_t, 0, 0, 1000), \ - CREATE_NAIVE_BENCHMARK(item_size, item_alignment, uint32_t, 1000, 1000, 1000) + #define BENCHMARK_TYPE(item_size, item_alignment) \ + CREATE_BENCHMARK(item_size, item_alignment, uint32_t, 100000, 0, 0), \ + CREATE_BENCHMARK(item_size, item_alignment, uint32_t, 0, 100000, 0), \ + CREATE_BENCHMARK(item_size, item_alignment, uint32_t, 0, 0, 1000), \ + CREATE_BENCHMARK(item_size, item_alignment, uint32_t, 1000, 1000, 1000), \ + CREATE_NAIVE_BENCHMARK(item_size, item_alignment, uint32_t, 100000, 0, 0), \ + CREATE_NAIVE_BENCHMARK(item_size, item_alignment, uint32_t, 0, 100000, 0), \ + CREATE_NAIVE_BENCHMARK(item_size, item_alignment, uint32_t, 0, 0, 1000), \ + CREATE_NAIVE_BENCHMARK(item_size, item_alignment, uint32_t, 1000, 1000, 1000), \ + CREATE_BENCHMARK(item_size, item_alignment, rocprim::uint128_t, 100000, 0, 0), \ + CREATE_BENCHMARK(item_size, item_alignment, rocprim::uint128_t, 0, 100000, 0), \ + CREATE_BENCHMARK(item_size, item_alignment, rocprim::uint128_t, 0, 0, 1000), \ + CREATE_BENCHMARK(item_size, item_alignment, rocprim::uint128_t, 1000, 1000, 1000), \ + CREATE_NAIVE_BENCHMARK(item_size, item_alignment, rocprim::uint128_t, 100000, 0, 0), \ + CREATE_NAIVE_BENCHMARK(item_size, item_alignment, rocprim::uint128_t, 0, 100000, 0), \ + CREATE_NAIVE_BENCHMARK(item_size, item_alignment, rocprim::uint128_t, 0, 0, 1000), \ + CREATE_NAIVE_BENCHMARK(item_size, \ + item_alignment, \ + rocprim::uint128_t, \ + 1000, \ + 1000, \ + 1000) #endif //BUILD_NAIVE_BENCHMARK int32_t main(int32_t argc, char* argv[]) @@ -618,15 +643,13 @@ int32_t main(int32_t argc, char* argv[]) benchmark::AddCustomContext("seed", seed_type); // Add benchmarks - std::vector benchmarks; - - benchmarks = {BENCHMARK_TYPE(1, 1), - BENCHMARK_TYPE(1, 2), - BENCHMARK_TYPE(1, 4), - BENCHMARK_TYPE(1, 8), - BENCHMARK_TYPE(2, 2), - BENCHMARK_TYPE(4, 4), - BENCHMARK_TYPE(8, 8)}; + std::vector benchmarks = {BENCHMARK_TYPE(1, 1), + BENCHMARK_TYPE(1, 2), + BENCHMARK_TYPE(1, 4), + BENCHMARK_TYPE(1, 8), + BENCHMARK_TYPE(2, 2), + BENCHMARK_TYPE(4, 4), + BENCHMARK_TYPE(8, 8)}; // Use manual timing for(auto& b : benchmarks) diff --git a/benchmark/benchmark_device_binary_search.cpp b/benchmark/benchmark_device_binary_search.cpp index 3b244795d..76d3e233f 100644 --- a/benchmark/benchmark_device_binary_search.cpp +++ b/benchmark/benchmark_device_binary_search.cpp @@ -34,23 +34,27 @@ // rocPRIM #include +#include +#include -#include -#include +#include +#include +#include #include +#include #include - -#include -#include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif #ifndef DEFAULT_N const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif -const unsigned int batch_size = 10; +const unsigned int batch_size = 10; const unsigned int warmup_size = 5; -template +template void run_benchmark(benchmark::State& state, size_t haystack_bytes, const managed_seed& seed, @@ -59,13 +63,16 @@ void run_benchmark(benchmark::State& state, bool sorted_needles) { using haystack_type = T; - using needle_type = T; - using output_type = size_t; - using compare_op_type = typename std::conditional::value, half_less, rocprim::less>::type; + using needle_type = T; + using output_type = size_t; + using compare_op_type = + typename std::conditional::value, + half_less, + rocprim::less>::type; // Calculate the number of elements from byte size size_t haystack_size = haystack_bytes / sizeof(haystack_type); - size_t needles_size = needles_bytes / sizeof(needle_type); + size_t needles_size = needles_bytes / sizeof(needle_type); compare_op_type compare_op; // Generate data @@ -83,28 +90,23 @@ void run_benchmark(benchmark::State& state, std::sort(needles.begin(), needles.end(), compare_op); } - haystack_type * d_haystack; - needle_type * d_needles; - output_type * d_output; - HIP_CHECK(hipMalloc(reinterpret_cast(&d_haystack), haystack_size * sizeof(haystack_type))); + haystack_type* d_haystack; + needle_type* d_needles; + output_type* d_output; + HIP_CHECK( + hipMalloc(reinterpret_cast(&d_haystack), haystack_size * sizeof(haystack_type))); HIP_CHECK(hipMalloc(reinterpret_cast(&d_needles), needles_size * sizeof(needle_type))); HIP_CHECK(hipMalloc(reinterpret_cast(&d_output), needles_size * sizeof(output_type))); - HIP_CHECK( - hipMemcpy( - d_haystack, haystack.data(), - haystack_size * sizeof(haystack_type), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_needles, needles.data(), - needles_size * sizeof(needle_type), - hipMemcpyHostToDevice - ) - ); - - void * d_temporary_storage = nullptr; + HIP_CHECK(hipMemcpy(d_haystack, + haystack.data(), + haystack_size * sizeof(haystack_type), + hipMemcpyHostToDevice)); + HIP_CHECK(hipMemcpy(d_needles, + needles.data(), + needles_size * sizeof(needle_type), + hipMemcpyHostToDevice)); + + void* d_temporary_storage = nullptr; size_t temporary_storage_bytes; auto dispatch_helper = dispatch_binary_search_helper(); HIP_CHECK(dispatch_helper.dispatch_binary_search(AlgorithmSelectorTag{}, @@ -121,7 +123,7 @@ void run_benchmark(benchmark::State& state, HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { HIP_CHECK(dispatch_helper.dispatch_binary_search(AlgorithmSelectorTag{}, d_temporary_storage, @@ -146,7 +148,7 @@ void run_benchmark(benchmark::State& state, // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { HIP_CHECK(dispatch_helper.dispatch_binary_search(AlgorithmSelectorTag{}, d_temporary_storage, @@ -199,7 +201,7 @@ void run_benchmark(benchmark::State& state, #define BENCHMARK_TYPE(type) \ BENCHMARK_ALGORITHMS(type, 10, true), BENCHMARK_ALGORITHMS(type, 10, false) -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { cli::Parser parser(argc, argv); parser.set_optional("size", "size", DEFAULT_BYTES, "number of bytes"); @@ -224,8 +226,8 @@ int main(int argc, char *argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); - const int trials = parser.get("trials"); + const size_t bytes = parser.get("size"); + const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); const managed_seed seed(seed_type); @@ -238,7 +240,7 @@ int main(int argc, char *argv[]) benchmark::AddCustomContext("bytes", std::to_string(bytes)); benchmark::AddCustomContext("seed", seed_type); - using custom_float2 = custom_type; + using custom_float2 = custom_type; using custom_double2 = custom_type; // Add benchmarks @@ -258,6 +260,8 @@ int main(int argc, char *argv[]) BENCHMARK_TYPE(int8_t), BENCHMARK_TYPE(uint8_t), BENCHMARK_TYPE(rocprim::half), + BENCHMARK_TYPE(rocprim::int128_t), + BENCHMARK_TYPE(rocprim::uint128_t), BENCHMARK_TYPE(custom_float2), BENCHMARK_TYPE(custom_double2)}; #endif // BENCHMARK_CONFIG_TUNING diff --git a/benchmark/benchmark_device_binary_search.parallel.cpp.in b/benchmark/benchmark_device_binary_search.parallel.cpp.in index e88bba55c..913cdb695 100644 --- a/benchmark/benchmark_device_binary_search.parallel.cpp.in +++ b/benchmark/benchmark_device_binary_search.parallel.cpp.in @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2023-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -23,6 +23,11 @@ #include "benchmark_utils.hpp" #include "benchmark_device_binary_search.parallel.hpp" +#include +#include + +#include + namespace { auto benchmarks = config_autotune_register::create +#include #include +#include #include #include +#include +#include #include +#include #include -#include - struct binary_search_subalgorithm { std::string name() const @@ -61,24 +64,24 @@ struct upper_bound_subalgorithm } }; -template +template struct dispatch_binary_search_helper { - template + template hipError_t dispatch_binary_search(binary_search_subalgorithm, Args&&... args) { using config = rocprim::binary_search_config; return rocprim::binary_search(std::forward(args)...); } - template + template hipError_t dispatch_binary_search(upper_bound_subalgorithm, Args&&... args) { using config = rocprim::upper_bound_config; return rocprim::upper_bound(std::forward(args)...); } - template + template hipError_t dispatch_binary_search(lower_bound_subalgorithm, Args&&... args) { using config = rocprim::lower_bound_config; @@ -89,26 +92,26 @@ struct dispatch_binary_search_helper template<> struct dispatch_binary_search_helper { - template + template hipError_t dispatch_binary_search(binary_search_subalgorithm, Args&&... args) { return rocprim::binary_search(std::forward(args)...); } - template + template hipError_t dispatch_binary_search(upper_bound_subalgorithm, Args&&... args) { return rocprim::upper_bound(std::forward(args)...); } - template + template hipError_t dispatch_binary_search(lower_bound_subalgorithm, Args&&... args) { return rocprim::lower_bound(std::forward(args)...); } }; -template +template struct device_binary_search_benchmark : public config_autotune_interface { std::string name() const override diff --git a/benchmark/benchmark_device_find_end.cpp b/benchmark/benchmark_device_find_end.cpp index 91ecc9c01..7cc52179a 100644 --- a/benchmark/benchmark_device_find_end.cpp +++ b/benchmark/benchmark_device_find_end.cpp @@ -32,8 +32,12 @@ // HIP API #include +#include + #include +#include #include +#include #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; @@ -53,10 +57,9 @@ const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; CREATE_BENCHMARK_FIND_END(TYPE, 10000, REPEATING) \ } -#define CREATE_BENCHMARK(TYPE) \ - { \ - CREATE_BENCHMARK_PATTERN(TYPE, true) \ - CREATE_BENCHMARK_PATTERN(TYPE, false) \ +#define CREATE_BENCHMARK(TYPE) \ + { \ + CREATE_BENCHMARK_PATTERN(TYPE, true) CREATE_BENCHMARK_PATTERN(TYPE, false) \ } int main(int argc, char* argv[]) @@ -96,6 +99,8 @@ int main(int argc, char* argv[]) CREATE_BENCHMARK(rocprim::half) CREATE_BENCHMARK(short) CREATE_BENCHMARK(float) + CREATE_BENCHMARK(rocprim::int128_t) + CREATE_BENCHMARK(rocprim::uint128_t) using custom_float2 = custom_type; using custom_double2 = custom_type; diff --git a/benchmark/benchmark_device_find_end.hpp b/benchmark/benchmark_device_find_end.hpp index e8ef9008c..214c70bbd 100644 --- a/benchmark/benchmark_device_find_end.hpp +++ b/benchmark/benchmark_device_find_end.hpp @@ -32,13 +32,15 @@ #include // rocPRIM +#include #include +#include +#include +#include #include #include -#include - template struct device_find_end_benchmark : public config_autotune_interface { @@ -87,7 +89,7 @@ struct device_find_end_benchmark : public config_autotune_interface { // Repeating similar pattern without early exits. keys_input[0] = 0; - for(size_t i = 0; i < size; i++) + for(size_t i = 0; i < size; ++i) { input[i] = keys_input[i % key_size]; } @@ -134,7 +136,7 @@ struct device_find_end_benchmark : public config_autotune_interface HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { HIP_CHECK(rocprim::find_end(d_temporary_storage, temporary_storage_bytes, @@ -159,7 +161,7 @@ struct device_find_end_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { HIP_CHECK(rocprim::find_end(d_temporary_storage, temporary_storage_bytes, diff --git a/benchmark/benchmark_device_find_first_of.cpp b/benchmark/benchmark_device_find_first_of.cpp index e48a603df..7eefab134 100644 --- a/benchmark/benchmark_device_find_first_of.cpp +++ b/benchmark/benchmark_device_find_first_of.cpp @@ -32,8 +32,16 @@ // HIP API #include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif + #include #include +#include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif #ifndef DEFAULT_BYTES constexpr size_t DEFAULT_BYTES = size_t{1} << 27; // 128 MiB @@ -120,6 +128,8 @@ int main(int argc, char* argv[]) CREATE_BENCHMARK(float) CREATE_BENCHMARK(int64_t) CREATE_BENCHMARK(double) + CREATE_BENCHMARK(rocprim::int128_t) + CREATE_BENCHMARK(rocprim::uint128_t) using custom_int2 = custom_type; using custom_longlong_double = custom_type; diff --git a/benchmark/benchmark_device_find_first_of.parallel.cpp.in b/benchmark/benchmark_device_find_first_of.parallel.cpp.in index 230f6640c..5bf3acfe0 100644 --- a/benchmark/benchmark_device_find_first_of.parallel.cpp.in +++ b/benchmark/benchmark_device_find_first_of.parallel.cpp.in @@ -23,6 +23,10 @@ #include "benchmark_device_find_first_of.parallel.hpp" #include "benchmark_utils.hpp" +#include + +#include + namespace { auto benchmarks = config_autotune_register::create_bulk( diff --git a/benchmark/benchmark_device_find_first_of.parallel.hpp b/benchmark/benchmark_device_find_first_of.parallel.hpp index 5681fe69e..82fbbe4eb 100644 --- a/benchmark/benchmark_device_find_first_of.parallel.hpp +++ b/benchmark/benchmark_device_find_first_of.parallel.hpp @@ -32,9 +32,14 @@ #include // rocPRIM +#include +#include #include +#include +#include #include +#include #include #include @@ -169,7 +174,7 @@ struct device_find_first_of_benchmark : public config_autotune_interface HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { for(size_t fi = 0; fi < first_occurrences.size(); ++fi) { @@ -191,7 +196,7 @@ struct device_find_first_of_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { for(size_t fi = 0; fi < first_occurrences.size(); ++fi) { diff --git a/benchmark/benchmark_device_histogram.cpp b/benchmark/benchmark_device_histogram.cpp index 4e7664ffb..01e26449e 100644 --- a/benchmark/benchmark_device_histogram.cpp +++ b/benchmark/benchmark_device_histogram.cpp @@ -30,24 +30,21 @@ #include // HIP API -#include #include // rocPRIM #include +#include -#include -#include -#include +#include #include +#include #include #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif -namespace rp = rocprim; - const unsigned int batch_size = 10; const unsigned int warmup_size = 5; @@ -66,7 +63,7 @@ int get_entropy_percents(int entropy_reduction) const int entropy_reductions[] = {0, 2, 4, 6}; -template +template void run_even_benchmark(benchmark::State& state, size_t bytes, const managed_seed&, @@ -96,33 +93,33 @@ void run_even_benchmark(benchmark::State& state, void* d_temporary_storage = nullptr; size_t temporary_storage_bytes = 0; - HIP_CHECK(rp::histogram_even(d_temporary_storage, - temporary_storage_bytes, - d_input, - size, - d_histogram, - bins + 1, - lower_level, - upper_level, - stream, - false)); + HIP_CHECK(rocprim::histogram_even(d_temporary_storage, + temporary_storage_bytes, + d_input, + size, + d_histogram, + bins + 1, + lower_level, + upper_level, + stream, + false)); HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { - HIP_CHECK(rp::histogram_even(d_temporary_storage, - temporary_storage_bytes, - d_input, - size, - d_histogram, - bins + 1, - lower_level, - upper_level, - stream, - false)); + HIP_CHECK(rocprim::histogram_even(d_temporary_storage, + temporary_storage_bytes, + d_input, + size, + d_histogram, + bins + 1, + lower_level, + upper_level, + stream, + false)); } HIP_CHECK(hipDeviceSynchronize()); @@ -136,18 +133,18 @@ void run_even_benchmark(benchmark::State& state, // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { - HIP_CHECK(rp::histogram_even(d_temporary_storage, - temporary_storage_bytes, - d_input, - size, - d_histogram, - bins + 1, - lower_level, - upper_level, - stream, - false)); + HIP_CHECK(rocprim::histogram_even(d_temporary_storage, + temporary_storage_bytes, + d_input, + size, + d_histogram, + bins + 1, + lower_level, + upper_level, + stream, + false)); } // Record stop event and wait until it completes @@ -171,7 +168,7 @@ void run_even_benchmark(benchmark::State& state, HIP_CHECK(hipFree(d_histogram)); } -template +template void run_multi_even_benchmark(benchmark::State& state, size_t bytes, const managed_seed&, @@ -190,7 +187,7 @@ void run_multi_even_benchmark(benchmark::State& state, unsigned int num_levels[ActiveChannels]; level_type lower_level[ActiveChannels]; level_type upper_level[ActiveChannels]; - for(unsigned int channel = 0; channel < ActiveChannels; channel++) + for(unsigned int channel = 0; channel < ActiveChannels; ++channel) { lower_level[channel] = 0; upper_level[channel] = bins * scale; @@ -204,7 +201,7 @@ void run_multi_even_benchmark(benchmark::State& state, T* d_input; counter_type* d_histogram[ActiveChannels]; HIP_CHECK(hipMalloc(&d_input, size * Channels * sizeof(T))); - for(unsigned int channel = 0; channel < ActiveChannels; channel++) + for(unsigned int channel = 0; channel < ActiveChannels; ++channel) { HIP_CHECK(hipMalloc(&d_histogram[channel], bins * sizeof(counter_type))); } @@ -212,33 +209,33 @@ void run_multi_even_benchmark(benchmark::State& state, void* d_temporary_storage = nullptr; size_t temporary_storage_bytes = 0; - HIP_CHECK((rp::multi_histogram_even(d_temporary_storage, - temporary_storage_bytes, - d_input, - size, - d_histogram, - num_levels, - lower_level, - upper_level, - stream, - false))); + HIP_CHECK((rocprim::multi_histogram_even(d_temporary_storage, + temporary_storage_bytes, + d_input, + size, + d_histogram, + num_levels, + lower_level, + upper_level, + stream, + false))); HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { - HIP_CHECK((rp::multi_histogram_even(d_temporary_storage, - temporary_storage_bytes, - d_input, - size, - d_histogram, - num_levels, - lower_level, - upper_level, - stream, - false))); + HIP_CHECK((rocprim::multi_histogram_even(d_temporary_storage, + temporary_storage_bytes, + d_input, + size, + d_histogram, + num_levels, + lower_level, + upper_level, + stream, + false))); } HIP_CHECK(hipDeviceSynchronize()); @@ -252,18 +249,19 @@ void run_multi_even_benchmark(benchmark::State& state, // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { - HIP_CHECK((rp::multi_histogram_even(d_temporary_storage, - temporary_storage_bytes, - d_input, - size, - d_histogram, - num_levels, - lower_level, - upper_level, - stream, - false))); + HIP_CHECK( + (rocprim::multi_histogram_even(d_temporary_storage, + temporary_storage_bytes, + d_input, + size, + d_histogram, + num_levels, + lower_level, + upper_level, + stream, + false))); } // Record stop event and wait until it completes @@ -284,15 +282,18 @@ void run_multi_even_benchmark(benchmark::State& state, HIP_CHECK(hipFree(d_temporary_storage)); HIP_CHECK(hipFree(d_input)); - for(unsigned int channel = 0; channel < ActiveChannels; channel++) + for(unsigned int channel = 0; channel < ActiveChannels; ++channel) { HIP_CHECK(hipFree(d_histogram[channel])); } } -template -void run_range_benchmark( - benchmark::State& state, size_t bytes, const managed_seed& seed, hipStream_t stream, size_t bins) +template +void run_range_benchmark(benchmark::State& state, + size_t bytes, + const managed_seed& seed, + hipStream_t stream, + size_t bins) { // Calculate the number of elements size_t size = bytes / sizeof(T); @@ -307,7 +308,7 @@ void run_range_benchmark( = get_random_data(size, random_range.first, random_range.second, seed.get_0()); std::vector levels(bins + 1); - for(size_t i = 0; i < levels.size(); i++) + for(size_t i = 0; i < levels.size(); ++i) { levels[i] = static_cast(i); } @@ -324,31 +325,31 @@ void run_range_benchmark( void* d_temporary_storage = nullptr; size_t temporary_storage_bytes = 0; - HIP_CHECK(rp::histogram_range(d_temporary_storage, - temporary_storage_bytes, - d_input, - size, - d_histogram, - bins + 1, - d_levels, - stream, - false)); + HIP_CHECK(rocprim::histogram_range(d_temporary_storage, + temporary_storage_bytes, + d_input, + size, + d_histogram, + bins + 1, + d_levels, + stream, + false)); HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { - HIP_CHECK(rp::histogram_range(d_temporary_storage, - temporary_storage_bytes, - d_input, - size, - d_histogram, - bins + 1, - d_levels, - stream, - false)); + HIP_CHECK(rocprim::histogram_range(d_temporary_storage, + temporary_storage_bytes, + d_input, + size, + d_histogram, + bins + 1, + d_levels, + stream, + false)); } HIP_CHECK(hipDeviceSynchronize()); @@ -362,17 +363,17 @@ void run_range_benchmark( // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { - HIP_CHECK(rp::histogram_range(d_temporary_storage, - temporary_storage_bytes, - d_input, - size, - d_histogram, - bins + 1, - d_levels, - stream, - false)); + HIP_CHECK(rocprim::histogram_range(d_temporary_storage, + temporary_storage_bytes, + d_input, + size, + d_histogram, + bins + 1, + d_levels, + stream, + false)); } // Record stop event and wait until it completes @@ -397,9 +398,12 @@ void run_range_benchmark( HIP_CHECK(hipFree(d_histogram)); } -template -void run_multi_range_benchmark( - benchmark::State& state, size_t bytes, const managed_seed& seed, hipStream_t stream, size_t bins) +template +void run_multi_range_benchmark(benchmark::State& state, + size_t bytes, + const managed_seed& seed, + hipStream_t stream, + size_t bins) { // Calculate the number of elements size_t size = bytes / sizeof(T); @@ -411,10 +415,10 @@ void run_multi_range_benchmark( const int num_levels_channel = bins + 1; unsigned int num_levels[ActiveChannels]; std::vector levels[ActiveChannels]; - for(unsigned int channel = 0; channel < ActiveChannels; channel++) + for(unsigned int channel = 0; channel < ActiveChannels; ++channel) { levels[channel].resize(num_levels_channel); - for(size_t i = 0; i < levels[channel].size(); i++) + for(size_t i = 0; i < levels[channel].size(); ++i) { levels[channel][i] = static_cast(i); } @@ -432,14 +436,14 @@ void run_multi_range_benchmark( level_type* d_levels[ActiveChannels]; counter_type* d_histogram[ActiveChannels]; HIP_CHECK(hipMalloc(&d_input, size * Channels * sizeof(T))); - for(unsigned int channel = 0; channel < ActiveChannels; channel++) + for(unsigned int channel = 0; channel < ActiveChannels; ++channel) { HIP_CHECK(hipMalloc(&d_levels[channel], num_levels_channel * sizeof(level_type))); HIP_CHECK(hipMalloc(&d_histogram[channel], size * sizeof(counter_type))); } HIP_CHECK(hipMemcpy(d_input, input.data(), size * Channels * sizeof(T), hipMemcpyHostToDevice)); - for(unsigned int channel = 0; channel < ActiveChannels; channel++) + for(unsigned int channel = 0; channel < ActiveChannels; ++channel) { HIP_CHECK(hipMemcpy(d_levels[channel], levels[channel].data(), @@ -449,31 +453,31 @@ void run_multi_range_benchmark( void* d_temporary_storage = nullptr; size_t temporary_storage_bytes = 0; - HIP_CHECK((rp::multi_histogram_range(d_temporary_storage, - temporary_storage_bytes, - d_input, - size, - d_histogram, - num_levels, - d_levels, - stream, - false))); + HIP_CHECK((rocprim::multi_histogram_range(d_temporary_storage, + temporary_storage_bytes, + d_input, + size, + d_histogram, + num_levels, + d_levels, + stream, + false))); HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { - HIP_CHECK((rp::multi_histogram_range(d_temporary_storage, - temporary_storage_bytes, - d_input, - size, - d_histogram, - num_levels, - d_levels, - stream, - false))); + HIP_CHECK((rocprim::multi_histogram_range(d_temporary_storage, + temporary_storage_bytes, + d_input, + size, + d_histogram, + num_levels, + d_levels, + stream, + false))); } HIP_CHECK(hipDeviceSynchronize()); @@ -487,17 +491,18 @@ void run_multi_range_benchmark( // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { - HIP_CHECK((rp::multi_histogram_range(d_temporary_storage, - temporary_storage_bytes, - d_input, - size, - d_histogram, - num_levels, - d_levels, - stream, - false))); + HIP_CHECK( + (rocprim::multi_histogram_range(d_temporary_storage, + temporary_storage_bytes, + d_input, + size, + d_histogram, + num_levels, + d_levels, + stream, + false))); } // Record stop event and wait until it completes @@ -518,7 +523,7 @@ void run_multi_range_benchmark( HIP_CHECK(hipFree(d_temporary_storage)); HIP_CHECK(hipFree(d_input)); - for(unsigned int channel = 0; channel < ActiveChannels; channel++) + for(unsigned int channel = 0; channel < ActiveChannels; ++channel) { HIP_CHECK(hipFree(d_levels[channel])); HIP_CHECK(hipFree(d_histogram[channel])); @@ -555,6 +560,10 @@ void add_even_benchmarks(std::vector& benchmark BENCHMARK_EVEN_TYPE(benchmarks, double, 1234); BENCHMARK_EVEN_TYPE(benchmarks, float, 1234); BENCHMARK_EVEN_TYPE(benchmarks, rocprim::half, 5); + CREATE_EVEN_BENCHMARK(benchmarks, rocprim::int128_t, 16, 16); + CREATE_EVEN_BENCHMARK(benchmarks, rocprim::int128_t, 256, 1); + CREATE_EVEN_BENCHMARK(benchmarks, rocprim::uint128_t, 16, 16); + CREATE_EVEN_BENCHMARK(benchmarks, rocprim::uint128_t, 256, 1); }; } @@ -569,7 +578,7 @@ void add_even_benchmarks(std::vector& benchmark [=](benchmark::State& state) \ { \ run_multi_even_benchmark(state, \ - bytes, \ + bytes, \ seed, \ stream, \ BINS, \ @@ -592,13 +601,16 @@ void add_multi_even_benchmarks(std::vector& ben { for(int entropy_reduction : entropy_reductions) { - std::vector bs = { - BENCHMARK_MULTI_EVEN_TYPE(4, 4, int, 1234), - BENCHMARK_MULTI_EVEN_TYPE(4, 3, short, 5), - CREATE_MULTI_EVEN_BENCHMARK(4, 3, unsigned char, 16, 16), - CREATE_MULTI_EVEN_BENCHMARK(4, 3, unsigned char, 256, 1), - BENCHMARK_MULTI_EVEN_TYPE(3, 3, float, 1234), - }; + std::vector bs + = {BENCHMARK_MULTI_EVEN_TYPE(4, 4, int, 1234), + BENCHMARK_MULTI_EVEN_TYPE(4, 3, short, 5), + CREATE_MULTI_EVEN_BENCHMARK(4, 3, unsigned char, 16, 16), + CREATE_MULTI_EVEN_BENCHMARK(4, 3, unsigned char, 256, 1), + BENCHMARK_MULTI_EVEN_TYPE(3, 3, float, 1234), + CREATE_MULTI_EVEN_BENCHMARK(4, 3, rocprim::int128_t, 16, 16), + CREATE_MULTI_EVEN_BENCHMARK(4, 3, rocprim::int128_t, 256, 1), + CREATE_MULTI_EVEN_BENCHMARK(4, 3, rocprim::uint128_t, 16, 16), + CREATE_MULTI_EVEN_BENCHMARK(4, 3, rocprim::uint128_t, 256, 1)}; benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); }; } @@ -608,7 +620,8 @@ void add_multi_even_benchmarks(std::vector& ben bench_naming::format_name("{lvl:device,algo:histogram_range,value_type:" #T ",bins:" \ + std::to_string(BINS) + ",cfg:default_config}") \ .c_str(), \ - [=](benchmark::State& state) { run_range_benchmark(state, bytes, seed, stream, BINS); }) + [=](benchmark::State& state) \ + { run_range_benchmark(state, bytes, seed, stream, BINS); }) // clang-format off #define BENCHMARK_RANGE_TYPE(T) \ @@ -623,16 +636,19 @@ void add_range_benchmarks(std::vector& benchmar const managed_seed& seed, hipStream_t stream) { - std::vector bs = { - BENCHMARK_RANGE_TYPE(long long), - BENCHMARK_RANGE_TYPE(int), - BENCHMARK_RANGE_TYPE(short), - CREATE_RANGE_BENCHMARK(unsigned char, 16), - CREATE_RANGE_BENCHMARK(unsigned char, 256), - BENCHMARK_RANGE_TYPE(double), - BENCHMARK_RANGE_TYPE(float), - BENCHMARK_RANGE_TYPE(rocprim::half), - }; + std::vector bs + = {BENCHMARK_RANGE_TYPE(long long), + BENCHMARK_RANGE_TYPE(int), + BENCHMARK_RANGE_TYPE(short), + CREATE_RANGE_BENCHMARK(unsigned char, 16), + CREATE_RANGE_BENCHMARK(unsigned char, 256), + BENCHMARK_RANGE_TYPE(double), + BENCHMARK_RANGE_TYPE(float), + BENCHMARK_RANGE_TYPE(rocprim::half), + CREATE_RANGE_BENCHMARK(rocprim::int128_t, 16), + CREATE_RANGE_BENCHMARK(rocprim::int128_t, 256), + CREATE_RANGE_BENCHMARK(rocprim::uint128_t, 16), + CREATE_RANGE_BENCHMARK(rocprim::uint128_t, 256)}; benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); } @@ -645,7 +661,7 @@ void add_range_benchmarks(std::vector& benchmar .c_str(), \ [=](benchmark::State& state) { \ run_multi_range_benchmark(state, \ - bytes, \ + bytes, \ seed, \ stream, \ BINS); \ @@ -664,14 +680,17 @@ void add_multi_range_benchmarks(std::vector& be const managed_seed& seed, hipStream_t stream) { - std::vector bs = { - BENCHMARK_MULTI_RANGE_TYPE(4, 4, int), - BENCHMARK_MULTI_RANGE_TYPE(4, 3, short), - CREATE_MULTI_RANGE_BENCHMARK(4, 3, unsigned char, 16), - CREATE_MULTI_RANGE_BENCHMARK(4, 3, unsigned char, 256), - BENCHMARK_MULTI_RANGE_TYPE(3, 3, float), - BENCHMARK_MULTI_RANGE_TYPE(2, 2, double), - }; + std::vector bs + = {BENCHMARK_MULTI_RANGE_TYPE(4, 4, int), + BENCHMARK_MULTI_RANGE_TYPE(4, 3, short), + CREATE_MULTI_RANGE_BENCHMARK(4, 3, unsigned char, 16), + CREATE_MULTI_RANGE_BENCHMARK(4, 3, unsigned char, 256), + BENCHMARK_MULTI_RANGE_TYPE(3, 3, float), + BENCHMARK_MULTI_RANGE_TYPE(2, 2, double), + CREATE_MULTI_RANGE_BENCHMARK(4, 3, rocprim::int128_t, 16), + CREATE_MULTI_RANGE_BENCHMARK(4, 3, rocprim::int128_t, 256), + CREATE_MULTI_RANGE_BENCHMARK(4, 3, rocprim::uint128_t, 16), + CREATE_MULTI_RANGE_BENCHMARK(4, 3, rocprim::uint128_t, 256)}; benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); } @@ -700,7 +719,7 @@ int main(int argc, char* argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); + const size_t bytes = parser.get("size"); const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); diff --git a/benchmark/benchmark_device_histogram.parallel.cpp.in b/benchmark/benchmark_device_histogram.parallel.cpp.in index fd60d57e6..bf45cd3c1 100644 --- a/benchmark/benchmark_device_histogram.parallel.cpp.in +++ b/benchmark/benchmark_device_histogram.parallel.cpp.in @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2023-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,11 +20,13 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include - #include "benchmark_utils.hpp" #include "benchmark_device_histogram.parallel.hpp" +#include + +#include + namespace { auto benchmarks = config_autotune_register::create_bulk( device_histogram_benchmark_generator<@DataType@, @BlockSize@>::create); diff --git a/benchmark/benchmark_device_histogram.parallel.hpp b/benchmark/benchmark_device_histogram.parallel.hpp index ff8406c1a..be9e8c8ea 100644 --- a/benchmark/benchmark_device_histogram.parallel.hpp +++ b/benchmark/benchmark_device_histogram.parallel.hpp @@ -32,17 +32,19 @@ #include // rocPRIM +#include #include #include +#include +#include +#include +#include #include -#include +#include #include -#include -#include - -template +template std::vector generate(size_t size, int entropy_reduction, int lower_level, int upper_level) { if(entropy_reduction >= 5) @@ -52,9 +54,9 @@ std::vector generate(size_t size, int entropy_reduction, int lower_level, int const size_t max_random_size = 1024 * 1024 + 4321; - const unsigned int seed = 123; - engine_type gen(seed); - std::vector data(size); + const unsigned int seed = 123; + engine_type gen(seed); + std::vector data(size); std::generate(data.begin(), data.begin() + std::min(size, max_random_size), [&]() @@ -63,7 +65,7 @@ std::vector generate(size_t size, int entropy_reduction, int lower_level, int // "An Improved Supercomputer Sorting Benchmark", 1992 // Kurt Thearling & Stephen Smith auto v = gen(); - for(int e = 0; e < entropy_reduction; e++) + for(int e = 0; e < entropy_reduction; ++e) { v &= gen(); } @@ -93,6 +95,7 @@ class input_cache { HIP_CHECK(hipFree(i.second)); } + total_cache_size = 0; cache.clear(); } @@ -105,6 +108,7 @@ class input_cache size_t size, F gen) { + static constexpr short max_default_bytes_count = 176; if(this->main_key != main_key) { // The main key (for example, data type) has been changed, clear the cache @@ -121,7 +125,18 @@ class input_cache // Generate a new buffer std::vector data = gen(); T* d_buffer; - HIP_CHECK(hipMalloc(&d_buffer, size * sizeof(T))); + if(total_cache_size >= max_default_bytes_count) + { + auto iter = cache.end(); + --iter; + d_buffer = reinterpret_cast(iter->second); + cache.erase(iter); + } + else + { + HIP_CHECK(hipMalloc(&d_buffer, size * sizeof(T))); + total_cache_size += sizeof(T); + } HIP_CHECK(hipMemcpy(d_buffer, data.data(), size * sizeof(T), hipMemcpyHostToDevice)); cache[additional_key] = d_buffer; return d_buffer; @@ -136,6 +151,7 @@ class input_cache private: std::string main_key; std::map cache; + short total_cache_size = 0; }; template @@ -185,10 +201,19 @@ struct device_histogram_benchmark : public config_autotune_interface struct case_data { + unsigned int bins; + int entropy_reduction; level_type lower_level[ActiveChannels]; level_type upper_level[ActiveChannels]; unsigned int num_levels[ActiveChannels]; - T* d_input; + T* get_d_input(size_t full_size) + { + return input_cache::instance().get_or_generate( + std::string(Traits::name()), + std::to_string(bins) + "_" + std::to_string(entropy_reduction), + full_size, + [&]() { return generate(full_size, entropy_reduction, 0, bins); }); + }; }; const std::size_t size = full_size / Channels; @@ -199,23 +224,18 @@ struct device_histogram_benchmark : public config_autotune_interface unsigned int max_bins = 0; std::vector cases_data; - for(auto& bins : cases) + for(const auto& bins : cases) { for(int entropy_reduction : {0, 2, 4, 6}) { - case_data data; + case_data data = {.bins = bins, .entropy_reduction = entropy_reduction}; // Reuse inputs for the same sample type. This autotune uses multipe inputs for all // combinations of bins and entropy, but the inputs do not depend on autotuned // params (bs, ipt, shared_impl_max_bins) and can be reused saving time needed for // generating and copying to device. - data.d_input = input_cache::instance().get_or_generate( - std::string(Traits::name()), - std::to_string(bins) + "_" + std::to_string(entropy_reduction), - full_size, - [&]() { return generate(full_size, entropy_reduction, 0, bins); }); - for(unsigned int channel = 0; channel < ActiveChannels; channel++) + for(unsigned int channel = 0; channel < ActiveChannels; ++channel) { data.lower_level[channel] = 0; data.upper_level[channel] = bins; @@ -227,7 +247,7 @@ struct device_histogram_benchmark : public config_autotune_interface HIP_CHECK((rocprim::multi_histogram_even( d_temporary_storage, current_temporary_storage_bytes, - data.d_input, + data.get_d_input(full_size), size, d_histogram, data.num_levels, @@ -243,21 +263,21 @@ struct device_histogram_benchmark : public config_autotune_interface } HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - for(unsigned int channel = 0; channel < ActiveChannels; channel++) + for(unsigned int channel = 0; channel < ActiveChannels; ++channel) { HIP_CHECK(hipMalloc(&d_histogram[channel], max_bins * sizeof(counter_type))); } HIP_CHECK(hipDeviceSynchronize()); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { for(auto& data : cases_data) { HIP_CHECK((rocprim::multi_histogram_even( d_temporary_storage, temporary_storage_bytes, - data.d_input, + data.get_d_input(full_size), size, d_histogram, data.num_levels, @@ -276,17 +296,20 @@ struct device_histogram_benchmark : public config_autotune_interface for(auto _ : state) { - // Record start event - HIP_CHECK(hipEventRecord(start, stream)); - + float elapsed_mseconds = 0; for(auto& data : cases_data) { - for(size_t i = 0; i < batch_size; i++) + T* d_input = data.get_d_input(full_size); + float partial_elapsed_mseconds; + // Record start event + HIP_CHECK(hipEventRecord(start, stream)); + + for(size_t i = 0; i < batch_size; ++i) { HIP_CHECK((rocprim::multi_histogram_even( d_temporary_storage, temporary_storage_bytes, - data.d_input, + d_input, size, d_histogram, data.num_levels, @@ -295,14 +318,14 @@ struct device_histogram_benchmark : public config_autotune_interface stream, false))); } - } - // Record stop event and wait until it completes - HIP_CHECK(hipEventRecord(stop, stream)); - HIP_CHECK(hipEventSynchronize(stop)); + // Record stop event and wait until it completes + HIP_CHECK(hipEventRecord(stop, stream)); + HIP_CHECK(hipEventSynchronize(stop)); - float elapsed_mseconds; - HIP_CHECK(hipEventElapsedTime(&elapsed_mseconds, start, stop)); + HIP_CHECK(hipEventElapsedTime(&partial_elapsed_mseconds, start, stop)); + elapsed_mseconds += partial_elapsed_mseconds; + } state.SetIterationTime(elapsed_mseconds / 1000); } @@ -316,7 +339,7 @@ struct device_histogram_benchmark : public config_autotune_interface * Channels); HIP_CHECK(hipFree(d_temporary_storage)); - for(unsigned int channel = 0; channel < ActiveChannels; channel++) + for(unsigned int channel = 0; channel < ActiveChannels; ++channel) { HIP_CHECK(hipFree(d_histogram[channel])); } diff --git a/benchmark/benchmark_device_memory.cpp b/benchmark/benchmark_device_memory.cpp index 061804548..720e35ced 100644 --- a/benchmark/benchmark_device_memory.cpp +++ b/benchmark/benchmark_device_memory.cpp @@ -28,13 +28,17 @@ #include // rocPRIM #include +#include #include +#include +#include +#include -#include +#include +#include #include - -#include -#include +#include +#include enum memory_operation_method { @@ -54,20 +58,19 @@ enum kernel_operation atomics_inter_warp_collision, }; -template< - kernel_operation Operation, - class T, - unsigned int ItemsPerThread, - unsigned int BlockSize = 0 -> +template struct operation; // no operation -template +template struct operation { - ROCPRIM_HOST_DEVICE inline - void operator()(T (&)[ItemsPerThread], void* = nullptr, unsigned int = 0, T* = nullptr) const + ROCPRIM_HOST_DEVICE + inline void + operator()(T (&)[ItemsPerThread], void* = nullptr, unsigned int = 0, T* = nullptr) const { // No operation } @@ -76,23 +79,25 @@ struct operation #define repeats 30 // custom operation -template +template struct operation { - ROCPRIM_HOST_DEVICE inline - void operator()(T (&input)[ItemsPerThread], - void* shared_storage = nullptr, unsigned int shared_storage_size = 0, - T* global_mem_output = nullptr) const + ROCPRIM_HOST_DEVICE + inline void + operator()(T (&input)[ItemsPerThread], + void* shared_storage = nullptr, + unsigned int shared_storage_size = 0, + T* global_mem_output = nullptr) const { - (void) shared_storage; - (void) shared_storage_size; - (void) global_mem_output; + (void)shared_storage; + (void)shared_storage_size; + (void)global_mem_output; ROCPRIM_UNROLL - for(unsigned int i = 0; i < ItemsPerThread; i++) + for(unsigned int i = 0; i < ItemsPerThread; ++i) { input[i] = input[i] + 666; ROCPRIM_UNROLL - for(unsigned int j = 0; j < repeats; j++) + for(unsigned int j = 0; j < repeats; ++j) { input[i] = input[i] * (input[j % ItemsPerThread]); } @@ -101,53 +106,57 @@ struct operation }; // block scan -template +template struct operation { - ROCPRIM_HOST_DEVICE inline - void operator()(T (&input)[ItemsPerThread], - void* shared_storage = nullptr, unsigned int shared_storage_size = 0, - T* global_mem_output = nullptr) const + ROCPRIM_HOST_DEVICE + inline void + operator()(T (&input)[ItemsPerThread], + void* shared_storage = nullptr, + unsigned int shared_storage_size = 0, + T* global_mem_output = nullptr) const { - (void) global_mem_output; - using block_scan_type = typename rocprim::block_scan< - T, BlockSize, rocprim::block_scan_algorithm::using_warp_scan>; + (void)global_mem_output; + using block_scan_type = typename rocprim:: + block_scan; block_scan_type bscan; // when using vectorized or striped functions // NOTE: This is not safe but it is the easiest way to prevent code repetition - if(shared_storage == nullptr || - shared_storage_size < sizeof(typename block_scan_type::storage_type)) + if(shared_storage == nullptr + || shared_storage_size < sizeof(typename block_scan_type::storage_type)) { __shared__ typename block_scan_type::storage_type storage; shared_storage = &storage; } bscan.inclusive_scan( - input, input, - *(reinterpret_cast(shared_storage)) - ); + input, + input, + *(reinterpret_cast(shared_storage))); __syncthreads(); } }; // atomics_no_collision -template +template struct operation { - ROCPRIM_HOST_DEVICE inline - void operator()(T (&input)[ItemsPerThread], - void* shared_storage = nullptr, unsigned int shared_storage_size = 0, - T* global_mem_output = nullptr) + ROCPRIM_HOST_DEVICE + inline void + operator()(T (&input)[ItemsPerThread], + void* shared_storage = nullptr, + unsigned int shared_storage_size = 0, + T* global_mem_output = nullptr) { - (void) shared_storage; - (void) shared_storage_size; - (void) input; - unsigned int index = threadIdx.x * ItemsPerThread + - blockIdx.x * blockDim.x * ItemsPerThread; + (void)shared_storage; + (void)shared_storage_size; + (void)input; + unsigned int index + = threadIdx.x * ItemsPerThread + blockIdx.x * blockDim.x * ItemsPerThread; ROCPRIM_UNROLL - for(unsigned int i = 0; i < ItemsPerThread; i++) + for(unsigned int i = 0; i < ItemsPerThread; ++i) { atomicAdd(&global_mem_output[index + i], T(666)); } @@ -155,21 +164,23 @@ struct operation }; // atomics_inter_block_collision -template +template struct operation { - ROCPRIM_HOST_DEVICE inline - void operator()(T (&input)[ItemsPerThread], - void* shared_storage = nullptr, unsigned int shared_storage_size = 0, - T* global_mem_output = nullptr) + ROCPRIM_HOST_DEVICE + inline void + operator()(T (&input)[ItemsPerThread], + void* shared_storage = nullptr, + unsigned int shared_storage_size = 0, + T* global_mem_output = nullptr) { - (void) shared_storage; - (void) shared_storage_size; - (void) input; + (void)shared_storage; + (void)shared_storage_size; + (void)input; unsigned int index = (threadIdx.x % rocprim::device_warp_size()) * ItemsPerThread + blockIdx.x * blockDim.x * ItemsPerThread; ROCPRIM_UNROLL - for(unsigned int i = 0; i < ItemsPerThread; i++) + for(unsigned int i = 0; i < ItemsPerThread; ++i) { atomicAdd(&global_mem_output[index + i], T(666)); } @@ -177,20 +188,22 @@ struct operation }; // atomics_inter_block_collision -template +template struct operation { - ROCPRIM_HOST_DEVICE inline - void operator()(T (&input)[ItemsPerThread], - void* shared_storage = nullptr, unsigned int shared_storage_size = 0, - T* global_mem_output = nullptr) + ROCPRIM_HOST_DEVICE + inline void + operator()(T (&input)[ItemsPerThread], + void* shared_storage = nullptr, + unsigned int shared_storage_size = 0, + T* global_mem_output = nullptr) { - (void) shared_storage; - (void) shared_storage_size; - (void) input; + (void)shared_storage; + (void)shared_storage_size; + (void)input; unsigned int index = threadIdx.x * ItemsPerThread; ROCPRIM_UNROLL - for(unsigned int i = 0; i < ItemsPerThread; i++) + for(unsigned int i = 0; i < ItemsPerThread; ++i) { atomicAdd(&global_mem_output[index + i], T(666)); } @@ -198,32 +211,28 @@ struct operation }; // block_primitive_direct method base kernel -template< - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - memory_operation_method MemOp, - class CustomOp = - typename operation::value_type, - typename std::enable_if::type = 0 -> -__global__ -__launch_bounds__(BlockSize) +template::value_type, + typename std::enable_if::type = 0> +__global__ __launch_bounds__(BlockSize) void operation_kernel(T* input, T* output, CustomOp op) { constexpr unsigned int items_per_block = BlockSize * ItemsPerThread; - using block_load_type = typename rocprim::block_load< - T, BlockSize, ItemsPerThread, rocprim::block_load_method::block_load_direct>; - using block_store_type = typename rocprim::block_store< - T, BlockSize, ItemsPerThread, rocprim::block_store_method::block_store_direct>; + using block_load_type = typename rocprim:: + block_load; + using block_store_type = typename rocprim:: + block_store; - block_load_type load; + block_load_type load; block_store_type store; __shared__ union { - typename block_load_type::storage_type load; + typename block_load_type::storage_type load; typename block_store_type::storage_type store; } storage; @@ -237,82 +246,75 @@ void operation_kernel(T* input, T* output, CustomOp op) } // vectorized method base kernel -template< - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - memory_operation_method MemOp, - class CustomOp = - typename operation::value_type, - typename std::enable_if::type = 0 -> -__global__ -__launch_bounds__(BlockSize) +template::value_type, + typename std::enable_if::type = 0> +__global__ __launch_bounds__(BlockSize) void operation_kernel(T* input, T* output, CustomOp op) { constexpr unsigned int items_per_block = BlockSize * ItemsPerThread; - int offset = blockIdx.x * items_per_block; - T items[ItemsPerThread]; + int offset = blockIdx.x * items_per_block; + T items[ItemsPerThread]; - rocprim::block_load_direct_blocked_vectorized - (threadIdx.x, input + offset, items); + rocprim::block_load_direct_blocked_vectorized(threadIdx.x, + input + offset, + items); __syncthreads(); op(items, nullptr, 0, output); - rocprim::block_store_direct_blocked_vectorized - (threadIdx.x, output + offset, items); + rocprim::block_store_direct_blocked_vectorized(threadIdx.x, + output + offset, + items); } // striped method base kernel -template< - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - memory_operation_method MemOp, - class CustomOp = - typename operation::value_type, - typename std::enable_if::type = 0 -> -__global__ -__launch_bounds__(BlockSize) +template::value_type, + typename std::enable_if::type = 0> +__global__ __launch_bounds__(BlockSize) void operation_kernel(T* input, T* output, CustomOp op) { - const unsigned int lid = threadIdx.x; + const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; - T items[ItemsPerThread]; + T items[ItemsPerThread]; rocprim::block_load_direct_striped(lid, input + block_offset, items); op(items, nullptr, 0, output); rocprim::block_store_direct_striped(lid, output + block_offset, items); } // block_primitives_transpose method base kernel -template< - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - memory_operation_method MemOp, - class CustomOp = - typename operation::value_type, - typename std::enable_if::type = 0 -> -__global__ -__launch_bounds__(BlockSize) +template::value_type, + typename std::enable_if::type = 0> +__global__ __launch_bounds__(BlockSize) void operation_kernel(T* input, T* output, CustomOp op) { constexpr unsigned int items_per_block = BlockSize * ItemsPerThread; - using block_load_type = typename rocprim::block_load< - T, BlockSize, ItemsPerThread, rocprim::block_load_method::block_load_transpose>; - using block_store_type = typename rocprim::block_store< - T, BlockSize, ItemsPerThread, rocprim::block_store_method::block_store_transpose>; + using block_load_type = typename rocprim:: + block_load; + using block_store_type = + typename rocprim::block_store; - block_load_type load; + block_load_type load; block_store_type store; __shared__ union { - typename block_load_type::storage_type load; + typename block_load_type::storage_type load; typename block_store_type::storage_type store; } storage; @@ -325,7 +327,7 @@ void operation_kernel(T* input, T* output, CustomOp op) store.store(output + offset, items, storage.store); } -template::max(), seed.get_0()); - T * d_input; - T * d_output; + T* d_input; + T* d_output; HIP_CHECK(hipMalloc(reinterpret_cast(&d_input), size * sizeof(T))); HIP_CHECK(hipMalloc(reinterpret_cast(&d_output), size * sizeof(T))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - size * sizeof(T), - hipMemcpyHostToDevice - ) - ); + HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); HIP_CHECK(hipDeviceSynchronize()); operation selected_operation; // Warm-up - for(size_t i = 0; i < 10; i++) + for(size_t i = 0; i < 10; ++i) { - hipLaunchKernelGGL( - HIP_KERNEL_NAME(operation_kernel), - dim3(grid_size), dim3(BlockSize), 0, stream, - d_input, d_output, selected_operation - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(operation_kernel), + dim3(grid_size), + dim3(BlockSize), + 0, + stream, + d_input, + d_output, + selected_operation); } HIP_CHECK(hipDeviceSynchronize()); @@ -378,13 +377,17 @@ void run_benchmark(benchmark::State& state, // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { hipLaunchKernelGGL( HIP_KERNEL_NAME(operation_kernel), - dim3(grid_size), dim3(BlockSize), 0, stream, - d_input, d_output, selected_operation - ); + dim3(grid_size), + dim3(BlockSize), + 0, + stream, + d_input, + d_output, + selected_operation); } // Record stop event and wait until it completes @@ -407,7 +410,7 @@ void run_benchmark(benchmark::State& state, HIP_CHECK(hipFree(d_output)); } -template +template void run_benchmark_memcpy(benchmark::State& state, size_t size, const managed_seed&, @@ -417,12 +420,12 @@ void run_benchmark_memcpy(benchmark::State& state, // Note: since this benchmark only tests performance by memcpying between device buffers, // we don't really need to transfer data into these from the host - whatever happens // to be in device memory will do. - T * d_input; - T * d_output; + T* d_input; + T* d_output; HIP_CHECK(hipMalloc(reinterpret_cast(&d_input), size * sizeof(T))); HIP_CHECK(hipMalloc(reinterpret_cast(&d_output), size * sizeof(T))); // Warm-up - for(size_t i = 0; i < 10; i++) + for(size_t i = 0; i < 10; ++i) { HIP_CHECK(hipMemcpy(d_output, d_input, size * sizeof(T), hipMemcpyDeviceToDevice)); } @@ -439,7 +442,7 @@ void run_benchmark_memcpy(benchmark::State& state, // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { HIP_CHECK(hipMemcpy(d_output, d_input, size * sizeof(T), hipMemcpyDeviceToDevice)); } @@ -485,13 +488,13 @@ void run_benchmark_memcpy(benchmark::State& state, seed, \ stream) -template +template constexpr unsigned int megabytes(unsigned int size) { - return(size * (1024 * 1024 / sizeof(T))); + return (size * (1024 * 1024 / sizeof(T))); } -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { cli::Parser parser(argc, argv); parser.set_optional("trials", "trials", -1, "number of iterations"); @@ -517,54 +520,404 @@ int main(int argc, char *argv[]) benchmark::AddCustomContext("seed", seed_type); // Add benchmarks - std::vector benchmarks = - { + std::vector benchmarks = { // simple memory copy not running kernel CREATE_BENCHMARK_MEMCPY(int, megabytes(128)), // simple memory copy - CREATE_BENCHMARK(block_primitives_transpose, no_operation, int, megabytes(128), 128, 1), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, int, megabytes(128), 128, 2), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, int, megabytes(128), 128, 4), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, int, megabytes(128), 128, 8), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, int, megabytes(128), 128, 16), - - CREATE_BENCHMARK(block_primitives_transpose, no_operation, int, megabytes(128), 256, 1), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, int, megabytes(128), 256, 2), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, int, megabytes(128), 256, 4), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, int, megabytes(128), 256, 8), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, int, megabytes(128), 256, 16), - - CREATE_BENCHMARK(block_primitives_transpose, no_operation, int, megabytes(128), 512, 1), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, int, megabytes(128), 512, 2), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, int, megabytes(128), 512, 4), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, int, megabytes(128), 512, 8), - - CREATE_BENCHMARK(block_primitives_transpose, no_operation, int, megabytes(128), 1024, 1), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, int, megabytes(128), 1024, 2), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, int, megabytes(128), 1024, 4), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, int, megabytes(128), 1024, 8), - - CREATE_BENCHMARK(block_primitives_transpose, no_operation, uint64_t, megabytes(128), 128, 1), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, uint64_t, megabytes(128), 128, 2), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, uint64_t, megabytes(128), 128, 4), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, uint64_t, megabytes(128), 128, 8), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, uint64_t, megabytes(128), 128, 16), - - CREATE_BENCHMARK(block_primitives_transpose, no_operation, uint64_t, megabytes(128), 256, 1), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, uint64_t, megabytes(128), 256, 2), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, uint64_t, megabytes(128), 256, 4), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, uint64_t, megabytes(128), 256, 8), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, uint64_t, megabytes(128), 256, 16), - - CREATE_BENCHMARK(block_primitives_transpose, no_operation, uint64_t, megabytes(128), 512, 1), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, uint64_t, megabytes(128), 512, 2), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, uint64_t, megabytes(128), 512, 4), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, uint64_t, megabytes(128), 512, 8), - - CREATE_BENCHMARK(block_primitives_transpose, no_operation, uint64_t, megabytes(128), 1024, 1), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, uint64_t, megabytes(128), 1024, 2), - CREATE_BENCHMARK(block_primitives_transpose, no_operation, uint64_t, megabytes(128), 1024, 4), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + int, + megabytes(128), + 128, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + int, + megabytes(128), + 128, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + int, + megabytes(128), + 128, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + int, + megabytes(128), + 128, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + int, + megabytes(128), + 128, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + int, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + int, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + int, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + int, + megabytes(128), + 256, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + int, + megabytes(128), + 256, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + int, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + int, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + int, + megabytes(128), + 512, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + int, + megabytes(128), + 512, + 8), + + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + int, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + int, + megabytes(128), + 1024, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + int, + megabytes(128), + 1024, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + int, + megabytes(128), + 1024, + 8), + + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + uint64_t, + megabytes(128), + 128, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + uint64_t, + megabytes(128), + 128, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + uint64_t, + megabytes(128), + 128, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + uint64_t, + megabytes(128), + 128, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + uint64_t, + megabytes(128), + 128, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + uint64_t, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + uint64_t, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + uint64_t, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + uint64_t, + megabytes(128), + 256, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + uint64_t, + megabytes(128), + 256, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + uint64_t, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + uint64_t, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + uint64_t, + megabytes(128), + 512, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + uint64_t, + megabytes(128), + 512, + 8), + + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + uint64_t, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + uint64_t, + megabytes(128), + 1024, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + uint64_t, + megabytes(128), + 1024, + 4), + + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::int128_t, + megabytes(128), + 128, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::int128_t, + megabytes(128), + 128, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::int128_t, + megabytes(128), + 128, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::int128_t, + megabytes(128), + 128, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::int128_t, + megabytes(128), + 128, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::int128_t, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::int128_t, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::int128_t, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::int128_t, + megabytes(128), + 256, + 8), + + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::int128_t, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::int128_t, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::int128_t, + megabytes(128), + 512, + 4), + + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::int128_t, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::int128_t, + megabytes(128), + 1024, + 2), + + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::uint128_t, + megabytes(128), + 128, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::uint128_t, + megabytes(128), + 128, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::uint128_t, + megabytes(128), + 128, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::uint128_t, + megabytes(128), + 128, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::uint128_t, + megabytes(128), + 128, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::uint128_t, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::uint128_t, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::uint128_t, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::uint128_t, + megabytes(128), + 256, + 8), + + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::uint128_t, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::uint128_t, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::uint128_t, + megabytes(128), + 512, + 4), + + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::uint128_t, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + no_operation, + rocprim::uint128_t, + megabytes(128), + 1024, + 2), // simple memory copy using vector type CREATE_BENCHMARK(vectorized, no_operation, int, megabytes(128), 128, 1), @@ -611,6 +964,230 @@ int main(int argc, char *argv[]) CREATE_BENCHMARK(vectorized, no_operation, uint64_t, megabytes(128), 1024, 4), CREATE_BENCHMARK(vectorized, no_operation, uint64_t, megabytes(128), 1024, 8), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::int128_t, + megabytes(128), + 128, + 1), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::int128_t, + megabytes(128), + 128, + 2), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::int128_t, + megabytes(128), + 128, + 4), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::int128_t, + megabytes(128), + 128, + 8), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::int128_t, + megabytes(128), + 128, + 16), + + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::int128_t, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::int128_t, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::int128_t, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::int128_t, + megabytes(128), + 256, + 8), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::int128_t, + megabytes(128), + 256, + 16), + + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::int128_t, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::int128_t, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::int128_t, + megabytes(128), + 512, + 4), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::int128_t, + megabytes(128), + 512, + 8), + + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::int128_t, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::int128_t, + megabytes(128), + 1024, + 2), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::int128_t, + megabytes(128), + 1024, + 4), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::int128_t, + megabytes(128), + 1024, + 8), + + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::uint128_t, + megabytes(128), + 128, + 1), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::uint128_t, + megabytes(128), + 128, + 2), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::uint128_t, + megabytes(128), + 128, + 4), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::uint128_t, + megabytes(128), + 128, + 8), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::uint128_t, + megabytes(128), + 128, + 16), + + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::uint128_t, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::uint128_t, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::uint128_t, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::uint128_t, + megabytes(128), + 256, + 8), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::uint128_t, + megabytes(128), + 256, + 16), + + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::uint128_t, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::uint128_t, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::uint128_t, + megabytes(128), + 512, + 4), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::uint128_t, + megabytes(128), + 512, + 8), + + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::uint128_t, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::uint128_t, + megabytes(128), + 1024, + 2), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::uint128_t, + megabytes(128), + 1024, + 4), + CREATE_BENCHMARK(vectorized, + no_operation, + rocprim::uint128_t, + megabytes(128), + 1024, + 8), + // simple memory copy using striped CREATE_BENCHMARK(striped, no_operation, int, megabytes(128), 128, 1), CREATE_BENCHMARK(striped, no_operation, int, megabytes(128), 128, 2), @@ -656,6 +1233,230 @@ int main(int argc, char *argv[]) CREATE_BENCHMARK(striped, no_operation, uint64_t, megabytes(128), 1024, 4), CREATE_BENCHMARK(striped, no_operation, uint64_t, megabytes(128), 1024, 8), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::int128_t, + megabytes(128), + 128, + 1), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::int128_t, + megabytes(128), + 128, + 2), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::int128_t, + megabytes(128), + 128, + 4), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::int128_t, + megabytes(128), + 128, + 8), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::int128_t, + megabytes(128), + 128, + 16), + + CREATE_BENCHMARK(striped, + no_operation, + rocprim::int128_t, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::int128_t, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::int128_t, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::int128_t, + megabytes(128), + 256, + 8), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::int128_t, + megabytes(128), + 256, + 16), + + CREATE_BENCHMARK(striped, + no_operation, + rocprim::int128_t, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::int128_t, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::int128_t, + megabytes(128), + 512, + 4), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::int128_t, + megabytes(128), + 512, + 8), + + CREATE_BENCHMARK(striped, + no_operation, + rocprim::int128_t, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::int128_t, + megabytes(128), + 1024, + 2), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::int128_t, + megabytes(128), + 1024, + 4), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::int128_t, + megabytes(128), + 1024, + 8), + + CREATE_BENCHMARK(striped, + no_operation, + rocprim::uint128_t, + megabytes(128), + 128, + 1), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::uint128_t, + megabytes(128), + 128, + 2), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::uint128_t, + megabytes(128), + 128, + 4), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::uint128_t, + megabytes(128), + 128, + 8), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::uint128_t, + megabytes(128), + 128, + 16), + + CREATE_BENCHMARK(striped, + no_operation, + rocprim::uint128_t, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::uint128_t, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::uint128_t, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::uint128_t, + megabytes(128), + 256, + 8), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::uint128_t, + megabytes(128), + 256, + 16), + + CREATE_BENCHMARK(striped, + no_operation, + rocprim::uint128_t, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::uint128_t, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::uint128_t, + megabytes(128), + 512, + 4), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::uint128_t, + megabytes(128), + 512, + 8), + + CREATE_BENCHMARK(striped, + no_operation, + rocprim::uint128_t, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::uint128_t, + megabytes(128), + 1024, + 2), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::uint128_t, + megabytes(128), + 1024, + 4), + CREATE_BENCHMARK(striped, + no_operation, + rocprim::uint128_t, + megabytes(128), + 1024, + 8), + // block_scan CREATE_BENCHMARK(block_primitives_transpose, block_scan, int, megabytes(128), 128, 1), CREATE_BENCHMARK(block_primitives_transpose, block_scan, int, megabytes(128), 128, 2), @@ -680,63 +1481,474 @@ int main(int argc, char *argv[]) CREATE_BENCHMARK(block_primitives_transpose, block_scan, int, megabytes(128), 1024, 4), CREATE_BENCHMARK(block_primitives_transpose, block_scan, int, megabytes(128), 1024, 8), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, float, megabytes(128), 256, 1), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, float, megabytes(128), 256, 2), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, float, megabytes(128), 256, 4), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, float, megabytes(128), 256, 8), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, float, megabytes(128), 256, 16), - - CREATE_BENCHMARK(block_primitives_transpose, block_scan, float, megabytes(128), 512, 1), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, float, megabytes(128), 512, 2), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, float, megabytes(128), 512, 4), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, float, megabytes(128), 512, 8), - - CREATE_BENCHMARK(block_primitives_transpose, block_scan, float, megabytes(128), 1024, 1), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, float, megabytes(128), 1024, 2), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, float, megabytes(128), 1024, 4), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, float, megabytes(128), 1024, 8), - - CREATE_BENCHMARK(block_primitives_transpose, block_scan, double, megabytes(128), 128, 1), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, double, megabytes(128), 128, 2), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, double, megabytes(128), 128, 4), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, double, megabytes(128), 128, 8), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, double, megabytes(128), 128, 16), - - CREATE_BENCHMARK(block_primitives_transpose, block_scan, double, megabytes(128), 256, 1), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, double, megabytes(128), 256, 2), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, double, megabytes(128), 256, 4), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, double, megabytes(128), 256, 8), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, double, megabytes(128), 256, 16), - - CREATE_BENCHMARK(block_primitives_transpose, block_scan, double, megabytes(128), 512, 1), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, double, megabytes(128), 512, 2), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, double, megabytes(128), 512, 4), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, double, megabytes(128), 512, 8), - - CREATE_BENCHMARK(block_primitives_transpose, block_scan, double, megabytes(128), 1024, 1), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, double, megabytes(128), 1024, 2), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, double, megabytes(128), 1024, 4), - - CREATE_BENCHMARK(block_primitives_transpose, block_scan, uint64_t, megabytes(128), 128, 1), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, uint64_t, megabytes(128), 128, 2), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, uint64_t, megabytes(128), 128, 4), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, uint64_t, megabytes(128), 128, 8), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, uint64_t, megabytes(128), 128, 16), - - CREATE_BENCHMARK(block_primitives_transpose, block_scan, uint64_t, megabytes(128), 256, 1), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, uint64_t, megabytes(128), 256, 2), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, uint64_t, megabytes(128), 256, 4), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, uint64_t, megabytes(128), 256, 8), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, uint64_t, megabytes(128), 256, 16), - - CREATE_BENCHMARK(block_primitives_transpose, block_scan, uint64_t, megabytes(128), 512, 1), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, uint64_t, megabytes(128), 512, 2), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, uint64_t, megabytes(128), 512, 4), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, uint64_t, megabytes(128), 512, 8), - - CREATE_BENCHMARK(block_primitives_transpose, block_scan, uint64_t, megabytes(128), 1024, 1), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, uint64_t, megabytes(128), 1024, 2), - CREATE_BENCHMARK(block_primitives_transpose, block_scan, uint64_t, megabytes(128), 1024, 4), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + float, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + float, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + float, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + float, + megabytes(128), + 256, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + float, + megabytes(128), + 256, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + float, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + float, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + float, + megabytes(128), + 512, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + float, + megabytes(128), + 512, + 8), + + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + float, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + float, + megabytes(128), + 1024, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + float, + megabytes(128), + 1024, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + float, + megabytes(128), + 1024, + 8), + + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + double, + megabytes(128), + 128, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + double, + megabytes(128), + 128, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + double, + megabytes(128), + 128, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + double, + megabytes(128), + 128, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + double, + megabytes(128), + 128, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + double, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + double, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + double, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + double, + megabytes(128), + 256, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + double, + megabytes(128), + 256, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + double, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + double, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + double, + megabytes(128), + 512, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + double, + megabytes(128), + 512, + 8), + + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + double, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + double, + megabytes(128), + 1024, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + double, + megabytes(128), + 1024, + 4), + + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + uint64_t, + megabytes(128), + 128, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + uint64_t, + megabytes(128), + 128, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + uint64_t, + megabytes(128), + 128, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + uint64_t, + megabytes(128), + 128, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + uint64_t, + megabytes(128), + 128, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + uint64_t, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + uint64_t, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + uint64_t, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + uint64_t, + megabytes(128), + 256, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + uint64_t, + megabytes(128), + 256, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + uint64_t, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + uint64_t, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + uint64_t, + megabytes(128), + 512, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + uint64_t, + megabytes(128), + 512, + 8), + + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + uint64_t, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + uint64_t, + megabytes(128), + 1024, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + uint64_t, + megabytes(128), + 1024, + 4), + + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::int128_t, + megabytes(128), + 128, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::int128_t, + megabytes(128), + 128, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::int128_t, + megabytes(128), + 128, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::int128_t, + megabytes(128), + 128, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::int128_t, + megabytes(128), + 128, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::int128_t, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::int128_t, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::int128_t, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::int128_t, + megabytes(128), + 256, + 8), + + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::int128_t, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::int128_t, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::int128_t, + megabytes(128), + 512, + 4), + + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::int128_t, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::int128_t, + megabytes(128), + 1024, + 2), + + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::uint128_t, + megabytes(128), + 128, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::uint128_t, + megabytes(128), + 128, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::uint128_t, + megabytes(128), + 128, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::uint128_t, + megabytes(128), + 128, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::uint128_t, + megabytes(128), + 128, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::uint128_t, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::uint128_t, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::uint128_t, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::uint128_t, + megabytes(128), + 256, + 8), + + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::uint128_t, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::uint128_t, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::uint128_t, + megabytes(128), + 512, + 4), + + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::uint128_t, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + block_scan, + rocprim::uint128_t, + megabytes(128), + 1024, + 2), // vectorized - block_scan CREATE_BENCHMARK(vectorized, block_scan, int, megabytes(128), 128, 1), @@ -825,157 +2037,1145 @@ int main(int argc, char *argv[]) CREATE_BENCHMARK(vectorized, block_scan, uint64_t, megabytes(128), 1024, 2), CREATE_BENCHMARK(vectorized, block_scan, uint64_t, megabytes(128), 1024, 4), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::int128_t, + megabytes(128), + 128, + 1), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::int128_t, + megabytes(128), + 128, + 2), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::int128_t, + megabytes(128), + 128, + 4), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::int128_t, + megabytes(128), + 128, + 8), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::int128_t, + megabytes(128), + 128, + 16), + + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::int128_t, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::int128_t, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::int128_t, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::int128_t, + megabytes(128), + 256, + 8), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::int128_t, + megabytes(128), + 256, + 16), + + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::int128_t, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::int128_t, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::int128_t, + megabytes(128), + 512, + 4), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::int128_t, + megabytes(128), + 512, + 8), + + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::int128_t, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::int128_t, + megabytes(128), + 1024, + 2), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::int128_t, + megabytes(128), + 1024, + 4), + + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::uint128_t, + megabytes(128), + 128, + 1), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::uint128_t, + megabytes(128), + 128, + 2), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::uint128_t, + megabytes(128), + 128, + 4), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::uint128_t, + megabytes(128), + 128, + 8), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::uint128_t, + megabytes(128), + 128, + 16), + + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::uint128_t, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::uint128_t, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::uint128_t, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::uint128_t, + megabytes(128), + 256, + 8), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::uint128_t, + megabytes(128), + 256, + 16), + + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::uint128_t, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::uint128_t, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::uint128_t, + megabytes(128), + 512, + 4), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::uint128_t, + megabytes(128), + 512, + 8), + + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::uint128_t, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::uint128_t, + megabytes(128), + 1024, + 2), + CREATE_BENCHMARK(vectorized, + block_scan, + rocprim::uint128_t, + megabytes(128), + 1024, + 4), + // custom_op - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, int, megabytes(128), 128, 1), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, int, megabytes(128), 128, 2), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, int, megabytes(128), 128, 4), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, int, megabytes(128), 128, 8), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, int, megabytes(128), 128, 16), - - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, int, megabytes(128), 256, 1), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, int, megabytes(128), 256, 2), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, int, megabytes(128), 256, 4), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, int, megabytes(128), 256, 8), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, int, megabytes(128), 256, 16), - - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, int, megabytes(128), 512, 1), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, int, megabytes(128), 512, 2), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, int, megabytes(128), 512, 4), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, int, megabytes(128), 512, 8), - - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, int, megabytes(128), 1024, 1), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, int, megabytes(128), 1024, 2), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, int, megabytes(128), 1024, 4), - - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, float, megabytes(128), 128, 1), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, float, megabytes(128), 128, 2), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, float, megabytes(128), 128, 4), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, float, megabytes(128), 128, 8), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, float, megabytes(128), 128, 16), - - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, float, megabytes(128), 256, 1), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, float, megabytes(128), 256, 2), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, float, megabytes(128), 256, 4), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, float, megabytes(128), 256, 8), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, float, megabytes(128), 256, 16), - - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, float, megabytes(128), 512, 1), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, float, megabytes(128), 512, 2), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, float, megabytes(128), 512, 4), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, float, megabytes(128), 512, 8), - - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, float, megabytes(128), 1024, 1), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, float, megabytes(128), 1024, 2), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, float, megabytes(128), 1024, 4), - - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, double, megabytes(128), 128, 1), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, double, megabytes(128), 128, 2), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, double, megabytes(128), 128, 4), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, double, megabytes(128), 128, 8), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, double, megabytes(128), 128, 16), - - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, double, megabytes(128), 256, 1), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, double, megabytes(128), 256, 2), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, double, megabytes(128), 256, 4), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, double, megabytes(128), 256, 8), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, double, megabytes(128), 256, 16), - - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, double, megabytes(128), 512, 1), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, double, megabytes(128), 512, 2), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, double, megabytes(128), 512, 4), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, double, megabytes(128), 512, 8), - - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, double, megabytes(128), 1024, 1), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, double, megabytes(128), 1024, 2), - - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, uint64_t, megabytes(128), 128, 1), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, uint64_t, megabytes(128), 128, 2), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, uint64_t, megabytes(128), 128, 4), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, uint64_t, megabytes(128), 128, 8), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, uint64_t, megabytes(128), 128, 16), - - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, uint64_t, megabytes(128), 256, 1), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, uint64_t, megabytes(128), 256, 2), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, uint64_t, megabytes(128), 256, 4), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, uint64_t, megabytes(128), 256, 8), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, uint64_t, megabytes(128), 256, 16), - - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, uint64_t, megabytes(128), 512, 1), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, uint64_t, megabytes(128), 512, 2), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, uint64_t, megabytes(128), 512, 4), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, uint64_t, megabytes(128), 512, 8), - - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, uint64_t, megabytes(128), 1024, 1), - CREATE_BENCHMARK(block_primitives_transpose, custom_operation, uint64_t, megabytes(128), 1024, 2), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + int, + megabytes(128), + 128, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + int, + megabytes(128), + 128, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + int, + megabytes(128), + 128, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + int, + megabytes(128), + 128, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + int, + megabytes(128), + 128, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + int, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + int, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + int, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + int, + megabytes(128), + 256, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + int, + megabytes(128), + 256, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + int, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + int, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + int, + megabytes(128), + 512, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + int, + megabytes(128), + 512, + 8), + + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + int, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + int, + megabytes(128), + 1024, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + int, + megabytes(128), + 1024, + 4), + + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + float, + megabytes(128), + 128, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + float, + megabytes(128), + 128, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + float, + megabytes(128), + 128, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + float, + megabytes(128), + 128, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + float, + megabytes(128), + 128, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + float, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + float, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + float, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + float, + megabytes(128), + 256, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + float, + megabytes(128), + 256, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + float, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + float, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + float, + megabytes(128), + 512, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + float, + megabytes(128), + 512, + 8), + + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + float, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + float, + megabytes(128), + 1024, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + float, + megabytes(128), + 1024, + 4), + + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + double, + megabytes(128), + 128, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + double, + megabytes(128), + 128, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + double, + megabytes(128), + 128, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + double, + megabytes(128), + 128, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + double, + megabytes(128), + 128, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + double, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + double, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + double, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + double, + megabytes(128), + 256, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + double, + megabytes(128), + 256, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + double, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + double, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + double, + megabytes(128), + 512, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + double, + megabytes(128), + 512, + 8), + + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + double, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + double, + megabytes(128), + 1024, + 2), + + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + uint64_t, + megabytes(128), + 128, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + uint64_t, + megabytes(128), + 128, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + uint64_t, + megabytes(128), + 128, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + uint64_t, + megabytes(128), + 128, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + uint64_t, + megabytes(128), + 128, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + uint64_t, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + uint64_t, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + uint64_t, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + uint64_t, + megabytes(128), + 256, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + uint64_t, + megabytes(128), + 256, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + uint64_t, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + uint64_t, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + uint64_t, + megabytes(128), + 512, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + uint64_t, + megabytes(128), + 512, + 8), + + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + uint64_t, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + uint64_t, + megabytes(128), + 1024, + 2), + + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::int128_t, + megabytes(128), + 128, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::int128_t, + megabytes(128), + 128, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::int128_t, + megabytes(128), + 128, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::int128_t, + megabytes(128), + 128, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::int128_t, + megabytes(128), + 128, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::int128_t, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::int128_t, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::int128_t, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::int128_t, + megabytes(128), + 256, + 8), + + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::int128_t, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::int128_t, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::int128_t, + megabytes(128), + 512, + 4), + + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::int128_t, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::int128_t, + megabytes(128), + 1024, + 2), + + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::uint128_t, + megabytes(128), + 128, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::uint128_t, + megabytes(128), + 128, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::uint128_t, + megabytes(128), + 128, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::uint128_t, + megabytes(128), + 128, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::uint128_t, + megabytes(128), + 128, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::uint128_t, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::uint128_t, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::uint128_t, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::uint128_t, + megabytes(128), + 256, + 8), + + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::uint128_t, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::uint128_t, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::uint128_t, + megabytes(128), + 512, + 4), + + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::uint128_t, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + custom_operation, + rocprim::uint128_t, + megabytes(128), + 1024, + 2), // block_primitives_transpose - atomics no collision - CREATE_BENCHMARK(block_primitives_transpose, atomics_no_collision, int, megabytes(128), 128, 1), - CREATE_BENCHMARK(block_primitives_transpose, atomics_no_collision, int, megabytes(128), 128, 2), - CREATE_BENCHMARK(block_primitives_transpose, atomics_no_collision, int, megabytes(128), 128, 4), - CREATE_BENCHMARK(block_primitives_transpose, atomics_no_collision, int, megabytes(128), 128, 8), - CREATE_BENCHMARK(block_primitives_transpose, atomics_no_collision, int, megabytes(128), 128, 16), - - CREATE_BENCHMARK(block_primitives_transpose, atomics_no_collision, int, megabytes(128), 256, 1), - CREATE_BENCHMARK(block_primitives_transpose, atomics_no_collision, int, megabytes(128), 256, 2), - CREATE_BENCHMARK(block_primitives_transpose, atomics_no_collision, int, megabytes(128), 256, 4), - CREATE_BENCHMARK(block_primitives_transpose, atomics_no_collision, int, megabytes(128), 256, 8), - CREATE_BENCHMARK(block_primitives_transpose, atomics_no_collision, int, megabytes(128), 256, 16), - - CREATE_BENCHMARK(block_primitives_transpose, atomics_no_collision, int, megabytes(128), 512, 1), - CREATE_BENCHMARK(block_primitives_transpose, atomics_no_collision, int, megabytes(128), 512, 2), - CREATE_BENCHMARK(block_primitives_transpose, atomics_no_collision, int, megabytes(128), 512, 4), - CREATE_BENCHMARK(block_primitives_transpose, atomics_no_collision, int, megabytes(128), 512, 8), - - CREATE_BENCHMARK(block_primitives_transpose, atomics_no_collision, int, megabytes(128), 1024, 1), - CREATE_BENCHMARK(block_primitives_transpose, atomics_no_collision, int, megabytes(128), 1024, 2), - CREATE_BENCHMARK(block_primitives_transpose, atomics_no_collision, int, megabytes(128), 1024, 4), - CREATE_BENCHMARK(block_primitives_transpose, atomics_no_collision, int, megabytes(128), 1024, 8), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_no_collision, + int, + megabytes(128), + 128, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_no_collision, + int, + megabytes(128), + 128, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_no_collision, + int, + megabytes(128), + 128, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_no_collision, + int, + megabytes(128), + 128, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_no_collision, + int, + megabytes(128), + 128, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + atomics_no_collision, + int, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_no_collision, + int, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_no_collision, + int, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_no_collision, + int, + megabytes(128), + 256, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_no_collision, + int, + megabytes(128), + 256, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + atomics_no_collision, + int, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_no_collision, + int, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_no_collision, + int, + megabytes(128), + 512, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_no_collision, + int, + megabytes(128), + 512, + 8), + + CREATE_BENCHMARK(block_primitives_transpose, + atomics_no_collision, + int, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_no_collision, + int, + megabytes(128), + 1024, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_no_collision, + int, + megabytes(128), + 1024, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_no_collision, + int, + megabytes(128), + 1024, + 8), // block_primitives_transpose - atomics inter block collision - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_block_collision, int, megabytes(128), 128, 1), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_block_collision, int, megabytes(128), 128, 2), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_block_collision, int, megabytes(128), 128, 4), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_block_collision, int, megabytes(128), 128, 8), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_block_collision, int, megabytes(128), 128, 16), - - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_block_collision, int, megabytes(128), 256, 1), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_block_collision, int, megabytes(128), 256, 2), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_block_collision, int, megabytes(128), 256, 4), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_block_collision, int, megabytes(128), 256, 8), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_block_collision, int, megabytes(128), 256, 16), - - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_block_collision, int, megabytes(128), 512, 1), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_block_collision, int, megabytes(128), 512, 2), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_block_collision, int, megabytes(128), 512, 4), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_block_collision, int, megabytes(128), 512, 8), - - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_block_collision, int, megabytes(128), 1024, 1), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_block_collision, int, megabytes(128), 1024, 2), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_block_collision, int, megabytes(128), 1024, 4), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_block_collision, int, megabytes(128), 1024, 8), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_block_collision, + int, + megabytes(128), + 128, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_block_collision, + int, + megabytes(128), + 128, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_block_collision, + int, + megabytes(128), + 128, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_block_collision, + int, + megabytes(128), + 128, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_block_collision, + int, + megabytes(128), + 128, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_block_collision, + int, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_block_collision, + int, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_block_collision, + int, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_block_collision, + int, + megabytes(128), + 256, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_block_collision, + int, + megabytes(128), + 256, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_block_collision, + int, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_block_collision, + int, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_block_collision, + int, + megabytes(128), + 512, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_block_collision, + int, + megabytes(128), + 512, + 8), + + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_block_collision, + int, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_block_collision, + int, + megabytes(128), + 1024, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_block_collision, + int, + megabytes(128), + 1024, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_block_collision, + int, + megabytes(128), + 1024, + 8), // block_primitives_transpose - atomics inter warp collision - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_warp_collision, int, megabytes(128), 128, 1), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_warp_collision, int, megabytes(128), 128, 2), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_warp_collision, int, megabytes(128), 128, 4), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_warp_collision, int, megabytes(128), 128, 8), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_warp_collision, int, megabytes(128), 128, 16), - - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_warp_collision, int, megabytes(128), 256, 1), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_warp_collision, int, megabytes(128), 256, 2), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_warp_collision, int, megabytes(128), 256, 4), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_warp_collision, int, megabytes(128), 256, 8), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_warp_collision, int, megabytes(128), 256, 16), - - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_warp_collision, int, megabytes(128), 512, 1), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_warp_collision, int, megabytes(128), 512, 2), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_warp_collision, int, megabytes(128), 512, 4), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_warp_collision, int, megabytes(128), 512, 8), - - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_warp_collision, int, megabytes(128), 1024, 1), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_warp_collision, int, megabytes(128), 1024, 2), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_warp_collision, int, megabytes(128), 1024, 4), - CREATE_BENCHMARK(block_primitives_transpose, atomics_inter_warp_collision, int, megabytes(128), 1024, 8), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_warp_collision, + int, + megabytes(128), + 128, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_warp_collision, + int, + megabytes(128), + 128, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_warp_collision, + int, + megabytes(128), + 128, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_warp_collision, + int, + megabytes(128), + 128, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_warp_collision, + int, + megabytes(128), + 128, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_warp_collision, + int, + megabytes(128), + 256, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_warp_collision, + int, + megabytes(128), + 256, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_warp_collision, + int, + megabytes(128), + 256, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_warp_collision, + int, + megabytes(128), + 256, + 8), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_warp_collision, + int, + megabytes(128), + 256, + 16), + + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_warp_collision, + int, + megabytes(128), + 512, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_warp_collision, + int, + megabytes(128), + 512, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_warp_collision, + int, + megabytes(128), + 512, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_warp_collision, + int, + megabytes(128), + 512, + 8), + + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_warp_collision, + int, + megabytes(128), + 1024, + 1), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_warp_collision, + int, + megabytes(128), + 1024, + 2), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_warp_collision, + int, + megabytes(128), + 1024, + 4), + CREATE_BENCHMARK(block_primitives_transpose, + atomics_inter_warp_collision, + int, + megabytes(128), + 1024, + 8) }; diff --git a/benchmark/benchmark_device_merge.cpp b/benchmark/benchmark_device_merge.cpp index fe90dea97..71b8860b0 100644 --- a/benchmark/benchmark_device_merge.cpp +++ b/benchmark/benchmark_device_merge.cpp @@ -32,15 +32,16 @@ #include // rocPRIM -#include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif -#include -#include +#include #include #include - -#include -#include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; @@ -67,7 +68,7 @@ const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; [=](benchmark::State& state) \ { run_merge_pairs_benchmark(state, bytes, seed, stream); }) -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { cli::Parser parser(argc, argv); parser.set_optional("size", "size", DEFAULT_BYTES, "number of bytes"); @@ -92,8 +93,8 @@ int main(int argc, char *argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); - const int trials = parser.get("trials"); + const size_t bytes = parser.get("size"); + const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); const managed_seed seed(seed_type); @@ -118,7 +119,7 @@ int main(int argc, char *argv[]) seed, stream); #else // BENCHMARK_CONFIG_TUNING - using custom_int2 = custom_type; + using custom_int2 = custom_type; using custom_double2 = custom_type; CREATE_BENCHMARK(int) @@ -129,6 +130,8 @@ int main(int argc, char *argv[]) CREATE_BENCHMARK(short) CREATE_BENCHMARK(custom_int2) CREATE_BENCHMARK(custom_double2) + CREATE_BENCHMARK(rocprim::int128_t) + CREATE_BENCHMARK(rocprim::uint128_t) CREATE_BENCHMARK(int, int) CREATE_BENCHMARK(long long, long long) @@ -138,6 +141,8 @@ int main(int argc, char *argv[]) CREATE_BENCHMARK(short, short) CREATE_BENCHMARK(custom_int2, custom_int2) CREATE_BENCHMARK(custom_double2, custom_double2) + CREATE_BENCHMARK(rocprim::int128_t, rocprim::int128_t) + CREATE_BENCHMARK(rocprim::uint128_t, rocprim::uint128_t) #endif // BENCHMARK_CONFIG_TUNING diff --git a/benchmark/benchmark_device_merge.parallel.cpp.in b/benchmark/benchmark_device_merge.parallel.cpp.in index 822f00c89..3147c3ac0 100644 --- a/benchmark/benchmark_device_merge.parallel.cpp.in +++ b/benchmark/benchmark_device_merge.parallel.cpp.in @@ -20,11 +20,13 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include - #include "benchmark_utils.hpp" #include "benchmark_device_merge.parallel.hpp" +#include + +#include + namespace { auto benchmarks = config_autotune_register::create_bulk( diff --git a/benchmark/benchmark_device_merge.parallel.hpp b/benchmark/benchmark_device_merge.parallel.hpp index 6aff37388..ec6df61fb 100644 --- a/benchmark/benchmark_device_merge.parallel.hpp +++ b/benchmark/benchmark_device_merge.parallel.hpp @@ -33,18 +33,21 @@ #include // rocPRIM HIP API +#include #include #include +#include +#include -#include -#include +#include +#include +#include #include +#include #include - -#include -#include - -namespace rp = rocprim; +#ifdef BENCHMARK_CONFIG_TUNING + #include +#endif template std::string config_name() @@ -128,32 +131,32 @@ struct device_merge_benchmark : public config_autotune_interface void* d_temporary_storage = nullptr; size_t temporary_storage_bytes = 0; - HIP_CHECK(rp::merge(d_temporary_storage, - temporary_storage_bytes, - d_keys_input1, - d_keys_input2, - d_keys_output, - size1, - size2, - compare_op, - stream, - false)); + HIP_CHECK(rocprim::merge(d_temporary_storage, + temporary_storage_bytes, + d_keys_input1, + d_keys_input2, + d_keys_output, + size1, + size2, + compare_op, + stream, + false)); HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { - HIP_CHECK(rp::merge(d_temporary_storage, - temporary_storage_bytes, - d_keys_input1, - d_keys_input2, - d_keys_output, - size1, - size2, - compare_op, - stream, - false)); + HIP_CHECK(rocprim::merge(d_temporary_storage, + temporary_storage_bytes, + d_keys_input1, + d_keys_input2, + d_keys_output, + size1, + size2, + compare_op, + stream, + false)); } HIP_CHECK(hipDeviceSynchronize()); @@ -167,18 +170,18 @@ struct device_merge_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { - HIP_CHECK(rp::merge(d_temporary_storage, - temporary_storage_bytes, - d_keys_input1, - d_keys_input2, - d_keys_output, - size1, - size2, - compare_op, - stream, - false)); + HIP_CHECK(rocprim::merge(d_temporary_storage, + temporary_storage_bytes, + d_keys_input1, + d_keys_input2, + d_keys_output, + size1, + size2, + compare_op, + stream, + false)); } // Record stop event and wait until it completes @@ -267,39 +270,39 @@ struct device_merge_benchmark : public config_autotune_interface void* d_temporary_storage = nullptr; size_t temporary_storage_bytes = 0; - HIP_CHECK(rp::merge(d_temporary_storage, - temporary_storage_bytes, - d_keys_input1, - d_keys_input2, - d_keys_output, - d_values_input1, - d_values_input2, - d_values_output, - size1, - size2, - compare_op, - stream, - false)); + HIP_CHECK(rocprim::merge(d_temporary_storage, + temporary_storage_bytes, + d_keys_input1, + d_keys_input2, + d_keys_output, + d_values_input1, + d_values_input2, + d_values_output, + size1, + size2, + compare_op, + stream, + false)); HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { - HIP_CHECK(rp::merge(d_temporary_storage, - temporary_storage_bytes, - d_keys_input1, - d_keys_input2, - d_keys_output, - d_values_input1, - d_values_input2, - d_values_output, - size1, - size2, - compare_op, - stream, - false)); + HIP_CHECK(rocprim::merge(d_temporary_storage, + temporary_storage_bytes, + d_keys_input1, + d_keys_input2, + d_keys_output, + d_values_input1, + d_values_input2, + d_values_output, + size1, + size2, + compare_op, + stream, + false)); } HIP_CHECK(hipDeviceSynchronize()); @@ -313,21 +316,21 @@ struct device_merge_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { - HIP_CHECK(rp::merge(d_temporary_storage, - temporary_storage_bytes, - d_keys_input1, - d_keys_input2, - d_keys_output, - d_values_input1, - d_values_input2, - d_values_output, - size1, - size2, - compare_op, - stream, - false)); + HIP_CHECK(rocprim::merge(d_temporary_storage, + temporary_storage_bytes, + d_keys_input1, + d_keys_input2, + d_keys_output, + d_values_input1, + d_values_input2, + d_values_output, + size1, + size2, + compare_op, + stream, + false)); } // Record stop event and wait until it completes diff --git a/benchmark/benchmark_device_merge_sort.cpp b/benchmark/benchmark_device_merge_sort.cpp index 00a339caa..a6347c3fd 100644 --- a/benchmark/benchmark_device_merge_sort.cpp +++ b/benchmark/benchmark_device_merge_sort.cpp @@ -32,20 +32,24 @@ // HIP API #include +#include + #include +#include #include +#include #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif -#define CREATE_BENCHMARK(...) \ - { \ - const device_merge_sort_benchmark<__VA_ARGS__> instance; \ +#define CREATE_BENCHMARK(...) \ + { \ + const device_merge_sort_benchmark<__VA_ARGS__> instance; \ REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { cli::Parser parser(argc, argv); parser.set_optional("size", "size", DEFAULT_BYTES, "number of bytes"); @@ -59,8 +63,8 @@ int main(int argc, char *argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); - const int trials = parser.get("trials"); + const size_t bytes = parser.get("size"); + const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); const managed_seed seed(seed_type); @@ -81,6 +85,8 @@ int main(int argc, char *argv[]) CREATE_BENCHMARK(uint8_t) CREATE_BENCHMARK(rocprim::half) CREATE_BENCHMARK(short) + CREATE_BENCHMARK(rocprim::int128_t) + CREATE_BENCHMARK(rocprim::uint128_t) using custom_float2 = custom_type; using custom_double2 = custom_type; @@ -100,6 +106,8 @@ int main(int argc, char *argv[]) CREATE_BENCHMARK(custom_int2, custom_double2) CREATE_BENCHMARK(custom_int2, custom_char_double) CREATE_BENCHMARK(custom_int2, custom_longlong_double) + CREATE_BENCHMARK(rocprim::int128_t, rocprim::int128_t) + CREATE_BENCHMARK(rocprim::uint128_t, rocprim::uint128_t) // Use manual timing for(auto& b : benchmarks) diff --git a/benchmark/benchmark_device_merge_sort.hpp b/benchmark/benchmark_device_merge_sort.hpp index c935730ae..a2ff75c03 100644 --- a/benchmark/benchmark_device_merge_sort.hpp +++ b/benchmark/benchmark_device_merge_sort.hpp @@ -34,16 +34,15 @@ // rocPRIM #include +#include +#include #include +#include #include -#include - namespace rp = rocprim; -template +template struct device_merge_sort_benchmark : public config_autotune_interface { std::string name() const override @@ -62,11 +61,11 @@ struct device_merge_sort_benchmark : public config_autotune_interface size_t bytes, const managed_seed& seed, hipStream_t stream) const -> - typename std::enable_if::value, void>::type + typename std::enable_if::value, void>::type { using key_type = Key; - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(key_type); // Generate data std::vector keys_input @@ -84,7 +83,7 @@ struct device_merge_sort_benchmark : public config_autotune_interface size * sizeof(key_type), hipMemcpyHostToDevice)); - ::rocprim::less lesser_op; + ::rp::less lesser_op; void* d_temporary_storage = nullptr; size_t temporary_storage_bytes = 0; @@ -101,7 +100,7 @@ struct device_merge_sort_benchmark : public config_autotune_interface HIP_CHECK(hipDeviceSynchronize()); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { HIP_CHECK(rp::merge_sort(d_temporary_storage, temporary_storage_bytes, @@ -124,7 +123,7 @@ struct device_merge_sort_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { HIP_CHECK(rp::merge_sort(d_temporary_storage, temporary_storage_bytes, @@ -163,12 +162,12 @@ struct device_merge_sort_benchmark : public config_autotune_interface size_t bytes, const managed_seed& seed, hipStream_t stream) const -> - typename std::enable_if::value, void>::type + typename std::enable_if::value, void>::type { using key_type = Key; using value_type = Value; - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(key_type); // Generate data std::vector keys_input @@ -198,7 +197,7 @@ struct device_merge_sort_benchmark : public config_autotune_interface size * sizeof(value_type), hipMemcpyHostToDevice)); - ::rocprim::less lesser_op; + ::rp::less lesser_op; void* d_temporary_storage = nullptr; size_t temporary_storage_bytes = 0; @@ -217,7 +216,7 @@ struct device_merge_sort_benchmark : public config_autotune_interface HIP_CHECK(hipDeviceSynchronize()); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { HIP_CHECK(rp::merge_sort(d_temporary_storage, temporary_storage_bytes, @@ -242,7 +241,7 @@ struct device_merge_sort_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { HIP_CHECK(rp::merge_sort(d_temporary_storage, temporary_storage_bytes, diff --git a/benchmark/benchmark_device_merge_sort_block_merge.cpp b/benchmark/benchmark_device_merge_sort_block_merge.cpp index 6138771da..61c283cfe 100644 --- a/benchmark/benchmark_device_merge_sort_block_merge.cpp +++ b/benchmark/benchmark_device_merge_sort_block_merge.cpp @@ -32,9 +32,16 @@ // HIP API #include -#include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif #include +#include +#include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif #ifndef DEFAULT_N const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; @@ -43,7 +50,7 @@ const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #define CREATE_BENCHMARK(...) \ { \ const device_merge_sort_block_merge_benchmark<__VA_ARGS__> instance; \ - REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ + REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } int main(int argc, char* argv[]) @@ -71,7 +78,7 @@ int main(int argc, char* argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); + const size_t bytes = parser.get("size"); const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); @@ -103,6 +110,8 @@ int main(int argc, char* argv[]) CREATE_BENCHMARK(uint8_t) CREATE_BENCHMARK(rocprim::half) CREATE_BENCHMARK(short) + CREATE_BENCHMARK(rocprim::int128_t) + CREATE_BENCHMARK(rocprim::uint128_t) using custom_float2 = custom_type; using custom_double2 = custom_type; @@ -122,6 +131,8 @@ int main(int argc, char* argv[]) CREATE_BENCHMARK(custom_int2, custom_double2) CREATE_BENCHMARK(custom_int2, custom_char_double) CREATE_BENCHMARK(custom_int2, custom_longlong_double) + CREATE_BENCHMARK(rocprim::int128_t, rocprim::int128_t) + CREATE_BENCHMARK(rocprim::uint128_t, rocprim::uint128_t) #endif // BENCHMARK_CONFIG_TUNING // Use manual timing diff --git a/benchmark/benchmark_device_merge_sort_block_merge.parallel.cpp.in b/benchmark/benchmark_device_merge_sort_block_merge.parallel.cpp.in index bc0682afe..5d34eacb3 100644 --- a/benchmark/benchmark_device_merge_sort_block_merge.parallel.cpp.in +++ b/benchmark/benchmark_device_merge_sort_block_merge.parallel.cpp.in @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,11 +20,13 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include - #include "benchmark_device_merge_sort_block_merge.parallel.hpp" #include "benchmark_utils.hpp" +#include + +#include + namespace { auto benchmarks = config_autotune_register::create_bulk( device_merge_sort_block_merge_benchmark_generator<@BlockSize@, @UseMergePath@, @KeyType@, @ValueType@>::create); diff --git a/benchmark/benchmark_device_merge_sort_block_merge.parallel.hpp b/benchmark/benchmark_device_merge_sort_block_merge.parallel.hpp index 960652e5d..512e913b7 100644 --- a/benchmark/benchmark_device_merge_sort_block_merge.parallel.hpp +++ b/benchmark/benchmark_device_merge_sort_block_merge.parallel.hpp @@ -32,15 +32,20 @@ #include // rocPRIM +#include +#include #include +#include +#include +#include +#include +#include +#include #include +#include #include -#include - -namespace rp = rocprim; - template std::string config_name() { @@ -112,44 +117,44 @@ struct device_merge_sort_block_merge_benchmark : public config_autotune_interfac // Merge_sort_block_merge algorithm expects partially sorted input: unsigned int sorted_block_size; - HIP_CHECK(rp::detail::merge_sort_block_sort(d_keys_input, - d_keys_input, - values_ptr, - values_ptr, - size, - sorted_block_size, - lesser_op, - stream, - false)); + HIP_CHECK(rocprim::detail::merge_sort_block_sort(d_keys_input, + d_keys_input, + values_ptr, + values_ptr, + size, + sorted_block_size, + lesser_op, + stream, + false)); void* d_temporary_storage = nullptr; size_t temporary_storage_bytes = 0; - HIP_CHECK(rp::detail::merge_sort_block_merge(d_temporary_storage, - temporary_storage_bytes, - d_keys, - values_ptr, - size, - sorted_block_size, - lesser_op, - stream, - false)); + HIP_CHECK(rocprim::detail::merge_sort_block_merge(d_temporary_storage, + temporary_storage_bytes, + d_keys, + values_ptr, + size, + sorted_block_size, + lesser_op, + stream, + false)); HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); hipError_t err; // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { - err = rp::detail::merge_sort_block_merge(d_temporary_storage, - temporary_storage_bytes, - d_keys, - values_ptr, - size, - sorted_block_size, - lesser_op, - stream, - false); + err = rocprim::detail::merge_sort_block_merge(d_temporary_storage, + temporary_storage_bytes, + d_keys, + values_ptr, + size, + sorted_block_size, + lesser_op, + stream, + false); } if(err == hipError_t::hipErrorAssert) { @@ -181,15 +186,15 @@ struct device_merge_sort_block_merge_benchmark : public config_autotune_interfac hipMemcpyDeviceToDevice, stream)); HIP_CHECK(hipEventRecord(start, stream)); - HIP_CHECK(rp::detail::merge_sort_block_merge(d_temporary_storage, - temporary_storage_bytes, - d_keys, - values_ptr, - size, - sorted_block_size, - lesser_op, - stream, - false)); + HIP_CHECK(rocprim::detail::merge_sort_block_merge(d_temporary_storage, + temporary_storage_bytes, + d_keys, + values_ptr, + size, + sorted_block_size, + lesser_op, + stream, + false)); // Record stop event and wait until it completes HIP_CHECK(hipEventRecord(stop, stream)); @@ -260,44 +265,44 @@ struct device_merge_sort_block_merge_benchmark : public config_autotune_interfac // Merge_sort_block_merge algorithm expects partially sorted input: unsigned int sorted_block_size; - HIP_CHECK(rp::detail::merge_sort_block_sort(d_keys_input, - d_keys_input, - d_values_input, - d_values_input, - size, - sorted_block_size, - lesser_op, - stream, - false)); + HIP_CHECK(rocprim::detail::merge_sort_block_sort(d_keys_input, + d_keys_input, + d_values_input, + d_values_input, + size, + sorted_block_size, + lesser_op, + stream, + false)); void* d_temporary_storage = nullptr; size_t temporary_storage_bytes = 0; - HIP_CHECK(rp::detail::merge_sort_block_merge(d_temporary_storage, - temporary_storage_bytes, - d_keys, - d_values, - size, - sorted_block_size, - lesser_op, - stream, - false)); + HIP_CHECK(rocprim::detail::merge_sort_block_merge(d_temporary_storage, + temporary_storage_bytes, + d_keys, + d_values, + size, + sorted_block_size, + lesser_op, + stream, + false)); HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); hipError_t err; // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { - err = rp::detail::merge_sort_block_merge(d_temporary_storage, - temporary_storage_bytes, - d_keys, - d_values, - size, - sorted_block_size, - lesser_op, - stream, - false); + err = rocprim::detail::merge_sort_block_merge(d_temporary_storage, + temporary_storage_bytes, + d_keys, + d_values, + size, + sorted_block_size, + lesser_op, + stream, + false); } if(err == hipError_t::hipErrorAssert) { @@ -336,15 +341,15 @@ struct device_merge_sort_block_merge_benchmark : public config_autotune_interfac hipMemcpyDeviceToDevice, stream)); HIP_CHECK(hipEventRecord(start, stream)); - HIP_CHECK(rp::detail::merge_sort_block_merge(d_temporary_storage, - temporary_storage_bytes, - d_keys, - d_values, - size, - sorted_block_size, - lesser_op, - stream, - false)); + HIP_CHECK(rocprim::detail::merge_sort_block_merge(d_temporary_storage, + temporary_storage_bytes, + d_keys, + d_values, + size, + sorted_block_size, + lesser_op, + stream, + false)); // Record stop event and wait until it completes HIP_CHECK(hipEventRecord(stop, stream)); diff --git a/benchmark/benchmark_device_merge_sort_block_sort.cpp b/benchmark/benchmark_device_merge_sort_block_sort.cpp index 46d6b2bdf..a97f0a3ba 100644 --- a/benchmark/benchmark_device_merge_sort_block_sort.cpp +++ b/benchmark/benchmark_device_merge_sort_block_sort.cpp @@ -32,9 +32,16 @@ // HIP API #include -#include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif #include +#include +#include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif #ifndef DEFAULT_N const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; @@ -43,7 +50,7 @@ const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #define CREATE_BENCHMARK(...) \ { \ const device_merge_sort_block_sort_benchmark<__VA_ARGS__> instance; \ - REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ + REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } int main(int argc, char* argv[]) @@ -71,7 +78,7 @@ int main(int argc, char* argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); + const size_t bytes = parser.get("size"); const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); @@ -103,6 +110,8 @@ int main(int argc, char* argv[]) CREATE_BENCHMARK(uint8_t) CREATE_BENCHMARK(rocprim::half) CREATE_BENCHMARK(short) + CREATE_BENCHMARK(rocprim::int128_t) + CREATE_BENCHMARK(rocprim::uint128_t) using custom_float2 = custom_type; using custom_double2 = custom_type; @@ -124,6 +133,8 @@ int main(int argc, char* argv[]) CREATE_BENCHMARK(custom_int2, custom_char_double) CREATE_BENCHMARK(custom_int2, custom_longlong_double) CREATE_BENCHMARK(int, custom_char_short) + CREATE_BENCHMARK(rocprim::int128_t, rocprim::int128_t) + CREATE_BENCHMARK(rocprim::uint128_t, rocprim::uint128_t) #endif // BENCHMARK_CONFIG_TUNING // Use manual timing diff --git a/benchmark/benchmark_device_merge_sort_block_sort.parallel.cpp.in b/benchmark/benchmark_device_merge_sort_block_sort.parallel.cpp.in index dab59927e..c0e241034 100644 --- a/benchmark/benchmark_device_merge_sort_block_sort.parallel.cpp.in +++ b/benchmark/benchmark_device_merge_sort_block_sort.parallel.cpp.in @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,11 +20,13 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include - #include "benchmark_device_merge_sort_block_sort.parallel.hpp" #include "benchmark_utils.hpp" +#include + +#include + namespace { auto benchmarks = config_autotune_register::create_bulk( device_merge_sort_block_sort_benchmark_generator<@BlockSize@, @BlockSortMethod@, @KeyType@, @ValueType@>::create); diff --git a/benchmark/benchmark_device_merge_sort_block_sort.parallel.hpp b/benchmark/benchmark_device_merge_sort_block_sort.parallel.hpp index 796c165e4..fed7af72a 100644 --- a/benchmark/benchmark_device_merge_sort_block_sort.parallel.hpp +++ b/benchmark/benchmark_device_merge_sort_block_sort.parallel.hpp @@ -32,22 +32,27 @@ #include // rocPRIM +#include +#include +#include #include +#include +#include +#include +#include +#include +#include #include +#include #include -#include - -namespace rp = rocprim; - constexpr const char* get_block_sort_method_name(rocprim::block_sort_algorithm alg) { switch(alg) { case rocprim::block_sort_algorithm::merge_sort: return "merge_sort"; - case rocprim::block_sort_algorithm::bitonic_sort: - return "bitonic_sort"; + case rocprim::block_sort_algorithm::bitonic_sort: return "bitonic_sort"; case rocprim::block_sort_algorithm::stable_merge_sort: return "stable_merge_sort"; // Not using `default: ...` because it kills effectiveness of -Wswitch @@ -96,7 +101,7 @@ struct device_merge_sort_block_sort_benchmark : public config_autotune_interface { using key_type = Key; - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(key_type); // Generate data std::vector keys_input @@ -118,17 +123,17 @@ struct device_merge_sort_block_sort_benchmark : public config_autotune_interface rocprim::empty_type* values_ptr = nullptr; unsigned int items_per_block; // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { - HIP_CHECK(rp::detail::merge_sort_block_sort(d_keys_input, - d_keys_output, - values_ptr, - values_ptr, - size, - items_per_block, - lesser_op, - stream, - false)); + HIP_CHECK(rocprim::detail::merge_sort_block_sort(d_keys_input, + d_keys_output, + values_ptr, + values_ptr, + size, + items_per_block, + lesser_op, + stream, + false)); } HIP_CHECK(hipDeviceSynchronize()); @@ -142,17 +147,17 @@ struct device_merge_sort_block_sort_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { - HIP_CHECK(rp::detail::merge_sort_block_sort(d_keys_input, - d_keys_output, - values_ptr, - values_ptr, - size, - items_per_block, - lesser_op, - stream, - false)); + HIP_CHECK(rocprim::detail::merge_sort_block_sort(d_keys_input, + d_keys_output, + values_ptr, + values_ptr, + size, + items_per_block, + lesser_op, + stream, + false)); } // Record stop event and wait until it completes @@ -186,7 +191,7 @@ struct device_merge_sort_block_sort_benchmark : public config_autotune_interface using key_type = Key; using value_type = Value; - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(key_type); // Generate data std::vector keys_input @@ -222,18 +227,18 @@ struct device_merge_sort_block_sort_benchmark : public config_autotune_interface HIP_CHECK(hipDeviceSynchronize()); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { - HIP_CHECK(rp::detail::merge_sort_block_sort(d_keys_input, - d_keys_output, - d_values_input, - d_values_output, - size, - items_per_block, - lesser_op, - stream, - false)); + HIP_CHECK(rocprim::detail::merge_sort_block_sort(d_keys_input, + d_keys_output, + d_values_input, + d_values_output, + size, + items_per_block, + lesser_op, + stream, + false)); } HIP_CHECK(hipDeviceSynchronize()); @@ -247,17 +252,17 @@ struct device_merge_sort_block_sort_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { - HIP_CHECK(rp::detail::merge_sort_block_sort(d_keys_input, - d_keys_output, - d_values_input, - d_values_output, - size, - items_per_block, - lesser_op, - stream, - false)); + HIP_CHECK(rocprim::detail::merge_sort_block_sort(d_keys_input, + d_keys_output, + d_values_input, + d_values_output, + size, + items_per_block, + lesser_op, + stream, + false)); } // Record stop event and wait until it completes diff --git a/benchmark/benchmark_device_nth_element.cpp b/benchmark/benchmark_device_nth_element.cpp index 0c0bcb2a2..549ccb1b1 100644 --- a/benchmark/benchmark_device_nth_element.cpp +++ b/benchmark/benchmark_device_nth_element.cpp @@ -32,23 +32,26 @@ // HIP API #include +#include + #include +#include #include +#include #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif -#define CREATE_BENCHMARK_NTH_ELEMENT(TYPE, SMALL_N) \ - { \ - const device_nth_element_benchmark instance(SMALL_N); \ +#define CREATE_BENCHMARK_NTH_ELEMENT(TYPE, SMALL_N) \ + { \ + const device_nth_element_benchmark instance(SMALL_N); \ REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } -#define CREATE_BENCHMARK(TYPE) \ - { \ - CREATE_BENCHMARK_NTH_ELEMENT(TYPE, true) \ - CREATE_BENCHMARK_NTH_ELEMENT(TYPE, false) \ +#define CREATE_BENCHMARK(TYPE) \ + { \ + CREATE_BENCHMARK_NTH_ELEMENT(TYPE, true) CREATE_BENCHMARK_NTH_ELEMENT(TYPE, false) \ } int main(int argc, char* argv[]) @@ -65,7 +68,7 @@ int main(int argc, char* argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); + const size_t bytes = parser.get("size"); const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); @@ -88,6 +91,8 @@ int main(int argc, char* argv[]) CREATE_BENCHMARK(rocprim::half) CREATE_BENCHMARK(short) CREATE_BENCHMARK(float) + CREATE_BENCHMARK(rocprim::int128_t) + CREATE_BENCHMARK(rocprim::uint128_t) using custom_float2 = custom_type; using custom_double2 = custom_type; diff --git a/benchmark/benchmark_device_nth_element.hpp b/benchmark/benchmark_device_nth_element.hpp index f517a1a77..d2b0cdddd 100644 --- a/benchmark/benchmark_device_nth_element.hpp +++ b/benchmark/benchmark_device_nth_element.hpp @@ -32,13 +32,14 @@ #include // rocPRIM +#include #include +#include +#include #include #include -#include - template struct device_nth_element_benchmark : public config_autotune_interface { @@ -67,9 +68,9 @@ struct device_nth_element_benchmark : public config_autotune_interface { using key_type = Key; - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(key_type); - size_t nth = 10; + size_t nth = 10; if(!small_n) { @@ -110,7 +111,7 @@ struct device_nth_element_benchmark : public config_autotune_interface HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { HIP_CHECK(rocprim::nth_element(d_temporary_storage, temporary_storage_bytes, @@ -134,7 +135,7 @@ struct device_nth_element_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { HIP_CHECK(rocprim::nth_element(d_temporary_storage, temporary_storage_bytes, diff --git a/benchmark/benchmark_device_partial_sort.cpp b/benchmark/benchmark_device_partial_sort.cpp index ea0a7b2b1..0d338c402 100644 --- a/benchmark/benchmark_device_partial_sort.cpp +++ b/benchmark/benchmark_device_partial_sort.cpp @@ -32,23 +32,26 @@ // HIP API #include +#include + #include +#include #include +#include #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif -#define CREATE_BENCHMARK_PARTIAL_SORT(TYPE, SMALL_N) \ - { \ - const device_partial_sort_benchmark instance(SMALL_N); \ +#define CREATE_BENCHMARK_PARTIAL_SORT(TYPE, SMALL_N) \ + { \ + const device_partial_sort_benchmark instance(SMALL_N); \ REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } -#define CREATE_BENCHMARK(TYPE) \ - { \ - CREATE_BENCHMARK_PARTIAL_SORT(TYPE, true) \ - CREATE_BENCHMARK_PARTIAL_SORT(TYPE, false) \ +#define CREATE_BENCHMARK(TYPE) \ + { \ + CREATE_BENCHMARK_PARTIAL_SORT(TYPE, true) CREATE_BENCHMARK_PARTIAL_SORT(TYPE, false) \ } int main(int argc, char* argv[]) @@ -65,7 +68,7 @@ int main(int argc, char* argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); + const size_t bytes = parser.get("size"); const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); @@ -88,6 +91,8 @@ int main(int argc, char* argv[]) CREATE_BENCHMARK(rocprim::half) CREATE_BENCHMARK(short) CREATE_BENCHMARK(float) + CREATE_BENCHMARK(rocprim::int128_t) + CREATE_BENCHMARK(rocprim::uint128_t) using custom_float2 = custom_type; using custom_double2 = custom_type; diff --git a/benchmark/benchmark_device_partial_sort.hpp b/benchmark/benchmark_device_partial_sort.hpp index efcb1c5d5..5c910a294 100644 --- a/benchmark/benchmark_device_partial_sort.hpp +++ b/benchmark/benchmark_device_partial_sort.hpp @@ -32,13 +32,14 @@ #include // rocPRIM +#include #include +#include +#include #include #include -#include - template struct device_partial_sort_benchmark : public config_autotune_interface { @@ -66,9 +67,9 @@ struct device_partial_sort_benchmark : public config_autotune_interface hipStream_t stream) const override { using key_type = Key; - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(key_type); - + size_t middle = 10; if(!small_n) @@ -108,7 +109,7 @@ struct device_partial_sort_benchmark : public config_autotune_interface HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { HIP_CHECK(hipMemcpy(d_keys_input, d_keys_new_data, @@ -133,7 +134,7 @@ struct device_partial_sort_benchmark : public config_autotune_interface for(auto _ : state) { float elapsed_mseconds = 0; - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { HIP_CHECK(hipMemcpy(d_keys_input, d_keys_new_data, diff --git a/benchmark/benchmark_device_partial_sort_copy.cpp b/benchmark/benchmark_device_partial_sort_copy.cpp index 2b5a1b840..62f912909 100644 --- a/benchmark/benchmark_device_partial_sort_copy.cpp +++ b/benchmark/benchmark_device_partial_sort_copy.cpp @@ -32,8 +32,12 @@ // HIP API #include +#include + #include +#include #include +#include #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; @@ -42,7 +46,7 @@ const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #define CREATE_BENCHMARK_PARTIAL_SORT_COPY(TYPE, SMALL_N) \ { \ const device_partial_sort_copy_benchmark instance(SMALL_N); \ - REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ + REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } #define CREATE_BENCHMARK(TYPE) \ @@ -65,7 +69,7 @@ int main(int argc, char* argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); + const size_t bytes = parser.get("size"); const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); @@ -88,6 +92,8 @@ int main(int argc, char* argv[]) CREATE_BENCHMARK(rocprim::half) CREATE_BENCHMARK(short) CREATE_BENCHMARK(float) + CREATE_BENCHMARK(rocprim::int128_t) + CREATE_BENCHMARK(rocprim::uint128_t) using custom_float2 = custom_type; using custom_double2 = custom_type; diff --git a/benchmark/benchmark_device_partial_sort_copy.hpp b/benchmark/benchmark_device_partial_sort_copy.hpp index 833c51599..b9898360a 100644 --- a/benchmark/benchmark_device_partial_sort_copy.hpp +++ b/benchmark/benchmark_device_partial_sort_copy.hpp @@ -32,13 +32,14 @@ #include // rocPRIM +#include #include +#include +#include #include #include -#include - template struct device_partial_sort_copy_benchmark : public config_autotune_interface { @@ -67,8 +68,8 @@ struct device_partial_sort_copy_benchmark : public config_autotune_interface { using key_type = Key; - // Calculate the number of elements - size_t size = bytes / sizeof(key_type); + // Calculate the number of elements + size_t size = bytes / sizeof(key_type); size_t middle = 10; if(!small_n) @@ -109,7 +110,7 @@ struct device_partial_sort_copy_benchmark : public config_autotune_interface HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { HIP_CHECK(rocprim::partial_sort_copy(d_temporary_storage, temporary_storage_bytes, @@ -133,7 +134,7 @@ struct device_partial_sort_copy_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { HIP_CHECK(rocprim::partial_sort_copy(d_temporary_storage, temporary_storage_bytes, diff --git a/benchmark/benchmark_device_partition.cpp b/benchmark/benchmark_device_partition.cpp index 46212e408..1acb23f0d 100644 --- a/benchmark/benchmark_device_partition.cpp +++ b/benchmark/benchmark_device_partition.cpp @@ -32,17 +32,17 @@ #include // rocPRIM -#include +#ifndef BENCHMARK_CONFIG_TUNING + #include + #include +#endif -#include -#include -#include -#include +#include #include #include - -#include -#include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; @@ -51,32 +51,32 @@ const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #define CREATE_PARTITION_FLAG_BENCHMARK(T, F, p) \ { \ const device_partition_flag_benchmark instance; \ - REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ + REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } #define CREATE_PARTITION_PREDICATE_BENCHMARK(T, p) \ { \ const device_partition_predicate_benchmark instance; \ - REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ + REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } #define CREATE_PARTITION_TWO_WAY_FLAG_BENCHMARK(T, F, p) \ { \ const device_partition_two_way_flag_benchmark instance; \ - REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ + REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } #define CREATE_PARTITION_TWO_WAY_PREDICATE_BENCHMARK(T, p) \ { \ const device_partition_two_way_predicate_benchmark \ instance; \ - REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ + REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } #define CREATE_PARTITION_THREE_WAY_BENCHMARK(T, p) \ { \ const device_partition_three_way_benchmark instance; \ - REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ + REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } #define BENCHMARK_FLAG_TYPE(type, flag_type) \ @@ -134,7 +134,7 @@ int main(int argc, char* argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); + const size_t bytes = parser.get("size"); const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); @@ -170,6 +170,8 @@ int main(int argc, char* argv[]) BENCHMARK_FLAG_TYPE(int8_t, int8_t); BENCHMARK_FLAG_TYPE(rocprim::half, int8_t); BENCHMARK_FLAG_TYPE(custom_double2, unsigned char); + BENCHMARK_FLAG_TYPE(rocprim::int128_t, int8_t); + BENCHMARK_FLAG_TYPE(rocprim::uint128_t, uint8_t); BENCHMARK_PREDICATE_TYPE(int); BENCHMARK_PREDICATE_TYPE(float); @@ -178,6 +180,8 @@ int main(int argc, char* argv[]) BENCHMARK_PREDICATE_TYPE(int8_t); BENCHMARK_PREDICATE_TYPE(rocprim::half); BENCHMARK_PREDICATE_TYPE(custom_int_double); + BENCHMARK_PREDICATE_TYPE(rocprim::int128_t); + BENCHMARK_PREDICATE_TYPE(rocprim::uint128_t); BENCHMARK_TWO_WAY_FLAG_TYPE(int, unsigned char); BENCHMARK_TWO_WAY_FLAG_TYPE(float, unsigned char); @@ -186,6 +190,8 @@ int main(int argc, char* argv[]) BENCHMARK_TWO_WAY_FLAG_TYPE(int8_t, int8_t); BENCHMARK_TWO_WAY_FLAG_TYPE(rocprim::half, int8_t); BENCHMARK_TWO_WAY_FLAG_TYPE(custom_double2, unsigned char); + BENCHMARK_TWO_WAY_FLAG_TYPE(rocprim::int128_t, int8_t); + BENCHMARK_TWO_WAY_FLAG_TYPE(rocprim::uint128_t, uint8_t); BENCHMARK_TWO_WAY_PREDICATE_TYPE(int); BENCHMARK_TWO_WAY_PREDICATE_TYPE(float); @@ -194,6 +200,8 @@ int main(int argc, char* argv[]) BENCHMARK_TWO_WAY_PREDICATE_TYPE(int8_t); BENCHMARK_TWO_WAY_PREDICATE_TYPE(rocprim::half); BENCHMARK_TWO_WAY_PREDICATE_TYPE(custom_int_double); + BENCHMARK_TWO_WAY_PREDICATE_TYPE(rocprim::int128_t); + BENCHMARK_TWO_WAY_PREDICATE_TYPE(rocprim::uint128_t); BENCHMARK_THREE_WAY_TYPE(int); BENCHMARK_THREE_WAY_TYPE(float); @@ -202,6 +210,8 @@ int main(int argc, char* argv[]) BENCHMARK_THREE_WAY_TYPE(int8_t); BENCHMARK_THREE_WAY_TYPE(rocprim::half); BENCHMARK_THREE_WAY_TYPE(custom_int_double); + BENCHMARK_THREE_WAY_TYPE(rocprim::int128_t); + BENCHMARK_THREE_WAY_TYPE(rocprim::uint128_t); #endif // Use manual timing diff --git a/benchmark/benchmark_device_partition.parallel.cpp.in b/benchmark/benchmark_device_partition.parallel.cpp.in index a94fbf54c..d06cdfabc 100644 --- a/benchmark/benchmark_device_partition.parallel.cpp.in +++ b/benchmark/benchmark_device_partition.parallel.cpp.in @@ -20,11 +20,13 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include - #include "benchmark_device_partition.parallel.hpp" #include "benchmark_utils.hpp" +#include + +#include + namespace { auto benchmarks = config_autotune_register::create_bulk( diff --git a/benchmark/benchmark_device_partition.parallel.hpp b/benchmark/benchmark_device_partition.parallel.hpp index afc39b669..7ebe2b8ee 100644 --- a/benchmark/benchmark_device_partition.parallel.hpp +++ b/benchmark/benchmark_device_partition.parallel.hpp @@ -28,17 +28,22 @@ #include -#include - #include -#include -#include -#include +#include +#include +#ifdef BENCHMARK_CONFIG_TUNING + #include +#endif // BENCHMARK_CONFIG_TUNING + +#include #include -#include #include #include +#ifdef BENCHMARK_CONFIG_TUNING + #include + #include +#endif // BENCHMARK_CONFIG_TUNING enum class partition_probability { @@ -114,9 +119,9 @@ inline const char* get_probability_name(partition_three_way_probability probabil constexpr int warmup_iter = 5; constexpr int batch_size = 10; -template struct device_partition_flag_benchmark : public config_autotune_interface { @@ -135,7 +140,7 @@ struct device_partition_flag_benchmark : public config_autotune_interface const managed_seed& seed, const hipStream_t stream) const override { - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(DataType); std::vector input = get_random_data(size, @@ -262,8 +267,8 @@ struct device_partition_flag_benchmark : public config_autotune_interface static constexpr bool is_tuning = Probability == partition_probability::tuning; }; -template struct device_partition_predicate_benchmark : public config_autotune_interface { @@ -281,7 +286,7 @@ struct device_partition_predicate_benchmark : public config_autotune_interface const managed_seed& seed, const hipStream_t stream) const override { - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(DataType); // all data types can represent [0, 127], -1 so a predicate can select all @@ -375,9 +380,9 @@ struct device_partition_predicate_benchmark : public config_autotune_interface static constexpr bool is_tuning = Probability == partition_probability::tuning; }; -template struct device_partition_two_way_flag_benchmark : public config_autotune_interface { @@ -396,9 +401,9 @@ struct device_partition_two_way_flag_benchmark : public config_autotune_interfac const managed_seed& seed, const hipStream_t stream) const override { - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(DataType); - + std::vector input = get_random_data(size, generate_limits::min(), generate_limits::max(), @@ -528,8 +533,8 @@ struct device_partition_two_way_flag_benchmark : public config_autotune_interfac static constexpr bool is_tuning = Probability == partition_probability::tuning; }; -template struct device_partition_two_way_predicate_benchmark : public config_autotune_interface { @@ -547,7 +552,7 @@ struct device_partition_two_way_predicate_benchmark : public config_autotune_int const managed_seed& seed, const hipStream_t stream) const override { - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(DataType); // all data types can represent [0, 127], -1 so a predicate can select all @@ -645,8 +650,8 @@ struct device_partition_two_way_predicate_benchmark : public config_autotune_int static constexpr bool is_tuning = Probability == partition_probability::tuning; }; -template struct device_partition_three_way_benchmark : public config_autotune_interface { @@ -664,7 +669,7 @@ struct device_partition_three_way_benchmark : public config_autotune_interface const managed_seed& seed, const hipStream_t stream) const override { - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(DataType); // all data types can represent [0, 127], -1 so a predicate can select all diff --git a/benchmark/benchmark_device_radix_sort.cpp b/benchmark/benchmark_device_radix_sort.cpp index afc021c46..bef154ba6 100644 --- a/benchmark/benchmark_device_radix_sort.cpp +++ b/benchmark/benchmark_device_radix_sort.cpp @@ -31,15 +31,15 @@ // HIP API #include -#include - #include +#include +#include #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { cli::Parser parser(argc, argv); parser.set_optional("size", "size", DEFAULT_BYTES, "number of bytes"); @@ -53,8 +53,8 @@ int main(int argc, char *argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); - const int trials = parser.get("trials"); + const size_t bytes = parser.get("size"); + const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); const managed_seed seed(seed_type); diff --git a/benchmark/benchmark_device_radix_sort.hpp b/benchmark/benchmark_device_radix_sort.hpp index 3f38d0975..3f8a90e3e 100644 --- a/benchmark/benchmark_device_radix_sort.hpp +++ b/benchmark/benchmark_device_radix_sort.hpp @@ -32,14 +32,16 @@ #include // rocPRIM +#include #include +#include +#include +#include #include #include #include -#include - namespace rp = rocprim; template keys_input @@ -101,7 +103,7 @@ struct device_radix_sort_benchmark : public config_autotune_interface HIP_CHECK(hipDeviceSynchronize()); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { HIP_CHECK(invoke_radix_sort(d_temporary_storage, temporary_storage_bytes, @@ -124,7 +126,7 @@ struct device_radix_sort_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { HIP_CHECK(invoke_radix_sort(d_temporary_storage, temporary_storage_bytes, @@ -168,7 +170,7 @@ struct device_radix_sort_benchmark : public config_autotune_interface using key_type = Key; using value_type = Value; - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(key_type); std::vector keys_input @@ -178,7 +180,7 @@ struct device_radix_sort_benchmark : public config_autotune_interface seed.get_0()); std::vector values_input(size); - for(size_t i = 0; i < size; i++) + for(size_t i = 0; i < size; ++i) { values_input[i] = value_type(i); } @@ -216,7 +218,7 @@ struct device_radix_sort_benchmark : public config_autotune_interface HIP_CHECK(hipDeviceSynchronize()); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { HIP_CHECK(invoke_radix_sort(d_temporary_storage, temporary_storage_bytes, @@ -239,7 +241,7 @@ struct device_radix_sort_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { HIP_CHECK(invoke_radix_sort(d_temporary_storage, temporary_storage_bytes, @@ -379,9 +381,9 @@ struct device_radix_sort_benchmark : public config_autotune_interface } }; -#define CREATE_RADIX_SORT_BENCHMARK(...) \ - { \ - const device_radix_sort_benchmark<__VA_ARGS__> instance; \ +#define CREATE_RADIX_SORT_BENCHMARK(...) \ + { \ + const device_radix_sort_benchmark<__VA_ARGS__> instance; \ REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } @@ -399,6 +401,8 @@ inline void add_sort_keys_benchmarks(std::vector& benchmarks, @@ -427,6 +431,8 @@ inline void add_sort_pairs_benchmarks(std::vector +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif #include +#include +#include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; @@ -43,7 +50,7 @@ const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #define CREATE_BENCHMARK(...) \ { \ const device_radix_sort_block_sort_benchmark<__VA_ARGS__> instance; \ - REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ + REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } int main(int argc, char* argv[]) @@ -71,7 +78,7 @@ int main(int argc, char* argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); + const size_t bytes = parser.get("size"); const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); @@ -103,6 +110,8 @@ int main(int argc, char* argv[]) CREATE_BENCHMARK(uint8_t) CREATE_BENCHMARK(rocprim::half) CREATE_BENCHMARK(short) + CREATE_BENCHMARK(rocprim::int128_t) + CREATE_BENCHMARK(rocprim::uint128_t) using custom_float2 = custom_type; using custom_double2 = custom_type; @@ -117,6 +126,8 @@ int main(int argc, char* argv[]) CREATE_BENCHMARK(int, custom_float2) CREATE_BENCHMARK(int, custom_char_double) CREATE_BENCHMARK(long long, custom_double2) + CREATE_BENCHMARK(rocprim::int128_t, rocprim::int128_t) + CREATE_BENCHMARK(rocprim::uint128_t, rocprim::uint128_t) #endif // BENCHMARK_CONFIG_TUNING // Use manual timing diff --git a/benchmark/benchmark_device_radix_sort_block_sort.parallel.cpp.in b/benchmark/benchmark_device_radix_sort_block_sort.parallel.cpp.in index d14348a8d..2e7c45ccc 100644 --- a/benchmark/benchmark_device_radix_sort_block_sort.parallel.cpp.in +++ b/benchmark/benchmark_device_radix_sort_block_sort.parallel.cpp.in @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,11 +20,13 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include - #include "benchmark_device_radix_sort_block_sort.parallel.hpp" #include "benchmark_utils.hpp" +#include + +#include + namespace { auto benchmarks = config_autotune_register::create_bulk( device_radix_sort_block_sort_benchmark_generator<@BlockSize@, @KeyType@, @ValueType@>::create); diff --git a/benchmark/benchmark_device_radix_sort_block_sort.parallel.hpp b/benchmark/benchmark_device_radix_sort_block_sort.parallel.hpp index 3c9c956e2..6578a28fe 100644 --- a/benchmark/benchmark_device_radix_sort_block_sort.parallel.hpp +++ b/benchmark/benchmark_device_radix_sort_block_sort.parallel.hpp @@ -32,15 +32,17 @@ #include // rocPRIM +#include #include +#include +#include +#include +#include #include +#include #include -#include - -namespace rp = rocprim; - template std::string config_name() { @@ -81,7 +83,7 @@ struct device_radix_sort_block_sort_benchmark : public config_autotune_interface { using key_type = Key; - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(key_type); // Generate data @@ -103,19 +105,20 @@ struct device_radix_sort_block_sort_benchmark : public config_autotune_interface rocprim::empty_type* values_ptr = nullptr; unsigned int items_per_block; // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { - HIP_CHECK((rp::detail::radix_sort_block_sort(d_keys_input, - d_keys_output, - values_ptr, - values_ptr, - size, - items_per_block, - rp::identity_decomposer{}, - 0, - sizeof(key_type) * 8, - stream, - false))); + HIP_CHECK((rocprim::detail::radix_sort_block_sort( + d_keys_input, + d_keys_output, + values_ptr, + values_ptr, + size, + items_per_block, + rocprim::identity_decomposer{}, + 0, + sizeof(key_type) * 8, + stream, + false))); } HIP_CHECK(hipDeviceSynchronize()); @@ -129,20 +132,20 @@ struct device_radix_sort_block_sort_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { - HIP_CHECK( - (rp::detail::radix_sort_block_sort(d_keys_input, - d_keys_output, - values_ptr, - values_ptr, - size, - items_per_block, - rp::identity_decomposer{}, - 0, - sizeof(key_type) * 8, - stream, - false))); + HIP_CHECK((rocprim::detail::radix_sort_block_sort( + d_keys_input, + d_keys_output, + values_ptr, + values_ptr, + size, + items_per_block, + rocprim::identity_decomposer{}, + 0, + sizeof(key_type) * 8, + stream, + false))); } // Record stop event and wait until it completes @@ -176,7 +179,7 @@ struct device_radix_sort_block_sort_benchmark : public config_autotune_interface using key_type = Key; using value_type = Value; - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(key_type); // Generate data @@ -187,7 +190,7 @@ struct device_radix_sort_block_sort_benchmark : public config_autotune_interface seed.get_0()); std::vector values_input(size); - for(size_t i = 0; i < size; i++) + for(size_t i = 0; i < size; ++i) { values_input[i] = value_type(i); } @@ -215,19 +218,20 @@ struct device_radix_sort_block_sort_benchmark : public config_autotune_interface HIP_CHECK(hipDeviceSynchronize()); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { - HIP_CHECK((rp::detail::radix_sort_block_sort(d_keys_input, - d_keys_output, - d_values_input, - d_values_output, - size, - items_per_block, - rp::identity_decomposer{}, - 0, - sizeof(key_type) * 8, - stream, - false))); + HIP_CHECK((rocprim::detail::radix_sort_block_sort( + d_keys_input, + d_keys_output, + d_values_input, + d_values_output, + size, + items_per_block, + rocprim::identity_decomposer{}, + 0, + sizeof(key_type) * 8, + stream, + false))); } HIP_CHECK(hipDeviceSynchronize()); @@ -241,20 +245,20 @@ struct device_radix_sort_block_sort_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { - HIP_CHECK( - (rp::detail::radix_sort_block_sort(d_keys_input, - d_keys_output, - d_values_input, - d_values_output, - size, - items_per_block, - rp::identity_decomposer{}, - 0, - sizeof(key_type) * 8, - stream, - false))); + HIP_CHECK((rocprim::detail::radix_sort_block_sort( + d_keys_input, + d_keys_output, + d_values_input, + d_values_output, + size, + items_per_block, + rocprim::identity_decomposer{}, + 0, + sizeof(key_type) * 8, + stream, + false))); } // Record stop event and wait until it completes diff --git a/benchmark/benchmark_device_radix_sort_onesweep.cpp b/benchmark/benchmark_device_radix_sort_onesweep.cpp index bd2b81800..3041cd5ea 100644 --- a/benchmark/benchmark_device_radix_sort_onesweep.cpp +++ b/benchmark/benchmark_device_radix_sort_onesweep.cpp @@ -32,9 +32,9 @@ #include "benchmark_device_radix_sort_onesweep.parallel.hpp" #include "benchmark_utils.hpp" -#include - #include +#include +#include #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; @@ -65,7 +65,7 @@ int main(int argc, char* argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); + const size_t bytes = parser.get("size"); const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); diff --git a/benchmark/benchmark_device_radix_sort_onesweep.parallel.cpp.in b/benchmark/benchmark_device_radix_sort_onesweep.parallel.cpp.in index ebe3c24e2..3560bf81e 100644 --- a/benchmark/benchmark_device_radix_sort_onesweep.parallel.cpp.in +++ b/benchmark/benchmark_device_radix_sort_onesweep.parallel.cpp.in @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,11 +20,13 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include - #include "benchmark_utils.hpp" #include "benchmark_device_radix_sort_onesweep.parallel.hpp" +#include + +#include + namespace { auto benchmarks = config_autotune_register::create_bulk( device_radix_sort_onesweep_benchmark_generator<@BlockSize@, @RadixBits@, @KeyType@, @ValueType@>::create); diff --git a/benchmark/benchmark_device_radix_sort_onesweep.parallel.hpp b/benchmark/benchmark_device_radix_sort_onesweep.parallel.hpp index c43696700..40db61f3c 100644 --- a/benchmark/benchmark_device_radix_sort_onesweep.parallel.hpp +++ b/benchmark/benchmark_device_radix_sort_onesweep.parallel.hpp @@ -32,23 +32,33 @@ #include // rocPRIM +#include +#include +#include #include +#include +#ifdef BENCHMARK_CONFIG_TUNING + #include +#endif +#include #include +#include #include +#ifdef BENCHMARK_CONFIG_TUNING + #include +#else + #include +#endif -#include - -namespace rp = rocprim; - -constexpr const char* radix_rank_algorithm_name(rp::block_radix_rank_algorithm algorithm) +constexpr const char* radix_rank_algorithm_name(rocprim::block_radix_rank_algorithm algorithm) { switch(algorithm) { - case rp::block_radix_rank_algorithm::basic: return "block_radix_rank_algorithm::basic"; - case rp::block_radix_rank_algorithm::basic_memoize: + case rocprim::block_radix_rank_algorithm::basic: return "block_radix_rank_algorithm::basic"; + case rocprim::block_radix_rank_algorithm::basic_memoize: return "block_radix_rank_algorithm::basic_memoize"; - case rp::block_radix_rank_algorithm::match: return "block_radix_rank_algorithm::match"; + case rocprim::block_radix_rank_algorithm::match: return "block_radix_rank_algorithm::match"; } return ""; // unknown algorithm @@ -98,7 +108,7 @@ struct device_radix_sort_onesweep_benchmark : public config_autotune_interface { using key_type = Key; - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(key_type); std::vector keys_input @@ -122,31 +132,8 @@ struct device_radix_sort_onesweep_benchmark : public config_autotune_interface bool is_result_in_output = true; rocprim::empty_type* d_values_ptr = nullptr; - HIP_CHECK((rp::detail::radix_sort_onesweep_impl(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - nullptr, - d_keys_output, - d_values_ptr, - nullptr, - d_values_ptr, - size, - is_result_in_output, - rp::identity_decomposer{}, - 0, - sizeof(key_type) * 8, - stream, - false, - false))); - - HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < warmup_size; i++) - { - HIP_CHECK( - (rp::detail::radix_sort_onesweep_impl(d_temporary_storage, + HIP_CHECK(( + rocprim::detail::radix_sort_onesweep_impl(d_temporary_storage, temporary_storage_bytes, d_keys_input, nullptr, @@ -156,12 +143,36 @@ struct device_radix_sort_onesweep_benchmark : public config_autotune_interface d_values_ptr, size, is_result_in_output, - rp::identity_decomposer{}, + rocprim::identity_decomposer{}, 0, sizeof(key_type) * 8, stream, false, false))); + + HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); + HIP_CHECK(hipDeviceSynchronize()); + + // Warm-up + for(size_t i = 0; i < warmup_size; ++i) + { + HIP_CHECK((rocprim::detail::radix_sort_onesweep_impl( + d_temporary_storage, + temporary_storage_bytes, + d_keys_input, + nullptr, + d_keys_output, + d_values_ptr, + nullptr, + d_values_ptr, + size, + is_result_in_output, + rocprim::identity_decomposer{}, + 0, + sizeof(key_type) * 8, + stream, + false, + false))); } HIP_CHECK(hipDeviceSynchronize()); @@ -175,25 +186,25 @@ struct device_radix_sort_onesweep_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { - HIP_CHECK( - (rp::detail::radix_sort_onesweep_impl(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - nullptr, - d_keys_output, - d_values_ptr, - nullptr, - d_values_ptr, - size, - is_result_in_output, - rp::identity_decomposer{}, - 0, - sizeof(key_type) * 8, - stream, - false, - false))); + HIP_CHECK((rocprim::detail::radix_sort_onesweep_impl( + d_temporary_storage, + temporary_storage_bytes, + d_keys_input, + nullptr, + d_keys_output, + d_values_ptr, + nullptr, + d_values_ptr, + size, + is_result_in_output, + rocprim::identity_decomposer{}, + 0, + sizeof(key_type) * 8, + stream, + false, + false))); } // Record stop event and wait until it completes @@ -228,7 +239,7 @@ struct device_radix_sort_onesweep_benchmark : public config_autotune_interface using key_type = Key; using value_type = Value; - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(key_type); std::vector keys_input @@ -238,7 +249,7 @@ struct device_radix_sort_onesweep_benchmark : public config_autotune_interface seed.get_0()); std::vector values_input(size); - for(size_t i = 0; i < size; i++) + for(size_t i = 0; i < size; ++i) { values_input[i] = value_type(i); } @@ -265,31 +276,8 @@ struct device_radix_sort_onesweep_benchmark : public config_autotune_interface size_t temporary_storage_bytes = 0; bool is_result_in_output = true; - HIP_CHECK((rp::detail::radix_sort_onesweep_impl(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - nullptr, - d_keys_output, - d_values_input, - nullptr, - d_values_output, - size, - is_result_in_output, - rp::identity_decomposer{}, - 0, - sizeof(key_type) * 8, - stream, - false, - false))); - - HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < warmup_size; i++) - { - HIP_CHECK( - (rp::detail::radix_sort_onesweep_impl(d_temporary_storage, + HIP_CHECK(( + rocprim::detail::radix_sort_onesweep_impl(d_temporary_storage, temporary_storage_bytes, d_keys_input, nullptr, @@ -299,12 +287,36 @@ struct device_radix_sort_onesweep_benchmark : public config_autotune_interface d_values_output, size, is_result_in_output, - rp::identity_decomposer{}, + rocprim::identity_decomposer{}, 0, sizeof(key_type) * 8, stream, false, false))); + + HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); + HIP_CHECK(hipDeviceSynchronize()); + + // Warm-up + for(size_t i = 0; i < warmup_size; ++i) + { + HIP_CHECK((rocprim::detail::radix_sort_onesweep_impl( + d_temporary_storage, + temporary_storage_bytes, + d_keys_input, + nullptr, + d_keys_output, + d_values_input, + nullptr, + d_values_output, + size, + is_result_in_output, + rocprim::identity_decomposer{}, + 0, + sizeof(key_type) * 8, + stream, + false, + false))); } HIP_CHECK(hipDeviceSynchronize()); @@ -318,25 +330,25 @@ struct device_radix_sort_onesweep_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { - HIP_CHECK( - (rp::detail::radix_sort_onesweep_impl(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - nullptr, - d_keys_output, - d_values_input, - nullptr, - d_values_output, - size, - is_result_in_output, - rp::identity_decomposer{}, - 0, - sizeof(key_type) * 8, - stream, - false, - false))); + HIP_CHECK((rocprim::detail::radix_sort_onesweep_impl( + d_temporary_storage, + temporary_storage_bytes, + d_keys_input, + nullptr, + d_keys_output, + d_values_input, + nullptr, + d_values_output, + size, + is_result_in_output, + rocprim::identity_decomposer{}, + 0, + sizeof(key_type) * 8, + stream, + false, + false))); } // Record stop event and wait until it completes @@ -386,16 +398,16 @@ struct device_radix_sort_onesweep_benchmark_generator // Calculation uses `rocprim::device_warp_size()`, which is 64 on host side unless overridden. // However, this does not affect the total size of shared memory for the current configuration space. // Were the implementation to change, causing retuning, this needs to be re-evaluated and possibly taken into account. - using sharedmem_storage = - typename rp::detail::onesweep_iteration_helper::storage_type; + using sharedmem_storage = typename rocprim::detail::onesweep_iteration_helper< + Key, + Value, + size_t, + BlockSize, + ItemsPerThread, + RadixBits, + false, + RadixRankAlgorithm, + rocprim::identity_decomposer>::storage_type; return sizeof(sharedmem_storage) < TUNING_SHARED_MEMORY_MAX; } @@ -455,7 +467,7 @@ struct device_radix_sort_onesweep_benchmark_generator #define CREATE_RADIX_SORT_BENCHMARK(...) \ { \ const device_radix_sort_onesweep_benchmark<__VA_ARGS__> instance; \ - REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ + REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } inline void add_sort_keys_benchmarks(std::vector& benchmarks, @@ -470,6 +482,8 @@ inline void add_sort_keys_benchmarks(std::vector& benchmarks, @@ -496,6 +510,8 @@ inline void add_sort_pairs_benchmarks(std::vector -#include +#ifndef BENCHMARK_CONFIG_TUNING + #include + #include +#endif #include +#include +#include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 128 * 4; #endif -#define CREATE_BENCHMARK(T, REDUCE_OP) \ - { \ - const device_reduce_benchmark instance; \ +#define CREATE_BENCHMARK(T, REDUCE_OP) \ + { \ + const device_reduce_benchmark instance; \ REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { cli::Parser parser(argc, argv); parser.set_optional("size", "size", DEFAULT_BYTES, "number of bytes"); @@ -70,8 +78,8 @@ int main(int argc, char *argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); - const int trials = parser.get("trials"); + const size_t bytes = parser.get("size"); + const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); const managed_seed seed(seed_type); @@ -87,7 +95,7 @@ int main(int argc, char *argv[]) // Add benchmarks std::vector benchmarks = {}; #ifdef BENCHMARK_CONFIG_TUNING - const int parallel_instance = parser.get("parallel_instance"); + const int parallel_instance = parser.get("parallel_instance"); const int parallel_instances = parser.get("parallel_instances"); config_autotune_register::register_benchmark_subset(benchmarks, parallel_instance, @@ -96,7 +104,7 @@ int main(int argc, char *argv[]) seed, stream); #else - using custom_float2 = custom_type; + using custom_float2 = custom_type; using custom_double2 = custom_type; CREATE_BENCHMARK(int, rocprim::plus) @@ -111,6 +119,9 @@ int main(int argc, char *argv[]) CREATE_BENCHMARK(custom_float2, rocprim::plus) CREATE_BENCHMARK(custom_double2, rocprim::plus) + + CREATE_BENCHMARK(rocprim::int128_t, rocprim::plus) + CREATE_BENCHMARK(rocprim::uint128_t, rocprim::plus) #endif // Use manual timing diff --git a/benchmark/benchmark_device_reduce.parallel.cpp.in b/benchmark/benchmark_device_reduce.parallel.cpp.in index b6599581d..a29fd2721 100644 --- a/benchmark/benchmark_device_reduce.parallel.cpp.in +++ b/benchmark/benchmark_device_reduce.parallel.cpp.in @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,11 +20,16 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include - #include "benchmark_utils.hpp" #include "benchmark_device_reduce.parallel.hpp" +#include +#include +#include +#include + +#include + namespace { auto benchmark = config_autotune_register::create, diff --git a/benchmark/benchmark_device_reduce.parallel.hpp b/benchmark/benchmark_device_reduce.parallel.hpp index 32a867051..9b0d4ef7e 100644 --- a/benchmark/benchmark_device_reduce.parallel.hpp +++ b/benchmark/benchmark_device_reduce.parallel.hpp @@ -32,14 +32,16 @@ #include // rocPRIM HIP API +#include +#include #include #include +#include +#include #include #include -#include - constexpr const char* get_reduce_method_name(rocprim::block_reduce_algorithm alg) { switch(alg) @@ -81,7 +83,7 @@ struct device_reduce_benchmark : public config_autotune_interface + ",cfg:" + config_name() + "}"); } - static constexpr unsigned int batch_size = 10; + static constexpr unsigned int batch_size = 10; static constexpr unsigned int warmup_size = 5; void run(benchmark::State& state, @@ -89,7 +91,7 @@ struct device_reduce_benchmark : public config_autotune_interface const managed_seed& seed, hipStream_t stream) const override { - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(T); BinaryFunction reduce_op{}; @@ -97,43 +99,39 @@ struct device_reduce_benchmark : public config_autotune_interface std::vector input = get_random_data(size, random_range.first, random_range.second, seed.get_0()); - T * d_input; - T * d_output; + T* d_input; + T* d_output; HIP_CHECK(hipMalloc(reinterpret_cast(&d_input), size * sizeof(T))); HIP_CHECK(hipMalloc(reinterpret_cast(&d_output), sizeof(T))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - size * sizeof(T), - hipMemcpyHostToDevice - ) - ); + HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); HIP_CHECK(hipDeviceSynchronize()); // Allocate temporary storage memory size_t temp_storage_size_bytes; - void * d_temp_storage = nullptr; + void* d_temp_storage = nullptr; // Get size of d_temp_storage - HIP_CHECK( - rocprim::reduce( - d_temp_storage, temp_storage_size_bytes, - d_input, d_output, T(), size, - reduce_op, stream - ) - ); - HIP_CHECK(hipMalloc(&d_temp_storage,temp_storage_size_bytes)); + HIP_CHECK(rocprim::reduce(d_temp_storage, + temp_storage_size_bytes, + d_input, + d_output, + T(), + size, + reduce_op, + stream)); + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_size_bytes)); HIP_CHECK(hipDeviceSynchronize()); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { - HIP_CHECK( - rocprim::reduce( - d_temp_storage, temp_storage_size_bytes, - d_input, d_output, T(), size, - reduce_op, stream - ) - ); + HIP_CHECK(rocprim::reduce(d_temp_storage, + temp_storage_size_bytes, + d_input, + d_output, + T(), + size, + reduce_op, + stream)); } HIP_CHECK(hipDeviceSynchronize()); @@ -147,15 +145,16 @@ struct device_reduce_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { - HIP_CHECK( - rocprim::reduce( - d_temp_storage, temp_storage_size_bytes, - d_input, d_output, T(), size, - reduce_op, stream - ) - ); + HIP_CHECK(rocprim::reduce(d_temp_storage, + temp_storage_size_bytes, + d_input, + d_output, + T(), + size, + reduce_op, + stream)); } HIP_CHECK(hipStreamSynchronize(stream)); diff --git a/benchmark/benchmark_device_reduce_by_key.cpp b/benchmark/benchmark_device_reduce_by_key.cpp index b2d01ec6f..f367acc87 100644 --- a/benchmark/benchmark_device_reduce_by_key.cpp +++ b/benchmark/benchmark_device_reduce_by_key.cpp @@ -31,9 +31,16 @@ // HIP API #include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif + #include #include #include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif #ifndef DEFAULT_BYTES constexpr size_t DEFAULT_BYTES = size_t{2} << 30; // 2 GiB @@ -50,21 +57,25 @@ constexpr size_t DEFAULT_BYTES = size_t{2} << 30; // 2 GiB CREATE_BENCHMARK(KEY, VALUE, 1000) // some of the tuned types -#define CREATE_BENCHMARK_TYPES(KEY) \ - CREATE_BENCHMARK_TYPE(KEY, int8_t); \ - CREATE_BENCHMARK_TYPE(KEY, rocprim::half); \ - CREATE_BENCHMARK_TYPE(KEY, int32_t); \ - CREATE_BENCHMARK_TYPE(KEY, float); \ +#define CREATE_BENCHMARK_TYPES(KEY) \ + CREATE_BENCHMARK_TYPE(KEY, int8_t); \ + CREATE_BENCHMARK_TYPE(KEY, rocprim::half); \ + CREATE_BENCHMARK_TYPE(KEY, int32_t); \ + CREATE_BENCHMARK_TYPE(KEY, rocprim::int128_t); \ + CREATE_BENCHMARK_TYPE(KEY, rocprim::uint128_t); \ + CREATE_BENCHMARK_TYPE(KEY, float); \ CREATE_BENCHMARK_TYPE(KEY, double) // all of the tuned types -#define CREATE_BENCHMARK_TYPE_TUNING(KEY) \ - CREATE_BENCHMARK_TYPE(KEY, int8_t); \ - CREATE_BENCHMARK_TYPE(KEY, int16_t); \ - CREATE_BENCHMARK_TYPE(KEY, int32_t); \ - CREATE_BENCHMARK_TYPE(KEY, int64_t); \ - CREATE_BENCHMARK_TYPE(KEY, rocprim::half); \ - CREATE_BENCHMARK_TYPE(KEY, float); \ +#define CREATE_BENCHMARK_TYPE_TUNING(KEY) \ + CREATE_BENCHMARK_TYPE(KEY, int8_t); \ + CREATE_BENCHMARK_TYPE(KEY, int16_t); \ + CREATE_BENCHMARK_TYPE(KEY, int32_t); \ + CREATE_BENCHMARK_TYPE(KEY, int64_t); \ + CREATE_BENCHMARK_TYPE(KEY, rocprim::int128_t); \ + CREATE_BENCHMARK_TYPE(KEY, rocprim::uint128_t); \ + CREATE_BENCHMARK_TYPE(KEY, rocprim::half); \ + CREATE_BENCHMARK_TYPE(KEY, float); \ CREATE_BENCHMARK_TYPE(KEY, double) int main(int argc, char* argv[]) @@ -126,6 +137,8 @@ int main(int argc, char* argv[]) CREATE_BENCHMARK_TYPES(rocprim::half); CREATE_BENCHMARK_TYPES(float); CREATE_BENCHMARK_TYPES(double); + CREATE_BENCHMARK_TYPES(rocprim::int128_t); + CREATE_BENCHMARK_TYPES(rocprim::uint128_t); // custom types using custom_float2 = custom_type; diff --git a/benchmark/benchmark_device_reduce_by_key.parallel.cpp.in b/benchmark/benchmark_device_reduce_by_key.parallel.cpp.in index 50c5fb2e2..e43d984f3 100644 --- a/benchmark/benchmark_device_reduce_by_key.parallel.cpp.in +++ b/benchmark/benchmark_device_reduce_by_key.parallel.cpp.in @@ -20,11 +20,13 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include - #include "benchmark_utils.hpp" #include "benchmark_device_reduce_by_key.parallel.hpp" +#include + +#include + namespace { auto benchmarks = config_autotune_register::create_bulk( device_reduce_by_key_benchmark_generator<@KeyType@, @ValueType@, @BlockSize@>::create); diff --git a/benchmark/benchmark_device_reduce_by_key.parallel.hpp b/benchmark/benchmark_device_reduce_by_key.parallel.hpp index edfd5e99f..d9e8a6be9 100644 --- a/benchmark/benchmark_device_reduce_by_key.parallel.hpp +++ b/benchmark/benchmark_device_reduce_by_key.parallel.hpp @@ -32,12 +32,26 @@ #include // rocPRIM HIP API +#include +#include #include #include +#include +#ifdef BENCHMARK_CONFIG_TUNING + #include + #include +#endif +#include #include +#include #include +#include #include +#ifdef BENCHMARK_CONFIG_TUNING + #include + #include +#endif template std::string config_name() @@ -226,7 +240,7 @@ struct device_reduce_by_key_benchmark : public config_autotune_interface template struct device_reduce_by_key_benchmark_generator { - template + template struct create_ipt { void operator()(std::vector>& storage) @@ -246,7 +260,11 @@ struct device_reduce_by_key_benchmark_generator static void create(std::vector>& storage) { - static_for_each, create_ipt>(storage); + static constexpr unsigned int max_items_per_thread = std::min( + TUNING_SHARED_MEMORY_MAX / std::max(sizeof(KeyType), sizeof(ValueType)) / BlockSize - 1, + size_t{15}); + static_for_each, create_ipt>( + storage); } }; diff --git a/benchmark/benchmark_device_reduce_by_key_deterministic.cpp b/benchmark/benchmark_device_reduce_by_key_deterministic.cpp index ea7c66b31..ae6c4b0a5 100644 --- a/benchmark/benchmark_device_reduce_by_key_deterministic.cpp +++ b/benchmark/benchmark_device_reduce_by_key_deterministic.cpp @@ -31,6 +31,8 @@ // HIP API #include +#include + #include #include #include @@ -50,22 +52,26 @@ constexpr size_t DEFAULT_BYTES = size_t{2} << 30; // 2 GiB CREATE_BENCHMARK(KEY, VALUE, 1000) // some of the tuned types -#define CREATE_BENCHMARK_TYPES(KEY) \ - CREATE_BENCHMARK_TYPE(KEY, int8_t); \ - CREATE_BENCHMARK_TYPE(KEY, rocprim::half); \ - CREATE_BENCHMARK_TYPE(KEY, int32_t); \ - CREATE_BENCHMARK_TYPE(KEY, float); \ - CREATE_BENCHMARK_TYPE(KEY, double) +#define CREATE_BENCHMARK_TYPES(KEY) \ + CREATE_BENCHMARK_TYPE(KEY, int8_t); \ + CREATE_BENCHMARK_TYPE(KEY, rocprim::half); \ + CREATE_BENCHMARK_TYPE(KEY, int32_t); \ + CREATE_BENCHMARK_TYPE(KEY, float); \ + CREATE_BENCHMARK_TYPE(KEY, double); \ + CREATE_BENCHMARK_TYPE(KEY, rocprim::int128_t); \ + CREATE_BENCHMARK_TYPE(KEY, rocprim::uint128_t) // all of the tuned types -#define CREATE_BENCHMARK_TYPE_TUNING(KEY) \ - CREATE_BENCHMARK_TYPE(KEY, int8_t); \ - CREATE_BENCHMARK_TYPE(KEY, int16_t); \ - CREATE_BENCHMARK_TYPE(KEY, int32_t); \ - CREATE_BENCHMARK_TYPE(KEY, int64_t); \ - CREATE_BENCHMARK_TYPE(KEY, rocprim::half); \ - CREATE_BENCHMARK_TYPE(KEY, float); \ - CREATE_BENCHMARK_TYPE(KEY, double) +#define CREATE_BENCHMARK_TYPE_TUNING(KEY) \ + CREATE_BENCHMARK_TYPE(KEY, int8_t); \ + CREATE_BENCHMARK_TYPE(KEY, int16_t); \ + CREATE_BENCHMARK_TYPE(KEY, int32_t); \ + CREATE_BENCHMARK_TYPE(KEY, int64_t); \ + CREATE_BENCHMARK_TYPE(KEY, rocprim::half); \ + CREATE_BENCHMARK_TYPE(KEY, float); \ + CREATE_BENCHMARK_TYPE(KEY, double); \ + CREATE_BENCHMARK_TYPE(KEY, rocprim::int128_t); \ + CREATE_BENCHMARK_TYPE(KEY, rocprim::uint128_t) int main(int argc, char* argv[]) { @@ -105,6 +111,8 @@ int main(int argc, char* argv[]) CREATE_BENCHMARK_TYPES(rocprim::half); CREATE_BENCHMARK_TYPES(float); CREATE_BENCHMARK_TYPES(double); + CREATE_BENCHMARK_TYPES(rocprim::int128_t); + CREATE_BENCHMARK_TYPES(rocprim::uint128_t); // custom types using custom_float2 = custom_type; diff --git a/benchmark/benchmark_device_run_length_encode.cpp b/benchmark/benchmark_device_run_length_encode.cpp index 80abe2d9b..660484969 100644 --- a/benchmark/benchmark_device_run_length_encode.cpp +++ b/benchmark/benchmark_device_run_length_encode.cpp @@ -33,9 +33,10 @@ #include // rocPRIM -#include +#include -#include +#include +#include #include #include @@ -63,6 +64,8 @@ void add_encode_benchmarks(std::vector& benchma CREATE_ENCODE_BENCHMARK(int16_t, MaxLength); CREATE_ENCODE_BENCHMARK(int32_t, MaxLength); CREATE_ENCODE_BENCHMARK(int64_t, MaxLength); + CREATE_ENCODE_BENCHMARK(rocprim::int128_t, MaxLength); + CREATE_ENCODE_BENCHMARK(rocprim::uint128_t, MaxLength); CREATE_ENCODE_BENCHMARK(rocprim::half, MaxLength); CREATE_ENCODE_BENCHMARK(float, MaxLength); CREATE_ENCODE_BENCHMARK(double, MaxLength); diff --git a/benchmark/benchmark_device_run_length_encode.parallel.cpp.in b/benchmark/benchmark_device_run_length_encode.parallel.cpp.in index 7515e7586..c3d6888fc 100644 --- a/benchmark/benchmark_device_run_length_encode.parallel.cpp.in +++ b/benchmark/benchmark_device_run_length_encode.parallel.cpp.in @@ -20,11 +20,13 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include - #include "benchmark_device_run_length_encode.parallel.hpp" #include "benchmark_utils.hpp" +#include + +#include + namespace { auto benchmarks = config_autotune_register::create_bulk( device_run_length_encode_benchmark_generator< diff --git a/benchmark/benchmark_device_run_length_encode.parallel.hpp b/benchmark/benchmark_device_run_length_encode.parallel.hpp index ccc61f55d..e0a84146a 100644 --- a/benchmark/benchmark_device_run_length_encode.parallel.hpp +++ b/benchmark/benchmark_device_run_length_encode.parallel.hpp @@ -32,12 +32,17 @@ #include // rocPRIM +#include #include #include +#ifdef BENCHMARK_CONFIG_TUNING + #include + #include +#endif #include +#include #include -#include #include #ifdef BENCHMARK_CONFIG_TUNING #include @@ -218,7 +223,10 @@ struct device_run_length_encode_benchmark_generator static void create(std::vector>& storage) { - static_for_each, create_ipt>(storage); + static constexpr unsigned int max_items_per_thread + = std::min(TUNING_SHARED_MEMORY_MAX / sizeof(T) / BlockSize - 1, size_t{15}); + static_for_each, create_ipt>( + storage); } }; diff --git a/benchmark/benchmark_device_run_length_encode_non_trivial_runs.cpp b/benchmark/benchmark_device_run_length_encode_non_trivial_runs.cpp index 56cb78777..5fad62914 100644 --- a/benchmark/benchmark_device_run_length_encode_non_trivial_runs.cpp +++ b/benchmark/benchmark_device_run_length_encode_non_trivial_runs.cpp @@ -33,9 +33,10 @@ #include // rocPRIM -#include +#include -#include +#include +#include #include #include @@ -63,6 +64,8 @@ void add_non_trivial_runs_benchmarks(std::vector - #include "benchmark_device_run_length_encode_non_trivial_runs.parallel.hpp" #include "benchmark_utils.hpp" +#include +#include + +#include + namespace { auto benchmarks = config_autotune_register::create_bulk( device_non_trivial_runs_benchmark_generator< diff --git a/benchmark/benchmark_device_run_length_encode_non_trivial_runs.parallel.hpp b/benchmark/benchmark_device_run_length_encode_non_trivial_runs.parallel.hpp index b057d7b6f..add6a686e 100644 --- a/benchmark/benchmark_device_run_length_encode_non_trivial_runs.parallel.hpp +++ b/benchmark/benchmark_device_run_length_encode_non_trivial_runs.parallel.hpp @@ -32,15 +32,24 @@ #include // rocPRIM +#include #include #include +#ifdef BENCHMARK_CONFIG_TUNING + #include + #include + #include + #include + #include +#endif -#include #include +#include #include #include #include #ifdef BENCHMARK_CONFIG_TUNING + #include #include #endif @@ -215,9 +224,11 @@ struct device_non_trivial_runs_benchmark_generator = std::max(sizeof(T), sizeof(OffsetCountPairT)); static constexpr unsigned int max_items_per_thread = max_shared_memory / (BlockSize * max_size_per_element); - static constexpr unsigned int max_items_per_thread_exponent - = rocprim::Log2::VALUE - 1; static constexpr unsigned int min_items_per_thread_exponent = 3u; + static constexpr unsigned int max_items_per_thread_exponent + = std::max(static_cast(rocprim::Log2::VALUE), + min_items_per_thread_exponent) + - 1u; static constexpr bool is_load_warp_transpose = BlockLoadMethod == ::rocprim::block_load_method::block_load_warp_transpose; diff --git a/benchmark/benchmark_device_scan.cpp b/benchmark/benchmark_device_scan.cpp index 7a8fce15f..c1cb0d5ee 100644 --- a/benchmark/benchmark_device_scan.cpp +++ b/benchmark/benchmark_device_scan.cpp @@ -31,17 +31,25 @@ // HIP API #include -#include +#ifndef BENCHMARK_CONFIG_TUNING + #include + #include +#endif #include +#include +#include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif -#define CREATE_EXCL_INCL_BENCHMARK(EXCL, T, SCAN_OP) \ - { \ - const device_scan_benchmark instance; \ +#define CREATE_EXCL_INCL_BENCHMARK(EXCL, T, SCAN_OP) \ + { \ + const device_scan_benchmark instance; \ REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } @@ -74,7 +82,7 @@ int main(int argc, char* argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); + const size_t bytes = parser.get("size"); const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); @@ -114,6 +122,8 @@ int main(int argc, char* argv[]) CREATE_BENCHMARK(int8_t, rocprim::plus) CREATE_BENCHMARK(uint8_t, rocprim::plus) CREATE_BENCHMARK(rocprim::half, rocprim::plus) + CREATE_BENCHMARK(rocprim::int128_t, rocprim::plus) + CREATE_BENCHMARK(rocprim::uint128_t, rocprim::plus) #endif // Use manual timing diff --git a/benchmark/benchmark_device_scan.parallel.cpp.in b/benchmark/benchmark_device_scan.parallel.cpp.in index b30545f8a..86bc54475 100644 --- a/benchmark/benchmark_device_scan.parallel.cpp.in +++ b/benchmark/benchmark_device_scan.parallel.cpp.in @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,11 +20,14 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include - #include "benchmark_device_scan.parallel.hpp" #include "benchmark_utils.hpp" +#include +#include + +#include + namespace { auto benchmarks = config_autotune_register::create_bulk( diff --git a/benchmark/benchmark_device_scan.parallel.hpp b/benchmark/benchmark_device_scan.parallel.hpp index e4ba20f2f..3dbc0b8b1 100644 --- a/benchmark/benchmark_device_scan.parallel.hpp +++ b/benchmark/benchmark_device_scan.parallel.hpp @@ -32,13 +32,24 @@ #include // rocPRIM +#include +#include #include #include +#include +#ifdef BENCHMARK_CONFIG_TUNING + #include + #include + #include +#endif +#include #include +#include #include - -#include +#ifdef BENCHMARK_CONFIG_TUNING + #include +#endif template std::string config_name() @@ -56,10 +67,10 @@ inline std::string config_name() } template, + typename T = int, + typename ScanOp = rocprim::plus, bool Deterministic = false, - class Config = rocprim::default_config> + typename Config = rocprim::default_config> struct device_scan_benchmark : public config_autotune_interface { std::string name() const override @@ -150,16 +161,16 @@ struct device_scan_benchmark : public config_autotune_interface const managed_seed& seed, hipStream_t stream) const override { - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(T); ScanOp scan_op{}; const auto random_range = limit_random_range(0, 1000); std::vector input = get_random_data(size, random_range.first, random_range.second, seed.get_0()); - T initial_value = T(123); - T* d_input; - T* d_output; + T initial_value = T(123); + T* d_input; + T* d_output; HIP_CHECK(hipMalloc(&d_input, size * sizeof(T))); HIP_CHECK(hipMalloc(&d_output, size * sizeof(T))); HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); @@ -181,7 +192,7 @@ struct device_scan_benchmark : public config_autotune_interface HIP_CHECK(hipDeviceSynchronize()); // Warm-up - for(size_t i = 0; i < 5; i++) + for(size_t i = 0; i < 5; ++i) { HIP_CHECK((run_device_scan(d_temp_storage, temp_storage_size_bytes, @@ -205,7 +216,7 @@ struct device_scan_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { HIP_CHECK((run_device_scan(d_temp_storage, temp_storage_size_bytes, @@ -273,7 +284,7 @@ struct device_scan_benchmark_generator { // Limit items per thread to not over-use shared memory static constexpr unsigned int max_items_per_thread - = ::rocprim::min(65536 / (block_size * sizeof(T)), 24); + = ::rocprim::min(65536 / (block_size * sizeof(T)) - 1, 24); static_for_each, create_ipt>(storage); } diff --git a/benchmark/benchmark_device_scan_by_key.cpp b/benchmark/benchmark_device_scan_by_key.cpp index 5528afa0d..e85164e0b 100644 --- a/benchmark/benchmark_device_scan_by_key.cpp +++ b/benchmark/benchmark_device_scan_by_key.cpp @@ -31,23 +31,31 @@ // HIP API #include -#include +#ifndef BENCHMARK_CONFIG_TUNING + #include + #include +#endif #include +#include +#include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif -#define CREATE_BY_KEY_BENCHMARK(EXCL, T, SCAN_OP, MAX_SEGMENT_LENGTH) \ - { \ - const device_scan_by_key_benchmark, \ - MAX_SEGMENT_LENGTH> \ - instance; \ +#define CREATE_BY_KEY_BENCHMARK(EXCL, T, SCAN_OP, MAX_SEGMENT_LENGTH) \ + { \ + const device_scan_by_key_benchmark, \ + MAX_SEGMENT_LENGTH> \ + instance; \ REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } @@ -87,7 +95,7 @@ int main(int argc, char* argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); + const size_t bytes = parser.get("size"); const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); @@ -127,6 +135,8 @@ int main(int argc, char* argv[]) CREATE_BENCHMARK(int8_t, rocprim::plus) CREATE_BENCHMARK(uint8_t, rocprim::plus) CREATE_BENCHMARK(rocprim::half, rocprim::plus) + CREATE_BENCHMARK(rocprim::int128_t, rocprim::plus) + CREATE_BENCHMARK(rocprim::uint128_t, rocprim::plus) #endif // Use manual timing diff --git a/benchmark/benchmark_device_scan_by_key.parallel.cpp.in b/benchmark/benchmark_device_scan_by_key.parallel.cpp.in index 1cf8d21f7..2e7b2b641 100644 --- a/benchmark/benchmark_device_scan_by_key.parallel.cpp.in +++ b/benchmark/benchmark_device_scan_by_key.parallel.cpp.in @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,11 +20,14 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include - #include "benchmark_device_scan_by_key.parallel.hpp" #include "benchmark_utils.hpp" +#include +#include + +#include + namespace { auto benchmarks = config_autotune_register::create_bulk( diff --git a/benchmark/benchmark_device_scan_by_key.parallel.hpp b/benchmark/benchmark_device_scan_by_key.parallel.hpp index 7d81fd8a0..089ff4cd3 100644 --- a/benchmark/benchmark_device_scan_by_key.parallel.hpp +++ b/benchmark/benchmark_device_scan_by_key.parallel.hpp @@ -32,13 +32,24 @@ #include // rocPRIM +#include +#include #include #include +#include +#ifdef BENCHMARK_CONFIG_TUNING + #include + #include + #include +#endif +#include #include +#include #include - -#include +#ifdef BENCHMARK_CONFIG_TUNING + #include +#endif template std::string config_name() @@ -56,13 +67,13 @@ inline std::string config_name() } template, - class CompareOp = rocprim::equal_to, + typename Key = int, + typename Value = int, + typename ScanOp = rocprim::plus, + typename CompareOp = rocprim::equal_to, unsigned int MaxSegmentLength = 1024, bool Deterministic = false, - class Config = rocprim::default_config> + typename Config = rocprim::default_config> struct device_scan_by_key_benchmark : public config_autotune_interface { std::string name() const override @@ -166,7 +177,7 @@ struct device_scan_by_key_benchmark : public config_autotune_interface const managed_seed& seed, hipStream_t stream) const override { - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(Value); constexpr bool debug = false; @@ -214,7 +225,7 @@ struct device_scan_by_key_benchmark : public config_autotune_interface HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_size_bytes)); // Warm-up - for(size_t i = 0; i < 5; i++) + for(size_t i = 0; i < 5; ++i) { HIP_CHECK((run_device_scan_by_key(d_temp_storage, temp_storage_size_bytes, @@ -241,7 +252,7 @@ struct device_scan_by_key_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { HIP_CHECK((run_device_scan_by_key(d_temp_storage, temp_storage_size_bytes, @@ -317,7 +328,10 @@ struct device_scan_by_key_benchmark_generator { // Limit items per thread to not over-use shared memory static constexpr unsigned int max_items_per_thread = ::rocprim::min( - 65536 / (block_size * (sizeof(KeyType) + sizeof(ValueType))), + 65536 + / (block_size + * (sizeof(KeyType) + sizeof(ValueType) + + (sizeof(KeyType) == 16 && sizeof(ValueType) == 1))), 24); static_for_each, create_ipt>(storage); diff --git a/benchmark/benchmark_device_scan_by_key_deterministic.cpp b/benchmark/benchmark_device_scan_by_key_deterministic.cpp index 126f99a02..ff803a2c9 100644 --- a/benchmark/benchmark_device_scan_by_key_deterministic.cpp +++ b/benchmark/benchmark_device_scan_by_key_deterministic.cpp @@ -31,24 +31,28 @@ // HIP API #include -#include +#include +#include #include +#include +#include +#include #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif -#define CREATE_BY_KEY_BENCHMARK(EXCL, T, SCAN_OP, MAX_SEGMENT_LENGTH) \ - { \ - const device_scan_by_key_benchmark, \ - MAX_SEGMENT_LENGTH, \ - true> \ - instance; \ +#define CREATE_BY_KEY_BENCHMARK(EXCL, T, SCAN_OP, MAX_SEGMENT_LENGTH) \ + { \ + const device_scan_by_key_benchmark, \ + MAX_SEGMENT_LENGTH, \ + true> \ + instance; \ REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } @@ -77,7 +81,7 @@ int main(int argc, char* argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); + const size_t bytes = parser.get("size"); const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); @@ -107,6 +111,8 @@ int main(int argc, char* argv[]) CREATE_BENCHMARK(int8_t, rocprim::plus) CREATE_BENCHMARK(uint8_t, rocprim::plus) CREATE_BENCHMARK(rocprim::half, rocprim::plus) + CREATE_BENCHMARK(rocprim::int128_t, rocprim::plus) + CREATE_BENCHMARK(rocprim::uint128_t, rocprim::plus) // Use manual timing for(auto& b : benchmarks) diff --git a/benchmark/benchmark_device_scan_deterministic.cpp b/benchmark/benchmark_device_scan_deterministic.cpp index b13c82bf9..3b5b05242 100644 --- a/benchmark/benchmark_device_scan_deterministic.cpp +++ b/benchmark/benchmark_device_scan_deterministic.cpp @@ -31,17 +31,21 @@ // HIP API #include -#include +#include +#include #include +#include +#include +#include #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif -#define CREATE_EXCL_INCL_BENCHMARK(EXCL, T, SCAN_OP) \ - { \ - const device_scan_benchmark instance; \ +#define CREATE_EXCL_INCL_BENCHMARK(EXCL, T, SCAN_OP) \ + { \ + const device_scan_benchmark instance; \ REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } @@ -63,7 +67,7 @@ int main(int argc, char* argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); + const size_t bytes = parser.get("size"); const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); @@ -93,6 +97,8 @@ int main(int argc, char* argv[]) CREATE_BENCHMARK(int8_t, rocprim::plus) CREATE_BENCHMARK(uint8_t, rocprim::plus) CREATE_BENCHMARK(rocprim::half, rocprim::plus) + CREATE_BENCHMARK(rocprim::int128_t, rocprim::plus) + CREATE_BENCHMARK(rocprim::uint128_t, rocprim::plus) // Use manual timing for(auto& b : benchmarks) diff --git a/benchmark/benchmark_device_search.cpp b/benchmark/benchmark_device_search.cpp index 5f00d9416..f3d5e0256 100644 --- a/benchmark/benchmark_device_search.cpp +++ b/benchmark/benchmark_device_search.cpp @@ -32,8 +32,12 @@ // HIP API #include +#include + #include +#include #include +#include #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; @@ -53,10 +57,9 @@ const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; CREATE_BENCHMARK_SEARCH(TYPE, 10000, REPEATING) \ } -#define CREATE_BENCHMARK(TYPE) \ - { \ - CREATE_BENCHMARK_PATTERN(TYPE, true) \ - CREATE_BENCHMARK_PATTERN(TYPE, false) \ +#define CREATE_BENCHMARK(TYPE) \ + { \ + CREATE_BENCHMARK_PATTERN(TYPE, true) CREATE_BENCHMARK_PATTERN(TYPE, false) \ } int main(int argc, char* argv[]) @@ -96,6 +99,8 @@ int main(int argc, char* argv[]) CREATE_BENCHMARK(rocprim::half) CREATE_BENCHMARK(short) CREATE_BENCHMARK(float) + CREATE_BENCHMARK(rocprim::int128_t) + CREATE_BENCHMARK(rocprim::uint128_t) using custom_float2 = custom_type; using custom_double2 = custom_type; diff --git a/benchmark/benchmark_device_search.hpp b/benchmark/benchmark_device_search.hpp index 7b6afa4f9..33a3cd6dc 100644 --- a/benchmark/benchmark_device_search.hpp +++ b/benchmark/benchmark_device_search.hpp @@ -32,13 +32,15 @@ #include // rocPRIM +#include #include +#include +#include +#include #include #include -#include - template struct device_search_benchmark : public config_autotune_interface { @@ -87,7 +89,7 @@ struct device_search_benchmark : public config_autotune_interface { // Repeating similar pattern without early exits. keys_input[key_size - 1] = 0; - for(size_t i = 0; i < size; i++) + for(size_t i = 0; i < size; ++i) { input[i] = keys_input[i % key_size]; } @@ -134,7 +136,7 @@ struct device_search_benchmark : public config_autotune_interface HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { HIP_CHECK(rocprim::search(d_temporary_storage, temporary_storage_bytes, @@ -159,7 +161,7 @@ struct device_search_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { HIP_CHECK(rocprim::search(d_temporary_storage, temporary_storage_bytes, diff --git a/benchmark/benchmark_device_search_n.cpp b/benchmark/benchmark_device_search_n.cpp index 6705ca0d9..ceaddfc20 100644 --- a/benchmark/benchmark_device_search_n.cpp +++ b/benchmark/benchmark_device_search_n.cpp @@ -21,6 +21,19 @@ // SOFTWARE. #include "benchmark_device_search_n.parallel.hpp" +#include "benchmark_utils.hpp" +// CmdParser +#include "cmdparser.hpp" + +// Google Benchmark +#include + +// HIP API +#include + +#include +#include +#include int main(int argc, char* argv[]) { diff --git a/benchmark/benchmark_device_search_n.parallel.cpp.in b/benchmark/benchmark_device_search_n.parallel.cpp.in index 6e3db47f0..a988f23df 100644 --- a/benchmark/benchmark_device_search_n.parallel.cpp.in +++ b/benchmark/benchmark_device_search_n.parallel.cpp.in @@ -20,14 +20,16 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include - #include "benchmark_device_search_n.parallel.hpp" #include "benchmark_utils.hpp" +#include + +#include + namespace { auto benchmarks = config_autotune_register::create_bulk( device_search_n_benchmark_generator< @InputType@, @BlockSize@>::create); -} \ No newline at end of file +} diff --git a/benchmark/benchmark_device_search_n.parallel.hpp b/benchmark/benchmark_device_search_n.parallel.hpp index fc2e0b7b6..8efe8a67c 100644 --- a/benchmark/benchmark_device_search_n.parallel.hpp +++ b/benchmark/benchmark_device_search_n.parallel.hpp @@ -33,17 +33,20 @@ #include // rocPRIM -#include +#include #include +#include +#include +#include // C++ Standard Library #include -#include -#include -#include +#include +#include +#include +#include #include #include -#include #include using custom_int2 = custom_type; @@ -70,7 +73,7 @@ constexpr bool is_type_arr_end = true; template constexpr bool is_type_arr_end> = false; -template +template inline unsigned int search_n_get_item_per_block() { using input_type = InputType; @@ -105,7 +108,7 @@ inline std::string to_string(benchmark_search_n_mode e) noexcept } // namespace -template +template class benchmark_search_n { public: @@ -229,7 +232,7 @@ class benchmark_search_n self.launch_search_n(); HIP_CHECK(hipMallocAsync(&self.d_temp_storage, self.temp_storage_size, self.stream)); // Warm-up - for(size_t i = 0; i < self.warmup_size; i++) + for(size_t i = 0; i < self.warmup_size; ++i) { self.launch_search_n(); } @@ -241,7 +244,7 @@ class benchmark_search_n // Record start event HIP_CHECK(hipEventRecord(self.start, self.stream)); - for(size_t i = 0; i < self.batch_size; i++) + for(size_t i = 0; i < self.batch_size; ++i) { self.launch_search_n(); } @@ -311,7 +314,7 @@ class benchmark_search_n { return benchmark::RegisterBenchmark( bench_naming::format_name( - "{lvl:device,algo:search_n,input_type:" + std::string(typeid(InputType).name()) + "{lvl:device,algo:search_n,input_type:" + std::string(Traits::name()) + ",size:" + std::to_string(size) + ",count:" + std::to_string(count) + ",mode:" + to_string(mode) + ",cfg:default_config}") .c_str(), @@ -332,7 +335,7 @@ static void clean_up_benchmarks_search_n() destructors = {}; } -template +template inline void add_one_benchmark_search_n(std::vector& benchmarks, const managed_seed _seed, const hipStream_t _stream, @@ -494,6 +497,8 @@ using benchmark_search_n_types = type_arr; diff --git a/benchmark/benchmark_device_segmented_radix_sort_keys.cpp b/benchmark/benchmark_device_segmented_radix_sort_keys.cpp index 0ec242d0b..e0594f822 100644 --- a/benchmark/benchmark_device_segmented_radix_sort_keys.cpp +++ b/benchmark/benchmark_device_segmented_radix_sort_keys.cpp @@ -32,19 +32,22 @@ // rocPRIM #include +#include -#include -#include -#include +#include +#include +#include +#include #include #include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif -namespace rp = rocprim; - namespace { @@ -58,7 +61,7 @@ constexpr std::array segment_lengths{30, 256, 3000, 300000}; // because the binary became too large to link. Runs into a "relocation R_X86_64_PC32 out of range" error. // This happens partially, because of the algorithm has 4 kernels, and decides at runtime which one to call. -template +template void run_sort_keys_benchmark(benchmark::State& state, size_t num_segments, size_t mean_segment_length, @@ -69,7 +72,7 @@ void run_sort_keys_benchmark(benchmark::State& state, using offset_type = int; using key_type = Key; - // Calculate the number of elements + // Calculate the number of elements size_t target_size = target_bytes / sizeof(key_type); std::vector offsets; @@ -124,37 +127,37 @@ void run_sort_keys_benchmark(benchmark::State& state, void* d_temporary_storage = nullptr; size_t temporary_storage_bytes = 0; - HIP_CHECK(rp::segmented_radix_sort_keys(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_keys_output, - size, - segments_count, - d_offsets, - d_offsets + 1, - 0, - sizeof(key_type) * 8, - stream, - false)); + HIP_CHECK(rocprim::segmented_radix_sort_keys(d_temporary_storage, + temporary_storage_bytes, + d_keys_input, + d_keys_output, + size, + segments_count, + d_offsets, + d_offsets + 1, + 0, + sizeof(key_type) * 8, + stream, + false)); HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { - HIP_CHECK(rp::segmented_radix_sort_keys(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_keys_output, - size, - segments_count, - d_offsets, - d_offsets + 1, - 0, - sizeof(key_type) * 8, - stream, - false)); + HIP_CHECK(rocprim::segmented_radix_sort_keys(d_temporary_storage, + temporary_storage_bytes, + d_keys_input, + d_keys_output, + size, + segments_count, + d_offsets, + d_offsets + 1, + 0, + sizeof(key_type) * 8, + stream, + false)); } HIP_CHECK(hipDeviceSynchronize()); @@ -168,20 +171,20 @@ void run_sort_keys_benchmark(benchmark::State& state, // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { - HIP_CHECK(rp::segmented_radix_sort_keys(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_keys_output, - size, - segments_count, - d_offsets, - d_offsets + 1, - 0, - sizeof(key_type) * 8, - stream, - false)); + HIP_CHECK(rocprim::segmented_radix_sort_keys(d_temporary_storage, + temporary_storage_bytes, + d_keys_input, + d_keys_output, + size, + segments_count, + d_offsets, + d_offsets + 1, + 0, + sizeof(key_type) * 8, + stream, + false)); } // Record stop event and wait until it completes @@ -206,7 +209,7 @@ void run_sort_keys_benchmark(benchmark::State& state, HIP_CHECK(hipFree(d_keys_output)); } -template +template void add_sort_keys_benchmarks(std::vector& benchmarks, size_t max_bytes, size_t min_size, @@ -214,7 +217,7 @@ void add_sort_keys_benchmarks(std::vector& benc const managed_seed& seed, hipStream_t stream) { - // Calculate the number of elements + // Calculate the number of elements size_t max_size = max_bytes / sizeof(KeyT); std::string key_name = Traits::name(); @@ -274,7 +277,7 @@ int main(int argc, char* argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); + const size_t bytes = parser.get("size"); const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); @@ -307,6 +310,18 @@ int main(int argc, char* argv[]) add_sort_keys_benchmarks(benchmarks, bytes, min_size, bytes / 2, seed, stream); add_sort_keys_benchmarks(benchmarks, bytes, min_size, bytes / 2, seed, stream); add_sort_keys_benchmarks(benchmarks, bytes, min_size, bytes / 2, seed, stream); + add_sort_keys_benchmarks(benchmarks, + bytes, + min_size, + bytes / 2, + seed, + stream); + add_sort_keys_benchmarks(benchmarks, + bytes, + min_size, + bytes / 2, + seed, + stream); #endif // Use manual timing diff --git a/benchmark/benchmark_device_segmented_radix_sort_keys.parallel.cpp.in b/benchmark/benchmark_device_segmented_radix_sort_keys.parallel.cpp.in index 3e506148a..6e2875ca1 100644 --- a/benchmark/benchmark_device_segmented_radix_sort_keys.parallel.cpp.in +++ b/benchmark/benchmark_device_segmented_radix_sort_keys.parallel.cpp.in @@ -20,11 +20,13 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include - #include "benchmark_device_segmented_radix_sort_keys.parallel.hpp" #include "benchmark_utils.hpp" +#include + +#include + namespace { auto benchmarks = config_autotune_register::create_bulk( diff --git a/benchmark/benchmark_device_segmented_radix_sort_keys.parallel.hpp b/benchmark/benchmark_device_segmented_radix_sort_keys.parallel.hpp index ad798af9f..34bd6db5d 100644 --- a/benchmark/benchmark_device_segmented_radix_sort_keys.parallel.hpp +++ b/benchmark/benchmark_device_segmented_radix_sort_keys.parallel.hpp @@ -32,14 +32,20 @@ #include // rocPRIM +#include #include #include +#include +#include +#include +#include +#include #include +#include +#include #include -#include - template std::string warp_sort_config_name(T const& warp_sort_config) { @@ -169,7 +175,7 @@ struct device_segmented_radix_sort_benchmark : public config_autotune_interface HIP_CHECK(hipDeviceSynchronize()); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { HIP_CHECK(rocprim::segmented_radix_sort_keys(d_temporary_storage, temporary_storage_bytes, @@ -196,7 +202,7 @@ struct device_segmented_radix_sort_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { HIP_CHECK(rocprim::segmented_radix_sort_keys(d_temporary_storage, temporary_storage_bytes, @@ -239,7 +245,7 @@ struct device_segmented_radix_sort_benchmark : public config_autotune_interface const managed_seed& seed, hipStream_t stream) const override { - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(Key); constexpr std::array @@ -280,7 +286,10 @@ template struct device_segmented_radix_sort_benchmark_generator { - static void create(std::vector>& storage) + template + static auto __create(std::vector>& storage) -> + typename std::enable_if<(key_size * BlockSize * ItemsPerThread < TUNING_SHARED_MEMORY_MAX), + void>::type { storage.emplace_back(std::make_unique, UnpartitionWarpAllowed>>>()); } + template + static auto __create(std::vector>& storage) -> + typename std::enable_if::type + {} + static void create(std::vector>& storage) + { + __create(storage); + } }; template class T, Tp... Idx> diff --git a/benchmark/benchmark_device_segmented_radix_sort_pairs.cpp b/benchmark/benchmark_device_segmented_radix_sort_pairs.cpp index d0666deeb..4ca1b0194 100644 --- a/benchmark/benchmark_device_segmented_radix_sort_pairs.cpp +++ b/benchmark/benchmark_device_segmented_radix_sort_pairs.cpp @@ -32,19 +32,25 @@ // rocPRIM #include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif -#include -#include -#include +#include +#include +#include +#include +#include #include #include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif -namespace rp = rocprim; - namespace { @@ -58,7 +64,7 @@ constexpr std::array segment_lengths{30, 256, 3000, 300000}; // because the binary became too large to link. Runs into a "relocation R_X86_64_PC32 out of range" error. // This happens partially, because of the algorithm has 4 kernels, and decides at runtime which one to call. -template +template void run_sort_pairs_benchmark(benchmark::State& state, size_t num_segments, size_t mean_segment_length, @@ -70,7 +76,7 @@ void run_sort_pairs_benchmark(benchmark::State& state, using key_type = Key; using value_type = Value; - // Calculate the number of elements + // Calculate the number of elements size_t target_size = target_bytes / sizeof(key_type); // Generate data @@ -138,41 +144,41 @@ void run_sort_pairs_benchmark(benchmark::State& state, void* d_temporary_storage = nullptr; size_t temporary_storage_bytes = 0; - HIP_CHECK(rp::segmented_radix_sort_pairs(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_keys_output, - d_values_input, - d_values_output, - size, - segments_count, - d_offsets, - d_offsets + 1, - 0, - sizeof(key_type) * 8, - stream, - false)); + HIP_CHECK(rocprim::segmented_radix_sort_pairs(d_temporary_storage, + temporary_storage_bytes, + d_keys_input, + d_keys_output, + d_values_input, + d_values_output, + size, + segments_count, + d_offsets, + d_offsets + 1, + 0, + sizeof(key_type) * 8, + stream, + false)); HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { - HIP_CHECK(rp::segmented_radix_sort_pairs(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_keys_output, - d_values_input, - d_values_output, - size, - segments_count, - d_offsets, - d_offsets + 1, - 0, - sizeof(key_type) * 8, - stream, - false)); + HIP_CHECK(rocprim::segmented_radix_sort_pairs(d_temporary_storage, + temporary_storage_bytes, + d_keys_input, + d_keys_output, + d_values_input, + d_values_output, + size, + segments_count, + d_offsets, + d_offsets + 1, + 0, + sizeof(key_type) * 8, + stream, + false)); } HIP_CHECK(hipDeviceSynchronize()); @@ -186,22 +192,22 @@ void run_sort_pairs_benchmark(benchmark::State& state, // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { - HIP_CHECK(rp::segmented_radix_sort_pairs(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_keys_output, - d_values_input, - d_values_output, - size, - segments_count, - d_offsets, - d_offsets + 1, - 0, - sizeof(key_type) * 8, - stream, - false)); + HIP_CHECK(rocprim::segmented_radix_sort_pairs(d_temporary_storage, + temporary_storage_bytes, + d_keys_input, + d_keys_output, + d_values_input, + d_values_output, + size, + segments_count, + d_offsets, + d_offsets + 1, + 0, + sizeof(key_type) * 8, + stream, + false)); } // Record stop event and wait until it completes @@ -229,7 +235,7 @@ void run_sort_pairs_benchmark(benchmark::State& state, HIP_CHECK(hipFree(d_values_output)); } -template +template void add_sort_pairs_benchmarks(std::vector& benchmarks, size_t max_bytes, size_t min_size, @@ -237,7 +243,7 @@ void add_sort_pairs_benchmarks(std::vector& ben const managed_seed& seed, hipStream_t stream) { - // Calculate the number of elements + // Calculate the number of elements size_t max_size = max_bytes / sizeof(KeyT); std::string key_name = Traits::name(); @@ -296,7 +302,7 @@ int main(int argc, char* argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); + const size_t bytes = parser.get("size"); const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); @@ -333,7 +339,12 @@ int main(int argc, char* argv[]) seed, stream); add_sort_pairs_benchmarks(benchmarks, bytes, min_size, bytes / 2, seed, stream); - add_sort_pairs_benchmarks(benchmarks, bytes, min_size, bytes / 2, seed, stream); + add_sort_pairs_benchmarks(benchmarks, + bytes, + min_size, + bytes / 2, + seed, + stream); add_sort_pairs_benchmarks(benchmarks, bytes, min_size, @@ -352,6 +363,18 @@ int main(int argc, char* argv[]) bytes / 2, seed, stream); + add_sort_pairs_benchmarks(benchmarks, + bytes, + min_size, + bytes / 2, + seed, + stream); + add_sort_pairs_benchmarks(benchmarks, + bytes, + min_size, + bytes / 2, + seed, + stream); #endif // Use manual timing diff --git a/benchmark/benchmark_device_segmented_radix_sort_pairs.parallel.cpp.in b/benchmark/benchmark_device_segmented_radix_sort_pairs.parallel.cpp.in index 37923ef01..5618b3b26 100644 --- a/benchmark/benchmark_device_segmented_radix_sort_pairs.parallel.cpp.in +++ b/benchmark/benchmark_device_segmented_radix_sort_pairs.parallel.cpp.in @@ -20,11 +20,13 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include - #include "benchmark_device_segmented_radix_sort_pairs.parallel.hpp" #include "benchmark_utils.hpp" +#include + +#include + namespace { auto benchmarks = config_autotune_register::create_bulk( diff --git a/benchmark/benchmark_device_segmented_radix_sort_pairs.parallel.hpp b/benchmark/benchmark_device_segmented_radix_sort_pairs.parallel.hpp index 3a5e3ecfd..88ae7baa0 100644 --- a/benchmark/benchmark_device_segmented_radix_sort_pairs.parallel.hpp +++ b/benchmark/benchmark_device_segmented_radix_sort_pairs.parallel.hpp @@ -32,14 +32,20 @@ #include // rocPRIM +#include #include #include +#include +#include +#include +#include +#include #include +#include +#include #include -#include - template std::string warp_sort_config_name(T const& warp_sort_config) { @@ -188,7 +194,7 @@ struct device_segmented_radix_sort_benchmark : public config_autotune_interface HIP_CHECK(hipDeviceSynchronize()); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { HIP_CHECK(rocprim::segmented_radix_sort_pairs(d_temporary_storage, temporary_storage_bytes, @@ -217,7 +223,7 @@ struct device_segmented_radix_sort_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { HIP_CHECK(rocprim::segmented_radix_sort_pairs(d_temporary_storage, temporary_storage_bytes, @@ -264,9 +270,9 @@ struct device_segmented_radix_sort_benchmark : public config_autotune_interface const managed_seed& seed, hipStream_t stream) const override { - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(Key); - + constexpr std::array segment_counts{10, 100, 1000, 2500, 5000, 7500, 10000, 100000}; constexpr std::array segment_lengths{30, 256, 3000, 300000}; @@ -306,7 +312,11 @@ template struct device_segmented_radix_sort_benchmark_generator { - static void create(std::vector>& storage) + template + static auto __create(std::vector>& storage) -> + typename std::enable_if<((key_size + value_type) * BlockSize * ItemsPerThread + <= TUNING_SHARED_MEMORY_MAX), + void>::type { storage.emplace_back(std::make_unique, UnpartitionWarpAllowed>>>()); } + template + static auto __create(std::vector>& storage) -> + typename std::enable_if::type + {} + static void create(std::vector>& storage) + { + __create(storage); + } }; template class T, Tp... Idx> diff --git a/benchmark/benchmark_device_segmented_reduce.cpp b/benchmark/benchmark_device_segmented_reduce.cpp index ddc5fff8f..91683a57b 100644 --- a/benchmark/benchmark_device_segmented_reduce.cpp +++ b/benchmark/benchmark_device_segmented_reduce.cpp @@ -32,10 +32,14 @@ // rocPRIM #include - -#include -#include -#include +#include +#include + +#include +#include +#include +#include +#include #include #include @@ -43,12 +47,10 @@ const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif -namespace rp = rocprim; - -const unsigned int batch_size = 10; +const unsigned int batch_size = 10; const unsigned int warmup_size = 5; -template +template void run_benchmark(benchmark::State& state, size_t desired_segments, size_t bytes, @@ -56,9 +58,9 @@ void run_benchmark(benchmark::State& state, hipStream_t stream) { using offset_type = int; - using value_type = T; + using value_type = T; - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(T); // Generate data @@ -68,13 +70,13 @@ void run_benchmark(benchmark::State& state, std::uniform_real_distribution segment_length_dis(0, avg_segment_length * 2); std::vector offsets; - unsigned int segments_count = 0; - size_t offset = 0; + unsigned int segments_count = 0; + size_t offset = 0; while(offset < size) { const size_t segment_length = std::round(segment_length_dis(gen)); offsets.push_back(offset); - segments_count++; + ++segments_count; offset += segment_length; } offsets.push_back(size); @@ -82,62 +84,58 @@ void run_benchmark(benchmark::State& state, std::vector values_input(size); std::iota(values_input.begin(), values_input.end(), 0); - offset_type * d_offsets; - HIP_CHECK(hipMalloc(reinterpret_cast(&d_offsets), (segments_count + 1) * sizeof(offset_type))); - HIP_CHECK( - hipMemcpy( - d_offsets, offsets.data(), - (segments_count + 1) * sizeof(offset_type), - hipMemcpyHostToDevice - ) - ); - - value_type * d_values_input; + offset_type* d_offsets; + HIP_CHECK(hipMalloc(reinterpret_cast(&d_offsets), + (segments_count + 1) * sizeof(offset_type))); + HIP_CHECK(hipMemcpy(d_offsets, + offsets.data(), + (segments_count + 1) * sizeof(offset_type), + hipMemcpyHostToDevice)); + + value_type* d_values_input; HIP_CHECK(hipMalloc(reinterpret_cast(&d_values_input), size * sizeof(value_type))); - HIP_CHECK( - hipMemcpy( - d_values_input, values_input.data(), - size * sizeof(value_type), - hipMemcpyHostToDevice - ) - ); + HIP_CHECK(hipMemcpy(d_values_input, + values_input.data(), + size * sizeof(value_type), + hipMemcpyHostToDevice)); - value_type * d_aggregates_output; - HIP_CHECK(hipMalloc(reinterpret_cast(&d_aggregates_output), segments_count * sizeof(value_type))); + value_type* d_aggregates_output; + HIP_CHECK(hipMalloc(reinterpret_cast(&d_aggregates_output), + segments_count * sizeof(value_type))); rocprim::plus reduce_op; - value_type init(0); + value_type init(0); - void * d_temporary_storage = nullptr; + void* d_temporary_storage = nullptr; size_t temporary_storage_bytes = 0; - HIP_CHECK( - rp::segmented_reduce( - d_temporary_storage, temporary_storage_bytes, - d_values_input, d_aggregates_output, - segments_count, - d_offsets, d_offsets + 1, - reduce_op, init, - stream - ) - ); + HIP_CHECK(rocprim::segmented_reduce(d_temporary_storage, + temporary_storage_bytes, + d_values_input, + d_aggregates_output, + segments_count, + d_offsets, + d_offsets + 1, + reduce_op, + init, + stream)); HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { - HIP_CHECK( - rp::segmented_reduce( - d_temporary_storage, temporary_storage_bytes, - d_values_input, d_aggregates_output, - segments_count, - d_offsets, d_offsets + 1, - reduce_op, init, - stream - ) - ); + HIP_CHECK(rocprim::segmented_reduce(d_temporary_storage, + temporary_storage_bytes, + d_values_input, + d_aggregates_output, + segments_count, + d_offsets, + d_offsets + 1, + reduce_op, + init, + stream)); } HIP_CHECK(hipDeviceSynchronize()); @@ -146,23 +144,23 @@ void run_benchmark(benchmark::State& state, HIP_CHECK(hipEventCreate(&start)); HIP_CHECK(hipEventCreate(&stop)); - for (auto _ : state) + for(auto _ : state) { // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { - HIP_CHECK( - rp::segmented_reduce( - d_temporary_storage, temporary_storage_bytes, - d_values_input, d_aggregates_output, - segments_count, - d_offsets, d_offsets + 1, - reduce_op, init, - stream - ) - ); + HIP_CHECK(rocprim::segmented_reduce(d_temporary_storage, + temporary_storage_bytes, + d_values_input, + d_aggregates_output, + segments_count, + d_offsets, + d_offsets + 1, + reduce_op, + init, + stream)); } // Record stop event and wait until it completes @@ -195,41 +193,37 @@ void run_benchmark(benchmark::State& state, .c_str(), \ run_benchmark, \ SEGMENTS, \ - bytes, \ + bytes, \ seed, \ stream) -#define BENCHMARK_TYPE(type) \ - CREATE_BENCHMARK(type, 1), \ - CREATE_BENCHMARK(type, 10), \ - CREATE_BENCHMARK(type, 100), \ - CREATE_BENCHMARK(type, 1000), \ - CREATE_BENCHMARK(type, 10000) +#define BENCHMARK_TYPE(type) \ + CREATE_BENCHMARK(type, 1), CREATE_BENCHMARK(type, 10), CREATE_BENCHMARK(type, 100), \ + CREATE_BENCHMARK(type, 1000), CREATE_BENCHMARK(type, 10000) void add_benchmarks(std::vector& benchmarks, size_t bytes, const managed_seed& seed, hipStream_t stream) { - using custom_float2 = custom_type; + using custom_float2 = custom_type; using custom_double2 = custom_type; - std::vector bs = - { - BENCHMARK_TYPE(float), - BENCHMARK_TYPE(double), - BENCHMARK_TYPE(int8_t), - BENCHMARK_TYPE(uint8_t), - BENCHMARK_TYPE(rocprim::half), - BENCHMARK_TYPE(int), - BENCHMARK_TYPE(custom_float2), - BENCHMARK_TYPE(custom_double2), - }; + std::vector bs = {BENCHMARK_TYPE(float), + BENCHMARK_TYPE(double), + BENCHMARK_TYPE(int8_t), + BENCHMARK_TYPE(uint8_t), + BENCHMARK_TYPE(rocprim::half), + BENCHMARK_TYPE(int), + BENCHMARK_TYPE(custom_float2), + BENCHMARK_TYPE(custom_double2), + BENCHMARK_TYPE(rocprim::int128_t), + BENCHMARK_TYPE(rocprim::uint128_t)}; benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); } -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { cli::Parser parser(argc, argv); parser.set_optional("size", "size", DEFAULT_BYTES, "number of bytes"); @@ -244,8 +238,8 @@ int main(int argc, char *argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); - const int trials = parser.get("trials"); + const size_t bytes = parser.get("size"); + const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); const managed_seed seed(seed_type); diff --git a/benchmark/benchmark_device_select.cpp b/benchmark/benchmark_device_select.cpp index c19e5835b..79002f3a5 100644 --- a/benchmark/benchmark_device_select.cpp +++ b/benchmark/benchmark_device_select.cpp @@ -32,16 +32,17 @@ #include // rocPRIM -#include +#ifndef BENCHMARK_CONFIG_TUNING + #include + #include +#endif -#include -#include -#include +#include #include #include - -#include -#include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; @@ -56,25 +57,25 @@ const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #define CREATE_SELECT_FLAG_BENCHMARK(T, F, p) \ { \ const device_select_flag_benchmark instance; \ - REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ + REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } #define CREATE_SELECT_PREDICATE_BENCHMARK(T, p) \ { \ const device_select_predicate_benchmark instance; \ - REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ + REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } #define CREATE_UNIQUE_BENCHMARK(T, p) \ { \ const device_select_unique_benchmark instance; \ - REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ + REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } #define CREATE_UNIQUE_BY_KEY_BENCHMARK(K, V, p) \ { \ const device_select_unique_by_key_benchmark instance; \ - REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ + REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } #define BENCHMARK_SELECT_PREDICATED_FLAG_TYPE(type, value) \ @@ -132,7 +133,7 @@ int main(int argc, char* argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); + const size_t bytes = parser.get("size"); const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); @@ -168,6 +169,8 @@ int main(int argc, char* argv[]) BENCHMARK_SELECT_FLAG_TYPE(int8_t, int8_t); BENCHMARK_SELECT_FLAG_TYPE(rocprim::half, int8_t); BENCHMARK_SELECT_FLAG_TYPE(custom_double2, unsigned char); + BENCHMARK_SELECT_FLAG_TYPE(rocprim::int128_t, unsigned char); + BENCHMARK_SELECT_FLAG_TYPE(rocprim::uint128_t, unsigned char); BENCHMARK_SELECT_PREDICATE_TYPE(int); BENCHMARK_SELECT_PREDICATE_TYPE(float); @@ -176,6 +179,8 @@ int main(int argc, char* argv[]) BENCHMARK_SELECT_PREDICATE_TYPE(int8_t); BENCHMARK_SELECT_PREDICATE_TYPE(rocprim::half); BENCHMARK_SELECT_PREDICATE_TYPE(custom_int_double); + BENCHMARK_SELECT_PREDICATE_TYPE(rocprim::int128_t); + BENCHMARK_SELECT_PREDICATE_TYPE(rocprim::uint128_t); BENCHMARK_SELECT_PREDICATED_FLAG_TYPE(int, unsigned char); BENCHMARK_SELECT_PREDICATED_FLAG_TYPE(float, unsigned char); @@ -184,6 +189,8 @@ int main(int argc, char* argv[]) BENCHMARK_SELECT_PREDICATED_FLAG_TYPE(int8_t, int8_t); BENCHMARK_SELECT_PREDICATED_FLAG_TYPE(rocprim::half, int8_t); BENCHMARK_SELECT_PREDICATED_FLAG_TYPE(custom_double2, unsigned char); + BENCHMARK_SELECT_PREDICATED_FLAG_TYPE(rocprim::int128_t, unsigned char); + BENCHMARK_SELECT_PREDICATED_FLAG_TYPE(rocprim::uint128_t, unsigned char); BENCHMARK_UNIQUE_TYPE(int); BENCHMARK_UNIQUE_TYPE(float); @@ -192,6 +199,8 @@ int main(int argc, char* argv[]) BENCHMARK_UNIQUE_TYPE(int8_t); BENCHMARK_UNIQUE_TYPE(rocprim::half); BENCHMARK_UNIQUE_TYPE(custom_int_double); + BENCHMARK_UNIQUE_TYPE(rocprim::int128_t); + BENCHMARK_UNIQUE_TYPE(rocprim::uint128_t); BENCHMARK_UNIQUE_BY_KEY_TYPE(int, int); BENCHMARK_UNIQUE_BY_KEY_TYPE(float, double); @@ -200,6 +209,8 @@ int main(int argc, char* argv[]) BENCHMARK_UNIQUE_BY_KEY_TYPE(int8_t, double); BENCHMARK_UNIQUE_BY_KEY_TYPE(rocprim::half, rocprim::half); BENCHMARK_UNIQUE_BY_KEY_TYPE(custom_int_double, custom_int_double); + BENCHMARK_UNIQUE_BY_KEY_TYPE(rocprim::int128_t, rocprim::int128_t); + BENCHMARK_UNIQUE_BY_KEY_TYPE(rocprim::uint128_t, rocprim::int128_t); #endif // Use manual timing diff --git a/benchmark/benchmark_device_select.parallel.cpp.in b/benchmark/benchmark_device_select.parallel.cpp.in index 4cef6dfd9..c9c505209 100644 --- a/benchmark/benchmark_device_select.parallel.cpp.in +++ b/benchmark/benchmark_device_select.parallel.cpp.in @@ -20,11 +20,13 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include - #include "benchmark_device_select.parallel.hpp" #include "benchmark_utils.hpp" +#include + +#include + namespace { auto benchmarks = config_autotune_register::create_bulk( diff --git a/benchmark/benchmark_device_select.parallel.hpp b/benchmark/benchmark_device_select.parallel.hpp index 793f5e130..5b7f42b71 100644 --- a/benchmark/benchmark_device_select.parallel.hpp +++ b/benchmark/benchmark_device_select.parallel.hpp @@ -28,16 +28,24 @@ #include -#include - #include -#include -#include -#include +#include +#include +#include +#include +#ifdef BENCHMARK_CONFIG_TUNING + #include +#endif + +#include #include #include #include +#ifdef BENCHMARK_CONFIG_TUNING + #include + #include +#endif enum class select_probability { @@ -77,9 +85,9 @@ inline const char* get_probability_name(select_probability probability) constexpr int warmup_iter = 5; constexpr int batch_size = 10; -template struct device_select_flag_benchmark : public config_autotune_interface { @@ -98,7 +106,7 @@ struct device_select_flag_benchmark : public config_autotune_interface const managed_seed& seed, hipStream_t stream) const override { - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(DataType); std::vector input = get_random_data(size, @@ -179,7 +187,7 @@ struct device_select_flag_benchmark : public config_autotune_interface void* d_temp_storage{}; HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_size_bytes)); - for(int i = 0; i < warmup_iter; i++) + for(int i = 0; i < warmup_iter; ++i) { dispatch(d_temp_storage, temp_storage_size_bytes); } @@ -225,8 +233,8 @@ struct device_select_flag_benchmark : public config_autotune_interface static constexpr bool is_tuning = Probability == select_probability::tuning; }; -template struct device_select_predicate_benchmark : public config_autotune_interface { @@ -244,7 +252,7 @@ struct device_select_predicate_benchmark : public config_autotune_interface const managed_seed& seed, hipStream_t stream) const override { - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(DataType); // all data types can represent [0, 127], -1 so a predicate can select all @@ -336,9 +344,9 @@ struct device_select_predicate_benchmark : public config_autotune_interface static constexpr bool is_tuning = Probability == select_probability::tuning; }; -template struct device_select_predicated_flag_benchmark : public config_autotune_interface { @@ -440,7 +448,7 @@ struct device_select_predicated_flag_benchmark : public config_autotune_interfac void* d_temp_storage{}; HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_size_bytes)); - for(int i = 0; i < warmup_iter; i++) + for(int i = 0; i < warmup_iter; ++i) { dispatch(d_temp_storage, temp_storage_size_bytes); } @@ -497,7 +505,7 @@ inline std::vector get_unique_input(size_t size, float probability, un auto input01 = get_random_data01(size, probability, seed); auto acc = input01[0]; input[0] = acc; - for(size_t i = 1; i < input01.size(); i++) + for(size_t i = 1; i < input01.size(); ++i) { input[i] = op(acc, input01[i]); } @@ -505,8 +513,8 @@ inline std::vector get_unique_input(size_t size, float probability, un return input; } -template struct device_select_unique_benchmark : public config_autotune_interface { @@ -524,7 +532,7 @@ struct device_select_unique_benchmark : public config_autotune_interface const managed_seed& seed, hipStream_t stream) const override { - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(DataType); std::vector input_0; @@ -641,9 +649,9 @@ struct device_select_unique_benchmark : public config_autotune_interface static constexpr bool is_tuning = Probability == select_probability::tuning; }; -template struct device_select_unique_by_key_benchmark : public config_autotune_interface { @@ -662,7 +670,7 @@ struct device_select_unique_by_key_benchmark : public config_autotune_interface const managed_seed& seed, hipStream_t stream) const override { - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(KeyType); std::vector input_keys_0; diff --git a/benchmark/benchmark_device_transform.cpp b/benchmark/benchmark_device_transform.cpp index f55715346..04a47c7d3 100644 --- a/benchmark/benchmark_device_transform.cpp +++ b/benchmark/benchmark_device_transform.cpp @@ -33,27 +33,28 @@ #include // rocPRIM -#include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif -#include -#include +#include #include #include - -#include -#include +#ifndef BENCHMARK_CONFIG_TUNING + #include +#endif #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 128 * 4; #endif -#define CREATE_BENCHMARK(T) \ - { \ - const device_transform_benchmark instance{}; \ +#define CREATE_BENCHMARK(T) \ + { \ + const device_transform_benchmark instance{}; \ REGISTER_BENCHMARK(benchmarks, bytes, seed, stream, instance); \ } -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { cli::Parser parser(argc, argv); parser.set_optional("size", "size", DEFAULT_BYTES, "number of bytes"); @@ -78,8 +79,8 @@ int main(int argc, char *argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); - const int trials = parser.get("trials"); + const size_t bytes = parser.get("size"); + const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); const managed_seed seed(seed_type); @@ -118,6 +119,9 @@ int main(int argc, char *argv[]) CREATE_BENCHMARK(custom_float2) CREATE_BENCHMARK(custom_double2) + + CREATE_BENCHMARK(rocprim::int128_t) + CREATE_BENCHMARK(rocprim::uint128_t) #endif // BENCHMARK_CONFIG_TUNING // Use manual timing diff --git a/benchmark/benchmark_device_transform.parallel.cpp.in b/benchmark/benchmark_device_transform.parallel.cpp.in index 196234c7a..f3a6a135c 100644 --- a/benchmark/benchmark_device_transform.parallel.cpp.in +++ b/benchmark/benchmark_device_transform.parallel.cpp.in @@ -20,11 +20,13 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include - #include "benchmark_utils.hpp" #include "benchmark_device_transform.parallel.hpp" +#include + +#include + namespace { auto benchmarks = config_autotune_register::create_bulk( device_transform_benchmark_generator< diff --git a/benchmark/benchmark_device_transform.parallel.hpp b/benchmark/benchmark_device_transform.parallel.hpp index 6bad9a392..0db664568 100644 --- a/benchmark/benchmark_device_transform.parallel.hpp +++ b/benchmark/benchmark_device_transform.parallel.hpp @@ -23,9 +23,7 @@ #ifndef ROCPRIM_BENCHMARK_DEVICE_TRANSFORM_PARALLEL_HPP_ #define ROCPRIM_BENCHMARK_DEVICE_TRANSFORM_PARALLEL_HPP_ -#include -#include -#include +#include "benchmark_utils.hpp" // Google Benchmark #include @@ -34,10 +32,15 @@ #include // rocPRIM -#include +#include +#include #include +#include -#include "benchmark_utils.hpp" +#include +#include +#include +#include template std::string transform_config_name() @@ -76,7 +79,7 @@ struct device_transform_benchmark : public config_autotune_interface { using output_type = T; - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(T); static constexpr bool debug_synchronous = false; @@ -108,7 +111,7 @@ struct device_transform_benchmark : public config_autotune_interface }; // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { HIP_CHECK(launch()); } @@ -125,7 +128,7 @@ struct device_transform_benchmark : public config_autotune_interface // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { HIP_CHECK(launch()); } diff --git a/benchmark/benchmark_predicate_iterator.cpp b/benchmark/benchmark_predicate_iterator.cpp index d38cf3a1d..4e3ba8117 100644 --- a/benchmark/benchmark_predicate_iterator.cpp +++ b/benchmark/benchmark_predicate_iterator.cpp @@ -31,15 +31,13 @@ #include #include #include +#include -#include -#include +#include +#include #include #include -#include -#include - #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 128 * 4; #endif @@ -47,34 +45,37 @@ const size_t DEFAULT_BYTES = 1024 * 1024 * 128 * 4; const unsigned int batch_size = 10; const unsigned int warmup_size = 5; -template +template struct identity { - __device__ T operator()(T value) + __device__ + T operator()(T value) { return value; } }; -template +template struct less_than { - __device__ bool operator()(T value) const + __device__ + bool operator()(T value) const { return value < T{C}; } }; -template +template struct increment { - __device__ T operator()(T value) const + __device__ + T operator()(T value) const { return value + T{I}; } }; -template +template struct transform_op { __device__ @@ -84,7 +85,7 @@ struct transform_op } }; -template +template struct transform_it { using value_type = T; @@ -97,7 +98,7 @@ struct transform_it } }; -template +template struct read_predicate_it { using value_type = T; @@ -110,7 +111,7 @@ struct read_predicate_it } }; -template +template struct write_predicate_it { using value_type = T; @@ -123,7 +124,7 @@ struct write_predicate_it } }; -template +template void run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& seed, @@ -131,21 +132,21 @@ void run_benchmark(benchmark::State& state, { using T = typename IteratorBenchmark::value_type; - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(T); const auto random_range = limit_random_range(0, 99); std::vector input = get_random_data(size, random_range.first, random_range.second, seed.get_0()); - T* d_input; - T* d_output; + T* d_input; + T* d_output; HIP_CHECK(hipMalloc(reinterpret_cast(&d_input), size * sizeof(T))); HIP_CHECK(hipMalloc(reinterpret_cast(&d_output), size * sizeof(T))); HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); HIP_CHECK(hipDeviceSynchronize()); // Warm-up - for(size_t i = 0; i < warmup_size; i++) + for(size_t i = 0; i < warmup_size; ++i) { IteratorBenchmark{}(d_input, d_output, size, stream); } @@ -161,7 +162,7 @@ void run_benchmark(benchmark::State& state, // Record start event HIP_CHECK(hipEventRecord(start, stream)); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 0; i < batch_size; ++i) { IteratorBenchmark{}(d_input, d_output, size, stream); } @@ -191,7 +192,7 @@ void run_benchmark(benchmark::State& state, ",key_type:" #T ",cfg:default_config}") \ .c_str(), \ run_benchmark, increment>>, \ - bytes, \ + bytes, \ seed, \ stream) @@ -228,7 +229,7 @@ int main(int argc, char* argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); + const size_t bytes = parser.get("size"); const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); @@ -245,11 +246,14 @@ int main(int argc, char* argv[]) using custom_128 = custom_type; // Add benchmarks - std::vector benchmarks = {CREATE_TYPED_BENCHMARK(int8_t), - CREATE_TYPED_BENCHMARK(int16_t), - CREATE_TYPED_BENCHMARK(int32_t), - CREATE_TYPED_BENCHMARK(int64_t), - CREATE_TYPED_BENCHMARK(custom_128)}; + std::vector benchmarks + = {CREATE_TYPED_BENCHMARK(int8_t), + CREATE_TYPED_BENCHMARK(int16_t), + CREATE_TYPED_BENCHMARK(int32_t), + CREATE_TYPED_BENCHMARK(int64_t), + CREATE_TYPED_BENCHMARK(custom_128), + CREATE_TYPED_BENCHMARK(rocprim::int128_t), + CREATE_TYPED_BENCHMARK(rocprim::uint128_t)}; // Use manual timing for(auto& b : benchmarks) diff --git a/benchmark/benchmark_utils.hpp b/benchmark/benchmark_utils.hpp index 5b7b4dc0b..cb6990bd6 100644 --- a/benchmark/benchmark_utils.hpp +++ b/benchmark/benchmark_utils.hpp @@ -26,18 +26,27 @@ // rocPRIM #include #include +#include #include #include // partition_config_params +#include +#include +#include #include +#include #include +#include +#include #include #include +#include #include -#include #include #include #include +#include +#include #include #include #include @@ -100,71 +109,169 @@ class managed_seed bool is_random; }; -ROCPRIM_HOST inline -rocprim::native_half half_to_native(const rocprim::half& x) +ROCPRIM_HOST +inline rocprim::native_half half_to_native(const rocprim::half& x) { - return *reinterpret_cast(&x); + return *reinterpret_cast(&x); } -ROCPRIM_HOST inline -rocprim::half native_to_half(const rocprim::native_half& x) +ROCPRIM_HOST +inline rocprim::half native_to_half(const rocprim::native_half& x) { - return *reinterpret_cast(&x); + return *reinterpret_cast(&x); } struct half_less { - ROCPRIM_HOST_DEVICE inline - bool operator()(const rocprim::half& a, const rocprim::half& b) const + ROCPRIM_HOST_DEVICE + inline bool + operator()(const rocprim::half& a, const rocprim::half& b) const { - #if __HIP_DEVICE_COMPILE__ +#if __HIP_DEVICE_COMPILE__ return a < b; - #else +#else return half_to_native(a) < half_to_native(b); - #endif +#endif } }; struct half_plus { - ROCPRIM_HOST_DEVICE inline - rocprim::half operator()(const rocprim::half& a, const rocprim::half& b) const + ROCPRIM_HOST_DEVICE + inline rocprim::half + operator()(const rocprim::half& a, const rocprim::half& b) const { - #if __HIP_DEVICE_COMPILE__ +#if __HIP_DEVICE_COMPILE__ return a + b; - #else +#else return native_to_half(half_to_native(a) + half_to_native(b)); - #endif +#endif } }; struct half_equal_to { - ROCPRIM_HOST_DEVICE inline - bool operator()(const rocprim::half& a, const rocprim::half& b) const + ROCPRIM_HOST_DEVICE + inline bool + operator()(const rocprim::half& a, const rocprim::half& b) const { - #if __HIP_DEVICE_COMPILE__ +#if __HIP_DEVICE_COMPILE__ return a == b; - #else +#else return half_to_native(a) == half_to_native(b); - #endif +#endif } }; // std::uniform_int_distribution is undefined for anything other than: -// short, int, long, long long, unsigned short, unsigned int, unsigned long, or unsigned long long -template -struct is_valid_for_int_distribution : - std::integral_constant::value || - std::is_same::value || - std::is_same::value || - std::is_same::value || - std::is_same::value || - std::is_same::value || - std::is_same::value || - std::is_same::value - > {}; +// short, int, long, long long, rocprim::int128_t, unsigned short, unsigned int, unsigned long, unsigned long long, or rocprim::uint128_t +template +struct is_valid_for_int_distribution + : std::integral_constant< + bool, + std::is_same::value || std::is_same::value + || std::is_same::value || std::is_same::value + || std::is_same::value || std::is_same::value + || std::is_same::value || std::is_same::value + || std::is_same::value + || std::is_same::value> +{}; + +// uniform_int_distribution is defined for supporting rocprim::int128_t and rocprim::uint128_t +template +class uniform_int_distribution +{ +public: + typedef IntType result_type; + + uniform_int_distribution() : uniform_int_distribution(0) {} + + explicit uniform_int_distribution(IntType _a, + IntType _b = rocprim::numeric_limits::max()) + : lower_bound{_a}, upper_bound{_b} + {} + + void reset() {} + + result_type a() const + { + return lower_bound; + } + + result_type b() const + { + return upper_bound; + } + + result_type min() const + { + return a(); + } + + result_type max() const + { + return b(); + } + + template + result_type operator()(Generator& urng) + { + rocprim::uint128_t range = upper_bound - lower_bound + 1; + auto offset = helper(urng, range); + return offset + lower_bound; + } + + friend bool operator==(const uniform_int_distribution& d1, const uniform_int_distribution& d2) + { + return d1.lower_bound == d2.lower_bound && d1.upper_bound == d2.upper_bound; + } + + friend bool operator!=(const uniform_int_distribution& d1, const uniform_int_distribution& d2) + { + return !(d1 == d2); + } + + // third constructor, param(), operator<< and operator>> are not defined + +private: + // Java approach in the reference below. + // Returns an unbiased random number from urng downscaled to [0, range) + template + static rocprim::uint128_t helper(Generator& urng, const rocprim::uint128_t& range) + { + // reference: Fast Random Integer Geeneration in an Interval + // ACM Transactions on Modeling and Computer Simulation 29 (1), 2019 + // https://arxiv.org/abs/1805.10941 + static std::uniform_int_distribution dists[2]; + auto random_number = rocprim::uint128_t{dists[0](urng)} << 64 | dists[1](urng); + if(!range) + { + return random_number; + } + auto result = random_number % range; + auto threshold = rocprim::numeric_limits::max() - range + 1; + while(random_number - result > threshold) + { + random_number = rocprim::uint128_t{dists[0](urng)} << 64 | dists[1](urng); + result = random_number % range; + } + return result; + } + + IntType lower_bound; + IntType upper_bound; +}; + +template +class uniform_int_distribution< + IntType, + std::enable_if_t<(!(std::is_same::value + || std::is_same::value))>> + : public std::uniform_int_distribution +{ +public: + using std::uniform_int_distribution::uniform_int_distribution; +}; template using it_value_t = typename std::iterator_traits::value_type; @@ -173,7 +280,7 @@ using engine_type = std::minstd_rand; // generate_random_data_n() generates only part of sequence and replicates it, // because benchmarks usually do not need "true" random sequence. -template +template inline auto generate_random_data_n( OutputIter it, size_t size, U min, V max, Generator& gen, size_t max_random_size = 1024 * 1024) -> typename std::enable_if_t>::value, OutputIter> @@ -183,11 +290,8 @@ inline auto generate_random_data_n( using dis_type = typename std::conditional< is_valid_for_int_distribution::value, T, - typename std::conditional::value, - int, - unsigned int>::type - >::type; - std::uniform_int_distribution distribution((T)min, (T)max); + typename std::conditional::value, int, unsigned int>::type>::type; + ::uniform_int_distribution distribution((T)min, (T)max); std::generate_n(it, std::min(size, max_random_size), [&]() { return distribution(gen); }); for(size_t i = max_random_size; i < size; i += max_random_size) { @@ -196,7 +300,7 @@ inline auto generate_random_data_n( return it + size; } -template +template inline auto generate_random_data_n(OutputIterator it, size_t size, U min, @@ -222,7 +326,7 @@ inline auto generate_random_data_n(OutputIterator it, return it + size; } -template +template inline std::vector get_random_data01(size_t size, float p, unsigned int seed, size_t max_random_size = 1024 * 1024) { @@ -239,44 +343,45 @@ inline std::vector return data; } -template +template struct custom_type { - using first_type = T; + using first_type = T; using second_type = U; T x; U y; - ROCPRIM_HOST_DEVICE inline - custom_type(T xx = 0, U yy = 0) : x(xx), y(yy) - { - } + ROCPRIM_HOST_DEVICE inline custom_type(T xx = 0, U yy = 0) : x(xx), y(yy) {} - ROCPRIM_HOST_DEVICE inline - ~custom_type() = default; + ROCPRIM_HOST_DEVICE inline ~custom_type() = default; - ROCPRIM_HOST_DEVICE inline - custom_type operator+(const custom_type& rhs) const + ROCPRIM_HOST_DEVICE + inline custom_type + operator+(const custom_type& rhs) const { return custom_type(x + rhs.x, y + rhs.y); } - ROCPRIM_HOST_DEVICE inline - bool operator<(const custom_type& rhs) const + ROCPRIM_HOST_DEVICE + inline bool + operator<(const custom_type& rhs) const { // intentionally suboptimal choice for short-circuting, // required to generate more performant device code return ((x == rhs.x && y < rhs.y) || x < rhs.x); } - ROCPRIM_HOST_DEVICE inline - bool operator==(const custom_type& rhs) const + ROCPRIM_HOST_DEVICE + inline bool + operator==(const custom_type& rhs) const { return x == rhs.x && y == rhs.y; } - ROCPRIM_HOST_DEVICE custom_type& operator+=(const custom_type& rhs) + ROCPRIM_HOST_DEVICE + custom_type& + operator+=(const custom_type& rhs) { this->x += rhs.x; this->y += rhs.y; @@ -288,7 +393,7 @@ template struct is_custom_type : std::false_type {}; -template +template struct is_custom_type> : std::true_type {}; @@ -318,7 +423,7 @@ struct is_comparable, T> std::true_type> {}; -template +template struct custom_type_decomposer { static_assert(is_custom_type::value, @@ -327,16 +432,18 @@ struct custom_type_decomposer using T = typename CustomType::first_type; using U = typename CustomType::second_type; - __host__ __device__ ::rocprim::tuple operator()(CustomType& key) const + __host__ __device__ + ::rocprim::tuple + operator()(CustomType& key) const { return ::rocprim::tuple{key.x, key.y}; } }; -template +template struct generate_limits; -template +template struct generate_limits::value>> { static inline T min() @@ -349,7 +456,7 @@ struct generate_limits::value>> } }; -template +template struct generate_limits::value>> { using F = typename T::first_type; @@ -364,7 +471,7 @@ struct generate_limits::value>> } }; -template +template struct generate_limits::value>> { static inline T min() @@ -377,7 +484,7 @@ struct generate_limits::value> } }; -template +template inline auto generate_random_data_n(OutputIterator it, size_t size, it_value_t min, @@ -388,7 +495,7 @@ inline auto generate_random_data_n(OutputIterator it, { using T = it_value_t; - using first_type = typename T::first_type; + using first_type = typename T::first_type; using second_type = typename T::second_type; std::vector fdata(size); @@ -396,14 +503,14 @@ inline auto generate_random_data_n(OutputIterator it, generate_random_data_n(fdata.begin(), size, min.x, max.x, gen, max_random_size); generate_random_data_n(sdata.begin(), size, min.y, max.y, gen, max_random_size); - for(size_t i = 0; i < size; i++) + for(size_t i = 0; i < size; ++i) { it[i] = T(fdata[i], sdata[i]); } return it + size; } -template +template inline auto generate_random_data_n(OutputIterator it, size_t size, it_value_t min, @@ -419,14 +526,14 @@ inline auto generate_random_data_n(OutputIterator it, using field_type = decltype(max.x); std::vector field_data(size); generate_random_data_n(field_data.begin(), size, min.x, max.x, gen, max_random_size); - for(size_t i = 0; i < size; i++) + for(size_t i = 0; i < size; ++i) { it[i] = T(field_data[i]); } return it + size; } -template +template inline std::vector get_random_data( size_t size, U min, V max, unsigned int seed, size_t max_random_size = 1024 * 1024) { @@ -482,7 +589,7 @@ auto limit_cast(U value) -> T } // This overload below is selected for non-standard float types, e.g. half, which cannot be compared with the limit types. -template +template inline auto limit_random_range(U range_start, V range_end) -> std::enable_if_t::value && (!is_comparable::value || !is_comparable::value), @@ -511,7 +618,7 @@ auto limit_random_range(U range_start, V range_end) }; } -template +template inline auto limit_random_range(U range_start, V range_end) -> std::enable_if_t::value && is_comparable::value && is_comparable::value, @@ -551,14 +658,14 @@ std::vector static_assert(rocprim::is_arithmetic::value, "Key type must be arithmetic"); engine_type prng(seed); - std::uniform_int_distribution segment_length_distribution( + ::uniform_int_distribution segment_length_distribution( std::numeric_limits::min(), max_segment_length); // std::uniform_real_distribution cannot handle rocprim::half, use float instead using dis_type = typename std::conditional::value, float, T>::type; using key_distribution_type = std::conditional_t::value, - std::uniform_int_distribution, + ::uniform_int_distribution, std::uniform_real_distribution>; key_distribution_type key_distribution(rocprim::numeric_limits::max()); std::vector keys(size); @@ -581,7 +688,7 @@ std::vector get_random_segments_iota(const size_t size, const size_t max_segment_length, unsigned int seed) { engine_type prng(seed); - std::uniform_int_distribution segment_length_distribution(1, max_segment_length); + ::uniform_int_distribution segment_length_distribution(1, max_segment_length); std::vector keys(size); @@ -598,7 +705,7 @@ std::vector return keys; } -template +template inline auto get_random_value(U min, V max, size_t seed_value) -> std::enable_if_t::value, T> { @@ -608,7 +715,7 @@ inline auto get_random_value(U min, V max, size_t seed_value) return result; } -template +template inline auto get_random_value(T min, T max, size_t seed_value) -> std::enable_if_t::value, T> { @@ -620,17 +727,17 @@ inline auto get_random_value(T min, T max, size_t seed_value) return T{result_first, result_second}; } -template +template struct make_index_range_impl; -template +template struct make_index_range_impl> { using type = std::integer_sequence; }; // make a std::integer_sequence with values from Start to End inclusive -template +template using make_index_range = typename make_index_range_impl>::type; @@ -663,7 +770,7 @@ void static_for_each(Args&&... args) struct config_autotune_interface { - virtual std::string name() const = 0; + virtual std::string name() const = 0; virtual std::string sort_key() const { return name(); @@ -674,13 +781,15 @@ struct config_autotune_interface struct config_autotune_register { - static std::vector>& vector() { + static std::vector>& vector() + { static std::vector> storage; return storage; } - template - static config_autotune_register create() { + template + static config_autotune_register create() + { vector().push_back(std::make_unique()); return config_autotune_register(); } @@ -709,7 +818,7 @@ struct config_autotune_register = (configs.size() + parallel_instance_count - 1) / parallel_instance_count; size_t start = std::min(parallel_instance_index * configs_per_instance, configs.size()); size_t end = std::min((parallel_instance_index + 1) * configs_per_instance, configs.size()); - for(size_t i = start; i < end; i++) + for(size_t i = start; i < end; ++i) { std::unique_ptr& uniq_ptr = configs.at(i); config_autotune_interface* tuning_benchmark = uniq_ptr.get(); @@ -800,7 +909,7 @@ struct bench_naming { int n = std::min(brackets_count, static_cast(m[3].length())); brackets_count -= n; - for(int c = 0; c < n; c++) + for(int c = 0; c < n; ++c) { result << "}"; } @@ -808,14 +917,14 @@ struct bench_naming } else { - brackets_count++; + ++brackets_count; result << "{"; insert_comma = false; } } while(brackets_count > 0) { - brackets_count--; + --brackets_count; result << "}"; } return result.str(); @@ -844,7 +953,7 @@ struct bench_naming { int n = std::min(brackets_count, static_cast(m[3].length())); brackets_count -= n; - for(int c = 0; c < n; c++) + for(int c = 0; c < n; ++c) { result << ">"; } @@ -852,14 +961,14 @@ struct bench_naming } else { - brackets_count++; + ++brackets_count; result << "<"; insert_comma = false; } } while(brackets_count > 0) { - brackets_count--; + --brackets_count; result << ">"; } return result.str(); @@ -869,7 +978,8 @@ struct bench_naming static std::string format_name(std::string string) { format format = get_format(); -std::regex r("([A-z0-9]*):\\s*((?:custom_type<[A-z0-9,]*>)|[A-z:\\(\\)\\.<>\\s0-9]*)(\\}*)"); + std::regex r( + "([A-z0-9]*):\\s*((?:custom_type<[A-z0-9,]*>)|[A-z:\\(\\)\\.<>\\s0-9]*)(\\}*)"); // First we perform some checks bool checks[4] = {false}; for(std::sregex_iterator i = std::sregex_iterator(string.begin(), string.end(), r); @@ -917,11 +1027,12 @@ std::regex r("([A-z0-9]*):\\s*((?:custom_type<[A-z0-9,]*>)|[A-z:\\(\\)\\.<>\\s0- } }; -template +template struct Traits { //static inline method instead of static inline attribute because that's only supported from C++17 onwards - static inline const char* name(){ + static inline const char* name() + { static_assert(sizeof(T) == 0, "Traits::name() unknown"); return "unknown"; } @@ -933,14 +1044,26 @@ inline const char* Traits::name() { return "char"; } -template <> -inline const char* Traits::name() { return "int"; } -template <> -inline const char* Traits::name() { return "short"; } -template <> -inline const char* Traits::name() { return "int8_t"; } -template <> -inline const char* Traits::name() { return "uint8_t"; } +template<> +inline const char* Traits::name() +{ + return "int"; +} +template<> +inline const char* Traits::name() +{ + return "short"; +} +template<> +inline const char* Traits::name() +{ + return "int8_t"; +} +template<> +inline const char* Traits::name() +{ + return "uint8_t"; +} template<> inline const char* Traits::name() { @@ -968,13 +1091,36 @@ inline const char* Traits::name() } // On MSVC `int64_t` and `long long` are the same, leading to multiple definition errors #ifndef _WIN32 -template <> -inline const char* Traits::name() { return "int64_t"; } +template<> +inline const char* Traits::name() +{ + return "int64_t"; +} #endif -template <> -inline const char* Traits::name() { return "float"; } -template <> -inline const char* Traits::name() { return "double"; } +// On MSVC `uint64_t` and `unsigned long long` are the same, leading to multiple definition errors +#ifndef _WIN32 +template<> +inline const char* Traits::name() +{ + return "uint64_t"; +} +#else +template<> +inline const char* Traits::name() +{ + return "unsigned long long"; +} +#endif +template<> +inline const char* Traits::name() +{ + return "float"; +} +template<> +inline const char* Traits::name() +{ + return "double"; +} template<> inline const char* Traits>::name() { @@ -1035,28 +1181,38 @@ inline const char* Traits>::name() { return "double2"; } +template<> +inline const char* Traits::name() +{ + return "rocprim::int128_t"; +} +template<> +inline const char* Traits::name() +{ + return "rocprim::uint128_t"; +} inline void add_common_benchmark_info() { - hipDeviceProp_t devProp; - int device_id = 0; + hipDeviceProp_t devProp; + int device_id = 0; HIP_CHECK(hipGetDevice(&device_id)); HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - auto str = [](const std::string& name, const std::string& val) { - benchmark::AddCustomContext(name, val); - }; + auto str = [](const std::string& name, const std::string& val) + { benchmark::AddCustomContext(name, val); }; - auto num = [](const std::string& name, const auto& value) { - benchmark::AddCustomContext(name, std::to_string(value)); - }; + auto num = [](const std::string& name, const auto& value) + { benchmark::AddCustomContext(name, std::to_string(value)); }; - auto dim2 = [num](const std::string& name, const auto* values) { + auto dim2 = [num](const std::string& name, const auto* values) + { num(name + "_x", values[0]); num(name + "_y", values[1]); }; - auto dim3 = [num, dim2](const std::string& name, const auto* values) { + auto dim3 = [num, dim2](const std::string& name, const auto* values) + { dim2(name, values); num(name + "_z", values[2]); }; @@ -1102,16 +1258,20 @@ inline void add_common_benchmark_info() num("hdp_ecc_enabled", devProp.ECCEnabled); num("hdp_tcc_driver", devProp.tccDriver); num("hdp_cooperative_multi_device_unmatched_func", devProp.cooperativeMultiDeviceUnmatchedFunc); - num("hdp_cooperative_multi_device_unmatched_grid_dim", devProp.cooperativeMultiDeviceUnmatchedGridDim); - num("hdp_cooperative_multi_device_unmatched_block_dim", devProp.cooperativeMultiDeviceUnmatchedBlockDim); - num("hdp_cooperative_multi_device_unmatched_shared_mem", devProp.cooperativeMultiDeviceUnmatchedSharedMem); + num("hdp_cooperative_multi_device_unmatched_grid_dim", + devProp.cooperativeMultiDeviceUnmatchedGridDim); + num("hdp_cooperative_multi_device_unmatched_block_dim", + devProp.cooperativeMultiDeviceUnmatchedBlockDim); + num("hdp_cooperative_multi_device_unmatched_shared_mem", + devProp.cooperativeMultiDeviceUnmatchedSharedMem); num("hdp_is_large_bar", devProp.isLargeBar); num("hdp_asic_revision", devProp.asicRevision); num("hdp_managed_memory", devProp.managedMemory); num("hdp_direct_managed_mem_access_from_host", devProp.directManagedMemAccessFromHost); num("hdp_concurrent_managed_access", devProp.concurrentManagedAccess); num("hdp_pageable_memory_access", devProp.pageableMemoryAccess); - num("hdp_pageable_memory_access_uses_host_page_tables", devProp.pageableMemoryAccessUsesHostPageTables); + num("hdp_pageable_memory_access_uses_host_page_tables", + devProp.pageableMemoryAccessUsesHostPageTables); const auto arch = devProp.arch; num("hdp_arch_has_global_int32_atomics", arch.hasGlobalInt32Atomics); diff --git a/benchmark/benchmark_warp_exchange.cpp b/benchmark/benchmark_warp_exchange.cpp index 5b7ec6013..361856695 100644 --- a/benchmark/benchmark_warp_exchange.cpp +++ b/benchmark/benchmark_warp_exchange.cpp @@ -29,17 +29,18 @@ // HIP API #include + +#include #include +#include +#include #include -#include -#include +#include #include +#include #include -#include -#include - #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif @@ -47,8 +48,8 @@ const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; struct BlockedToStripedOp { template< - class warp_exchange_type, - class T, + typename warp_exchange_type, + typename T, unsigned int ItemsPerThread > ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE @@ -63,8 +64,8 @@ struct BlockedToStripedOp struct StripedToBlockedOp { template< - class warp_exchange_type, - class T, + typename warp_exchange_type, + typename T, unsigned int ItemsPerThread > ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE @@ -79,8 +80,8 @@ struct StripedToBlockedOp struct BlockedToStripedShuffleOp { template< - class warp_exchange_type, - class T, + typename warp_exchange_type, + typename T, unsigned int ItemsPerThread > ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE @@ -95,8 +96,8 @@ struct BlockedToStripedShuffleOp struct StripedToBlockedShuffleOp { template< - class warp_exchange_type, - class T, + typename warp_exchange_type, + typename T, unsigned int ItemsPerThread > ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE @@ -110,12 +111,7 @@ struct StripedToBlockedShuffleOp struct ScatterToStripedOp { - template< - class T, - class OffsetT, - class warp_exchange_type, - unsigned int ItemsPerThread - > + template ROCPRIM_DEVICE ROCPRIM_INLINE void operator()(warp_exchange_type warp_exchange, T (&thread_data)[ItemsPerThread], @@ -129,16 +125,17 @@ struct ScatterToStripedOp template -__device__ auto warp_exchange_benchmark(T* d_output, unsigned int trials) + typename Op, + typename T> +__device__ +auto warp_exchange_benchmark(T* d_output, unsigned int trials) -> std::enable_if_t && !std::is_same::value> { T thread_data[ItemsPerThread]; ROCPRIM_UNROLL - for(unsigned int i = 0; i < ItemsPerThread; i++) + for(unsigned int i = 0; i < ItemsPerThread; ++i) { // generate unique value each data-element thread_data[i] = static_cast(threadIdx.x * ItemsPerThread + i); @@ -146,103 +143,105 @@ __device__ auto warp_exchange_benchmark(T* d_output, unsigned int trials) using warp_exchange_type = ::rocprim::warp_exchange; constexpr unsigned int warps_in_block = BlockSize / LogicalWarpSize; - const unsigned int warp_id = threadIdx.x / LogicalWarpSize; + const unsigned int warp_id = threadIdx.x / LogicalWarpSize; ROCPRIM_SHARED_MEMORY typename warp_exchange_type::storage_type storage[warps_in_block]; ROCPRIM_NO_UNROLL - for(unsigned int i = 0; i < trials; i++) + for(unsigned int i = 0; i < trials; ++i) { Op{}(warp_exchange_type(), thread_data, storage[warp_id]); ::rocprim::wave_barrier(); } ROCPRIM_UNROLL - for(unsigned int i = 0; i < ItemsPerThread; i++) + for(unsigned int i = 0; i < ItemsPerThread; ++i) { const unsigned int global_idx = (BlockSize * blockIdx.x + threadIdx.x) * ItemsPerThread + i; - d_output[global_idx] = thread_data[i]; + d_output[global_idx] = thread_data[i]; } } template -__device__ auto warp_exchange_benchmark(T* d_output, unsigned int trials) + typename Op, + typename T> +__device__ +auto warp_exchange_benchmark(T* d_output, unsigned int trials) -> std::enable_if_t && std::is_same::value> { - T thread_data[ItemsPerThread]; - unsigned int thread_ranks[ItemsPerThread]; + T thread_data[ItemsPerThread]; + unsigned int thread_ranks[ItemsPerThread]; constexpr unsigned int warps_in_block = BlockSize / LogicalWarpSize; const unsigned int warp_id = threadIdx.x / LogicalWarpSize; const unsigned int lane_id = threadIdx.x % LogicalWarpSize; ROCPRIM_UNROLL - for(unsigned int i = 0; i < ItemsPerThread; i++) + for(unsigned int i = 0; i < ItemsPerThread; ++i) { // generate unique value each data-element thread_data[i] = static_cast(threadIdx.x * ItemsPerThread + i); // generate unique destination location for each data-element const unsigned int s_lane_id = i % 2 == 0 ? LogicalWarpSize - 1 - lane_id : lane_id; - thread_ranks[i] = s_lane_id*ItemsPerThread+i; // scatter values in warp across whole storage + thread_ranks[i] + = s_lane_id * ItemsPerThread + i; // scatter values in warp across whole storage } using warp_exchange_type = ::rocprim::warp_exchange; ROCPRIM_SHARED_MEMORY typename warp_exchange_type::storage_type storage[warps_in_block]; ROCPRIM_NO_UNROLL - for(unsigned int i = 0; i < trials; i++) + for(unsigned int i = 0; i < trials; ++i) { Op{}(warp_exchange_type(), thread_data, thread_ranks, storage[warp_id]); ::rocprim::wave_barrier(); } ROCPRIM_UNROLL - for(unsigned int i = 0; i < ItemsPerThread; i++) + for(unsigned int i = 0; i < ItemsPerThread; ++i) { const unsigned int global_idx = (BlockSize * blockIdx.x + threadIdx.x) * ItemsPerThread + i; - d_output[global_idx] = thread_data[i]; + d_output[global_idx] = thread_data[i]; } } template -__device__ auto warp_exchange_benchmark(T* /*d_output*/, unsigned int /*trials*/) + typename Op, + typename T> +__device__ +auto warp_exchange_benchmark(T* /*d_output*/, unsigned int /*trials*/) -> std::enable_if_t> {} template -__global__ __launch_bounds__(BlockSize) void warp_exchange_kernel(T* d_output, unsigned int trials) + typename Op, + typename T> +__global__ __launch_bounds__(BlockSize) +void warp_exchange_kernel(T* d_output, unsigned int trials) { warp_exchange_benchmark(d_output, trials); } -template< - class T, - unsigned int BlockSize, - unsigned int ItemsPerThread, - unsigned int LogicalWarpSize, - class Op -> +template void run_benchmark(benchmark::State& state, hipStream_t stream, size_t bytes) { - // Calculate the number of elements + // Calculate the number of elements size_t N = bytes / sizeof(T); - constexpr unsigned int trials = 200; + constexpr unsigned int trials = 200; constexpr unsigned int items_per_block = BlockSize * ItemsPerThread; - const unsigned int size = items_per_block * ((N + items_per_block - 1) / items_per_block); + const unsigned int size = items_per_block * ((N + items_per_block - 1) / items_per_block); - T * d_output; + T* d_output; HIP_CHECK(hipMalloc(&d_output, size * sizeof(T))); // HIP events creation @@ -288,7 +287,7 @@ void run_benchmark(benchmark::State& state, hipStream_t stream, size_t bytes) stream, \ bytes) -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { cli::Parser parser(argc, argv); parser.set_optional("size", "size", DEFAULT_BYTES, "number of bytes"); @@ -301,8 +300,8 @@ int main(int argc, char *argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); - const int trials = parser.get("trials"); + const size_t bytes = parser.get("size"); + const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); // HIP @@ -351,7 +350,77 @@ int main(int argc, char *argv[]) CREATE_BENCHMARK(int, 256, 4, 16, ScatterToStripedOp), CREATE_BENCHMARK(int, 256, 4, 32, ScatterToStripedOp), CREATE_BENCHMARK(int, 256, 16, 16, ScatterToStripedOp), - CREATE_BENCHMARK(int, 256, 16, 32, ScatterToStripedOp)}; + CREATE_BENCHMARK(int, 256, 16, 32, ScatterToStripedOp), + + CREATE_BENCHMARK(rocprim::int128_t, 256, 1, 16, BlockedToStripedOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 1, 32, BlockedToStripedOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 4, 16, BlockedToStripedOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 4, 32, BlockedToStripedOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 16, 16, BlockedToStripedOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 16, 32, BlockedToStripedOp), + + CREATE_BENCHMARK(rocprim::int128_t, 256, 1, 16, StripedToBlockedOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 1, 32, StripedToBlockedOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 4, 16, StripedToBlockedOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 4, 32, StripedToBlockedOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 16, 16, StripedToBlockedOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 16, 32, StripedToBlockedOp), + + CREATE_BENCHMARK(rocprim::int128_t, 256, 1, 16, BlockedToStripedShuffleOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 1, 32, BlockedToStripedShuffleOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 4, 16, BlockedToStripedShuffleOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 4, 32, BlockedToStripedShuffleOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 16, 16, BlockedToStripedShuffleOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 16, 32, BlockedToStripedShuffleOp), + + CREATE_BENCHMARK(rocprim::int128_t, 256, 1, 16, StripedToBlockedShuffleOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 1, 32, StripedToBlockedShuffleOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 4, 16, StripedToBlockedShuffleOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 4, 32, StripedToBlockedShuffleOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 16, 16, StripedToBlockedShuffleOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 16, 32, StripedToBlockedShuffleOp), + + CREATE_BENCHMARK(rocprim::int128_t, 256, 1, 16, ScatterToStripedOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 1, 32, ScatterToStripedOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 4, 16, ScatterToStripedOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 4, 32, ScatterToStripedOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 16, 16, ScatterToStripedOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 16, 32, ScatterToStripedOp), + + CREATE_BENCHMARK(rocprim::uint128_t, 256, 1, 16, BlockedToStripedOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 1, 32, BlockedToStripedOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 4, 16, BlockedToStripedOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 4, 32, BlockedToStripedOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 16, 16, BlockedToStripedOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 16, 32, BlockedToStripedOp), + + CREATE_BENCHMARK(rocprim::uint128_t, 256, 1, 16, StripedToBlockedOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 1, 32, StripedToBlockedOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 4, 16, StripedToBlockedOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 4, 32, StripedToBlockedOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 16, 16, StripedToBlockedOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 16, 32, StripedToBlockedOp), + + CREATE_BENCHMARK(rocprim::uint128_t, 256, 1, 16, BlockedToStripedShuffleOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 1, 32, BlockedToStripedShuffleOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 4, 16, BlockedToStripedShuffleOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 4, 32, BlockedToStripedShuffleOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 16, 16, BlockedToStripedShuffleOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 16, 32, BlockedToStripedShuffleOp), + + CREATE_BENCHMARK(rocprim::uint128_t, 256, 1, 16, StripedToBlockedShuffleOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 1, 32, StripedToBlockedShuffleOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 4, 16, StripedToBlockedShuffleOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 4, 32, StripedToBlockedShuffleOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 16, 16, StripedToBlockedShuffleOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 16, 32, StripedToBlockedShuffleOp), + + CREATE_BENCHMARK(rocprim::uint128_t, 256, 1, 16, ScatterToStripedOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 1, 32, ScatterToStripedOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 4, 16, ScatterToStripedOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 4, 32, ScatterToStripedOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 16, 16, ScatterToStripedOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 16, 32, ScatterToStripedOp)}; int hip_device = 0; HIP_CHECK(::rocprim::detail::get_device_from_stream(stream, hip_device)); @@ -380,12 +449,50 @@ int main(int argc, char *argv[]) CREATE_BENCHMARK(int, 256, 1, 64, ScatterToStripedOp), CREATE_BENCHMARK(int, 256, 4, 64, ScatterToStripedOp), - CREATE_BENCHMARK(int, 256, 16, 64, ScatterToStripedOp)}; - benchmarks.insert( - benchmarks.end(), - additional_benchmarks.begin(), - additional_benchmarks.end() - ); + CREATE_BENCHMARK(int, 256, 16, 64, ScatterToStripedOp), + + CREATE_BENCHMARK(rocprim::int128_t, 256, 1, 64, BlockedToStripedOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 4, 64, BlockedToStripedOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 16, 64, BlockedToStripedOp), + + CREATE_BENCHMARK(rocprim::int128_t, 256, 1, 64, StripedToBlockedOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 4, 64, StripedToBlockedOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 16, 64, StripedToBlockedOp), + + CREATE_BENCHMARK(rocprim::int128_t, 256, 1, 64, BlockedToStripedShuffleOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 4, 64, BlockedToStripedShuffleOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 16, 64, BlockedToStripedShuffleOp), + + CREATE_BENCHMARK(rocprim::int128_t, 256, 1, 64, StripedToBlockedShuffleOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 4, 64, StripedToBlockedShuffleOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 16, 64, StripedToBlockedShuffleOp), + + CREATE_BENCHMARK(rocprim::int128_t, 256, 1, 64, ScatterToStripedOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 4, 64, ScatterToStripedOp), + CREATE_BENCHMARK(rocprim::int128_t, 256, 16, 64, ScatterToStripedOp), + + CREATE_BENCHMARK(rocprim::uint128_t, 256, 1, 64, BlockedToStripedOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 4, 64, BlockedToStripedOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 16, 64, BlockedToStripedOp), + + CREATE_BENCHMARK(rocprim::uint128_t, 256, 1, 64, StripedToBlockedOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 4, 64, StripedToBlockedOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 16, 64, StripedToBlockedOp), + + CREATE_BENCHMARK(rocprim::uint128_t, 256, 1, 64, BlockedToStripedShuffleOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 4, 64, BlockedToStripedShuffleOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 16, 64, BlockedToStripedShuffleOp), + + CREATE_BENCHMARK(rocprim::uint128_t, 256, 1, 64, StripedToBlockedShuffleOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 4, 64, StripedToBlockedShuffleOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 16, 64, StripedToBlockedShuffleOp), + + CREATE_BENCHMARK(rocprim::uint128_t, 256, 1, 64, ScatterToStripedOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 4, 64, ScatterToStripedOp), + CREATE_BENCHMARK(rocprim::uint128_t, 256, 16, 64, ScatterToStripedOp)}; + benchmarks.insert(benchmarks.end(), + additional_benchmarks.begin(), + additional_benchmarks.end()); } // Use manual timing @@ -398,7 +505,7 @@ int main(int argc, char *argv[]) // Force number of iterations if(trials > 0) { - for (auto& b : benchmarks) + for(auto& b : benchmarks) { b->Iterations(trials); } diff --git a/benchmark/benchmark_warp_reduce.cpp b/benchmark/benchmark_warp_reduce.cpp index 6c63e9af4..87a909225 100644 --- a/benchmark/benchmark_warp_reduce.cpp +++ b/benchmark/benchmark_warp_reduce.cpp @@ -31,23 +31,22 @@ #include // rocPRIM +#include +#include #include -#include -#include +#include +#include #include +#include #include -#include -#include - #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif -template -__global__ -__launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) +template +__global__ __launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) void warp_reduce_kernel(const T* d_input, T* d_output) { const unsigned int i = blockIdx.x * blockDim.x + threadIdx.x; @@ -57,7 +56,7 @@ void warp_reduce_kernel(const T* d_input, T* d_output) using wreduce_t = rocprim::warp_reduce; __shared__ typename wreduce_t::storage_type storage; ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < Trials; trial++) + for(unsigned int trial = 0; trial < Trials; ++trial) { wreduce_t().reduce(value, value, storage); } @@ -65,20 +64,19 @@ void warp_reduce_kernel(const T* d_input, T* d_output) d_output[i] = value; } -template -__global__ -__launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) +template +__global__ __launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) void segmented_warp_reduce_kernel(const T* d_input, Flag* d_flags, T* d_output) { const unsigned int i = blockIdx.x * blockDim.x + threadIdx.x; auto value = d_input[i]; - auto flag = d_flags[i]; + auto flag = d_flags[i]; using wreduce_t = rocprim::warp_reduce; __shared__ typename wreduce_t::storage_type storage; ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < Trials; trial++) + for(unsigned int trial = 0; trial < Trials; ++trial) { wreduce_t().head_segmented_reduce(value, value, flag, storage); } @@ -91,17 +89,19 @@ template + typename T, + typename Flag> inline auto execute_warp_reduce_kernel( T* input, T* output, Flag* /* flags */, size_t size, hipStream_t stream) -> typename std::enable_if::type { - hipLaunchKernelGGL( - HIP_KERNEL_NAME(warp_reduce_kernel), - dim3(size/BlockSize), dim3(BlockSize), 0, stream, - input, output - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(warp_reduce_kernel), + dim3(size / BlockSize), + dim3(BlockSize), + 0, + stream, + input, + output); HIP_CHECK(hipGetLastError()); } @@ -110,59 +110,53 @@ template + typename T, + typename Flag> inline auto execute_warp_reduce_kernel(T* input, T* output, Flag* flags, size_t size, hipStream_t stream) -> typename std::enable_if::type { - hipLaunchKernelGGL( - HIP_KERNEL_NAME(segmented_warp_reduce_kernel), - dim3(size/BlockSize), dim3(BlockSize), 0, stream, - input, flags, output - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(segmented_warp_reduce_kernel), + dim3(size / BlockSize), + dim3(BlockSize), + 0, + stream, + input, + flags, + output); HIP_CHECK(hipGetLastError()); } template -void run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& seed, hipStream_t stream) +void run_benchmark(benchmark::State& state, + size_t bytes, + const managed_seed& seed, + hipStream_t stream) { using flag_type = unsigned char; - // Calculate the number of elements + // Calculate the number of elements size_t N = bytes / sizeof(T); - const auto size = BlockSize * ((N + BlockSize - 1)/BlockSize); + const auto size = BlockSize * ((N + BlockSize - 1) / BlockSize); const auto random_range = limit_random_range(0, 10); std::vector input = get_random_data(size, random_range.first, random_range.second, seed.get_0()); std::vector flags = get_random_data(size, 0, 1, seed.get_1()); - T * d_input; - flag_type * d_flags; - T * d_output; + T* d_input; + flag_type* d_flags; + T* d_output; HIP_CHECK(hipMalloc(reinterpret_cast(&d_input), size * sizeof(T))); HIP_CHECK(hipMalloc(reinterpret_cast(&d_flags), size * sizeof(flag_type))); HIP_CHECK(hipMalloc(reinterpret_cast(&d_output), size * sizeof(T))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - size * sizeof(T), - hipMemcpyHostToDevice - ) - ); - HIP_CHECK( - hipMemcpy( - d_flags, flags.data(), - size * sizeof(flag_type), - hipMemcpyHostToDevice - ) - ); + HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); + HIP_CHECK(hipMemcpy(d_flags, flags.data(), size * sizeof(flag_type), hipMemcpyHostToDevice)); HIP_CHECK(hipDeviceSynchronize()); // HIP events creation @@ -210,7 +204,7 @@ void run_benchmark(benchmark::State& state, size_t bytes, const managed_seed& se + ",ws:" #WS ",cfg:{bs:" #BS "}}") \ .c_str(), \ run_benchmark, \ - bytes, \ + bytes, \ seed, \ stream) @@ -229,12 +223,14 @@ void add_benchmarks(std::vector& benchmarks, BENCHMARK_TYPE(double), BENCHMARK_TYPE(int8_t), BENCHMARK_TYPE(uint8_t), - BENCHMARK_TYPE(rocprim::half)}; + BENCHMARK_TYPE(rocprim::half), + BENCHMARK_TYPE(rocprim::int128_t), + BENCHMARK_TYPE(rocprim::uint128_t)}; benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); } -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { cli::Parser parser(argc, argv); parser.set_optional("size", "size", DEFAULT_BYTES, "number of bytes"); @@ -248,8 +244,8 @@ int main(int argc, char *argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); - const int trials = parser.get("trials"); + const size_t bytes = parser.get("size"); + const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); const managed_seed seed(seed_type); diff --git a/benchmark/benchmark_warp_scan.cpp b/benchmark/benchmark_warp_scan.cpp index bfa87924a..d934d962e 100644 --- a/benchmark/benchmark_warp_scan.cpp +++ b/benchmark/benchmark_warp_scan.cpp @@ -29,34 +29,30 @@ // HIP API #include // rocPRIM +#include +#include #include -#include -#include +#include +#include #include #include -#include -#include - #ifndef DEFAULT_BYTES const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif -namespace rp = rocprim; - -template -__global__ -__launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) +template +__global__ __launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) void warp_inclusive_scan_kernel(const T* input, T* output) { - const unsigned int i = blockIdx.x * blockDim.x + threadIdx.x; - auto value = input[i]; + const unsigned int i = blockIdx.x * blockDim.x + threadIdx.x; + auto value = input[i]; - using wscan_t = rp::warp_scan; + using wscan_t = rocprim::warp_scan; __shared__ typename wscan_t::storage_type storage; ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < Trials; trial++) + for(unsigned int trial = 0; trial < Trials; ++trial) { wscan_t().inclusive_scan(value, value, storage); } @@ -64,18 +60,17 @@ void warp_inclusive_scan_kernel(const T* input, T* output) output[i] = value; } -template -__global__ -__launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) +template +__global__ __launch_bounds__(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) void warp_exclusive_scan_kernel(const T* input, T* output, const T init) { - const unsigned int i = blockIdx.x * blockDim.x + threadIdx.x; - auto value = input[i]; + const unsigned int i = blockIdx.x * blockDim.x + threadIdx.x; + auto value = input[i]; - using wscan_t = rp::warp_scan; + using wscan_t = rocprim::warp_scan; __shared__ typename wscan_t::storage_type storage; ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < Trials; trial++) + for(unsigned int trial = 0; trial < Trials; ++trial) { wscan_t().exclusive_scan(value, value, init, storage); } @@ -83,33 +78,25 @@ void warp_exclusive_scan_kernel(const T* input, T* output, const T init) output[i] = value; } -template< - class T, - unsigned int BlockSize, - unsigned int WarpSize, - bool Inclusive = true, - unsigned int Trials = 100 -> +template void run_benchmark(benchmark::State& state, hipStream_t stream, size_t bytes) { - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(T); // Make sure size is a multiple of BlockSize - size = BlockSize * ((size + BlockSize - 1)/BlockSize); + size = BlockSize * ((size + BlockSize - 1) / BlockSize); // Allocate and fill memory std::vector input(size, (T)1); - T * d_input; - T * d_output; + T* d_input; + T* d_output; HIP_CHECK(hipMalloc(reinterpret_cast(&d_input), size * sizeof(T))); HIP_CHECK(hipMalloc(reinterpret_cast(&d_output), size * sizeof(T))); - HIP_CHECK( - hipMemcpy( - d_input, input.data(), - size * sizeof(T), - hipMemcpyHostToDevice - ) - ); + HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); HIP_CHECK(hipDeviceSynchronize()); // HIP events creation @@ -117,26 +104,31 @@ void run_benchmark(benchmark::State& state, hipStream_t stream, size_t bytes) HIP_CHECK(hipEventCreate(&start)); HIP_CHECK(hipEventCreate(&stop)); - for (auto _ : state) + for(auto _ : state) { // Record start event HIP_CHECK(hipEventRecord(start, stream)); if(Inclusive) { - hipLaunchKernelGGL( - HIP_KERNEL_NAME(warp_inclusive_scan_kernel), - dim3(size/BlockSize), dim3(BlockSize), 0, stream, - d_input, d_output - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(warp_inclusive_scan_kernel), + dim3(size / BlockSize), + dim3(BlockSize), + 0, + stream, + d_input, + d_output); } else { - hipLaunchKernelGGL( - HIP_KERNEL_NAME(warp_exclusive_scan_kernel), - dim3(size/BlockSize), dim3(BlockSize), 0, stream, - d_input, d_output, input[0] - ); + hipLaunchKernelGGL(HIP_KERNEL_NAME(warp_exclusive_scan_kernel), + dim3(size / BlockSize), + dim3(BlockSize), + 0, + stream, + d_input, + d_output, + input[0]); } HIP_CHECK(hipGetLastError()); @@ -181,7 +173,7 @@ void add_benchmarks(std::vector& benchmarks, hipStream_t stream, size_t bytes) { - using custom_double2 = custom_type; + using custom_double2 = custom_type; using custom_int_double = custom_type; std::vector new_benchmarks @@ -192,11 +184,13 @@ void add_benchmarks(std::vector& benchmarks, BENCHMARK_TYPE(uint8_t), BENCHMARK_TYPE(rocprim::half), BENCHMARK_TYPE(custom_double2), - BENCHMARK_TYPE(custom_int_double)}; + BENCHMARK_TYPE(custom_int_double), + BENCHMARK_TYPE(rocprim::int128_t), + BENCHMARK_TYPE(rocprim::uint128_t)}; benchmarks.insert(benchmarks.end(), new_benchmarks.begin(), new_benchmarks.end()); } -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { cli::Parser parser(argc, argv); parser.set_optional("size", "size", DEFAULT_BYTES, "number of bytes"); @@ -209,8 +203,8 @@ int main(int argc, char *argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); - const int trials = parser.get("trials"); + const size_t bytes = parser.get("size"); + const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); // HIP diff --git a/benchmark/benchmark_warp_sort.cpp b/benchmark/benchmark_warp_sort.cpp index c99ebc9d6..226a5ebe5 100644 --- a/benchmark/benchmark_warp_sort.cpp +++ b/benchmark/benchmark_warp_sort.cpp @@ -31,71 +31,66 @@ // rocPRIM #include #include +#include +#include #include -#include -#include +#include +#include #include #include -#include -#include - #ifndef DEFAULT_BYTES -const size_t DEFAULT_BYTES = 1024 * 1024 * 32 *4; +const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif -namespace rp = rocprim; - -template -__global__ -__launch_bounds__(BlockSize) +template +__global__ __launch_bounds__(BlockSize) void warp_sort_kernel(K* input_keys, K* output_keys) { - const unsigned int flat_tid = threadIdx.x; + const unsigned int flat_tid = threadIdx.x; const unsigned int items_per_block = BlockSize * ItemsPerThread; - const unsigned int block_offset = blockIdx.x * items_per_block; + const unsigned int block_offset = blockIdx.x * items_per_block; K keys[ItemsPerThread]; - rp::block_load_direct_striped(flat_tid, input_keys + block_offset, keys); + rocprim::block_load_direct_striped(flat_tid, input_keys + block_offset, keys); - rp::warp_sort wsort; + rocprim::warp_sort wsort; wsort.sort(keys); - rp::block_store_direct_blocked(flat_tid, output_keys + block_offset, keys); + rocprim::block_store_direct_blocked(flat_tid, output_keys + block_offset, keys); } -template -__global__ -__launch_bounds__(BlockSize) +__global__ __launch_bounds__(BlockSize) void warp_sort_by_key_kernel(K* input_keys, V* input_values, K* output_keys, V* output_values) { - const unsigned int flat_tid = threadIdx.x; + const unsigned int flat_tid = threadIdx.x; const unsigned int items_per_block = BlockSize * ItemsPerThread; - const unsigned int block_offset = blockIdx.x * items_per_block; + const unsigned int block_offset = blockIdx.x * items_per_block; K keys[ItemsPerThread]; V values[ItemsPerThread]; - rp::block_load_direct_striped(flat_tid, input_keys + block_offset, keys); - rp::block_load_direct_striped(flat_tid, input_values + block_offset, values); + rocprim::block_load_direct_striped(flat_tid, input_keys + block_offset, keys); + rocprim::block_load_direct_striped(flat_tid, input_values + block_offset, values); - rp::warp_sort wsort; + rocprim::warp_sort wsort; wsort.sort(keys, values); - rp::block_store_direct_blocked(flat_tid, output_keys + block_offset, keys); - rp::block_store_direct_blocked(flat_tid, output_values + block_offset, values); + rocprim::block_store_direct_blocked(flat_tid, output_keys + block_offset, keys); + rocprim::block_store_direct_blocked(flat_tid, output_values + block_offset, values); } -template void run_benchmark(benchmark::State& state, @@ -103,12 +98,12 @@ void run_benchmark(benchmark::State& state, const managed_seed& seed, hipStream_t stream) { - // Calculate the number of elements + // Calculate the number of elements size_t size = bytes / sizeof(Key); // Make sure size is a multiple of items_per_block constexpr auto items_per_block = BlockSize * ItemsPerThread; - size = BlockSize * ((size + items_per_block - 1) / items_per_block); + size = BlockSize * ((size + items_per_block - 1) / items_per_block); // Allocate and fill memory const auto random_range = limit_random_range(0, 10'000); std::vector input_key @@ -120,30 +115,23 @@ void run_benchmark(benchmark::State& state, input_value = get_random_data(size, random_range.first, random_range.second, seed.get_1()); } - Key * d_input_key = nullptr; - Key * d_output_key = nullptr; - Value * d_input_value = nullptr; - Value * d_output_value = nullptr; + Key* d_input_key = nullptr; + Key* d_output_key = nullptr; + Value* d_input_value = nullptr; + Value* d_output_value = nullptr; HIP_CHECK(hipMalloc(reinterpret_cast(&d_input_key), size * sizeof(Key))); HIP_CHECK(hipMalloc(reinterpret_cast(&d_output_key), size * sizeof(Key))); - if(SortByKey) { + if(SortByKey) + { HIP_CHECK(hipMalloc(reinterpret_cast(&d_input_value), size * sizeof(Value))); HIP_CHECK(hipMalloc(reinterpret_cast(&d_output_value), size * sizeof(Value))); } - HIP_CHECK( - hipMemcpy( - d_input_key, input_key.data(), - size * sizeof(Key), - hipMemcpyHostToDevice - ) - ); - if(SortByKey) HIP_CHECK( - hipMemcpy( - d_input_value, input_value.data(), - size * sizeof(Value), - hipMemcpyHostToDevice - ) - ); + HIP_CHECK(hipMemcpy(d_input_key, input_key.data(), size * sizeof(Key), hipMemcpyHostToDevice)); + if(SortByKey) + HIP_CHECK(hipMemcpy(d_input_value, + input_value.data(), + size * sizeof(Value), + hipMemcpyHostToDevice)); HIP_CHECK(hipDeviceSynchronize()); // HIP events creation @@ -159,23 +147,34 @@ void run_benchmark(benchmark::State& state, if(SortByKey) { ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < Trials; ++trial) { + for(unsigned int trial = 0; trial < Trials; ++trial) + { hipLaunchKernelGGL( - HIP_KERNEL_NAME(warp_sort_by_key_kernel), - dim3(size/items_per_block), dim3(BlockSize), 0, stream, - d_input_key, d_input_value, d_output_key, d_output_value - ); + HIP_KERNEL_NAME( + warp_sort_by_key_kernel), + dim3(size / items_per_block), + dim3(BlockSize), + 0, + stream, + d_input_key, + d_input_value, + d_output_key, + d_output_value); } } else { ROCPRIM_NO_UNROLL - for(unsigned int trial = 0; trial < Trials; ++trial) { + for(unsigned int trial = 0; trial < Trials; ++trial) + { hipLaunchKernelGGL( HIP_KERNEL_NAME(warp_sort_kernel), - dim3(size/items_per_block), dim3(BlockSize), 0, stream, - d_input_key, d_output_key - ); + dim3(size / items_per_block), + dim3(BlockSize), + 0, + stream, + d_input_key, + d_output_key); } } HIP_CHECK(hipGetLastError()); @@ -195,7 +194,8 @@ void run_benchmark(benchmark::State& state, // SortByKey also transfers values auto sorted_type_size = sizeof(Key); - if(SortByKey) sorted_type_size += sizeof(Value); + if(SortByKey) + sorted_type_size += sizeof(Value); state.SetBytesProcessed(state.iterations() * size * sorted_type_size * Trials); state.SetItemsProcessed(state.iterations() * size * Trials); @@ -212,7 +212,7 @@ void run_benchmark(benchmark::State& state, + ",ws:" #WS ",cfg:{bs:" #BS ",ipt:" #IPT "}}") \ .c_str(), \ run_benchmark, \ - bytes, \ + bytes, \ seed, \ stream) @@ -222,7 +222,7 @@ void run_benchmark(benchmark::State& state, ",cfg:{bs:" #BS ",ipt:" #IPT "}}") \ .c_str(), \ run_benchmark, \ - bytes, \ + bytes, \ seed, \ stream) @@ -243,7 +243,7 @@ void run_benchmark(benchmark::State& state, CREATE_SORTBYKEY_BENCHMARK(type, value, 256, 64, 2), \ CREATE_SORTBYKEY_BENCHMARK(type, value, 256, 64, 4) -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { cli::Parser parser(argc, argv); parser.set_optional("size", "size", DEFAULT_BYTES, "number of bytes"); @@ -257,8 +257,8 @@ int main(int argc, char *argv[]) // Parse argv benchmark::Initialize(&argc, argv); - const size_t bytes = parser.get("size"); - const int trials = parser.get("trials"); + const size_t bytes = parser.get("size"); + const int trials = parser.get("trials"); bench_naming::set_format(parser.get("name_format")); const std::string seed_type = parser.get("seed"); const managed_seed seed(seed_type); @@ -271,7 +271,7 @@ int main(int argc, char *argv[]) benchmark::AddCustomContext("bytes", std::to_string(bytes)); benchmark::AddCustomContext("seed", seed_type); - using custom_double2 = custom_type; + using custom_double2 = custom_type; using custom_int_double = custom_type; using custom_int2 = custom_type; @@ -285,6 +285,8 @@ int main(int argc, char *argv[]) BENCHMARK_TYPE(int8_t), BENCHMARK_TYPE(uint8_t), BENCHMARK_TYPE(rocprim::half), + BENCHMARK_TYPE(rocprim::int128_t), + BENCHMARK_TYPE(rocprim::uint128_t), BENCHMARK_KEY_TYPE(float, float), BENCHMARK_KEY_TYPE(unsigned int, int), @@ -295,7 +297,9 @@ int main(int argc, char *argv[]) BENCHMARK_KEY_TYPE(custom_int2, custom_longlong_double), BENCHMARK_KEY_TYPE(int8_t, int8_t), BENCHMARK_KEY_TYPE(uint8_t, uint8_t), - BENCHMARK_KEY_TYPE(rocprim::half, rocprim::half)}; + BENCHMARK_KEY_TYPE(rocprim::half, rocprim::half), + BENCHMARK_KEY_TYPE(rocprim::int128_t, rocprim::int128_t), + BENCHMARK_KEY_TYPE(rocprim::uint128_t, rocprim::uint128_t)}; // Use manual timing for(auto& b : benchmarks) diff --git a/scripts/autotune/fallback_config.json b/scripts/autotune/fallback_config.json index 510082089..c820cd84c 100644 --- a/scripts/autotune/fallback_config.json +++ b/scripts/autotune/fallback_config.json @@ -19,8 +19,7 @@ "is_floating_point": true }, { - "algo_regex": "^device_merge_sort", - "based_on_type": "custom_type", + "based_on_type": "rocprim::int128_t", "sizeof_max_inclusive": 16, "sizeof_min_exclusive": 8, "is_floating_point": false diff --git a/scripts/autotune/templates/config_template b/scripts/autotune/templates/config_template index e93bf26ae..11443aae7 100644 --- a/scripts/autotune/templates/config_template +++ b/scripts/autotune/templates/config_template @@ -21,7 +21,9 @@ #ifndef {{ get_header_guard() }} #define {{ get_header_guard() }} -#include "../../../type_traits.hpp" +#include "../../../config.hpp" +#include "../../../type_traits_interface.hpp" +#include "../../config_types.hpp" #include "../device_config_helper.hpp" #include diff --git a/scripts/autotune/templates/merge_config_template b/scripts/autotune/templates/merge_config_template index 49ce9b233..e89592f68 100644 --- a/scripts/autotune/templates/merge_config_template +++ b/scripts/autotune/templates/merge_config_template @@ -10,7 +10,7 @@ merge_config<{{ measurement['cfg']['bs'] }}, {{ measurement['cfg']['ipt'] }}> { {% macro general_case() -%} template -struct default_merge_config : default_merge_base::type +struct default_merge_config : default_merge_config_base::type {}; {%- endmacro %} From 2f140cbf8ea1b780c11d116c39e46a97b519743b Mon Sep 17 00:00:00 2001 From: Cenxuan Tian Date: Mon, 6 Jan 2025 09:06:07 +0000 Subject: [PATCH 59/84] Resolve "Verify sorting on device" --- test/rocprim/CMakeLists.txt | 1 + test/rocprim/test_device_radix_sort.hpp | 60 +++++-- test/rocprim/test_utils_sort_checker.cpp | 135 ++++++++++++++++ test/rocprim/test_utils_sort_checker.hpp | 164 ++++++++++++++++++++ test/rocprim/test_utils_sort_comparator.hpp | 11 +- 5 files changed, 355 insertions(+), 16 deletions(-) create mode 100644 test/rocprim/test_utils_sort_checker.cpp create mode 100644 test/rocprim/test_utils_sort_checker.hpp diff --git a/test/rocprim/CMakeLists.txt b/test/rocprim/CMakeLists.txt index 07d81d3fe..b281c429c 100644 --- a/test/rocprim/CMakeLists.txt +++ b/test/rocprim/CMakeLists.txt @@ -306,6 +306,7 @@ add_rocprim_test("rocprim.reverse_iterator" test_reverse_iterator.cpp) add_rocprim_test("rocprim.texture_cache_iterator" test_texture_cache_iterator.cpp) add_rocprim_test("rocprim.thread" test_thread.cpp) add_rocprim_test("rocprim.thread_algos" test_thread_algos.cpp) +add_rocprim_test("rocprim.utils_sort_checker" test_utils_sort_checker.cpp) add_rocprim_test("rocprim.transform_iterator" test_transform_iterator.cpp) add_rocprim_cpp_standard_test(14 OFF "_cpp14" "rocprim.type_traits_interface" test_type_traits_interface.cpp) add_rocprim_cpp_standard_test(14 ON "_gnupp14" "rocprim.type_traits_interface" test_type_traits_interface.cpp) diff --git a/test/rocprim/test_device_radix_sort.hpp b/test/rocprim/test_device_radix_sort.hpp index a299efe6f..2377745b3 100644 --- a/test/rocprim/test_device_radix_sort.hpp +++ b/test/rocprim/test_device_radix_sort.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -34,6 +34,7 @@ #include "test_utils_custom_float_type.hpp" #include "test_utils_custom_test_types.hpp" #include "test_utils_device_ptr.hpp" +#include "test_utils_sort_checker.hpp" #include "test_utils_sort_comparator.hpp" #include "test_utils_types.hpp" @@ -280,13 +281,6 @@ void sort_keys() = in_place ? d_keys_input : d_keys_output_alloc; d_keys_output_alloc.resize(in_place ? 0 : size); - // Calculate expected results on host - std::vector expected(keys_input.get(), keys_input.get() + size); - std::stable_sort( - expected.begin(), - expected.end(), - test_utils::key_comparator()); - // Use arbitrary custom config to increase test coverage without making more test cases using config = rocprim::radix_sort_config 4096) + { + bool is_sorted = test_utils::device_sort_check( + d_keys_output.get(), + size, + test_utils::key_comparator{}, + stream, + debug_synchronous); + if(!is_sorted) + { + // Load output to host + const auto keys_output = d_keys_output.load_to_unique_ptr(); + + // Calculate expected results on host + std::vector expected(keys_input.get(), keys_input.get() + size); + std::stable_sort( + expected.begin(), + expected.end(), + test_utils::key_comparator{}); + + ASSERT_NO_FATAL_FAILURE(test_utils::assert_bit_eq(keys_output.get(), + keys_output.get() + size, + expected.begin(), + expected.end())); + FAIL(); + } + } + else + { + // Load output to host + const auto keys_output = d_keys_output.load_to_unique_ptr(); + + // Calculate expected results on host + std::vector expected(keys_input.get(), keys_input.get() + size); + std::stable_sort( + expected.begin(), + expected.end(), + test_utils::key_comparator{}); + + ASSERT_NO_FATAL_FAILURE(test_utils::assert_bit_eq(keys_output.get(), + keys_output.get() + size, + expected.begin(), + expected.end())); + } } } diff --git a/test/rocprim/test_utils_sort_checker.cpp b/test/rocprim/test_utils_sort_checker.cpp new file mode 100644 index 000000000..a070bd81e --- /dev/null +++ b/test/rocprim/test_utils_sort_checker.cpp @@ -0,0 +1,135 @@ +// MIT License +// +// Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#ifndef TEST_UTILS_SORT_CHECKER_CPP_ +#define TEST_UTILS_SORT_CHECKER_CPP_ + +#include "test_utils_sort_checker.hpp" +#include "test_utils_device_ptr.hpp" +#include "test_utils_sort_comparator.hpp" +#include "test_utils_types.hpp" + +template> +struct RocprimSortCheckerParams +{ + using input_type = InputType; + using op_type = OpType; +}; + +template +class RocprimSortCheckerTests : public ::testing::Test +{ +public: + using input_type = typename Params::input_type; + using op_type = typename Params::op_type; +}; + +using RocprimSortCheckerTestParams = ::testing::Types< + RocprimSortCheckerParams>, + RocprimSortCheckerParams>, + RocprimSortCheckerParams>; + +TYPED_TEST_SUITE(RocprimSortCheckerTests, RocprimSortCheckerTestParams); + +TYPED_TEST(RocprimSortCheckerTests, TrueTest) +{ + using input_type = typename TestFixture::input_type; + using op_type = typename TestFixture::op_type; + + int device_id = test_common_utils::obtain_device_from_ctest(); + HIP_CHECK(hipSetDevice(device_id)); + + const auto op = op_type{}; + + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + { + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + + for(const auto size : test_utils::get_sizes(seed_value)) + { + if(!size) + { + continue; + } + std::vector input = test_utils::get_random_data( + size, + 0, + test_utils::numeric_limits::max(), + ++seed_value); + + std::sort(input.begin(), input.end(), op); + + test_utils::device_ptr d_input(input); + ASSERT_TRUE(test_utils::device_sort_check(d_input.get(), size, op)); + } + } + SUCCEED(); +} + +TEST(RocprimSortCheckerTests, FalseTest) +{ + using input_type = unsigned int; + + int device_id = test_common_utils::obtain_device_from_ctest(); + HIP_CHECK(hipSetDevice(device_id)); + + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + { + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + + for(const auto size : test_utils::get_sizes(seed_value)) + { + if(size > 20) + { + return; + } + if(!size) + { + continue; + } + std::vector input = test_utils::get_random_data( + size, + 0, + test_utils::numeric_limits::max(), + ++seed_value); + bool all_equal = true; + for(const auto i : input) + { + all_equal = all_equal && (i == input[0]); + } + if(all_equal) + { + continue; + } + std::sort(input.begin(), input.end(), rocprim::less{}); + + test_utils::device_ptr d_input(input); + ASSERT_FALSE( + test_utils::device_sort_check(d_input.get(), size, rocprim::greater{})); + } + } + SUCCEED(); +} + +#endif // TEST_UTILS_SORT_CHECKER_CPP_ diff --git a/test/rocprim/test_utils_sort_checker.hpp b/test/rocprim/test_utils_sort_checker.hpp new file mode 100644 index 000000000..36f73e57a --- /dev/null +++ b/test/rocprim/test_utils_sort_checker.hpp @@ -0,0 +1,164 @@ +// MIT License +// +// Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#ifndef TEST_UTILS_SORT_CHECKER_HPP_ +#define TEST_UTILS_SORT_CHECKER_HPP_ + +#include + +#include "../common_test_header.hpp" +#include "test_utils_device_ptr.hpp" + +#include +#include +#include + +namespace test_utils +{ +namespace detail +{ +template +ROCPRIM_HOST_DEVICE +inline bool sort_check_compare(const InputType& i, + const InputType& next, + const BinaryFunction binary_function) +{ + auto ret_1 = binary_function(i, next); + auto ret_2 = binary_function(next, i); + if(!((ret_1 && ret_2) || (!ret_1 && !ret_2)) && !ret_1) + { + return false; + } + return true; +} + +template +struct transform_output_placeholder +{ + + using value_type = ValueType; + using reference = const value_type&; + using pointer = const value_type*; + using difference_type = typename std::iterator_traits::difference_type; + using iterator_category = std::output_iterator_tag; + + struct fake_ref + { + constexpr fake_ref operator=(const value_type&) const + { + return fake_ref{}; + } + }; + + constexpr fake_ref operator[](difference_type) const + { + return fake_ref{}; + } + constexpr transform_output_placeholder operator+(difference_type) const + { + return transform_output_placeholder{}; + } +}; + +template +inline bool host_sort_check(const InputIterator input, + const size_t size, + const BinaryFunction binary_function, + const hipStream_t stream = hipStreamDefault) +{ + using input_type = typename std::iterator_traits::value_type; + + if(size < 0) + { + return false; + } + else if(size <= 1) + { + return true; + } + + std::vector host_input(size); + HIP_CHECK(hipMemcpyAsync(host_input.data(), + input, + sizeof(input_type) * size, + hipMemcpyDeviceToHost, + stream)); + for(size_t i = 0; i < size - 1; ++i) + { + if(!detail::sort_check_compare(host_input[i], host_input[i + 1], binary_function)) + { + return false; + } + } + return true; +} + +} // namespace detail +template +inline bool device_sort_check(const InputIterator input, + const size_t size, + const BinaryFunction binary_function, + const hipStream_t stream = hipStreamDefault, + const bool debug_synchronous = false) +{ + using input_type = typename std::iterator_traits::value_type; + + if(size < 0) + { + return false; + } + else if(size <= 1) + { + return true; + } + + test_utils::device_ptr d_success_flag(std::vector({1})); + int* const d_success_flag_pointer = d_success_flag.get(); + + const auto deref_op = [=](const input_type& i) -> input_type + { + // Obtaining the address is not suitable for all kinds of iterators + // but not sure if there is a better way to do this. + auto addr = &i; + const auto& next = *(++addr); // next value + if(!detail::sort_check_compare(i, next, binary_function)) + { + *d_success_flag_pointer = 0; + } + return i; + }; + const auto d_iter + = rocprim::transform_iterator(input, deref_op); + + HIP_CHECK(rocprim::transform(d_iter, + detail::transform_output_placeholder{}, + size - 1, + rocprim::identity(), + stream, + debug_synchronous)); + + return static_cast(d_success_flag.load()[0]); +} + +} // namespace test_utils + +#endif // TEST_UTILS_SORT_CHECKER_HPP_ diff --git a/test/rocprim/test_utils_sort_comparator.hpp b/test/rocprim/test_utils_sort_comparator.hpp index f9e26e373..026d873df 100644 --- a/test/rocprim/test_utils_sort_comparator.hpp +++ b/test/rocprim/test_utils_sort_comparator.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -47,6 +47,7 @@ template::value, int> = 0> +ROCPRIM_HOST_DEVICE auto to_bits(const Key key) -> typename rocprim::get_unsigned_bits_type::unsigned_type { using unsigned_bits_type = typename rocprim::get_unsigned_bits_type::unsigned_type; @@ -74,11 +75,12 @@ template::value, int> = 0> +ROCPRIM_HOST_DEVICE auto to_bits(const Key key) -> typename rocprim::get_unsigned_bits_type::unsigned_type { using unsigned_bits_type = typename rocprim::get_unsigned_bits_type::unsigned_type; unsigned_bits_type bit_key; - std::memcpy(&bit_key, &key, sizeof(bit_key)); + memcpy(&bit_key, &key, sizeof(bit_key)); return to_bits(bit_key); } @@ -94,6 +96,7 @@ template::value, int> = 0> +ROCPRIM_HOST_DEVICE auto to_bits(const Key key) -> typename rocprim::get_unsigned_bits_type::unsigned_type { using unsigned_bits_type = typename rocprim::get_unsigned_bits_type::unsigned_type; @@ -133,6 +136,7 @@ template::value, int> = 0> +ROCPRIM_HOST_DEVICE auto to_bits(const Key& key) -> typename rocprim::get_unsigned_bits_type::unsigned_type { using inner_t = typename inner_type::type; @@ -161,6 +165,7 @@ template::value, int> = 0> +ROCPRIM_HOST_DEVICE auto to_bits(const Key key) -> typename rocprim::get_unsigned_bits_type::unsigned_type { return to_bits(key.x); @@ -170,6 +175,7 @@ template::value, int> = 0> +ROCPRIM_HOST_DEVICE auto to_bits(const Key key) -> typename rocprim::get_unsigned_bits_type::unsigned_type { return to_bits(key.x); @@ -186,6 +192,7 @@ constexpr bool is_floating_nan_host(const T& a) template struct key_comparator { + ROCPRIM_HOST_DEVICE bool operator()(const Key lhs, const Key rhs) const { const auto l = detail::to_bits(lhs); From d8101b428751b5a615d8dde085fd31a48e452014 Mon Sep 17 00:00:00 2001 From: Cenxuan Tian Date: Mon, 6 Jan 2025 14:39:25 +0000 Subject: [PATCH 60/84] Resolve "Expose lookback_scan_state::create and lookback_scan_state::get_storage_size as ROCPRIM_HOST_DEVICE." --- .../device/detail/lookback_scan_state.hpp | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/rocprim/include/rocprim/device/detail/lookback_scan_state.hpp b/rocprim/include/rocprim/device/detail/lookback_scan_state.hpp index dddab29c9..59b7d70e1 100644 --- a/rocprim/include/rocprim/device/detail/lookback_scan_state.hpp +++ b/rocprim/include/rocprim/device/detail/lookback_scan_state.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -202,29 +202,32 @@ struct lookback_scan_state static constexpr bool use_sleep = UseSleep; // temp_storage must point to allocation of get_storage_size(number_of_blocks) bytes - ROCPRIM_HOST static inline hipError_t create(lookback_scan_state& state, - void* temp_storage, - const unsigned int number_of_blocks, - const hipStream_t /*stream*/) + ROCPRIM_HOST_DEVICE + static inline hipError_t create(lookback_scan_state& state, + void* temp_storage, + const unsigned int number_of_blocks, + const hipStream_t /*stream*/) { (void)number_of_blocks; state.prefixes = reinterpret_cast(temp_storage); return hipSuccess; } - [[deprecated( - "Please use the overload returns an error code, this function assumes the default" - " stream and silently ignores errors.")]] ROCPRIM_HOST static inline lookback_scan_state - create(void* temp_storage, const unsigned int number_of_blocks) + [[deprecated("Please use the overload returns an error code, this function assumes the default" + " stream and silently ignores errors.")]] + ROCPRIM_HOST_DEVICE + static inline lookback_scan_state create(void* temp_storage, + const unsigned int number_of_blocks) { lookback_scan_state result; (void)create(result, temp_storage, number_of_blocks, /*default stream*/ 0); return result; } - ROCPRIM_HOST static inline hipError_t get_storage_size(const unsigned int number_of_blocks, - const hipStream_t stream, - size_t& storage_size) + ROCPRIM_HOST_DEVICE + static inline hipError_t get_storage_size(const unsigned int number_of_blocks, + const hipStream_t stream, + size_t& storage_size) { unsigned int warp_size; hipError_t error = ::rocprim::host_warp_size(stream, warp_size); @@ -235,8 +238,9 @@ struct lookback_scan_state } [[deprecated("Please use the overload returns an error code, this function assumes the default" - " stream and silently ignores errors.")]] ROCPRIM_HOST static inline size_t - get_storage_size(const unsigned int number_of_blocks) + " stream and silently ignores errors.")]] + ROCPRIM_HOST_DEVICE + static inline size_t get_storage_size(const unsigned int number_of_blocks) { size_t result; (void)get_storage_size(number_of_blocks, /*default stream*/ 0, result); From b80bba6d2c7a4d43b858a3b5c8085d497dd45bd0 Mon Sep 17 00:00:00 2001 From: Nara Prasetya Date: Thu, 9 Jan 2025 18:16:20 +0000 Subject: [PATCH 61/84] add naive in-place api for device merge --- CHANGELOG.md | 1 + benchmark/CMakeLists.txt | 1 + benchmark/benchmark_device_merge_inplace.cpp | 284 ++++++++++++ docs/device_ops/merge.rst | 21 +- .../rocprim/device/device_merge_inplace.hpp | 185 ++++++++ rocprim/include/rocprim/rocprim.hpp | 3 +- test/rocprim/CMakeLists.txt | 1 + test/rocprim/test_device_merge_inplace.cpp | 412 ++++++++++++++++++ 8 files changed, 903 insertions(+), 5 deletions(-) create mode 100644 benchmark/benchmark_device_merge_inplace.cpp create mode 100644 rocprim/include/rocprim/device/device_merge_inplace.hpp create mode 100644 test/rocprim/test_device_merge_inplace.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 885dc1e5d..f21d41eb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ Full documentation for rocPRIM is available at [https://rocm.docs.amd.com/projec * Added `rocprim::unrolled_copy` thread function to copy multiple items inside the thread. * Added `rocprim::int128_t` and `rocprim::uint128_t` to benchmarks for improved performance evaluation on 128-bit integers. * Added `rocprim::int128_t` to autotuning supported types to improve performance for 128-bit integers. +* Added `rocprim::merge_inplace` function that allows merging in-place. ### Resolved issues diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index fd8ee7804..472b7df54 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -147,6 +147,7 @@ add_rocprim_benchmark(benchmark_device_find_first_of.cpp) add_rocprim_benchmark(benchmark_device_find_end.cpp) add_rocprim_benchmark(benchmark_device_histogram.cpp) add_rocprim_benchmark(benchmark_device_merge.cpp) +add_rocprim_benchmark(benchmark_device_merge_inplace.cpp) add_rocprim_benchmark(benchmark_device_merge_sort.cpp) add_rocprim_benchmark(benchmark_device_merge_sort_block_sort.cpp) add_rocprim_benchmark(benchmark_device_merge_sort_block_merge.cpp) diff --git a/benchmark/benchmark_device_merge_inplace.cpp b/benchmark/benchmark_device_merge_inplace.cpp new file mode 100644 index 000000000..919c7e8c0 --- /dev/null +++ b/benchmark/benchmark_device_merge_inplace.cpp @@ -0,0 +1,284 @@ +// MIT License +// +// Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "benchmark_utils.hpp" +#include "cmdparser.hpp" + +#include +#include + +#include + +#include +#include + +#include +#include +#include +#include +#include + +#ifndef DEFAULT_BYTES +const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; +#endif + +template +struct random_monotonic_iterator +{ + const unsigned int seed; + random_monotonic_iterator(unsigned int seed) : seed(seed) {} + + using difference_type = std::ptrdiff_t; + using value_type = T; + + // not all integral types are valid for int distribution + using dist_value_type = std::conditional_t::value + && !is_valid_for_int_distribution::value, + int, + T>; + + using dist_type = std::conditional_t::value, + uniform_int_distribution, + std::uniform_real_distribution>; + + std::mt19937 engine{seed}; + dist_type dist{dist_value_type{0}, dist_value_type{increment}}; + + dist_value_type value = dist_value_type{0}; + + int operator*() const + { + return limit_cast(value); + } + + random_monotonic_iterator& operator++() + { + // prefix + value += dist(engine); + return *this; + } + + random_monotonic_iterator operator++(int) + { + // postfix + random_monotonic_iterator retval{*this}; + value += dist(engine); + return retval; + } +}; + +template +struct inplace_runner +{ + using value_type = ValueT; + using compare_op_type = + typename std::conditional::value, + half_less, + rocprim::less>::type; + + value_type* d_data; + size_t left_size; + size_t right_size; + hipStream_t stream; + + void* d_temporary_storage = nullptr; + size_t temporary_storage_bytes = 0; + + compare_op_type compare_op{}; + + inplace_runner(value_type* data, size_t left_size, size_t right_size, hipStream_t stream) + : d_data(data), left_size(left_size), right_size(right_size), stream(stream) + {} + + size_t prepare() + { + HIP_CHECK(rocprim::merge_inplace(d_temporary_storage, + temporary_storage_bytes, + d_data, + left_size, + right_size, + compare_op, + stream)); + HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); + + return temporary_storage_bytes; + } + + void run() + { + HIP_CHECK(rocprim::merge_inplace(d_temporary_storage, + temporary_storage_bytes, + d_data, + left_size, + right_size, + compare_op, + stream)); + } + + void clean() + { + HIP_CHECK(hipFree(d_temporary_storage)); + } +}; + +template +void run_merge_inplace_benchmarks(benchmark::State& state, + size_t size_a, + size_t size_b, + const managed_seed& seed, + hipStream_t stream) +{ + using value_type = ValueT; + using runner_type = RunnerT; + + size_t total_size = size_a + size_b; + std::vector h_data(total_size); + + auto gen_a_it = random_monotonic_iterator{seed.get_0()}; + auto gen_b_it = random_monotonic_iterator{seed.get_1()}; + + // generate left array + for(size_t i = 0; i < size_a; ++i) + { + h_data[i] = static_cast(*(gen_a_it++)); + } + + // generate right array + for(size_t i = 0; i < size_b; ++i) + { + h_data[size_a + i] = static_cast(*(gen_b_it++)); + } + + size_t num_bytes = total_size * sizeof(value_type); + + value_type* d_data; + + HIP_CHECK(hipMalloc(&d_data, num_bytes)); + + runner_type runner{d_data, size_a, size_b, stream}; + + size_t temp_storage_size = runner.prepare(); + + hipEvent_t start, stop; + HIP_CHECK(hipEventCreate(&start)); + HIP_CHECK(hipEventCreate(&stop)); + + for(auto _ : state) + { + HIP_CHECK(hipMemcpy(d_data, h_data.data(), num_bytes, hipMemcpyHostToDevice)); + + HIP_CHECK(hipEventRecord(start, stream)); + runner.run(); + HIP_CHECK(hipEventRecord(stop, stream)); + HIP_CHECK(hipEventSynchronize(stop)); + + float elapsed_mseconds; + HIP_CHECK(hipEventElapsedTime(&elapsed_mseconds, start, stop)); + state.SetIterationTime(elapsed_mseconds / 1000); + + std::stringstream label; + label << "temp_storage=" << temp_storage_size; + + state.SetLabel(label.str()); + } + + HIP_CHECK(hipEventDestroy(start)); + HIP_CHECK(hipEventDestroy(stop)); + + state.SetBytesProcessed(state.iterations() * num_bytes); + state.SetItemsProcessed(state.iterations() * total_size); + + HIP_CHECK(hipFree(d_data)); + runner.clean(); +} + +#define CREATE_MERGE_INPLACE_BENCHMARK(Value) \ + benchmark::RegisterBenchmark( \ + bench_naming::format_name("{lvl:device,algo:merge_inplace,value_type:" #Value \ + ",cfg:default_config}") \ + .c_str(), \ + [=](benchmark::State& state) { \ + run_merge_inplace_benchmarks>(state, \ + size, \ + size, \ + seed, \ + stream); \ + }) + +int main(int argc, char* argv[]) +{ + cli::Parser parser(argc, argv); + parser.set_optional("size", "size", DEFAULT_BYTES, "number of values"); + parser.set_optional("trials", "trials", -1, "number of iterations"); + parser.set_optional("name_format", + "name_format", + "human", + "either: json,human,txt"); + parser.set_optional("seed", "seed", "random", get_seed_message()); + parser.run_and_exit_if_error(); + + // Parse argv + benchmark::Initialize(&argc, argv); + const size_t size = parser.get("size"); + const int trials = parser.get("trials"); + bench_naming::set_format(parser.get("name_format")); + const std::string seed_type = parser.get("seed"); + const managed_seed seed(seed_type); + + // HIP + hipStream_t stream = hipStreamDefault; // default + + // Benchmark info + add_common_benchmark_info(); + benchmark::AddCustomContext("size", std::to_string(size)); + benchmark::AddCustomContext("seed", seed_type); + + // Add benchmarks + std::vector benchmarks = { + CREATE_MERGE_INPLACE_BENCHMARK(int8_t), + CREATE_MERGE_INPLACE_BENCHMARK(int16_t), + CREATE_MERGE_INPLACE_BENCHMARK(int32_t), + CREATE_MERGE_INPLACE_BENCHMARK(int64_t), + CREATE_MERGE_INPLACE_BENCHMARK(rocprim::int128_t), + }; + + // Use manual timing + for(auto& b : benchmarks) + { + b->UseManualTime(); + b->Unit(benchmark::kMillisecond); + } + + // Force number of iterations + if(trials > 0) + { + for(auto& b : benchmarks) + { + b->Iterations(trials); + } + } + + // Run benchmarks + benchmark::RunSpecifiedBenchmarks(); + + return 0; +} diff --git a/docs/device_ops/merge.rst b/docs/device_ops/merge.rst index 9c088af39..35bb39b3f 100644 --- a/docs/device_ops/merge.rst +++ b/docs/device_ops/merge.rst @@ -9,12 +9,25 @@ ******************************************************************** Configuring the kernel -======================= +====================== -.. doxygenstruct:: rocprim::merge_config +Merge +----- -merge -========== +.. doxygentypedef:: rocprim::merge_config + +Merge inplace +------------- + +.. doxygenstruct:: rocprim::merge_inplace_config + +Merge +===== .. doxygenfunction:: rocprim::merge (void *temporary_storage, size_t &storage_size, InputIterator1 input1, InputIterator2 input2, OutputIterator output, const size_t input1_size, const size_t input2_size, BinaryFunction compare_function=BinaryFunction(), const hipStream_t stream=0, bool debug_synchronous=false) .. doxygenfunction:: rocprim::merge (void *temporary_storage, size_t &storage_size, KeysInputIterator1 keys_input1, KeysInputIterator2 keys_input2, KeysOutputIterator keys_output, ValuesInputIterator1 values_input1, ValuesInputIterator2 values_input2, ValuesOutputIterator values_output, const size_t input1_size, const size_t input2_size, BinaryFunction compare_function=BinaryFunction(), const hipStream_t stream=0, bool debug_synchronous=false) + +Merge inplace +============= + +.. doxygenfunction:: rocprim::merge_inplace (void *temporary_storage, size_t &storage_size, Iterator data, size_t left_size, size_t right_size, BinaryFunction compare_function=BinaryFunction(), const hipStream_t stream=0, bool debug_synchronous=false) diff --git a/rocprim/include/rocprim/device/device_merge_inplace.hpp b/rocprim/include/rocprim/device/device_merge_inplace.hpp new file mode 100644 index 000000000..01a65a991 --- /dev/null +++ b/rocprim/include/rocprim/device/device_merge_inplace.hpp @@ -0,0 +1,185 @@ +// Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef ROCPRIM_DEVICE_DEVICE_MERGE_INPLACE_HPP_ +#define ROCPRIM_DEVICE_DEVICE_MERGE_INPLACE_HPP_ + +#include "../config.hpp" +#include "../functional.hpp" + +#include "../detail/temp_storage.hpp" +#include "../device/config_types.hpp" +#include "../device/device_merge.hpp" +#include "../device/device_transform.hpp" + +#include + +#include +#include + +BEGIN_ROCPRIM_NAMESPACE + +/// \addtogroup devicemodule +/// @{ + +/// \brief Parallel merge inplace primitive for device level. +/// +/// The `merge_inplace` function performs a device-wide merge in place. It merges two ordered sets +/// of input values based on a comparison function using significantly less temporary storage +/// compared to `merge`. +/// +/// \warning This function prioritizes temporary storage over speed. In most cases using `merge` +/// and a device copy is significantly faster. +/// +/// \warning This function uses cooperative groups. Please make sure that the device and platform +/// supports cooperative kernel launches. +/// +/// \par Overview +/// * The function can write intermediate values to the data array while the algorithm is running. +/// * Returns the required size of `temporary_storage` in `storage_size` if `temporary_storage` is a +/// null pointer. +/// * Accepts a custom `compare_function`. +/// +/// \tparam Config Configuration of the primitive, must be `default_config` or `merge_inplace_config`. +/// \tparam Iterator Random access iterator type for the input and output range. Must meet the +/// requirements of `std::random_access_iterator`. +/// \tparam BinaryFunction Binary function type that is used for the comparison. +/// +/// \param [in] temporary_storage Pointer to a device-accessible temporary storage. When a null +/// pointer is passed the required allocation size in bytes is written to `storage_size` and the +/// function returns `hipSuccess` without performing the merge operation. +/// \param [in,out] storage_size Reference to size in bytes of `temporary_storage`. +/// \param [in,out] data Iterator to the first value to merge. +/// \param [in] left_size Number of elements in the first input range. +/// \param [in] right_size Number of elements in the second input range. +/// \param [in] compare_function Binary operation function that will be used for comparison. The +/// signature of the function should be equivalent to the following: `bool f(const T &a, const T &b);`. +/// The signature does not need to have `const &`, but the function object must not modify +/// the objects passed to it. The default value is `BinaryFunction()`. +/// \param [in] stream The HIP stream object. Default is `0` (`hipDefaultStream`). +/// \param [in] debug_synchronous If `true`, forces a device synchronization after every kernel +/// launch in order to check for errors. Default value is `false`. +/// +/// \returns `hipSuccess` (`0`) after succesful merge; otherwise a HIP runtime error of type +/// `hipError_t`. +/// +/// \par Example +/// \parblock +/// \code{.cpp} +/// #include +/// +/// size_t left_size; // e.g. 4 +/// size_t right_size; // e.g. 4 +/// int* data; // e.g. [1, 3, 5, 7, 0, 2, 4, 6] +/// // output: [0, 1, 2, 3, 4, 5, 6, 7] +/// +/// size_t temporary_storage_size_bytes; +/// void * temporary_storage_ptr = nullptr; +/// +/// rocprim::merge_inplace( +/// temporary_storage_ptr, +/// temporary_storage_size_bytes, +/// data, +/// left_size, +/// right_size); +/// +/// hipMalloc(&temporary_storage_ptr, temporary_storage_size_bytes); +/// +/// rocprim::merge_inplace( +/// temporary_storage_ptr, +/// temporary_storage_size_bytes, +/// data, +/// left_size, +/// right_size); +/// \endcode +/// \endparblock +template::value_type>> +inline hipError_t merge_inplace(void* temporary_storage, + size_t& storage_size, + Iterator data, + size_t left_size, + size_t right_size, + BinaryFunction compare_function = BinaryFunction(), + const hipStream_t stream = 0, + bool debug_synchronous = false) +{ + using value_type = typename std::iterator_traits::value_type; + + // Get temporary size of normal merge. + size_t merge_storage_size = 0; + ROCPRIM_RETURN_ON_ERROR(::rocprim::merge(nullptr, + merge_storage_size, + data, + data + left_size, + data, + left_size, + right_size, + compare_function, + stream, + debug_synchronous)); + + // Create temporary storage for duplicated input and merge. + void* merge_storage; + value_type* copied_input; + ROCPRIM_RETURN_ON_ERROR(detail::temp_storage::partition( + temporary_storage, + storage_size, + detail::temp_storage::make_linear_partition( + detail::temp_storage::ptr_aligned_array(&copied_input, left_size + right_size), + detail::temp_storage::make_partition(&merge_storage, merge_storage_size)))); + + // Return required temporary storage if no temporary storage is given. + if(temporary_storage == nullptr) + { + return hipSuccess; + } + + // Duplicate the input so we can use data as output. + ROCPRIM_RETURN_ON_ERROR(rocprim::transform(data, + copied_input, + left_size + right_size, + ::rocprim::identity{}, + stream, + debug_synchronous)); + + // Merge from copied input to data. + ROCPRIM_RETURN_ON_ERROR(rocprim::merge(merge_storage, + merge_storage_size, + copied_input, + copied_input + left_size, + data, + left_size, + right_size, + compare_function, + stream, + debug_synchronous)); + + return hipSuccess; +} + +/// @} +// end of group devicemodule + +END_ROCPRIM_NAMESPACE + +#endif // ROCPRIM_DEVICE_DEVICE_MERGE_INPLACE_HPP_ diff --git a/rocprim/include/rocprim/rocprim.hpp b/rocprim/include/rocprim/rocprim.hpp index 2cd772532..5fa4410dc 100644 --- a/rocprim/include/rocprim/rocprim.hpp +++ b/rocprim/include/rocprim/rocprim.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -68,6 +68,7 @@ #include "device/device_histogram.hpp" #include "device/device_memcpy.hpp" #include "device/device_merge.hpp" +#include "device/device_merge_inplace.hpp" #include "device/device_merge_sort.hpp" #include "device/device_nth_element.hpp" #include "device/device_partial_sort.hpp" diff --git a/test/rocprim/CMakeLists.txt b/test/rocprim/CMakeLists.txt index b281c429c..87542b75b 100644 --- a/test/rocprim/CMakeLists.txt +++ b/test/rocprim/CMakeLists.txt @@ -282,6 +282,7 @@ add_rocprim_test("rocprim.device_adjacent_find" test_device_adjacent_find.cpp) add_rocprim_test("rocprim.device_find_end" test_device_find_end.cpp) add_rocprim_test("rocprim.device_histogram" test_device_histogram.cpp) add_rocprim_test("rocprim.device_merge" test_device_merge.cpp) +add_rocprim_test("rocprim.device_merge_inplace" test_device_merge_inplace.cpp) add_rocprim_test("rocprim.device_merge_sort" test_device_merge_sort.cpp) add_rocprim_cpp17_test("rocprim.nth_element" test_device_nth_element.cpp) add_rocprim_cpp17_test("rocprim.device_partial_sort" test_device_partial_sort.cpp) diff --git a/test/rocprim/test_device_merge_inplace.cpp b/test/rocprim/test_device_merge_inplace.cpp new file mode 100644 index 000000000..1da4373d1 --- /dev/null +++ b/test/rocprim/test_device_merge_inplace.cpp @@ -0,0 +1,412 @@ +// MIT License +// +// Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "../common_test_header.hpp" +#include "rocprim/device/config_types.hpp" +#include "test_seed.hpp" +#include "test_utils_assertions.hpp" +#include "test_utils_data_generation.hpp" + +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include +#include + +TEST(RocprimDeviceMergeInplaceTests, Basic) +{ + using value_type = int; + size_t storage_size = 0; + + int left_size = 128 * 1024; + int right_size = 128 * 1024; + + int total_size = left_size + right_size; + + std::vector h_data(total_size); + + // fill all even values + for(int i = 0; i < left_size; ++i) + { + h_data[i] = i * 2; + } + + // fill all odd + for(int i = 0; i < right_size; ++i) + { + h_data[left_size + i] = i * 2 + 1; + } + + test_utils::device_ptr d_data(h_data); + std::vector h_expected(h_data); + + // get temporary storage + HIP_CHECK(rocprim::merge_inplace(nullptr, storage_size, d_data.get(), left_size, right_size)); + + test_utils::device_ptr d_temp_storage(storage_size); + + HIP_CHECK(rocprim::merge_inplace(d_temp_storage.get(), + storage_size, + d_data.get(), + left_size, + right_size)); + + std::inplace_merge(h_expected.begin(), h_expected.begin() + left_size, h_expected.end()); + + h_data = d_data.load(); + d_data.free_manually(); + + test_utils::assert_eq(h_expected, h_data); +} + +struct small_sizes +{ + std::vector> operator()() + { + return { + std::make_tuple(0, 0), + std::make_tuple(2, 1), + std::make_tuple(10, 10), + std::make_tuple(111, 111), + std::make_tuple(128, 1289), + std::make_tuple(12, 1000), + std::make_tuple(123, 3000), + std::make_tuple(1024, 512), + std::make_tuple(2345, 49), + std::make_tuple(17867, 41), + std::make_tuple(17867, 34567), + std::make_tuple(924353, 1723454), + std::make_tuple(123, 33554432), + std::make_tuple(33554432, 123), + std::make_tuple(33554432, 33554432), + std::make_tuple(34567, (1 << 17) - 1220), + }; + } +}; + +struct large_sizes +{ + std::vector> operator()() + { + return { + std::make_tuple((1ULL << 14) - 1652, (1ULL << 27) - 5839), + std::make_tuple((1ULL << 27) - 2459, (1ULL << 14) - 2134), + std::make_tuple((1ULL << 27) - 9532, (1ULL << 27) - 8421), + std::make_tuple((1ULL << 32) + 5327, (1ULL << 32) + 9682), + }; + } +}; + +template +struct linear_data_generator +{ + static constexpr bool is_random = false; + + auto get_iterator(seed_type /* seed */) + { + return rocprim::make_transform_iterator(rocprim::make_counting_iterator(0), + [](T v) { return v * increment + start; }); + } + + auto get_max_size() + { + return increment == 0 + ? std::numeric_limits::max() + : static_cast((std::numeric_limits::max() - start) / abs(increment)); + } +}; + +template +struct random_data_generator +{ + static constexpr bool is_random = true; + + struct random_monotonic + { + using difference_type = std::ptrdiff_t; + using value_type = T; + + // not all integral types are valid for int distribution + using dist_value_type = std::conditional_t< + std::is_integral::value + && !test_utils::is_valid_for_int_distribution::value, + int, + value_type>; + + using val_dist_type = std::conditional_t::value, + std::uniform_int_distribution, + std::uniform_real_distribution>; + using dup_dist_type = std::uniform_int_distribution; + + seed_type seed; + int duplicates; + + std::mt19937 engine{std::random_device{}()}; + val_dist_type val_dist{dist_value_type{1}, dist_value_type{increment}}; + dup_dist_type dup_dist{1, max_duplicates}; + + dist_value_type value = dist_value_type{0}; + + random_monotonic(seed_type seed) : seed(seed) {} + + int operator*() const + { + return test_utils::saturate_cast(value); + } + + void next() + { + // consume a duplicate + duplicates--; + + // if we have duplicates left over, do nothing + if(duplicates > 0 || value >= std::numeric_limits::max() - increment) + return; + + // get new duplicates + duplicates = max_duplicates > 1 ? dup_dist(engine) : 1; + value += val_dist(engine); + } + + random_monotonic& operator++() + { + // prefix + next(); + return *this; + } + + random_monotonic operator++(int) + { + // postfix + random_monotonic retval{*this}; + next(); + return retval; + } + }; + + auto get_iterator(seed_type seed) + { + return random_monotonic{seed}; + } + + auto get_max_size() + { + return static_cast(std::numeric_limits::max()); + } +}; + +template, + bool UseGraphs = false> +struct DeviceMergeInplaceParams +{ + using value_type = ValueType; + using gen_a_type = GenAType; + using gen_b_type = GenBType; + using sizes = Sizes; +}; + +typedef ::testing::Types< + // linear even-odd + DeviceMergeInplaceParams, + linear_data_generator>, + DeviceMergeInplaceParams, + linear_data_generator>, + DeviceMergeInplaceParams, + linear_data_generator>, + // linear edge cases + DeviceMergeInplaceParams, + linear_data_generator>, + DeviceMergeInplaceParams, + linear_data_generator>, + DeviceMergeInplaceParams, + linear_data_generator>, + DeviceMergeInplaceParams, + linear_data_generator>, + // random data + DeviceMergeInplaceParams, + random_data_generator>, + DeviceMergeInplaceParams, + random_data_generator>, + DeviceMergeInplaceParams, + random_data_generator>, + DeviceMergeInplaceParams, + random_data_generator>, + // large input sizes + DeviceMergeInplaceParams, + random_data_generator, + large_sizes>, + DeviceMergeInplaceParams, + random_data_generator, + large_sizes>> + DeviceMergeInplaceTestsParams; + +template +struct DeviceMergeInplaceTests : public testing::Test +{ + using value_type = typename Params::value_type; + using gen_a_type = typename Params::gen_a_type; + using gen_b_type = typename Params::gen_b_type; + using sizes = typename Params::sizes; +}; + +TYPED_TEST_SUITE(DeviceMergeInplaceTests, DeviceMergeInplaceTestsParams); + +TYPED_TEST(DeviceMergeInplaceTests, MergeInplace) +{ + using value_type = typename TestFixture::value_type; + using gen_a_type = typename TestFixture::gen_a_type; + using gen_b_type = typename TestFixture::gen_b_type; + using binary_op = rocprim::less; + + auto sizes = typename TestFixture::sizes{}(); + + binary_op compare_op{}; + + hipStream_t stream = hipStreamDefault; + + for(auto size : sizes) + { + size_t num_seeds + = gen_a_type::is_random || gen_b_type::is_random ? (random_seeds_count + seed_size) : 1; + + size_t size_a = std::get<0>(size); + size_t size_b = std::get<1>(size); + size_t size_total = size_a + size_b; + + // hipMallocManaged() currently doesnt support zero byte allocation + if((size_a == 0 || size_b == 0) && test_common_utils::use_hmm()) + { + continue; + } + + auto gen_a = gen_a_type{}; + auto gen_b = gen_b_type{}; + + // don't test sizes more than we can actually generate + if(size_a > gen_a.get_max_size() || size_b > gen_b.get_max_size()) + { + continue; + } + + std::vector h_data(size_total); + + size_t total_bytes = sizeof(value_type) * size_total; + size_t storage_size = 0; + + HIP_CHECK(rocprim::merge_inplace(nullptr, + storage_size, + static_cast(nullptr), + size_a, + size_b, + compare_op, + stream)); + + // ensure tests always fit on device + HIP_CHECK(hipSetDevice(hipGetStreamDeviceId(stream))); + size_t free_vram; + size_t available_vram; + HIP_CHECK(hipMemGetInfo(&free_vram, &available_vram)); + if(available_vram < total_bytes + storage_size) + { + continue; + } + + // We only allocate on the device *after* ensuring we have enough available vram. + test_utils::device_ptr d_data = test_utils::device_ptr(h_data); + test_utils::device_ptr d_temp_storage(storage_size); + + for(size_t seed_index = 0; seed_index < num_seeds; seed_index++) + { + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); + auto gen_a_it = gen_a.get_iterator(seed_value); + auto gen_b_it = gen_b.get_iterator(seed_value + 1); + + // generate left array + for(size_t i = 0; i < size_a; ++i) + { + h_data[i] = static_cast(*(gen_a_it++)); + } + + // generate right array + for(size_t i = 0; i < size_b; ++i) + { + h_data[size_a + i] = static_cast(*(gen_b_it++)); + } + + // move input to device + d_data.store_async(h_data, stream); + + // run merge in place + HIP_CHECK(rocprim::merge_inplace(d_temp_storage.get(), + storage_size, + d_data.get(), + size_a, + size_b, + compare_op, + stream)); + + // compare with reference + auto h_output = d_data.load_async(stream); + + // compute reference + std::vector h_reference(size_a + size_b); + std::merge(h_data.begin(), + h_data.begin() + size_a, + h_data.begin() + size_a, + h_data.end(), + h_reference.begin()); + + ASSERT_NO_FATAL_FAILURE((test_utils::assert_eq(h_output, h_reference))); + } + + d_data.free_manually(); + d_temp_storage.free_manually(); + } +} From 1189d00203fae76a8af44ae43ee66e64602452a0 Mon Sep 17 00:00:00 2001 From: Nick Breed Date: Thu, 9 Jan 2025 14:45:13 +0000 Subject: [PATCH 62/84] To remove error in CI change it to AMDGPU_TARGETS --- .gitlab-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2dff23450..f640ad150 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ # MIT License # -# Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -120,7 +120,7 @@ copyright-date: -D BUILD_TEST=ON -D BUILD_EXAMPLE=ON -D BUILD_BENCHMARK=ON - -D GPU_TARGETS=$GPU_TARGETS + -D AMDGPU_TARGETS=$GPU_TARGETS -D CMAKE_C_COMPILER_LAUNCHER=phc_sccache_c -D CMAKE_CXX_COMPILER_LAUNCHER=phc_sccache_cxx -D CMAKE_CXX_STANDARD=14 @@ -190,7 +190,7 @@ build:cmake-minimum-apt: -D CMAKE_C_COMPILER_LAUNCHER=phc_sccache_c -D CMAKE_CXX_COMPILER_LAUNCHER=phc_sccache_cxx -D BUILD_EXAMPLE=ON - -D GPU_TARGETS=$GPU_TARGETS + -D AMDGPU_TARGETS=$GPU_TARGETS -D CMAKE_CXX_STANDARD="$BUILD_VERSION" -S $CI_PROJECT_DIR -B $BUILD_DIR @@ -277,7 +277,7 @@ build:windows: -S $CI_PROJECT_DIR -B $CI_PROJECT_DIR/build -D BUILD_$BUILD_TARGET=ON - -D GPU_TARGETS=$GPU_TARGET + -D AMDGPU_TARGETS=$GPU_TARGET -D CMAKE_CXX_COMPILER:PATH="${env:HIP_PATH}\bin\clang++.exe" -D CMAKE_PREFIX_PATH:PATH="${env:HIP_PATH}" -D CMAKE_BUILD_TYPE="$BUILD_TYPE" @@ -325,7 +325,7 @@ autotune:build: -D BUILD_EXAMPLE=OFF -D BUILD_BENCHMARK=ON -D BENCHMARK_CONFIG_TUNING=ON - -D GPU_TARGETS=$GPU_TARGETS + -D AMDGPU_TARGETS=$GPU_TARGETS -D CMAKE_C_COMPILER_LAUNCHER=phc_sccache_c -D CMAKE_CXX_COMPILER_LAUNCHER=phc_sccache_cxx -D CMAKE_CXX_STANDARD=14 @@ -433,7 +433,7 @@ test-windows-release: -G Ninja -D CMAKE_CXX_COMPILER="$AMDCLANG" -D CMAKE_BUILD_TYPE=Release - -D GPU_TARGETS=$GPU_TARGETS + -D AMDGPU_TARGETS=$GPU_TARGETS -D CMAKE_CXX_STANDARD=14 -S "$CI_PROJECT_DIR/test/extra" -B "$CI_PROJECT_DIR/package_test" From e3b35ce6e1d5a742752280e6014de3fbd3acca87 Mon Sep 17 00:00:00 2001 From: Nara Prasetya Date: Mon, 13 Jan 2025 09:56:45 +0000 Subject: [PATCH 63/84] fix device_ptr assign move and void types --- test/rocprim/test_device_scan.cpp | 5 +- test/rocprim/test_utils_device_ptr.hpp | 180 ++++++++++++------------- 2 files changed, 91 insertions(+), 94 deletions(-) diff --git a/test/rocprim/test_device_scan.cpp b/test/rocprim/test_device_scan.cpp index c98827f18..bc77f4525 100644 --- a/test/rocprim/test_device_scan.cpp +++ b/test/rocprim/test_device_scan.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -1604,7 +1604,8 @@ TYPED_TEST(RocprimDeviceScanFutureTests, ExclusiveScan) stream)); // allocate temporary storage - test_utils::device_ptr d_temp_storage(temp_storage_size_bytes + // we use a char pointer as we need to offset it + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes + temp_storage_reduce); test_utils::GraphHelper gHelper;; diff --git a/test/rocprim/test_utils_device_ptr.hpp b/test/rocprim/test_utils_device_ptr.hpp index 4ec4de6a2..4a0fa4ac0 100644 --- a/test/rocprim/test_utils_device_ptr.hpp +++ b/test/rocprim/test_utils_device_ptr.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2021-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,36 +21,42 @@ #ifndef ROCPRIM_TEST_UTILS_DEVICE_PTR_HPP #define ROCPRIM_TEST_UTILS_DEVICE_PTR_HPP +#include "../common_test_header.hpp" + #include #include #include -#include #include -#include "../common_test_header.hpp" - namespace test_utils { /// \brief An RAII friendly class to manage the memory allocated on device. /// /// \tparam A Template type used by the class. -template +template class device_ptr { public: - using decay_type = std::decay_t; + using decay_type = std::decay_t; using size_type = std::size_t; - using value_type = typename std:: - conditional_t::value, unsigned char, PointerType>; + using value_type = ValueType; + +private: + // If value_type is void we want to emulate allocating bytes (uchar). + using value_type_proxy + = std::conditional_t::value, unsigned char, ValueType>; + +public: + static constexpr size_t value_size = sizeof(value_type_proxy); - device_ptr() : device_raw_ptr_(nullptr), number_of_ele_(0) {}; + device_ptr() : device_raw_ptr_(nullptr), number_of_ele_(0){}; /// \brief Construct with a pre-allocated memory space. device_ptr(size_type pre_alloc_number_of_ele) : device_raw_ptr_(nullptr), number_of_ele_(pre_alloc_number_of_ele) { - size_type storage_size = number_of_ele_ * sizeof(value_type); + size_type storage_size = number_of_ele_ * value_size; HIP_CHECK(test_common_utils::hipMallocHelper(&device_raw_ptr_, storage_size)); }; @@ -67,10 +73,10 @@ class device_ptr explicit device_ptr(std::vector const& data) : device_raw_ptr_(nullptr), number_of_ele_(data.size()) { - static_assert(sizeof(InValueType) == sizeof(value_type), + static_assert(sizeof(InValueType) == value_size, "value_type of input must have the same size with device_ptr::value_type"); - size_type storage_size = number_of_ele_ * sizeof(value_type); + size_type storage_size = number_of_ele_ * value_size; HIP_CHECK(test_common_utils::hipMallocHelper(&device_raw_ptr_, storage_size)); HIP_CHECK(hipMemcpy(device_raw_ptr_, data.data(), storage_size, hipMemcpyHostToDevice)); } @@ -79,10 +85,10 @@ class device_ptr explicit device_ptr(std::vector const& data, hipStream_t stream) : device_raw_ptr_(nullptr), number_of_ele_(data.size()) { - static_assert(sizeof(InValueType) == sizeof(value_type), + static_assert(sizeof(InValueType) == value_size, "value_type of input must have the same size with device_ptr::value_type"); - size_type storage_size = number_of_ele_ * sizeof(value_type); + size_type storage_size = number_of_ele_ * value_size; HIP_CHECK(test_common_utils::hipMallocHelper(&device_raw_ptr_, storage_size)); HIP_CHECK(hipMemcpyAsync(device_raw_ptr_, data.data(), @@ -95,10 +101,10 @@ class device_ptr explicit device_ptr(std::array const& data) : device_raw_ptr_(nullptr), number_of_ele_(Size) { - static_assert(sizeof(InValueType) == sizeof(value_type), + static_assert(sizeof(InValueType) == value_size, "value_type of input must have the same size with device_ptr::value_type"); - size_type storage_size = Size * sizeof(value_type); + size_type storage_size = Size * value_size; HIP_CHECK(test_common_utils::hipMallocHelper(&device_raw_ptr_, storage_size)); HIP_CHECK(hipMemcpy(device_raw_ptr_, data.data(), storage_size, hipMemcpyHostToDevice)); } @@ -107,10 +113,10 @@ class device_ptr explicit device_ptr(std::array const& data, hipStream_t stream) : device_raw_ptr_(nullptr), number_of_ele_(Size) { - static_assert(sizeof(InValueType) == sizeof(value_type), + static_assert(sizeof(InValueType) == value_size, "value_type of input must have the same size with device_ptr::value_type"); - size_type storage_size = Size * sizeof(value_type); + size_type storage_size = Size * value_size; HIP_CHECK(test_common_utils::hipMallocHelper(&device_raw_ptr_, storage_size)); HIP_CHECK(hipMemcpyAsync(device_raw_ptr_, data.data(), @@ -123,10 +129,10 @@ class device_ptr explicit device_ptr(std::unique_ptr const& uptr, size_type size) : device_raw_ptr_(nullptr), number_of_ele_(size) { - static_assert(sizeof(InValueType) == sizeof(value_type), + static_assert(sizeof(InValueType) == value_size, "value_type of input must have the same size with device_ptr::value_type"); - size_type storage_size = size * sizeof(value_type); + size_type storage_size = size * value_size; HIP_CHECK(test_common_utils::hipMallocHelper(&device_raw_ptr_, storage_size)); HIP_CHECK(hipMemcpy(device_raw_ptr_, uptr.get(), storage_size, hipMemcpyHostToDevice)); } @@ -137,10 +143,10 @@ class device_ptr hipStream_t stream) : device_raw_ptr_(nullptr), number_of_ele_(size) { - static_assert(sizeof(InValueType) == sizeof(value_type), + static_assert(sizeof(InValueType) == value_size, "value_type of input must have the same size with device_ptr::value_type"); - size_type storage_size = size * sizeof(value_type); + size_type storage_size = size * value_size; HIP_CHECK(test_common_utils::hipMallocHelper(&device_raw_ptr_, storage_size)); HIP_CHECK(hipMemcpyAsync(device_raw_ptr_, uptr.get(), @@ -159,9 +165,12 @@ class device_ptr device_ptr& operator=(device_ptr&& other) noexcept { free_manually(); + device_raw_ptr_ = other.device_raw_ptr_; number_of_ele_ = other.number_of_ele_; other.leak(); + + return *this; }; /// \brief Do copy on the device. @@ -171,7 +180,7 @@ class device_ptr { device_ptr ret; ret.number_of_ele_ = number_of_ele_; - size_type storage_size = number_of_ele_ * sizeof(value_type); + size_type storage_size = number_of_ele_ * value_size; HIP_CHECK(test_common_utils::hipMallocHelper(&ret.device_raw_ptr_, storage_size)); HIP_CHECK( hipMemcpy(ret.device_raw_ptr_, device_raw_ptr_, storage_size, hipMemcpyDeviceToDevice)); @@ -182,7 +191,7 @@ class device_ptr { device_ptr ret; ret.number_of_ele_ = number_of_ele_; - size_type storage_size = number_of_ele_ * sizeof(value_type); + size_type storage_size = number_of_ele_ * value_size; HIP_CHECK(test_common_utils::hipMallocHelper(&ret.device_raw_ptr_, storage_size)); HIP_CHECK(hipMemcpyAsync(ret.device_raw_ptr_, device_raw_ptr_, @@ -202,7 +211,7 @@ class device_ptr auto ret_deivce_raw_ptr_ = static_cast(static_cast(device_raw_ptr_)); - auto ret_number_of_ele_ = sizeof(value_type) * number_of_ele_ / sizeof(target_value_t); + auto ret_number_of_ele_ = value_size * number_of_ele_ / sizeof(target_value_t); leak(); return {ret_deivce_raw_ptr_, ret_number_of_ele_}; } @@ -223,7 +232,7 @@ class device_ptr /// \brief Call this function to garbage the memory in advance void free_manually() { - if(device_raw_ptr_) + if(device_raw_ptr_ != nullptr) { HIP_CHECK(hipFree(device_raw_ptr_)); } @@ -240,10 +249,10 @@ class device_ptr { value_type* device_temp_ptr = nullptr; HIP_CHECK(test_common_utils::hipMallocHelper(&device_temp_ptr, - new_number_of_ele * sizeof(value_type))); + new_number_of_ele * value_size)); HIP_CHECK(hipMemcpy(device_temp_ptr, device_raw_ptr_, - std::min(new_number_of_ele, number_of_ele_) * sizeof(value_type), + std::min(new_number_of_ele, number_of_ele_) * value_size, hipMemcpyDeviceToDevice)); free_manually(); device_raw_ptr_ = device_temp_ptr; @@ -261,13 +270,12 @@ class device_ptr { value_type* device_temp_ptr = nullptr; HIP_CHECK(test_common_utils::hipMallocHelper(&device_temp_ptr, - new_number_of_ele * sizeof(value_type))); - HIP_CHECK( - hipMemcpyAsync(device_temp_ptr, - device_raw_ptr_, - std::min(new_number_of_ele, number_of_ele_) * sizeof(value_type), - hipMemcpyDeviceToDevice, - stream)); + new_number_of_ele * value_size)); + HIP_CHECK(hipMemcpyAsync(device_temp_ptr, + device_raw_ptr_, + std::min(new_number_of_ele, number_of_ele_) * value_size, + hipMemcpyDeviceToDevice, + stream)); free_manually(); device_raw_ptr_ = device_temp_ptr; number_of_ele_ = new_number_of_ele; @@ -284,9 +292,8 @@ class device_ptr else { value_type* device_temp_ptr = nullptr; - const auto err - = test_common_utils::hipMallocHelper(&device_temp_ptr, - new_number_of_ele * sizeof(value_type)); + const auto err = test_common_utils::hipMallocHelper(&device_temp_ptr, + new_number_of_ele * value_size); if(err == hipErrorOutOfMemory) { return false; @@ -294,7 +301,7 @@ class device_ptr HIP_CHECK(err); HIP_CHECK(hipMemcpy(device_temp_ptr, device_raw_ptr_, - std::min(new_number_of_ele, number_of_ele_) * sizeof(value_type), + std::min(new_number_of_ele, number_of_ele_) * value_size, hipMemcpyDeviceToDevice)); free_manually(); device_raw_ptr_ = device_temp_ptr; @@ -312,20 +319,18 @@ class device_ptr else { value_type* device_temp_ptr = nullptr; - const auto err - = test_common_utils::hipMallocHelper(&device_temp_ptr, - new_number_of_ele * sizeof(value_type)); + const auto err = test_common_utils::hipMallocHelper(&device_temp_ptr, + new_number_of_ele * value_size); if(err == hipErrorOutOfMemory) { return false; } HIP_CHECK(err); - HIP_CHECK( - hipMemcpyAsync(device_temp_ptr, - device_raw_ptr_, - std::min(new_number_of_ele, number_of_ele_) * sizeof(value_type), - hipMemcpyDeviceToDevice, - stream)); + HIP_CHECK(hipMemcpyAsync(device_temp_ptr, + device_raw_ptr_, + std::min(new_number_of_ele, number_of_ele_) * value_size, + hipMemcpyDeviceToDevice, + stream)); free_manually(); device_raw_ptr_ = device_temp_ptr; number_of_ele_ = new_number_of_ele; @@ -336,7 +341,7 @@ class device_ptr /// \brief Get the size of this memory space size_type msize() const noexcept { - return number_of_ele_ * sizeof(value_type); + return number_of_ele_ * value_size; } /// \brief Get the number of elements @@ -349,7 +354,7 @@ class device_ptr template void store(std::vector const& host_vec, size_type offset = 0) { - static_assert(sizeof(InValueType) == sizeof(value_type), + static_assert(sizeof(InValueType) == value_size, "value_type of input must have the same size with device_ptr::value_type"); if(host_vec.size() + offset > number_of_ele_) @@ -359,14 +364,14 @@ class device_ptr HIP_CHECK(hipMemcpy(device_raw_ptr_ + offset, host_vec.data(), - host_vec.size() * sizeof(value_type), + host_vec.size() * value_size, hipMemcpyHostToDevice)); } template void store(std::array const& host_arr) { - static_assert(sizeof(InValueType) == sizeof(value_type), + static_assert(sizeof(InValueType) == value_size, "value_type of input must have the same size with device_ptr::value_type"); if(Size > number_of_ele_) @@ -374,10 +379,8 @@ class device_ptr resize(Size); } - HIP_CHECK(hipMemcpy(device_raw_ptr_, - host_arr.data(), - Size * sizeof(value_type), - hipMemcpyHostToDevice)); + HIP_CHECK( + hipMemcpy(device_raw_ptr_, host_arr.data(), Size * value_size, hipMemcpyHostToDevice)); } template @@ -385,7 +388,7 @@ class device_ptr store(std::unique_ptr const& uptr, size_type offset, size_type number_of_ele) { static_assert( - sizeof(InValueType) == sizeof(value_type), + sizeof(InValueType) == value_size, "value_type of input unique_ptr must have the same size with device_ptr::value_type"); if(offset + number_of_ele > number_of_ele_) @@ -394,7 +397,7 @@ class device_ptr } HIP_CHECK(hipMemcpy(device_raw_ptr_ + offset, uptr.get(), - number_of_ele * sizeof(value_type), + number_of_ele * value_size, hipMemcpyHostToDevice)); } @@ -402,7 +405,7 @@ class device_ptr void store_async(std::vector const& host_vec, hipStream_t stream) { static_assert( - sizeof(InValueType) == sizeof(value_type), + sizeof(InValueType) == value_size, "value_type of input vector must have the same size with device_ptr::value_type"); if(host_vec.size() > number_of_ele_) @@ -412,7 +415,7 @@ class device_ptr HIP_CHECK(hipMemcpyAsync(device_raw_ptr_, host_vec.data(), - host_vec.size() * sizeof(value_type), + host_vec.size() * value_size, hipMemcpyHostToDevice, stream)); } @@ -420,7 +423,7 @@ class device_ptr template void store_async(std::array const& host_arr, hipStream_t stream) { - static_assert(sizeof(InValueType) == sizeof(value_type), + static_assert(sizeof(InValueType) == value_size, "value_type of input must have the same size with device_ptr::value_type"); if(Size > number_of_ele_) @@ -430,7 +433,7 @@ class device_ptr HIP_CHECK(hipMemcpyAsync(device_raw_ptr_, host_arr.data(), - Size * sizeof(value_type), + Size * value_size, hipMemcpyHostToDevice, stream)); } @@ -442,7 +445,7 @@ class device_ptr hipStream_t stream) { static_assert( - sizeof(InValueType) == sizeof(value_type), + sizeof(InValueType) == value_size, "value_type of input unique_ptr must have the same size with device_ptr::value_type"); if(offset + number_of_ele > number_of_ele_) @@ -451,34 +454,31 @@ class device_ptr } HIP_CHECK(hipMemcpyAsync(device_raw_ptr_ + offset, uptr.get(), - number_of_ele * sizeof(value_type), + number_of_ele * value_size, hipMemcpyHostToDevice, stream)); } // will not check the boundary - void store_value_at(size_type pos, value_type const& value) + void store_value_at(size_type pos, value_type_proxy const& value) { - HIP_CHECK( - hipMemcpy(device_raw_ptr_ + pos, &value, sizeof(value_type), hipMemcpyHostToDevice)); + HIP_CHECK(hipMemcpy(device_raw_ptr_ + pos, &value, value_size, hipMemcpyHostToDevice)); } // will not check the boundary - void store_value_at_async(size_type pos, value_type const& value, hipStream_t stream) + template + void store_value_at_async(size_type pos, value_type_proxy const& value, hipStream_t stream) { - HIP_CHECK(hipMemcpy(device_raw_ptr_ + pos, - &value, - sizeof(value_type), - hipMemcpyHostToDevice, - stream)); + HIP_CHECK( + hipMemcpy(device_raw_ptr_ + pos, &value, value_size, hipMemcpyHostToDevice, stream)); } /// \brief Copy from device to device template void replace(device_ptr const& device_ptr) { - static_assert(sizeof(InPtrValueType) == sizeof(value_type), - "sizeof(InPtrValueType) must equal to sizeof(value_type)"); + static_assert(sizeof(InPtrValueType) == value_size, + "sizeof(InPtrValueType) must equal to value_size"); if(device_ptr.number_of_ele_ > number_of_ele_) { @@ -487,15 +487,15 @@ class device_ptr HIP_CHECK(hipMemcpy(device_raw_ptr_, device_ptr.device_raw_ptr_, - device_ptr.number_of_ele_ * sizeof(value_type), + device_ptr.number_of_ele_ * value_size, hipMemcpyDeviceToDevice)); } template void replace_async(device_ptr const& device_ptr, hipStream_t stream) { - static_assert(sizeof(InPtrValueType) == sizeof(value_type), - "sizeof(InPtrValueType) must equal to sizeof(value_type)"); + static_assert(sizeof(InPtrValueType) == value_size, + "sizeof(InPtrValueType) must equal to value_size"); if(device_ptr.number_of_ele_ > number_of_ele_) { @@ -504,7 +504,7 @@ class device_ptr HIP_CHECK(hipMemcpyAsync(device_raw_ptr_, device_ptr.device_raw_ptr_, - device_ptr.number_of_ele_ * sizeof(value_type), + device_ptr.number_of_ele_ * value_size, hipMemcpyDeviceToDevice, stream)); } @@ -531,7 +531,7 @@ class device_ptr std::vector ret(number_of_ele_); HIP_CHECK(hipMemcpy(ret.data(), device_raw_ptr_, - number_of_ele_ * sizeof(value_type), + number_of_ele_ * value_size, hipMemcpyDeviceToHost)); return ret; } @@ -541,7 +541,7 @@ class device_ptr std::vector ret(number_of_ele_); HIP_CHECK(hipMemcpyAsync(ret.data(), device_raw_ptr_, - number_of_ele_ * sizeof(value_type), + number_of_ele_ * value_size, hipMemcpyDeviceToHost, stream)); return ret; @@ -553,7 +553,7 @@ class device_ptr std::array ret; HIP_CHECK(hipMemcpy(ret.data(), device_raw_ptr_, - std::min(number_of_ele_, Size) * sizeof(value_type), + std::min(number_of_ele_, Size) * value_size, hipMemcpyDeviceToHost)); return ret; } @@ -564,7 +564,7 @@ class device_ptr std::array ret; HIP_CHECK(hipMemcpyAsync(ret.data(), device_raw_ptr_, - std::min(number_of_ele_, Size) * sizeof(value_type), + std::min(number_of_ele_, Size) * value_size, hipMemcpyDeviceToHost, stream)); return ret; @@ -575,7 +575,7 @@ class device_ptr std::unique_ptr ret(new value_type[number_of_ele_]); HIP_CHECK(hipMemcpy(ret.get(), device_raw_ptr_, - number_of_ele_ * sizeof(value_type), + number_of_ele_ * value_size, hipMemcpyDeviceToHost)); return ret; } @@ -585,7 +585,7 @@ class device_ptr std::unique_ptr ret(new value_type[number_of_ele_]); HIP_CHECK(hipMemcpyAsync(ret.get(), device_raw_ptr_, - number_of_ele_ * sizeof(value_type), + number_of_ele_ * value_size, hipMemcpyDeviceToHost, stream)); return ret; @@ -594,19 +594,15 @@ class device_ptr auto load_value_at(size_type pos) const { value_type ret; - HIP_CHECK( - hipMemcpy(&ret, device_raw_ptr_ + pos, sizeof(value_type), hipMemcpyDeviceToHost)); + HIP_CHECK(hipMemcpy(&ret, device_raw_ptr_ + pos, value_size, hipMemcpyDeviceToHost)); return ret; } auto load_value_at_async(size_type pos, hipStream_t stream) const { value_type ret; - HIP_CHECK(hipMemcpyAsync(&ret, - device_raw_ptr_ + pos, - sizeof(value_type), - hipMemcpyDeviceToHost, - stream)); + HIP_CHECK( + hipMemcpyAsync(&ret, device_raw_ptr_ + pos, value_size, hipMemcpyDeviceToHost, stream)); return ret; } From d4a70926f9a0792b0281b96ebd4f629f2ee74d9e Mon Sep 17 00:00:00 2001 From: Beatriz Navidad Vilches Date: Mon, 13 Jan 2025 10:28:40 +0000 Subject: [PATCH 64/84] Resolve "Add block- and warp-level inclusive_scan with initial value support" --- CHANGELOG.md | 1 + rocprim/include/rocprim/block/block_scan.hpp | 191 ++++++++++++- .../detail/block_scan_reduce_then_scan.hpp | 145 +++++++++- .../block/detail/block_scan_warp_scan.hpp | 192 ++++++++++++- .../rocprim/warp/detail/warp_scan_dpp.hpp | 39 ++- .../warp/detail/warp_scan_shared_mem.hpp | 42 ++- .../rocprim/warp/detail/warp_scan_shuffle.hpp | 49 +++- rocprim/include/rocprim/warp/warp_scan.hpp | 157 ++++++++++- test/rocprim/test_block_scan.hpp | 151 ++++++++++- test/rocprim/test_block_scan.kernels.hpp | 42 ++- test/rocprim/test_warp_scan.hpp | 254 +++++++++++++++++- test/rocprim/test_warp_scan.kernels.hpp | 46 +++- 12 files changed, 1298 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f21d41eb7..4becb366d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Full documentation for rocPRIM is available at [https://rocm.docs.amd.com/projec * Added `rocprim::int128_t` and `rocprim::uint128_t` to benchmarks for improved performance evaluation on 128-bit integers. * Added `rocprim::int128_t` to autotuning supported types to improve performance for 128-bit integers. * Added `rocprim::merge_inplace` function that allows merging in-place. +* Added initial value support for warp- and block-level inclusive scan. ### Resolved issues diff --git a/rocprim/include/rocprim/block/block_scan.hpp b/rocprim/include/rocprim/block/block_scan.hpp index aed0fcefe..a7958fc75 100644 --- a/rocprim/include/rocprim/block/block_scan.hpp +++ b/rocprim/include/rocprim/block/block_scan.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2021 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -507,6 +507,97 @@ class block_scan } } + /// \brief Performs seeded inclusive scan across threads in a block. + /// + /// \tparam ItemsPerThread - number of items in the \p input array. + /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// is rocprim::plus. + /// + /// \param [in] input - reference to an array containing thread input values. + /// \param [in] init - initial value to seed the inclusive scan. + /// \param [out] output - reference to a thread output array. May be aliased with \p input. + /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] scan_op - binary operation function object that will be used for scan. + /// The signature of the function should be equivalent to the following: + /// T f(const T &a, const T &b);. The signature does not need to have + /// const &, but function object must not modify the objects passed to it. + /// + /// \par Storage reusage + /// Synchronization barrier should be placed before \p storage is reused + /// or repurposed: \p __syncthreads() or \p rocprim::syncthreads(). + /// + /// \par Examples + /// \parblock + /// The examples present seeded inclusive maximum scan operations performed on a block of 128 threads, + /// each provides two \p long value. + /// + /// \code{.cpp} + /// __global__ void example_kernel(...) // blockDim.x = 128 + /// { + /// // specialize block_scan for long and block of 128 threads + /// using block_scan_f = rocprim::block_scan; + /// // allocate storage in shared memory for the block + /// __shared__ block_scan_long::storage_type storage; + /// + /// long input[2] = ...; + /// long init = ...; + /// long output[2]; + /// // execute inclusive min scan + /// block_scan_long().inclusive_scan( + /// input, + /// init, + /// output, + /// storage, + /// rocprim::maximum() + /// ); + /// ... + /// } + /// \endcode + /// If the \p input values across threads in a block are {-1, 2, -3, 4, ..., -255, 256} + /// and the value for seeding the scan is 1, then the \p output values will be + /// {1, 2, 2, 4, ..., 254, 256}. + /// \endparblock + template> + ROCPRIM_DEVICE ROCPRIM_INLINE + void inclusive_scan(T (&input)[ItemsPerThread], + T init, + T (&output)[ItemsPerThread], + storage_type& storage, + BinaryFunction scan_op = BinaryFunction()) + { + base_type::inclusive_scan(input, init, output, storage, scan_op); + } + + /// \overload + /// \brief Performs seeded inclusive scan across threads in a block. + /// + /// * This overload does not accept storage argument. Required shared memory is + /// allocated by the method itself. + /// + /// \tparam ItemsPerThread - number of items in the \p input array. + /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// is rocprim::plus. + /// + /// \param [in] input - reference to an array containing thread input values. + /// \param [in] init - initial value to seed the inclusive scan. + /// \param [out] output - reference to a thread output array. May be aliased with \p input. + /// \param [in] scan_op - binary operation function object that will be used for scan. + /// The signature of the function should be equivalent to the following: + /// T f(const T &a, const T &b);. The signature does not need to have + /// const &, but function object must not modify the objects passed to it. + template< + unsigned int ItemsPerThread, + class BinaryFunction = ::rocprim::plus + > + ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE + void inclusive_scan(T (&input)[ItemsPerThread], + T init, + T (&output)[ItemsPerThread], + BinaryFunction scan_op = BinaryFunction()) + { + base_type::inclusive_scan(input, init, output, scan_op); + } + /// \brief Performs inclusive scan and reduction across threads in a block. /// /// \tparam ItemsPerThread - number of items in the \p input array. @@ -615,6 +706,104 @@ class block_scan } } + /// \brief Performs seeded inclusive scan and reduction across threads in a block. + /// + /// \tparam ItemsPerThread - number of items in the \p input array. + /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// is rocprim::plus. + /// + /// \param [in] input - reference to an array containing thread input values. + /// \param [in] init - initial value to seed the inclusive scan. + /// \param [out] output - reference to a thread output array. May be aliased with \p input. + /// \param [out] reduction - result of reducing of all \p input values in a block. + /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] scan_op - binary operation function object that will be used for scan. + /// The signature of the function should be equivalent to the following: + /// T f(const T &a, const T &b);. The signature does not need to have + /// const &, but function object must not modify the objects passed to it. + /// + /// \par Storage reusage + /// Synchronization barrier should be placed before \p storage is reused + /// or repurposed: \p __syncthreads() or \p rocprim::syncthreads(). + /// + /// \par Examples + /// \parblock + /// The examples present seeded inclusive maximum scan operations performed on a block of 128 threads, + /// each provides two \p long value. + /// + /// \code{.cpp} + /// __global__ void example_kernel(...) // blockDim.x = 128 + /// { + /// // specialize block_scan for long and block of 128 threads + /// using block_scan_f = rocprim::block_scan; + /// // allocate storage in shared memory for the block + /// __shared__ block_scan_long::storage_type storage; + /// + /// long input[2] = ...; + /// long init = ...; + /// long output[2]; + /// long reduction; + /// // execute inclusive min scan + /// block_scan_long().inclusive_scan( + /// input, + /// init, + /// output, + /// reduction, + /// storage, + /// rocprim::maximum() + /// ); + /// ... + /// } + /// \endcode + /// + /// If the \p input values across threads in a block are {-1, 2, -3, 4, ..., -255, 256} + /// and the value for seeding the scan is 1, then the \p output values will be + /// {1, 2, 2, 4, ..., 254, 256} and the \p reduction will be \p 256. + /// \endparblock + template> + ROCPRIM_DEVICE ROCPRIM_INLINE + void inclusive_scan(T (&input)[ItemsPerThread], + T init, + T (&output)[ItemsPerThread], + T& reduction, + storage_type& storage, + BinaryFunction scan_op = BinaryFunction()) + { + base_type::inclusive_scan(input, init, output, reduction, storage, scan_op); + } + + /// \overload + /// \brief Performs seeded inclusive scan and reduction across threads in a block. + /// + /// * This overload does not accept storage argument. Required shared memory is + /// allocated by the method itself. + /// + /// \tparam ItemsPerThread - number of items in the \p input array. + /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// is rocprim::plus. + /// + /// \param [in] input - reference to an array containing thread input values. + /// \param [in] init - initial value to seed the inclusive scan. + /// \param [out] output - reference to a thread output array. May be aliased with \p input. + /// \param [out] reduction - result of reducing of all \p input values in a block. + /// \param [in] scan_op - binary operation function object that will be used for scan. + /// The signature of the function should be equivalent to the following: + /// T f(const T &a, const T &b);. The signature does not need to have + /// const &, but function object must not modify the objects passed to it. + template< + unsigned int ItemsPerThread, + class BinaryFunction = ::rocprim::plus + > + ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE + void inclusive_scan(T (&input)[ItemsPerThread], + T init, + T (&output)[ItemsPerThread], + T& reduction, + BinaryFunction scan_op = BinaryFunction()) + { + base_type::inclusive_scan(input, init, output, reduction, scan_op); + } + /// \brief Performs inclusive scan across threads in a block, and uses /// \p prefix_callback_op to generate prefix value for the whole block. /// diff --git a/rocprim/include/rocprim/block/detail/block_scan_reduce_then_scan.hpp b/rocprim/include/rocprim/block/detail/block_scan_reduce_then_scan.hpp index be986573b..66f9e95e8 100644 --- a/rocprim/include/rocprim/block/detail/block_scan_reduce_then_scan.hpp +++ b/rocprim/include/rocprim/block/detail/block_scan_reduce_then_scan.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -183,6 +183,54 @@ class block_scan_reduce_then_scan this->inclusive_scan(input, output, storage, scan_op); } + template + ROCPRIM_DEVICE ROCPRIM_INLINE + void inclusive_scan(T (&input)[ItemsPerThread], + T init, + T (&output)[ItemsPerThread], + storage_type& storage, + BinaryFunction scan_op) + { + // Reduce thread items + T thread_input = input[0]; + ROCPRIM_UNROLL + for(unsigned int i = 1; i < ItemsPerThread; i++) + { + thread_input = scan_op(thread_input, input[i]); + } + + // Scan of reduced values to get prefixes + const auto flat_tid = ::rocprim::flat_block_thread_id(); + // Calculates inclusive scan, result for each thread is stored in storage_.threads[flat_tid] + this->exclusive_scan_init_impl(flat_tid, + thread_input, + thread_input, // input, output + init, + storage, + scan_op); + + // Include prefix (first thread has init as prefix) + output[0] = scan_op(thread_input, input[0]); + + // Final thread-local scan + ROCPRIM_UNROLL + for(unsigned int i = 1; i < ItemsPerThread; i++) + { + output[i] = scan_op(output[i - 1], input[i]); + } + } + + template + ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE + void inclusive_scan(T (&input)[ItemsPerThread], + T init, + T (&output)[ItemsPerThread], + BinaryFunction scan_op) + { + ROCPRIM_SHARED_MEMORY storage_type storage; + this->inclusive_scan(input, init, output, storage, scan_op); + } + template ROCPRIM_DEVICE ROCPRIM_INLINE void inclusive_scan(T (&input)[ItemsPerThread], @@ -208,6 +256,33 @@ class block_scan_reduce_then_scan this->inclusive_scan(input, output, reduction, storage, scan_op); } + template + ROCPRIM_DEVICE ROCPRIM_INLINE + void inclusive_scan(T (&input)[ItemsPerThread], + T init, + T (&output)[ItemsPerThread], + T& reduction, + storage_type& storage, + BinaryFunction scan_op) + { + storage_type_& storage_ = storage.get(); + this->inclusive_scan(input, init, output, storage, scan_op); + // Save reduction result + reduction = storage_.threads[index(BlockSize - 1)]; + } + + template + ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE + void inclusive_scan(T (&input)[ItemsPerThread], + T init, + T (&output)[ItemsPerThread], + T& reduction, + BinaryFunction scan_op) + { + ROCPRIM_SHARED_MEMORY storage_type storage; + this->inclusive_scan(input, init, output, reduction, storage, scan_op); + } + template< class PrefixCallback, unsigned int ItemsPerThread, @@ -536,6 +611,54 @@ class block_scan_reduce_then_scan ::rocprim::syncthreads(); } + // Calculates inclusive scan results and stores them in storage_.threads, + // result for each thread is stored in storage_.threads[flat_tid]. + // It uses an `init` value to seed the inclusive scan. + template + ROCPRIM_DEVICE ROCPRIM_INLINE + void inclusive_scan_base( + const unsigned int flat_tid, T input, T init, storage_type& storage, BinaryFunction scan_op) + { + storage_type_& storage_ = storage.get(); + storage_.threads[index(flat_tid)] = input; + if(flat_tid == 0) + { + storage_.threads[index(flat_tid)] = scan_op(init, input); + } + ::rocprim::syncthreads(); + if(flat_tid < warp_size_) + { + const unsigned int idx_start = index(flat_tid * thread_reduction_size_); + + T thread_reduction = storage_.threads[idx_start]; + ROCPRIM_UNROLL + for(unsigned int i = 1; i < thread_reduction_size_; i++) + { + thread_reduction = scan_op(thread_reduction, storage_.threads[idx_start + i]); + } + + // Calculate warp prefixes + warp_scan_prefix_type().inclusive_scan(thread_reduction, thread_reduction, scan_op); + thread_reduction = warp_shuffle_up(thread_reduction, 1, warp_size_); + + // Include warp prefix + thread_reduction = scan_op(thread_reduction, storage_.threads[idx_start]); + if(flat_tid == 0) + { + thread_reduction = scan_op(init, input); + } + + storage_.threads[idx_start] = thread_reduction; + ROCPRIM_UNROLL + for(unsigned int i = 1; i < thread_reduction_size_; i++) + { + thread_reduction = scan_op(thread_reduction, storage_.threads[idx_start + i]); + storage_.threads[idx_start + i] = thread_reduction; + } + } + ::rocprim::syncthreads(); + } + template ROCPRIM_DEVICE ROCPRIM_INLINE void exclusive_scan_impl(const unsigned int flat_tid, @@ -552,6 +675,26 @@ class block_scan_reduce_then_scan if(flat_tid != 0) output = scan_op(init, storage_.threads[index(flat_tid-1)]); } + template + ROCPRIM_DEVICE ROCPRIM_INLINE + void exclusive_scan_init_impl(const unsigned int flat_tid, + T input, + T& output, + T init, + storage_type& storage, + BinaryFunction scan_op) + { + storage_type_& storage_ = storage.get(); + // Calculates inclusive scan, result for each thread is stored in storage_.threads[flat_tid] + // This sets storage_.threads[index(flat_tid-1)] = op(... op(op(init, input[0]), input[1]), ...), input[i]) with i up to tid + this->inclusive_scan_base(flat_tid, input, init, storage, scan_op); + output = init; + if(flat_tid != 0) + { + output = storage_.threads[index(flat_tid - 1)]; + } + } + template ROCPRIM_DEVICE ROCPRIM_INLINE void exclusive_scan_impl(const unsigned int flat_tid, diff --git a/rocprim/include/rocprim/block/detail/block_scan_warp_scan.hpp b/rocprim/include/rocprim/block/detail/block_scan_warp_scan.hpp index 91faf00da..4c2c9b490 100644 --- a/rocprim/include/rocprim/block/detail/block_scan_warp_scan.hpp +++ b/rocprim/include/rocprim/block/detail/block_scan_warp_scan.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -199,6 +199,53 @@ class block_scan_warp_scan this->inclusive_scan(input, output, storage, scan_op); } + template + ROCPRIM_DEVICE ROCPRIM_INLINE + void inclusive_scan(T (&input)[ItemsPerThread], + T init, + T (&output)[ItemsPerThread], + storage_type& storage, + BinaryFunction scan_op) + { + // Reduce thread items + T thread_input = input[0]; + ROCPRIM_UNROLL + for(unsigned int i = 1; i < ItemsPerThread; i++) + { + thread_input = scan_op(thread_input, input[i]); + } + + // Scan of reduced values to get prefixes + const auto flat_tid = ::rocprim::flat_block_thread_id(); + this->exclusive_scan_init_impl(flat_tid, + thread_input, + thread_input, // input, output + init, + storage, + scan_op); + + // Include prefix (first thread has init as prefix) + output[0] = scan_op(thread_input, input[0]); + + // Final thread-local scan + ROCPRIM_UNROLL + for(unsigned int i = 1; i < ItemsPerThread; i++) + { + output[i] = scan_op(output[i - 1], input[i]); + } + } + + template + ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE + void inclusive_scan(T (&input)[ItemsPerThread], + T init, + T (&output)[ItemsPerThread], + BinaryFunction scan_op) + { + ROCPRIM_SHARED_MEMORY storage_type storage; + this->inclusive_scan(input, init, output, storage, scan_op); + } + template ROCPRIM_DEVICE ROCPRIM_INLINE void inclusive_scan(T (&input)[ItemsPerThread], @@ -224,6 +271,33 @@ class block_scan_warp_scan this->inclusive_scan(input, output, reduction, storage, scan_op); } + template + ROCPRIM_DEVICE ROCPRIM_INLINE + void inclusive_scan(T (&input)[ItemsPerThread], + T init, + T (&output)[ItemsPerThread], + T& reduction, + storage_type& storage, + BinaryFunction scan_op) + { + storage_type_& storage_ = storage.get(); + this->inclusive_scan(input, init, output, storage, scan_op); + // Save reduction result + reduction = storage_.warp_prefixes[warps_no_ - 1]; + } + + template + ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE + void inclusive_scan(T (&input)[ItemsPerThread], + T init, + T (&output)[ItemsPerThread], + T& reduction, + BinaryFunction scan_op) + { + ROCPRIM_SHARED_MEMORY storage_type storage; + this->inclusive_scan(input, init, output, reduction, storage, scan_op); + } + template< class PrefixCallback, unsigned int ItemsPerThread, @@ -627,6 +701,87 @@ class block_scan_warp_scan } } + // Exclusive scan with initial value when BlockSize is bigger than warp_size + // Warp prefixes stored in storage_.warp_prefixes include the initial value + template + ROCPRIM_DEVICE ROCPRIM_INLINE + auto exclusive_scan_init_impl(const unsigned int flat_tid, + T input, + T& output, + T init, + storage_type& storage, + BinaryFunction scan_op) -> + typename std::enable_if<(BlockSize_ > ::rocprim::device_warp_size())>::type + { + storage_type_& storage_ = storage.get(); + // Perform warp scan on input values with init seed + warp_scan_input_type().inclusive_scan( + // not using shared mem, see note in storage_type + input, + output, + scan_op); + + // i-th warp will have its prefix stored in storage_.warp_prefixes[i-1] + const auto warp_id = ::rocprim::warp_id(flat_tid); + this->calculate_warp_prefixes(flat_tid, warp_id, output, init, storage, scan_op); + + // Include initial value in warp prefixes, and fix warp prefixes + // for exclusive scan (first warp prefix is init) + auto warp_prefix = init; + if(warp_id != 0) + { + warp_prefix = storage_.warp_prefixes[warp_id - 1]; + } + + // Use warp prefix to calculate the final scan results for every thread + output = scan_op(warp_prefix, output); // include warp prefix in scan results + output = warp_shuffle_up(output, 1, warp_size_); // shift to get exclusive results + if(::rocprim::lane_id() == 0) + { + output = warp_prefix; + } + } + + // Exclusive scan with initial value when BlockSize is less than warp_size. + // Warp prefixes stored in storage_.warp_prefixes include the initial value + // When BlockSize is less than warp_size we dont need the extra prefix calculations. + template + ROCPRIM_DEVICE ROCPRIM_INLINE + auto exclusive_scan_init_impl(const unsigned int flat_tid, + T input, + T& output, + T init, + storage_type& storage, + BinaryFunction scan_op) -> + typename std::enable_if ::rocprim::device_warp_size())>::type + { + (void)flat_tid; + (void)storage; + (void)init; + storage_type_& storage_ = storage.get(); + // Perform warp scan on input values with init seed + warp_scan_input_type().inclusive_scan( + // not using shared mem, see note in storage_type + input, + output, + scan_op, + init); + + if(flat_tid == BlockSize_ - 1) + { + storage_.warp_prefixes[0] = output; + } + ::rocprim::syncthreads(); + + // Use warp prefix to calculate the final scan results for every thread + // output = scan_op(init, output); // include warp prefix in scan results + output = warp_shuffle_up(output, 1, warp_size_); // shift to get exclusive results + if(::rocprim::lane_id() == 0) + { + output = init; + } + } + // Exclusive scan with unknown initial value template ROCPRIM_DEVICE ROCPRIM_INLINE @@ -721,6 +876,41 @@ class block_scan_warp_scan ::rocprim::syncthreads(); } + // i-th warp will have its prefix stored in storage_.warp_prefixes[i-1] and includes init + template + ROCPRIM_DEVICE ROCPRIM_INLINE + void calculate_warp_prefixes(const unsigned int flat_tid, + const unsigned int warp_id, + T inclusive_input, + T init, + storage_type& storage, + BinaryFunction scan_op) + { + storage_type_& storage_ = storage.get(); + // Save the warp reduction result, that is the scan result + // for last element in each warp + if(flat_tid == ::rocprim::min((warp_id + 1) * warp_size_, BlockSize_) - 1) + { + storage_.warp_prefixes[warp_id] = inclusive_input; + } + ::rocprim::syncthreads(); + + // Scan the warp reduction results and store in storage_.warp_prefixes + // Notice that in this step we seed the scan + if(flat_tid < warps_no_) + { + auto warp_prefix = storage_.warp_prefixes[flat_tid]; + warp_scan_prefix_type().inclusive_scan( + // not using shared mem, see note in storage_type + warp_prefix, + warp_prefix, + scan_op, + init); + storage_.warp_prefixes[flat_tid] = warp_prefix; + } + ::rocprim::syncthreads(); + } + // THIS OVERWRITES storage_.warp_prefixes[warps_no_ - 1] template ROCPRIM_DEVICE ROCPRIM_INLINE diff --git a/rocprim/include/rocprim/warp/detail/warp_scan_dpp.hpp b/rocprim/include/rocprim/warp/detail/warp_scan_dpp.hpp index f53458818..df00a5455 100644 --- a/rocprim/include/rocprim/warp/detail/warp_scan_dpp.hpp +++ b/rocprim/include/rocprim/warp/detail/warp_scan_dpp.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -106,6 +106,23 @@ class warp_scan_dpp inclusive_scan(input, output, scan_op); } + template + ROCPRIM_DEVICE ROCPRIM_INLINE + void inclusive_scan(T input, T& output, BinaryFunction scan_op, T init) + { + inclusive_scan(input, output, scan_op); + // Include init value in scan results + output = scan_op(init, output); + } + + template + ROCPRIM_DEVICE ROCPRIM_INLINE + void inclusive_scan(T input, T& output, storage_type& storage, BinaryFunction scan_op, T init) + { + (void)storage; // disables unused parameter warning + inclusive_scan(input, output, scan_op, init); + } + template ROCPRIM_DEVICE ROCPRIM_INLINE void inclusive_scan(T input, T& output, T& reduction, @@ -125,6 +142,26 @@ class warp_scan_dpp inclusive_scan(input, output, reduction, scan_op); } + template + ROCPRIM_DEVICE ROCPRIM_INLINE + void inclusive_scan(T input, T& output, T& reduction, BinaryFunction scan_op, T init) + { + inclusive_scan(input, output, scan_op); + // Include init value in scan results + output = scan_op(init, output); + // Broadcast value from the last thread in warp + reduction = warp_shuffle(output, WarpSize - 1, WarpSize); + } + + template + ROCPRIM_DEVICE ROCPRIM_INLINE + void inclusive_scan( + T input, T& output, T& reduction, storage_type& storage, BinaryFunction scan_op, T init) + { + (void)storage; + inclusive_scan(input, output, reduction, scan_op, init); + } + template ROCPRIM_DEVICE ROCPRIM_INLINE void exclusive_scan(T input, T& output, T init, BinaryFunction scan_op) diff --git a/rocprim/include/rocprim/warp/detail/warp_scan_shared_mem.hpp b/rocprim/include/rocprim/warp/detail/warp_scan_shared_mem.hpp index d1300584e..4ae09ea7b 100644 --- a/rocprim/include/rocprim/warp/detail/warp_scan_shared_mem.hpp +++ b/rocprim/include/rocprim/warp/detail/warp_scan_shared_mem.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -78,6 +78,35 @@ class warp_scan_shared_mem output = me; } + template + ROCPRIM_DEVICE ROCPRIM_INLINE + void inclusive_scan(T input, T& output, storage_type& storage, BinaryFunction scan_op, T init) + { + const unsigned int lid = detail::logical_lane_id(); + storage_type_& storage_ = storage.get(); + + T me = input; + storage_.threads[lid] = me; + ::rocprim::wave_barrier(); + for(unsigned int i = 1; i < WarpSize; i *= 2) + { + const bool do_op = lid >= i; + if(do_op) + { + T other = storage_.threads[lid - i]; + me = scan_op(other, me); + } + ::rocprim::wave_barrier(); + if(do_op) + { + storage_.threads[lid] = me; + } + ::rocprim::wave_barrier(); + } + output = scan_op(init, me); + storage_.threads[lid] = output; + } + template ROCPRIM_DEVICE ROCPRIM_INLINE void inclusive_scan(T input, T& output, T& reduction, @@ -88,6 +117,17 @@ class warp_scan_shared_mem reduction = storage_.threads[WarpSize - 1]; } + template + ROCPRIM_DEVICE ROCPRIM_INLINE + void inclusive_scan( + T input, T& output, T& reduction, storage_type& storage, BinaryFunction scan_op, T init) + { + storage_type_& storage_ = storage.get(); + inclusive_scan(input, output, storage, scan_op, init); + ::rocprim::wave_barrier(); + reduction = storage_.threads[WarpSize - 1]; + } + template ROCPRIM_DEVICE ROCPRIM_INLINE void exclusive_scan(T input, T& output, T init, diff --git a/rocprim/include/rocprim/warp/detail/warp_scan_shuffle.hpp b/rocprim/include/rocprim/warp/detail/warp_scan_shuffle.hpp index 106888c4a..95a58a5b2 100644 --- a/rocprim/include/rocprim/warp/detail/warp_scan_shuffle.hpp +++ b/rocprim/include/rocprim/warp/detail/warp_scan_shuffle.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -70,6 +70,33 @@ class warp_scan_shuffle inclusive_scan(input, output, scan_op); } + template + ROCPRIM_DEVICE ROCPRIM_INLINE + void inclusive_scan(T input, T& output, BinaryFunction scan_op, T init) + { + output = input; + + T value; + const unsigned int id = detail::logical_lane_id(); + ROCPRIM_UNROLL + for(unsigned int offset = 1; offset < WarpSize; offset *= 2) + { + value = warp_shuffle_up(output, offset, WarpSize); + if(id >= offset) + output = scan_op(value, output); + } + // Include init value in scan results + output = scan_op(init, output); + } + + template + ROCPRIM_DEVICE ROCPRIM_INLINE + void inclusive_scan(T input, T& output, storage_type& storage, BinaryFunction scan_op, T init) + { + (void)storage; // disables unused parameter warning + inclusive_scan(input, output, scan_op, init); + } + template ROCPRIM_DEVICE ROCPRIM_INLINE void inclusive_scan(T input, T& output, T& reduction, @@ -89,6 +116,26 @@ class warp_scan_shuffle inclusive_scan(input, output, reduction, scan_op); } + template + ROCPRIM_DEVICE ROCPRIM_INLINE + void inclusive_scan(T input, T& output, T& reduction, BinaryFunction scan_op, T init) + { + inclusive_scan(input, output, scan_op); + // Include init value in scan results + output = scan_op(init, output); + // Broadcast value from the last thread in warp + reduction = warp_shuffle(output, WarpSize - 1, WarpSize); + } + + template + ROCPRIM_DEVICE ROCPRIM_INLINE + void inclusive_scan( + T input, T& output, T& reduction, storage_type& storage, BinaryFunction scan_op, T init) + { + (void)storage; + inclusive_scan(input, output, reduction, scan_op, init); + } + template ROCPRIM_DEVICE ROCPRIM_INLINE void exclusive_scan(T input, T& output, T init, BinaryFunction scan_op) diff --git a/rocprim/include/rocprim/warp/warp_scan.hpp b/rocprim/include/rocprim/warp/warp_scan.hpp index 73ae72af2..e692ddd05 100644 --- a/rocprim/include/rocprim/warp/warp_scan.hpp +++ b/rocprim/include/rocprim/warp/warp_scan.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -202,6 +202,82 @@ class warp_scan return; } + /// \brief Performs seeded inclusive scan across threads in a logical warp. + /// + /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// is rocprim::plus. + /// + /// \param [in] input - thread input value. + /// \param [out] output - reference to a thread output value. May be aliased with \p input. + /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] init - initial value to seed the inclusive scan. + /// \param [in] scan_op - binary operation function object that will be used for scan. + /// The signature of the function should be equivalent to the following: + /// T f(const T &a, const T &b);. The signature does not need to have + /// const &, but function object must not modify the objects passed to it. + /// + /// \par Storage reusage + /// Synchronization barrier should be placed before \p storage is reused + /// or repurposed: \p __syncthreads() or \p rocprim::syncthreads(). + /// + /// \par Examples + /// \parblock + /// The examples present seeded inclusive min scan operations performed on groups of 32 threads, + /// each provides one \p float value, result is returned using the same variable as for input. + /// Hardware warp size is 64. Block (tile) size is 256. + /// + /// \code{.cpp} + /// __global__ void example_kernel(...) // blockDim.x = 256 + /// { + /// // specialize warp_scan for float and logical warp of 32 threads + /// using warp_scan_f = rocprim::warp_scan; + /// // allocate storage in shared memory + /// __shared__ warp_scan_float::storage_type temp[8]; // 256/32 = 8 + /// + /// int logical_warp_id = threadIdx.x/32; + /// float value = ...; + /// float init = ...; + /// // execute inclusive min scan + /// warp_scan_float().inclusive_scan( + /// value, // input + /// value, // output + /// temp[logical_warp_id], + /// init, + /// rocprim::minimum() + /// ); + /// ... + /// } + /// \endcode + /// + /// If the input values across threads in a block/tile are {1, -2, 3, -4, ..., 255, -256} + /// and the value for seeding the scan is -1, then output values in the first logical warp will be + /// {-1, -2, -2, -4, ..., -32},, in the second: {-1, -34, -34, -36, ..., -64}, etc. + /// \endparblock + template, unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto inclusive_scan(T input, + T& output, + storage_type& storage, + T init, + BinaryFunction scan_op = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type + { + base_type::inclusive_scan(input, output, storage, scan_op, init); + } + + /// \brief Performs seeded inclusive scan across threads in a logical warp. + /// Invalid Warp Size + template, unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto inclusive_scan(T, T&, storage_type&, T, BinaryFunction scan_op = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type + { + (void)scan_op; + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size " + ". Aborting warp sort."); + return; + } + /// \brief Performs inclusive scan and reduction across threads in a logical warp. /// /// \tparam BinaryFunction - type of binary function used for scan. Default type @@ -276,6 +352,85 @@ class warp_scan return; } + /// \brief Performs seeded inclusive scan and reduction across threads in a logical warp. + /// + /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// is rocprim::plus. + /// + /// \param [in] input - thread input value. + /// \param [out] output - reference to a thread output value. May be aliased with \p input. + /// \param [out] reduction - result of reducing of all \p input values in logical warp. + /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] init - initial value to seed the inclusive scan. + /// \param [in] scan_op - binary operation function object that will be used for scan. + /// The signature of the function should be equivalent to the following: + /// T f(const T &a, const T &b);. The signature does not need to have + /// const &, but function object must not modify the objects passed to it. + /// + /// \par Storage reusage + /// Synchronization barrier should be placed before \p storage is reused + /// or repurposed: \p __syncthreads() or \p rocprim::syncthreads(). + /// + /// \par Examples + /// \parblock + /// The examples present seeded inclusive prefix sum operations performed on groups of 64 threads, + /// each thread provides one \p int value. Hardware warp size is 64. Block (tile) size is 256. + /// + /// \code{.cpp} + /// __global__ void example_kernel(...) // blockDim.x = 256 + /// { + /// // specialize warp_scan for int and logical warp of 64 threads + /// using warp_scan_int = rocprim::warp_scan; + /// // allocate storage in shared memory + /// __shared__ warp_scan_int::storage_type temp[4]; // 256/64 = 4 + /// + /// int logical_warp_id = threadIdx.x/64; + /// int input = ...; + /// int init = ...; + /// int output, reduction; + /// // inclusive prefix sum + /// warp_scan_int().inclusive_scan( + /// input, + /// output, + /// reduction, + /// temp[logical_warp_id], + /// init + /// ); + /// ... + /// } + /// \endcode + /// + /// If the \p input values across threads in a block/tile are {1, 1, 1, 1, ..., 1, 1} + /// and the value for seeding the scan is 1, then + /// \p output values in the every logical warp will be {2, 3, 4, 5, ..., 65}. + /// The \p reduction will be equal \p 65. + /// \endparblock + template, unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto inclusive_scan(T input, + T& output, + T& reduction, + storage_type& storage, + T init, + BinaryFunction scan_op = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize <= device_warp_size()), void>::type + { + base_type::inclusive_scan(input, output, reduction, storage, scan_op, init); + } + + /// \brief Performs seeded inclusive scan and reduction across threads in a logical warp. + /// Invalid Warp Size + template, unsigned int FunctionWarpSize = WarpSize> + ROCPRIM_DEVICE ROCPRIM_INLINE + auto inclusive_scan(T, T&, T&, storage_type&, T, BinaryFunction scan_op = BinaryFunction()) -> + typename std::enable_if<(FunctionWarpSize > device_warp_size()), void>::type + { + (void)scan_op; + ROCPRIM_PRINT_ERROR_ONCE("Specified warp size exceeds current hardware supported warp size " + ". Aborting warp sort."); + return; + } + /// \brief Performs exclusive scan across threads in a logical warp. /// /// \tparam BinaryFunction - type of binary function used for scan. Default type diff --git a/test/rocprim/test_block_scan.hpp b/test/rocprim/test_block_scan.hpp index b6eefd886..0cf2a8590 100644 --- a/test/rocprim/test_block_scan.hpp +++ b/test/rocprim/test_block_scan.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -94,7 +94,79 @@ typed_test_def(suite_name_single, name_suffix, InclusiveScan) T(0), grid_size); } +} + +typed_test_def(suite_name_single, name_suffix, InclusiveScanInitialValue) +{ + using T = typename TestFixture::input_type; + // for bfloat16 and half we use double for host-side accumulation + using binary_op_type_host = typename test_utils::select_plus_operator_host::type; + binary_op_type_host binary_op_host; + using acc_type = typename test_utils::select_plus_operator_host::acc_type; + + constexpr size_t block_size = TestFixture::block_size; + + int device_id = test_common_utils::obtain_device_from_ctest(); + SCOPED_TRACE(testing::Message() << "with device_id = " << device_id); + HIP_CHECK(hipSetDevice(device_id)); + + // Given block size not supported + if(block_size > test_utils::get_max_block_size()) + { + return; + } + + const size_t size = block_size * 58; + const size_t grid_size = size / block_size; + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) + { + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); + + // Generate data + std::vector output = test_utils::get_random_data(size, 2, 50, seed_value); + std::vector output2 = output; + T initial_value = test_utils::get_random_value(2, 200, seed_value); + SCOPED_TRACE(testing::Message() << "with initial_value = " << initial_value); + + // Calculate expected results on host + std::vector expected(output.size(), T(0)); + for(size_t i = 0; i < output.size() / block_size; i++) + { + acc_type accumulator(initial_value); + for(size_t j = 0; j < block_size; j++) + { + auto idx = i * block_size + j; + accumulator = binary_op_host(output[idx], accumulator); + expected[idx] = static_cast(accumulator); + } + } + + // Writing to device memory + test_utils::device_ptr device_output(output.size()); + + static_run_algo::run( + output, + output, + expected, + expected, + device_output.get(), + NULL, + initial_value, + grid_size); + + static_run_algo::run( + output2, + output2, + expected, + expected, + device_output.get(), + NULL, + initial_value, + grid_size); + } } typed_test_def(suite_name_single, name_suffix, InclusiveScanReduce) @@ -172,6 +244,83 @@ typed_test_def(suite_name_single, name_suffix, InclusiveScanReduce) } +typed_test_def(suite_name_single, name_suffix, InclusiveScanReduceInitialValue) +{ + using T = typename TestFixture::input_type; + // for bfloat16 and half we use double for host-side accumulation + using binary_op_type_host = typename test_utils::select_plus_operator_host::type; + binary_op_type_host binary_op_host; + using acc_type = typename test_utils::select_plus_operator_host::acc_type; + + constexpr size_t block_size = TestFixture::block_size; + + int device_id = test_common_utils::obtain_device_from_ctest(); + SCOPED_TRACE(testing::Message() << "with device_id = " << device_id); + HIP_CHECK(hipSetDevice(device_id)); + + // Given block size not supported + if(block_size > test_utils::get_max_block_size()) + { + return; + } + + const size_t size = block_size * 58; + const size_t grid_size = size / block_size; + + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) + { + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); + + // Generate data + std::vector output = test_utils::get_random_data(size, 2, 50, seed_value); + std::vector output2 = output; + std::vector output_reductions(size / block_size); + T initial_value = test_utils::get_random_value(2, 200, seed_value); + SCOPED_TRACE(testing::Message() << "with initial_value = " << initial_value); + + // Calculate expected results on host + std::vector expected(output.size(), T(0)); + std::vector expected_reductions(output_reductions.size(), T(0)); + for(size_t i = 0; i < output.size() / block_size; i++) + { + acc_type accumulator(initial_value); + for(size_t j = 0; j < block_size; j++) + { + auto idx = i * block_size + j; + accumulator = binary_op_host(output[idx], accumulator); + expected[idx] = static_cast(accumulator); + } + expected_reductions[i] = expected[(i + 1) * block_size - 1]; + } + + // Writing to device memory + test_utils::device_ptr device_output(output.size()); + test_utils::device_ptr device_output_reductions(output_reductions.size()); + + static_run_algo::run( + output, + output_reductions, + expected, + expected_reductions, + device_output.get(), + device_output_reductions.get(), + initial_value, + grid_size); + + static_run_algo::run( + output2, + output_reductions, + expected, + expected_reductions, + device_output.get(), + device_output_reductions.get(), + initial_value, + grid_size); + } +} + typed_test_def(suite_name_single, name_suffix, InclusiveScanPrefixCallback) { using T = typename TestFixture::input_type; diff --git a/test/rocprim/test_block_scan.kernels.hpp b/test/rocprim/test_block_scan.kernels.hpp index bcb2f34fc..e35518acd 100644 --- a/test/rocprim/test_block_scan.kernels.hpp +++ b/test/rocprim/test_block_scan.kernels.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -175,6 +175,46 @@ void scan_kernel(T* device_output, T* device_output_b, T init) } } +template::type* = nullptr> +__global__ +__launch_bounds__(BlockSize) +void scan_kernel(T* device_output, T* device_output_b, T init) +{ + (void)device_output_b; + const unsigned int index = (blockIdx.x * BlockSize) + threadIdx.x; + T input[1] = {device_output[index]}; + T output[1]; + rocprim::block_scan bscan; + bscan.inclusive_scan(input, init, output); + device_output[index] = output[0]; +} + +template::type* = nullptr> +__global__ +__launch_bounds__(BlockSize) +void scan_kernel(T* device_output, T* device_output_b, T init) +{ + const unsigned int index = (blockIdx.x * BlockSize) + threadIdx.x; + T input[1] = {device_output[index]}; + T output[1]; + T reduction; + rocprim::block_scan bscan; + bscan.inclusive_scan(input, init, output, reduction); + device_output[index] = output[0]; + if(threadIdx.x == 0) + { + device_output_b[blockIdx.x] = reduction; + } +} + template < class T, unsigned int BlockSize, diff --git a/test/rocprim/test_warp_scan.hpp b/test/rocprim/test_warp_scan.hpp index 1330103ee..f208d47ec 100644 --- a/test/rocprim/test_warp_scan.hpp +++ b/test/rocprim/test_warp_scan.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -133,6 +133,127 @@ typed_test_def(RocprimWarpScanTests, name_suffix, InclusiveScan) } +typed_test_def(RocprimWarpScanTests, name_suffix, InclusiveScanInitialValue) +{ + int device_id = test_common_utils::obtain_device_from_ctest(); + SCOPED_TRACE(testing::Message() << "with device_id = " << device_id); + HIP_CHECK(hipSetDevice(device_id)); + + using T = typename TestFixture::params::type; + // for bfloat16 and half we use double for host-side accumulation + using binary_op_type_host = typename test_utils::select_plus_operator_host::type; + binary_op_type_host binary_op_host; + using acc_type = typename test_utils::select_plus_operator_host::acc_type; + + // logical warp side for warp primitive, execution warp size is always rocprim::warp_size() + static constexpr size_t logical_warp_size = TestFixture::params::warp_size; + + // The different warp sizes + static constexpr size_t ws32 = size_t(ROCPRIM_WARP_SIZE_32); + static constexpr size_t ws64 = size_t(ROCPRIM_WARP_SIZE_64); + + // Block size of warp size 32 + static constexpr size_t block_size_ws32 + = rocprim::detail::is_power_of_two(logical_warp_size) + ? rocprim::max(ws32, logical_warp_size * 4) + : rocprim::max((ws32 / logical_warp_size), 1) * logical_warp_size; + + // Block size of warp size 64 + static constexpr size_t block_size_ws64 + = rocprim::detail::is_power_of_two(logical_warp_size) + ? rocprim::max(ws64, logical_warp_size * 4) + : rocprim::max((ws64 / logical_warp_size), 1) * logical_warp_size; + + unsigned int current_device_warp_size; + HIP_CHECK(::rocprim::host_warp_size(device_id, current_device_warp_size)); + + const size_t block_size = current_device_warp_size == ws32 ? block_size_ws32 : block_size_ws64; + const unsigned int grid_size = 4; + const size_t size = block_size * grid_size; + + // Check if warp size is supported + if((logical_warp_size > current_device_warp_size) + || (current_device_warp_size != ws32 + && current_device_warp_size != ws64)) // Only WarpSize 32 and 64 is supported + { + printf("Unsupported test warp size/computed block size: %zu/%zu. Current device warp size: " + "%u. Skipping test\n", + logical_warp_size, + block_size, + current_device_warp_size); + GTEST_SKIP(); + } + + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) + { + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); + + // Generate data + std::vector input = test_utils::get_random_data(size, 2, 50, seed_value); + std::vector output(size); + std::vector expected(output.size(), T(0)); + T initial_value = test_utils::get_random_data(1, 2, 50, seed_value)[0]; + SCOPED_TRACE(testing::Message() << "with initial_value = " << initial_value); + + // Calculate expected results on host + for(size_t i = 0; i < input.size() / logical_warp_size; i++) + { + acc_type accumulator(initial_value); + for(size_t j = 0; j < logical_warp_size; j++) + { + auto idx = i * logical_warp_size + j; + accumulator = binary_op_host(input[idx], accumulator); + expected[idx] = static_cast(accumulator); + } + } + + // Writing to device memory + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(output.size()); + + // Launching kernel + if(current_device_warp_size == ws32) + { + hipLaunchKernelGGL( + HIP_KERNEL_NAME(warp_inclusive_scan_initial_value_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_input.get(), + device_output.get(), + initial_value); + } + else if(current_device_warp_size == ws64) + { + hipLaunchKernelGGL( + HIP_KERNEL_NAME(warp_inclusive_scan_initial_value_kernel), + dim3(grid_size), + dim3(block_size), + 0, + 0, + device_input.get(), + device_output.get(), + initial_value); + } + + HIP_CHECK(hipGetLastError()); + HIP_CHECK(hipDeviceSynchronize()); + + // Read from device memory + output = device_output.load(); + + // Validating results + test_utils::assert_near(output, expected, test_utils::precision * logical_warp_size); + } +} + typed_test_def(RocprimWarpScanTests, name_suffix, InclusiveScanReduce) { int device_id = test_common_utils::obtain_device_from_ctest(); @@ -254,6 +375,137 @@ typed_test_def(RocprimWarpScanTests, name_suffix, InclusiveScanReduce) } +typed_test_def(RocprimWarpScanTests, name_suffix, InclusiveScanReduceInitialValue) +{ + int device_id = test_common_utils::obtain_device_from_ctest(); + SCOPED_TRACE(testing::Message() << "with device_id = " << device_id); + HIP_CHECK(hipSetDevice(device_id)); + + using T = typename TestFixture::params::type; + // for bfloat16 and half we use double for host-side accumulation + using binary_op_type_host = typename test_utils::select_plus_operator_host::type; + binary_op_type_host binary_op_host; + using acc_type = typename test_utils::select_plus_operator_host::acc_type; + + // logical warp side for warp primitive, execution warp size is always rocprim::warp_size() + static constexpr size_t logical_warp_size = TestFixture::params::warp_size; + + // The different warp sizes + static constexpr size_t ws32 = size_t(ROCPRIM_WARP_SIZE_32); + static constexpr size_t ws64 = size_t(ROCPRIM_WARP_SIZE_64); + + // Block size of warp size 32 + static constexpr size_t block_size_ws32 + = rocprim::detail::is_power_of_two(logical_warp_size) + ? rocprim::max(ws32, logical_warp_size * 4) + : rocprim::max((ws32 / logical_warp_size), 1) * logical_warp_size; + + // Block size of warp size 64 + static constexpr size_t block_size_ws64 + = rocprim::detail::is_power_of_two(logical_warp_size) + ? rocprim::max(ws64, logical_warp_size * 4) + : rocprim::max((ws64 / logical_warp_size), 1) * logical_warp_size; + + unsigned int current_device_warp_size; + HIP_CHECK(::rocprim::host_warp_size(device_id, current_device_warp_size)); + + const size_t block_size = current_device_warp_size == ws32 ? block_size_ws32 : block_size_ws64; + const unsigned int grid_size = 4; + const size_t size = block_size * grid_size; + + // Check if warp size is supported + if((logical_warp_size > current_device_warp_size) + || (current_device_warp_size != ws32 + && current_device_warp_size != ws64)) // Only WarpSize 32 and 64 is supported + { + printf("Unsupported test warp size/computed block size: %zu/%zu. Current device warp size: " + "%u. Skipping test\n", + logical_warp_size, + block_size, + current_device_warp_size); + GTEST_SKIP(); + } + + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) + { + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); + + // Generate data + std::vector input = test_utils::get_random_data(size, 2, 50, seed_value); + std::vector output(size); + std::vector output_reductions(size / logical_warp_size); + std::vector expected(output.size(), T(0)); + std::vector expected_reductions(output_reductions.size(), T(0)); + T initial_value = test_utils::get_random_data(1, 2, 50, seed_value)[0]; + SCOPED_TRACE(testing::Message() << "with initial_value = " << initial_value); + + // Calculate expected results on host + for(size_t i = 0; i < output.size() / logical_warp_size; i++) + { + acc_type accumulator(initial_value); + for(size_t j = 0; j < logical_warp_size; j++) + { + auto idx = i * logical_warp_size + j; + accumulator = binary_op_host(input[idx], accumulator); + expected[idx] = static_cast(accumulator); + } + expected_reductions[i] = expected[(i + 1) * logical_warp_size - 1]; + } + + // Writing to device memory + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(output.size()); + test_utils::device_ptr device_output_reductions(output_reductions.size()); + + // Launching kernel + if(current_device_warp_size == ws32) + { + hipLaunchKernelGGL( + HIP_KERNEL_NAME(warp_inclusive_scan_reduce_initial_value_kernel), + dim3(grid_size), + dim3(block_size_ws32), + 0, + 0, + device_input.get(), + device_output.get(), + device_output_reductions.get(), + initial_value); + } + else if(current_device_warp_size == ws64) + { + hipLaunchKernelGGL( + HIP_KERNEL_NAME(warp_inclusive_scan_reduce_initial_value_kernel), + dim3(grid_size), + dim3(block_size_ws64), + 0, + 0, + device_input.get(), + device_output.get(), + device_output_reductions.get(), + initial_value); + } + + HIP_CHECK(hipGetLastError()); + HIP_CHECK(hipDeviceSynchronize()); + + // Read from device memory + output = device_output.load(); + output_reductions = device_output_reductions.load(); + + // Validating results + test_utils::assert_near(output, expected, test_utils::precision * logical_warp_size); + test_utils::assert_near(output_reductions, + expected_reductions, + test_utils::precision * logical_warp_size); + } +} + typed_test_def(RocprimWarpScanTests, name_suffix, ExclusiveScan) { int device_id = test_common_utils::obtain_device_from_ctest(); diff --git a/test/rocprim/test_warp_scan.kernels.hpp b/test/rocprim/test_warp_scan.kernels.hpp index 9047e437c..1a5a6dc61 100644 --- a/test/rocprim/test_warp_scan.kernels.hpp +++ b/test/rocprim/test_warp_scan.kernels.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -41,6 +41,24 @@ void warp_inclusive_scan_kernel(T* device_input, T* device_output) device_output[index] = value; } +template +__global__ +__launch_bounds__(BlockSize) +void warp_inclusive_scan_initial_value_kernel(T* device_input, T* device_output, T initial_value) +{ + constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; + const unsigned int warp_id = rocprim::detail::logical_warp_id(); + unsigned int index = threadIdx.x + (blockIdx.x * blockDim.x); + + T value = device_input[index]; + + using wscan_t = rocprim::warp_scan; + __shared__ typename wscan_t::storage_type storage[warps_no]; + wscan_t().inclusive_scan(value, value, storage[warp_id], initial_value); + + device_output[index] = value; +} + template __global__ __launch_bounds__(BlockSize) @@ -66,6 +84,32 @@ void warp_inclusive_scan_reduce_kernel(T* device_input, } } +template +__global__ +__launch_bounds__(BlockSize) +void warp_inclusive_scan_reduce_initial_value_kernel(T* device_input, + T* device_output, + T* device_output_reductions, + T initial_value) +{ + constexpr unsigned int warps_no = BlockSize / LogicalWarpSize; + const unsigned int warp_id = rocprim::detail::logical_warp_id(); + unsigned int index = threadIdx.x + (blockIdx.x * BlockSize); + + T value = device_input[index]; + T reduction; + + using wscan_t = rocprim::warp_scan; + __shared__ typename wscan_t::storage_type storage[warps_no]; + wscan_t().inclusive_scan(value, value, reduction, storage[warp_id], initial_value); + + device_output[index] = value; + if((threadIdx.x % LogicalWarpSize) == 0) + { + device_output_reductions[index / LogicalWarpSize] = reduction; + } +} + template __global__ __launch_bounds__(BlockSize) From ae1ce5696834919e82ceefc5bf767d4130d5ddaa Mon Sep 17 00:00:00 2001 From: Sander Bos Date: Mon, 13 Jan 2025 10:31:50 +0000 Subject: [PATCH 65/84] Fix C++ includes showing up in the docs --- docs/concepts/type_traits.rst | 50 ++++++++++++++++--- .../include/rocprim/type_traits_interface.hpp | 29 +---------- 2 files changed, 44 insertions(+), 35 deletions(-) diff --git a/docs/concepts/type_traits.rst b/docs/concepts/type_traits.rst index 2fe17362f..525519959 100644 --- a/docs/concepts/type_traits.rst +++ b/docs/concepts/type_traits.rst @@ -26,30 +26,66 @@ Traits should be implemented as required by specific algorithms. Some traits can Interface ========= -.. doxygengroup:: type_traits_interfaces - :content-only: +.. doxygenstruct:: rocprim::traits::define + +.. doxygenstruct:: rocprim::traits::get :members: Available traits ================ -.. doxygengroup:: available_traits - :content-only: +.. doxygenstruct:: rocprim::traits::is_arithmetic + :members: + +.. doxygenstruct:: rocprim::traits::is_scalar + :members: + +.. doxygenstruct:: rocprim::traits::number_format + :members: + +.. doxygenstruct:: rocprim::traits::integral_sign + :members: + +.. doxygenstruct:: rocprim::traits::float_bit_mask + :members: + +.. doxygenstruct:: rocprim::traits::is_fundamental :members: Type traits wrappers ==================== .. doxygengroup:: rocprim_type_traits_wrapper :content-only: + +.. doxygenstruct:: rocprim::is_floating_point + :no-link: + +.. doxygenstruct:: rocprim::is_integral + :no-link: + +.. doxygenstruct:: rocprim::is_arithmetic + :no-link: + +.. doxygenstruct:: rocprim::is_fundamental + :no-link: + +.. doxygenstruct:: rocprim::is_unsigned + :no-link: + +.. doxygenstruct:: rocprim::is_signed + :no-link: + +.. doxygenstruct:: rocprim::is_scalar + :no-link: + +.. doxygenstruct:: rocprim::is_compound :no-link: Types with predefined traits ============================ -.. doxygengroup:: rocprim_pre_defined_traits +.. doxygengroup:: rocprim_pre_defined_traits :content-only: :members: :outline: :no-link: - - diff --git a/rocprim/include/rocprim/type_traits_interface.hpp b/rocprim/include/rocprim/type_traits_interface.hpp index 25eccc628..f34a8c43c 100644 --- a/rocprim/include/rocprim/type_traits_interface.hpp +++ b/rocprim/include/rocprim/type_traits_interface.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -61,10 +61,6 @@ struct [[deprecated]] float_bit_mask; namespace traits { -/// \defgroup type_traits_interfaces Interfaces for defining and obtaining the traits -/// \addtogroup type_traits_interfaces -/// @{ - /// \par Overview /// This template struct provides an interface for downstream libraries to implement type traits for /// their custom types. Users can utilize this template struct to define traits for these types. Users @@ -107,16 +103,6 @@ template struct define {}; -/// @} - -/// predef -template -struct get; - -/// \defgroup available_traits Traits that can be used -/// \addtogroup available_traits -/// @{ - /// \par Definability /// * **Undefinable**: For types with `predefined traits`. /// * **Optional**: For other types. @@ -517,11 +503,6 @@ struct is_fundamental #endif }; -/// @} - -/// \addtogroup type_traits_interfaces -/// @{ - /// \par Overview /// This template struct is designed to allow rocPRIM algorithms to retrieve trait information from C++ /// build-in arithmetic types, rocPRIM types, and custom types. This API is not static because of ODR. @@ -627,8 +608,6 @@ struct get }; }; -/// @} - } // namespace traits /// \defgroup rocprim_pre_defined_traits Trait definitions for rocPRIM arithmetic types and additional traits for @@ -735,10 +714,6 @@ struct traits::define /// @} -/// \defgroup rocprim_type_traits_wrapper Handy wrappers for obtaining type traits -/// \addtogroup rocprim_type_traits_wrapper -/// @{ - /// \brief An extension of `std::is_floating_point` that supports additional arithmetic types, /// including `rocprim::half`, `rocprim::bfloat16`, and any types with trait /// `rocprim::traits::number_format::values` implemented. @@ -791,8 +766,6 @@ template struct is_compound : std::integral_constant().is_compound()> {}; -/// @} - END_ROCPRIM_NAMESPACE #endif From 9995fa43d7d85bb4b0a457c3ad61b65445c9f925 Mon Sep 17 00:00:00 2001 From: Nick Breed Date: Tue, 14 Jan 2025 07:54:06 +0000 Subject: [PATCH 66/84] Resolve "Performance regressions during tuning: Adjacent find" --- ...enchmark_device_adjacent_find.parallel.hpp | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/benchmark/benchmark_device_adjacent_find.parallel.hpp b/benchmark/benchmark_device_adjacent_find.parallel.hpp index 30ce1450b..24ddc7812 100644 --- a/benchmark/benchmark_device_adjacent_find.parallel.hpp +++ b/benchmark/benchmark_device_adjacent_find.parallel.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -145,17 +145,17 @@ struct device_adjacent_find_benchmark : public config_autotune_interface void* d_tmp_storage = nullptr; auto launch_adjacent_find = [&]() { - HIP_CHECK(::rocprim::adjacent_find(d_tmp_storage, - tmp_storage_size, - d_input, - d_output, - size, - rocprim::equal_to{}, - stream, - false)); + HIP_CHECK(::rocprim::adjacent_find(d_tmp_storage, + tmp_storage_size, + d_input, + d_output, + size, + rocprim::equal_to{}, + stream, + false)); }; - // Get size of tmporary storage + // Get size of temporary storage launch_adjacent_find(); HIP_CHECK(hipMalloc(&d_tmp_storage, tmp_storage_size)); @@ -211,9 +211,8 @@ struct device_adjacent_find_benchmark : public config_autotune_interface template struct device_adjacent_find_benchmark_generator { - static constexpr unsigned int min_items_per_thread = 1; - static constexpr unsigned int max_items_per_thread_arg - = TUNING_SHARED_MEMORY_MAX / (BlockSize * sizeof(InputT) * 2); + static constexpr unsigned int min_items_per_thread = 1; + static constexpr unsigned int max_items_per_thread_exponent = rocprim::Log2<32>::VALUE; template struct create_pos @@ -234,8 +233,6 @@ struct device_adjacent_find_benchmark_generator }; void operator()(std::vector>& storage) { - static constexpr unsigned int max_items_per_thread_exponent - = rocprim::Log2::VALUE - 1; static_for_each< make_index_range, create_ipt>(storage); From 42b55181d02c3224b4c718da51788ba3206f6ebf Mon Sep 17 00:00:00 2001 From: Nick Breed Date: Fri, 10 Jan 2025 09:48:50 +0000 Subject: [PATCH 67/84] CI skip label fix --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f640ad150..9abd34ecb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,13 +44,14 @@ stages: workflow: rules: + - if: $CI_MERGE_REQUEST_LABELS =~ /CI Skip/ + when: never - if: $CI_MERGE_REQUEST_TITLE !~ /Draft:/ variables: ROCPRIM_TEST_RUNS: 3 - if: $CI_MERGE_REQUEST_TITLE =~ /Draft:/ variables: ROCPRIM_TEST_RUNS: 1 - - if: $CI_MERGE_REQUEST_LABELS !~ /CI Skip/ variables: PACKAGE_DIR: $BUILD_DIR/package From 2764058f28ddd494e257b8c1893883652172ed16 Mon Sep 17 00:00:00 2001 From: Nara Prasetya Date: Wed, 15 Jan 2025 08:57:02 +0000 Subject: [PATCH 68/84] fix doxygen warning in merge.rst --- docs/device_ops/merge.rst | 7 +------ rocprim/include/rocprim/device/device_merge_inplace.hpp | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/docs/device_ops/merge.rst b/docs/device_ops/merge.rst index 35bb39b3f..494fef1c5 100644 --- a/docs/device_ops/merge.rst +++ b/docs/device_ops/merge.rst @@ -14,12 +14,7 @@ Configuring the kernel Merge ----- -.. doxygentypedef:: rocprim::merge_config - -Merge inplace -------------- - -.. doxygenstruct:: rocprim::merge_inplace_config +.. doxygenstruct:: rocprim::merge_config Merge ===== diff --git a/rocprim/include/rocprim/device/device_merge_inplace.hpp b/rocprim/include/rocprim/device/device_merge_inplace.hpp index 01a65a991..08f7815bf 100644 --- a/rocprim/include/rocprim/device/device_merge_inplace.hpp +++ b/rocprim/include/rocprim/device/device_merge_inplace.hpp @@ -57,7 +57,7 @@ BEGIN_ROCPRIM_NAMESPACE /// null pointer. /// * Accepts a custom `compare_function`. /// -/// \tparam Config Configuration of the primitive, must be `default_config` or `merge_inplace_config`. +/// \tparam Config Configuration of the primitive, must be `default_config`. /// \tparam Iterator Random access iterator type for the input and output range. Must meet the /// requirements of `std::random_access_iterator`. /// \tparam BinaryFunction Binary function type that is used for the comparison. From 2add4a301215e8830a489e2f2fba9a7df5e27490 Mon Sep 17 00:00:00 2001 From: Enrico Degregori Date: Wed, 15 Jan 2025 11:19:28 +0000 Subject: [PATCH 69/84] Resolve "Autotuning for segmented reduce" --- benchmark/ConfigAutotuneSettings.cmake | 9 +- .../benchmark_device_segmented_reduce.cpp | 216 +++++----------- ...rk_device_segmented_reduce.parallel.cpp.in | 32 +++ ...hmark_device_segmented_reduce.parallel.hpp | 238 ++++++++++++++++++ docs/concepts/tuning.rst | 2 +- .../device/detail/device_segmented_reduce.hpp | 4 +- .../device/device_segmented_reduce.hpp | 6 +- .../device/device_segmented_reduce_config.hpp | 79 ++++++ scripts/autotune/create_optimization.py | 12 +- .../segmented_reduce_config_template | 19 ++ 10 files changed, 449 insertions(+), 168 deletions(-) create mode 100644 benchmark/benchmark_device_segmented_reduce.parallel.cpp.in create mode 100644 benchmark/benchmark_device_segmented_reduce.parallel.hpp create mode 100644 rocprim/include/rocprim/device/device_segmented_reduce_config.hpp create mode 100644 scripts/autotune/templates/segmented_reduce_config_template diff --git a/benchmark/ConfigAutotuneSettings.cmake b/benchmark/ConfigAutotuneSettings.cmake index af1cb9798..25d73f710 100644 --- a/benchmark/ConfigAutotuneSettings.cmake +++ b/benchmark/ConfigAutotuneSettings.cmake @@ -1,6 +1,6 @@ # MIT License # -# Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -97,7 +97,12 @@ KeyType;ValueType;LongBits;BlockSize;ItemsPerThread;WarpSmallLWS;WarpSmallIPT;Wa set(list_across "${TUNING_TYPES};${LIMITED_TUNING_TYPES};8;256;4 8 16;8;4;256;64;16;8;256" PARENT_SCOPE) set(output_pattern_suffix "\ @KeyType@_@ValueType@_@LongBits@_@BlockSize@_@ItemsPerThread@_@WarpSmallLWS@_@WarpSmallIPT@_@WarpSmallBS@_@WarpPartition@_@WarpMediumLWS@_@WarpMediumIPT@_@WarpMediumBS@" PARENT_SCOPE) - elseif(file STREQUAL "benchmark_device_transform") +elseif(file STREQUAL "benchmark_device_segmented_reduce") + set(list_across_names "DataType;BlockSize;ItemsPerThread" PARENT_SCOPE) + set(list_across "\ +${TUNING_TYPES};64 128 256;1 2 4 8 16" PARENT_SCOPE) + set(output_pattern_suffix "@DataType@_@BlockSize@_@ItemsPerThread@" PARENT_SCOPE) +elseif(file STREQUAL "benchmark_device_transform") set(list_across_names "\ DataType;BlockSize;" PARENT_SCOPE) set(list_across "${TUNING_TYPES};64 128 256 512 1024" PARENT_SCOPE) diff --git a/benchmark/benchmark_device_segmented_reduce.cpp b/benchmark/benchmark_device_segmented_reduce.cpp index 91683a57b..a37d06c85 100644 --- a/benchmark/benchmark_device_segmented_reduce.cpp +++ b/benchmark/benchmark_device_segmented_reduce.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,6 +20,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +#include "benchmark_device_segmented_reduce.parallel.hpp" #include "benchmark_utils.hpp" // CmdParser #include "cmdparser.hpp" @@ -47,159 +48,36 @@ const size_t DEFAULT_BYTES = 1024 * 1024 * 32 * 4; #endif -const unsigned int batch_size = 10; -const unsigned int warmup_size = 5; - -template -void run_benchmark(benchmark::State& state, - size_t desired_segments, - size_t bytes, - const managed_seed& seed, - hipStream_t stream) -{ - using offset_type = int; - using value_type = T; - - // Calculate the number of elements - size_t size = bytes / sizeof(T); - - // Generate data - engine_type gen(seed.get_0()); - - const double avg_segment_length = static_cast(size) / desired_segments; - std::uniform_real_distribution segment_length_dis(0, avg_segment_length * 2); - - std::vector offsets; - unsigned int segments_count = 0; - size_t offset = 0; - while(offset < size) - { - const size_t segment_length = std::round(segment_length_dis(gen)); - offsets.push_back(offset); - ++segments_count; - offset += segment_length; - } - offsets.push_back(size); - - std::vector values_input(size); - std::iota(values_input.begin(), values_input.end(), 0); - - offset_type* d_offsets; - HIP_CHECK(hipMalloc(reinterpret_cast(&d_offsets), - (segments_count + 1) * sizeof(offset_type))); - HIP_CHECK(hipMemcpy(d_offsets, - offsets.data(), - (segments_count + 1) * sizeof(offset_type), - hipMemcpyHostToDevice)); - - value_type* d_values_input; - HIP_CHECK(hipMalloc(reinterpret_cast(&d_values_input), size * sizeof(value_type))); - HIP_CHECK(hipMemcpy(d_values_input, - values_input.data(), - size * sizeof(value_type), - hipMemcpyHostToDevice)); - - value_type* d_aggregates_output; - HIP_CHECK(hipMalloc(reinterpret_cast(&d_aggregates_output), - segments_count * sizeof(value_type))); - - rocprim::plus reduce_op; - value_type init(0); - - void* d_temporary_storage = nullptr; - size_t temporary_storage_bytes = 0; - - HIP_CHECK(rocprim::segmented_reduce(d_temporary_storage, - temporary_storage_bytes, - d_values_input, - d_aggregates_output, - segments_count, - d_offsets, - d_offsets + 1, - reduce_op, - init, - stream)); - - HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < warmup_size; ++i) - { - HIP_CHECK(rocprim::segmented_reduce(d_temporary_storage, - temporary_storage_bytes, - d_values_input, - d_aggregates_output, - segments_count, - d_offsets, - d_offsets + 1, - reduce_op, - init, - stream)); - } - HIP_CHECK(hipDeviceSynchronize()); - - // HIP events creation - hipEvent_t start, stop; - HIP_CHECK(hipEventCreate(&start)); - HIP_CHECK(hipEventCreate(&stop)); - - for(auto _ : state) - { - // Record start event - HIP_CHECK(hipEventRecord(start, stream)); - - for(size_t i = 0; i < batch_size; ++i) - { - HIP_CHECK(rocprim::segmented_reduce(d_temporary_storage, - temporary_storage_bytes, - d_values_input, - d_aggregates_output, - segments_count, - d_offsets, - d_offsets + 1, - reduce_op, - init, - stream)); - } - - // Record stop event and wait until it completes - HIP_CHECK(hipEventRecord(stop, stream)); - HIP_CHECK(hipEventSynchronize(stop)); - - float elapsed_mseconds; - HIP_CHECK(hipEventElapsedTime(&elapsed_mseconds, start, stop)); - state.SetIterationTime(elapsed_mseconds / 1000); - } - - // Destroy HIP events - HIP_CHECK(hipEventDestroy(start)); - HIP_CHECK(hipEventDestroy(stop)); - - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(value_type)); - state.SetItemsProcessed(state.iterations() * batch_size * size); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_offsets)); - HIP_CHECK(hipFree(d_values_input)); - HIP_CHECK(hipFree(d_aggregates_output)); -} - -#define CREATE_BENCHMARK(T, SEGMENTS) \ - benchmark::RegisterBenchmark( \ +#define ADD_BENCHMARK(T, SEGMENTS, INSTANCE) \ + benchmark::internal::Benchmark* benchmark = benchmark::RegisterBenchmark( \ bench_naming::format_name("{lvl:device,algo:reduce_segmented,key_type:" #T \ ",segment_count:" \ + std::to_string(SEGMENTS) + ",cfg:default_config}") \ .c_str(), \ - run_benchmark, \ + [INSTANCE](benchmark::State& state, \ + size_t _desired_segments, \ + size_t _size, \ + const managed_seed& _seed, \ + hipStream_t _stream) \ + { INSTANCE.run_benchmark(state, _desired_segments, _size, _seed, _stream); }, \ SEGMENTS, \ bytes, \ seed, \ - stream) + stream); -#define BENCHMARK_TYPE(type) \ - CREATE_BENCHMARK(type, 1), CREATE_BENCHMARK(type, 10), CREATE_BENCHMARK(type, 100), \ - CREATE_BENCHMARK(type, 1000), CREATE_BENCHMARK(type, 10000) +#define CREATE_BENCHMARK(T, SEGMENTS) \ + { \ + const device_segmented_reduce_benchmark instance; \ + ADD_BENCHMARK(T, SEGMENTS, instance) \ + benchmarks.emplace_back(benchmark); \ + } + +#define BENCHMARK_TYPE(type) \ + CREATE_BENCHMARK(type, 1) \ + CREATE_BENCHMARK(type, 10) \ + CREATE_BENCHMARK(type, 100) \ + CREATE_BENCHMARK(type, 1000) \ + CREATE_BENCHMARK(type, 10000) void add_benchmarks(std::vector& benchmarks, size_t bytes, @@ -209,18 +87,16 @@ void add_benchmarks(std::vector& benchmarks, using custom_float2 = custom_type; using custom_double2 = custom_type; - std::vector bs = {BENCHMARK_TYPE(float), - BENCHMARK_TYPE(double), - BENCHMARK_TYPE(int8_t), - BENCHMARK_TYPE(uint8_t), - BENCHMARK_TYPE(rocprim::half), - BENCHMARK_TYPE(int), - BENCHMARK_TYPE(custom_float2), - BENCHMARK_TYPE(custom_double2), - BENCHMARK_TYPE(rocprim::int128_t), - BENCHMARK_TYPE(rocprim::uint128_t)}; - - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); + BENCHMARK_TYPE(float) + BENCHMARK_TYPE(double) + BENCHMARK_TYPE(int8_t) + BENCHMARK_TYPE(uint8_t) + BENCHMARK_TYPE(rocprim::half) + BENCHMARK_TYPE(int) + BENCHMARK_TYPE(custom_float2) + BENCHMARK_TYPE(custom_double2) + BENCHMARK_TYPE(rocprim::int128_t) + BENCHMARK_TYPE(rocprim::uint128_t) } int main(int argc, char* argv[]) @@ -234,6 +110,17 @@ int main(int argc, char* argv[]) "either: json,human,txt"); // fixed seed as a random seed adds a lot of variance parser.set_optional("seed", "seed", "321", get_seed_message()); +#ifdef BENCHMARK_CONFIG_TUNING + // optionally run an evenly split subset of benchmarks, when making multiple program invocations + parser.set_optional("parallel_instance", + "parallel_instance", + 0, + "parallel instance index"); + parser.set_optional("parallel_instances", + "parallel_instances", + 1, + "total parallel instances"); +#endif parser.run_and_exit_if_error(); // Parse argv @@ -253,8 +140,19 @@ int main(int argc, char* argv[]) benchmark::AddCustomContext("seed", seed_type); // Add benchmarks - std::vector benchmarks; + std::vector benchmarks = {}; +#ifdef BENCHMARK_CONFIG_TUNING + const int parallel_instance = parser.get("parallel_instance"); + const int parallel_instances = parser.get("parallel_instances"); + config_autotune_register::register_benchmark_subset(benchmarks, + parallel_instance, + parallel_instances, + bytes, + seed, + stream); +#else add_benchmarks(benchmarks, bytes, seed, stream); +#endif // Use manual timing for(auto& b : benchmarks) diff --git a/benchmark/benchmark_device_segmented_reduce.parallel.cpp.in b/benchmark/benchmark_device_segmented_reduce.parallel.cpp.in new file mode 100644 index 000000000..ebb05ca7b --- /dev/null +++ b/benchmark/benchmark_device_segmented_reduce.parallel.cpp.in @@ -0,0 +1,32 @@ +// MIT License +// +// Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include + +#include "benchmark_utils.hpp" +#include "benchmark_device_segmented_reduce.parallel.hpp" + +namespace { + auto benchmark = config_autotune_register::create, + rocprim::reduce_config<@BlockSize@u, @ItemsPerThread@u, rocprim::block_reduce_algorithm::using_warp_reduce>>>(); +} diff --git a/benchmark/benchmark_device_segmented_reduce.parallel.hpp b/benchmark/benchmark_device_segmented_reduce.parallel.hpp new file mode 100644 index 000000000..c42b0347e --- /dev/null +++ b/benchmark/benchmark_device_segmented_reduce.parallel.hpp @@ -0,0 +1,238 @@ +// MIT License +// +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#ifndef ROCPRIM_BENCHMARK_DEVICE_SEGMENTED_REDUCE_PARALLEL_HPP_ +#define ROCPRIM_BENCHMARK_DEVICE_SEGMENTED_REDUCE_PARALLEL_HPP_ + +#include "benchmark_utils.hpp" + +// Google Benchmark +#include + +// HIP API +#include + +// rocPRIM +#include + +#include +#include +#include +#include +#include + +namespace rp = rocprim; + +constexpr const char* get_reduce_method_name(rocprim::block_reduce_algorithm alg) +{ + switch(alg) + { + case rocprim::block_reduce_algorithm::raking_reduce: return "raking_reduce"; + case rocprim::block_reduce_algorithm::raking_reduce_commutative_only: + return "raking_reduce_commutative_only"; + case rocprim::block_reduce_algorithm::using_warp_reduce: + return "using_warp_reduce"; + // Not using `default: ...` because it kills effectiveness of -Wswitch + } + return "unknown_algorithm"; +} + +template +std::string config_name() +{ + const rocprim::detail::reduce_config_params config = Config(); + return "{bs:" + std::to_string(config.reduce_config.block_size) + + ",ipt:" + std::to_string(config.reduce_config.items_per_thread) + + ",method:" + std::string(get_reduce_method_name(config.block_reduce_method)) + "}"; +} + +template<> +inline std::string config_name() +{ + return "default_config"; +} + +template, + typename Config = rocprim::default_config> +struct device_segmented_reduce_benchmark : public config_autotune_interface +{ + + std::string name() const override + { + return bench_naming::format_name("{lvl:device,algo:segmented_reduce,key_type:" + + std::string(Traits::name()) + + ",cfg:" + config_name() + "}"); + } + + static constexpr unsigned int batch_size = 10; + static constexpr unsigned int warmup_size = 5; + + void run_benchmark(benchmark::State& state, + size_t desired_segment, + size_t bytes, + const managed_seed& seed, + hipStream_t stream) const + { + using offset_type = int; + using value_type = T; + + // Calculate the number of elements + size_t size = bytes / sizeof(T); + + // Generate data + engine_type gen(seed.get_0()); + + const double avg_segment_length = static_cast(size) / desired_segment; + std::uniform_real_distribution segment_length_dis(0, avg_segment_length * 2); + + std::vector offsets; + unsigned int segments_count = 0; + size_t offset = 0; + while(offset < size) + { + const size_t segment_length = std::round(segment_length_dis(gen)); + offsets.push_back(offset); + segments_count++; + offset += segment_length; + } + offsets.push_back(size); + + std::vector values_input(size); + std::iota(values_input.begin(), values_input.end(), 0); + + offset_type* d_offsets; + HIP_CHECK(hipMalloc(reinterpret_cast(&d_offsets), + (segments_count + 1) * sizeof(offset_type))); + HIP_CHECK(hipMemcpy(d_offsets, + offsets.data(), + (segments_count + 1) * sizeof(offset_type), + hipMemcpyHostToDevice)); + + value_type* d_values_input; + HIP_CHECK(hipMalloc(reinterpret_cast(&d_values_input), size * sizeof(value_type))); + HIP_CHECK(hipMemcpy(d_values_input, + values_input.data(), + size * sizeof(value_type), + hipMemcpyHostToDevice)); + + value_type* d_aggregates_output; + HIP_CHECK(hipMalloc(reinterpret_cast(&d_aggregates_output), + segments_count * sizeof(value_type))); + + rocprim::plus reduce_op; + value_type init(0); + + void* d_temporary_storage = nullptr; + size_t temporary_storage_bytes = 0; + + HIP_CHECK(rp::segmented_reduce(d_temporary_storage, + temporary_storage_bytes, + d_values_input, + d_aggregates_output, + segments_count, + d_offsets, + d_offsets + 1, + reduce_op, + init, + stream)); + + HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); + HIP_CHECK(hipDeviceSynchronize()); + + // Warm-up + for(size_t i = 0; i < warmup_size; i++) + { + HIP_CHECK(rp::segmented_reduce(d_temporary_storage, + temporary_storage_bytes, + d_values_input, + d_aggregates_output, + segments_count, + d_offsets, + d_offsets + 1, + reduce_op, + init, + stream)); + } + HIP_CHECK(hipDeviceSynchronize()); + + // HIP events creation + hipEvent_t start, stop; + HIP_CHECK(hipEventCreate(&start)); + HIP_CHECK(hipEventCreate(&stop)); + + for(auto _ : state) + { + // Record start event + HIP_CHECK(hipEventRecord(start, stream)); + + for(size_t i = 0; i < batch_size; i++) + { + HIP_CHECK(rp::segmented_reduce(d_temporary_storage, + temporary_storage_bytes, + d_values_input, + d_aggregates_output, + segments_count, + d_offsets, + d_offsets + 1, + reduce_op, + init, + stream)); + } + + // Record stop event and wait until it completes + HIP_CHECK(hipEventRecord(stop, stream)); + HIP_CHECK(hipEventSynchronize(stop)); + + float elapsed_mseconds; + HIP_CHECK(hipEventElapsedTime(&elapsed_mseconds, start, stop)); + state.SetIterationTime(elapsed_mseconds / 1000); + } + + // Destroy HIP events + HIP_CHECK(hipEventDestroy(start)); + HIP_CHECK(hipEventDestroy(stop)); + + state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(value_type)); + state.SetItemsProcessed(state.iterations() * batch_size * size); + + HIP_CHECK(hipFree(d_temporary_storage)); + HIP_CHECK(hipFree(d_offsets)); + HIP_CHECK(hipFree(d_values_input)); + HIP_CHECK(hipFree(d_aggregates_output)); + } + + void run(benchmark::State& state, + size_t bytes, + const managed_seed& seed, + hipStream_t stream) const override + { + constexpr std::array desired_segments{1, 10, 100, 1000, 10000}; + + for(const auto desired_segment : desired_segments) + { + run_benchmark(state, desired_segment, bytes, seed, stream); + } + } +}; + +#endif \ No newline at end of file diff --git a/docs/concepts/tuning.rst b/docs/concepts/tuning.rst index 673214146..f2b16fb03 100644 --- a/docs/concepts/tuning.rst +++ b/docs/concepts/tuning.rst @@ -16,7 +16,7 @@ What we call *autotuning* is a method of generating the above-mentioned architec 1. Configure the project for autotuning. Autotune is an extension on top of the regular benchmarking process and it is enabled with a CMake option ``BENCHMARK_CONFIG_TUNING``, which doubles as a C++ macro to determine whether autotuning is enabled. 2. When the project is configured, a large amount of C++ benchmark files are generated with variation in parameters such as block size, items per thread, and method. The files are generated based on a template (``benchmark/benchmark_*.parallel.cpp.in``) and arguments defined in ``ConfigAutotuneSettings.cmake``. CMake will automatically detect when the input template changes and will reconfigure the required files as necessary. 3. Compile results in one executable based on all generated files for an algorithm. -4. Run the executable and gather the JSON output files. The generation of output files is triggered by the use of ``--benchmark_out_format=json`` and ``--benchmark_out=.json`` options when running the executable. +4. Run the executable and gather the JSON output files. The generation of output files is triggered by the use of ``--benchmark_out_format=json --name_format json`` and ``--benchmark_out=.json`` options when running the executable. 5. Convert the benchmark results into a config with ``scripts/autotune/create_optimization.py``. This python script injects the optimal configurations into the templates in ``scripts/autotune/templates``. * The option ``--out_basedir`` can be used to place the output config(s) in a specific path, otherwise the config(s) will be placed in the current directory. diff --git a/rocprim/include/rocprim/device/detail/device_segmented_reduce.hpp b/rocprim/include/rocprim/device/detail/device_segmented_reduce.hpp index 20b75ffc9..6b242ba12 100644 --- a/rocprim/include/rocprim/device/detail/device_segmented_reduce.hpp +++ b/rocprim/include/rocprim/device/detail/device_segmented_reduce.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -33,7 +33,7 @@ #include "../../block/block_load_func.hpp" #include "../../block/block_reduce.hpp" #include "../config_types.hpp" -#include "../device_reduce_config.hpp" +#include "../device_segmented_reduce_config.hpp" BEGIN_ROCPRIM_NAMESPACE diff --git a/rocprim/include/rocprim/device/device_segmented_reduce.hpp b/rocprim/include/rocprim/device/device_segmented_reduce.hpp index 6b396d8f6..8278643ea 100644 --- a/rocprim/include/rocprim/device/device_segmented_reduce.hpp +++ b/rocprim/include/rocprim/device/device_segmented_reduce.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -30,7 +30,7 @@ #include "../detail/various.hpp" #include "../functional.hpp" -#include "detail/config/device_reduce.hpp" +#include "detail/config/device_segmented_reduce.hpp" #include "detail/device_segmented_reduce.hpp" #include "rocprim/type_traits.hpp" @@ -88,7 +88,7 @@ hipError_t segmented_reduce_impl(void * temporary_storage, using result_type = typename ::rocprim::invoke_result_binary_op::type; - using config = wrapped_reduce_config; + using config = wrapped_segmented_reduce_config; detail::target_arch target_arch; hipError_t result = host_target_arch(stream, target_arch); diff --git a/rocprim/include/rocprim/device/device_segmented_reduce_config.hpp b/rocprim/include/rocprim/device/device_segmented_reduce_config.hpp new file mode 100644 index 000000000..f64800af0 --- /dev/null +++ b/rocprim/include/rocprim/device/device_segmented_reduce_config.hpp @@ -0,0 +1,79 @@ +// Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef ROCPRIM_DEVICE_DEVICE_SEGMENTED_REDUCE_CONFIG_HPP_ +#define ROCPRIM_DEVICE_DEVICE_SEGMENTED_REDUCE_CONFIG_HPP_ + +#include "../block/block_reduce.hpp" +#include "config_types.hpp" +#include "detail/config/device_segmented_reduce.hpp" + +/// \addtogroup primitivesmodule_deviceconfigs +/// @{ + +BEGIN_ROCPRIM_NAMESPACE + +namespace detail +{ + +template +struct wrapped_segmented_reduce_config +{ + static_assert(std::is_same::value, + "Config must be a specialization of struct template reduce_config"); + + template + struct architecture_config + { + static constexpr reduce_config_params params = ReduceConfig(); + }; +}; + +template +struct wrapped_segmented_reduce_config +{ + template + struct architecture_config + { + static constexpr reduce_config_params params + = default_segmented_reduce_config(Arch), Value>(); + }; +}; + +#ifndef DOXYGEN_SHOULD_SKIP_THIS +template +template +constexpr reduce_config_params + wrapped_segmented_reduce_config::architecture_config::params; + +template +template +constexpr reduce_config_params + wrapped_segmented_reduce_config::architecture_config::params; +#endif // DOXYGEN_SHOULD_SKIP_THIS + +} // namespace detail + +END_ROCPRIM_NAMESPACE + +/// @} +// end of group primitivesmodule_deviceconfigs + +#endif // ROCPRIM_DEVICE_DEVICE_SEGMENTED_REDUCE_CONFIG_HPP_ diff --git a/scripts/autotune/create_optimization.py b/scripts/autotune/create_optimization.py index f29e00f5a..1042bfb6d 100755 --- a/scripts/autotune/create_optimization.py +++ b/scripts/autotune/create_optimization.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -461,6 +461,14 @@ class AlgorithmDeviceReduce(Algorithm): def __init__(self, fallback_entries): Algorithm.__init__(self, fallback_entries) +class AlgorithmDeviceSegmentedReduce(Algorithm): + algorithm_name = "device_segmented_reduce" + config_selection_params = [ + SelectionType(name="key_type", is_optional=False, select_on_size_only=False)] + cpp_configuration_template_name = "segmented_reduce_config_template" + def __init__(self, fallback_entries): + Algorithm.__init__(self, fallback_entries) + class AlgorithmDeviceScan(Algorithm): algorithm_name = "device_scan" cpp_configuration_template_name = "scan_config_template" @@ -687,6 +695,8 @@ def create_algorithm(algorithm_name: str, fallback_entries: List[FallbackCase]): return AlgorithmDeviceRadixSortOnesweep(fallback_entries) elif algorithm_name == 'device_reduce': return AlgorithmDeviceReduce(fallback_entries) + elif algorithm_name == 'device_segmented_reduce': + return AlgorithmDeviceSegmentedReduce(fallback_entries) elif algorithm_name == 'device_scan': return AlgorithmDeviceScan(fallback_entries) elif algorithm_name == 'device_scan_by_key': diff --git a/scripts/autotune/templates/segmented_reduce_config_template b/scripts/autotune/templates/segmented_reduce_config_template new file mode 100644 index 000000000..3b6ed10bc --- /dev/null +++ b/scripts/autotune/templates/segmented_reduce_config_template @@ -0,0 +1,19 @@ +{% extends "config_template" %} + +{% macro get_header_guard() %} +ROCPRIM_DEVICE_DETAIL_CONFIG_DEVICE_SEGMENTED_REDUCE_HPP_ +{%- endmacro %} + +{% macro kernel_configuration(measurement) -%} +reduce_config<{{ measurement['cfg']['bs'] }}, {{ measurement['cfg']['ipt'] }}, ::rocprim::block_reduce_algorithm::{{ measurement['cfg']['method'] }}> { }; +{%- endmacro %} + +{% macro general_case() -%} +template struct default_segmented_reduce_config : +default_reduce_config_base::type { }; +{%- endmacro %} + +{% macro configuration_fallback(benchmark_of_architecture, based_on_type, fallback_selection_criteria) -%} +// Based on {{ based_on_type }} +template struct default_segmented_reduce_config({{ benchmark_of_architecture.name }}), key_type, {{ fallback_selection_criteria }}> : +{%- endmacro %} From e0617cb1b1c4da59b981fe5d6635f74ca0490ab2 Mon Sep 17 00:00:00 2001 From: Nick Breed Date: Wed, 15 Jan 2025 21:02:49 +0000 Subject: [PATCH 70/84] Resolve "Update thread_store for parity" --- .../include/rocprim/thread/thread_copy.hpp | 218 ++++++++++++++++++ .../include/rocprim/thread/thread_load.hpp | 91 +++----- .../include/rocprim/thread/thread_store.hpp | 45 +++- test/rocprim/test_thread_algos.cpp | 134 ++++++++++- 4 files changed, 430 insertions(+), 58 deletions(-) create mode 100644 rocprim/include/rocprim/thread/thread_copy.hpp diff --git a/rocprim/include/rocprim/thread/thread_copy.hpp b/rocprim/include/rocprim/thread/thread_copy.hpp new file mode 100644 index 000000000..317051830 --- /dev/null +++ b/rocprim/include/rocprim/thread/thread_copy.hpp @@ -0,0 +1,218 @@ +// MIT License +// +// Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#ifndef ROCPRIM_THREAD_THREAD_COPY_HPP_ +#define ROCPRIM_THREAD_THREAD_COPY_HPP_ + +#include +#include +#include + +#include "../functional.hpp" +#include "../types.hpp" + +BEGIN_ROCPRIM_NAMESPACE + +namespace detail +{ + +template +struct tuple_prepend; + +template +struct tuple_prepend> +{ + using type = std::tuple; +}; + +template +constexpr bool can_load = Alignment % sizeof(T) == 0 && Size >= sizeof(T); + +/// This struct defines the chain of types that will be tried +/// (in order) to load types with. Be sure to define these types +/// in order of descending "efficiency" (higher size & alignment first). +template +struct next_type; + +template<> +struct next_type +{ + using type = uint64_t; +}; + +template<> +struct next_type +{ + using type = uint32_t; +}; + +template<> +struct next_type +{ + using type = uint16_t; +}; + +template<> +struct next_type +{ + using type = uint8_t; +}; + +template<> +struct next_type +{ + using type = void; +}; + +/// \brief This struct is used to build a tuple of primitive types which +/// a copy can be broken down in. +/// +/// When performing a non-trivial copy, such as a non-temporal, volatile, or atomic +/// load, the compiler may not accept non-primitive types. In the case of volatile +/// copies, this is because a `const volatile T&` constructor must be defined. +/// For non-temporal loads, only trivial types are allowed. +/// This struct builds a tuple of primitive types in which such a copy can be broken +/// down in. The types in the resulting tuple are understood to be packed in memory, +/// with total alignment and size as passed to this structure. `T` is the first +/// type that we try to load with (leave this as the default); if it doesn't apply, +/// the chain defined by `next_type` is followed until one is found that applies. +/// +/// \tparam Alignment - Total alignment of the memory to be copied. +/// \tparam Size - Total size of the memory to be copied. +template +struct fused_copy; + +template +struct fused_copy 0 && can_load)>> +{ + using type = + typename tuple_prepend::type>::type; +}; + +template +struct fused_copy 0 && !can_load)>> +{ + using type = typename fused_copy::type>::type; +}; + +template +struct fused_copy +{ + using type = std::tuple<>; +}; + +template +using fused_copy_t = typename fused_copy::type; + +template +ROCPRIM_DEVICE ROCPRIM_INLINE +void unrolled_tuple_copy_impl(std::tuple* __restrict__ dst, + const std::tuple* __restrict__ src, + F copy_op, + std::index_sequence) +{ + // TODO: Replace with fold expression when we switch to C++17. + int dummy[] = {(copy_op(std::get(*dst), std::get(*src)), 0)...}; + (void)dummy; +} + +template +ROCPRIM_DEVICE ROCPRIM_INLINE +void unrolled_tuple_copy(std::tuple* __restrict__ dst, + const std::tuple* __restrict__ src, + F copy_op) +{ + unrolled_tuple_copy_impl(dst, src, copy_op, std::make_index_sequence{}); +} + +template +ROCPRIM_DEVICE ROCPRIM_INLINE +void thread_fused_copy(T* __restrict__ dst, const U* __restrict__ src, F copy_op) +{ + static_assert(sizeof(T) == sizeof(U), "thread_fused_copy types must have the same size"); + + using fused_copy_tuple = detail::fused_copy_t; + static_assert( + sizeof(T) == sizeof(fused_copy_tuple), + "internal error: fused copy tuple size should be the same as input/output data type"); + + unrolled_tuple_copy(reinterpret_cast(dst), + reinterpret_cast(src), + copy_op); +} + +} // namespace detail + +/// \brief Thread copy with fused loads. +/// +/// This function performs a "fused" copy: The copy operation +/// is explicitly unrolled into primitive copy operations +/// of the highest alignment and size possible, depending +/// on the alignment and size of the input types. +/// +/// \tparam T - The destination data type. +/// \tparam U - The source data type. +/// \tparam Alignment - If given, explicit alignment of the input types. Both `dst` and `src` +/// must be aligned to this size. +/// \param dst - Target memory, where the result is written. +/// \param src - Source memory, where the data is read from. +template +ROCPRIM_DEVICE ROCPRIM_INLINE +void thread_fused_copy(T* __restrict__ dst, const U* __restrict__ src) +{ + detail::thread_fused_copy(dst, + src, + [](auto& dst, const auto& src) { dst = src; }); +} + +namespace detail +{ + +template +ROCPRIM_DEVICE ROCPRIM_INLINE +void unrolled_copy_impl(InputIteratorT src, T* dst, std::integer_sequence) +{ + int dummy[] = {(dst[Is] = src[Is], 0)...}; + (void)dummy; +} + +} // namespace detail + +/// \brief Copy Count number of items from src to dst. +/// \tparam Count number of items to copy +/// \tparam InputIteratorT the input iterator type +/// \tparam T Type of Data to be copied to +/// \param src [in] - Input iterator for data that will be copied +/// \param dst [out] - The pointer the data will be copied to. +template +ROCPRIM_DEVICE ROCPRIM_INLINE +void unrolled_copy(InputIteratorT src, T* dst) +{ + detail::unrolled_copy_impl(src, dst, std::make_integer_sequence{}); +} + +END_ROCPRIM_NAMESPACE + +#endif diff --git a/rocprim/include/rocprim/thread/thread_load.hpp b/rocprim/include/rocprim/thread/thread_load.hpp index 0dd9e8a50..6185b61cc 100644 --- a/rocprim/include/rocprim/thread/thread_load.hpp +++ b/rocprim/include/rocprim/thread/thread_load.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2024, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -33,6 +33,7 @@ #include "../config.hpp" #include "../detail/various.hpp" #include "../type_traits.hpp" +#include "thread_copy.hpp" #include @@ -163,70 +164,50 @@ template return detail::AsmThreadLoad(ptr); } -namespace detail -{ - -template -ROCPRIM_DEVICE ROCPRIM_INLINE -void unrolled_copy_impl(InputIteratorT src, T* dst, std::integer_sequence) -{ - int dummy[] = {(dst[Is] = src[Is], 0)...}; - (void)dummy; -} - -} // namespace detail - -/// \brief Copy Count number of items from src to dst. -/// \tparam Count number of items to copy -/// \tparam InputIteratorT the input iterator type -/// \tparam T Type of Data to be copied to -/// \param src [in] - Input iterator for data that will be copied -/// \param dst [out] - The pointer the data will be copied to. -template -ROCPRIM_DEVICE ROCPRIM_INLINE -void unrolled_copy(InputIteratorT src, T* dst) -{ - detail::unrolled_copy_impl(src, dst, std::make_integer_sequence{}); -} - -/// \brief Thread loads with volatile_pointer for fundamental types -/// \tparam T Type of Data to be copied -/// \param ptr [in] - Input pointer for data that will be copied -/// \return returns loaded value -template -ROCPRIM_DEVICE ROCPRIM_INLINE -T thread_load_volatile(const T* ptr, Int2Type /*is_fundamental*/) -{ - return *reinterpret_cast(ptr); -} - -/// \brief Thread loads with volatile_pointer for non-fundamental types +/// \brief Volatile thread load. +/// /// \tparam T Type of Data to be copied /// \param ptr [in] - Input pointer for data that will be copied /// \return returns loaded value template ROCPRIM_DEVICE ROCPRIM_INLINE -T thread_load_volatile(const T* ptr, Int2Type /*is_fundamental*/) +T thread_load_volatile(const T* ptr) { - using device_word = typename detail::word_type::type; - constexpr int volatile_multiple = sizeof(T) / sizeof(device_word); - - T retval; - device_word* words = reinterpret_cast(&retval); - unrolled_copy(reinterpret_cast(ptr), words); - return retval; + T result; + detail::thread_fused_copy(&result, + ptr, + [](auto& dst, const auto& src) + { + using U = std::remove_reference_t; + dst = *static_cast(&src); + }); + return result; } -/// \brief Thread loads with volatile_pointer -/// \tparam T Type of Data to be copied -/// \param ptr [in] - Input pointer for data that will be copied -/// \return returns loaded value -template +/// \brief Load with non-temporal hint. +/// +/// Non-temporal loads help the compiler and hardware to optimize loading +/// data which is not expected to be re-used, for example by bypassing +/// the data cache. +/// +/// \tparam T - Type of data to be loaded. +/// \tparam Alignment - Explicit alignment of the source data. +/// \param ptr [in] - Pointer to data to be loaded. +/// \return Returns loaded value. +template ROCPRIM_DEVICE ROCPRIM_INLINE -T thread_load_volatile(const T* ptr) +T thread_load_nontemporal(const T* ptr) { - static constexpr bool is_constructible = std::is_constructible::value; - return thread_load_volatile(ptr, Int2Type()); +#if __has_builtin(__builtin_nontemporal_load) + alignas(Alignment) T result; + detail::thread_fused_copy(&result, + ptr, + [](auto& dst, const auto& src) + { dst = __builtin_nontemporal_load(&src); }); + return result; +#else + return *ptr; +#endif } /// @} diff --git a/rocprim/include/rocprim/thread/thread_store.hpp b/rocprim/include/rocprim/thread/thread_store.hpp index 27b00a64e..fd2ec5ec8 100644 --- a/rocprim/include/rocprim/thread/thread_store.hpp +++ b/rocprim/include/rocprim/thread/thread_store.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2024, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -30,9 +30,9 @@ #ifndef ROCPRIM_THREAD_THREAD_STORE_HPP_ #define ROCPRIM_THREAD_THREAD_STORE_HPP_ - #include "../config.hpp" #include "../detail/various.hpp" +#include "thread_copy.hpp" BEGIN_ROCPRIM_NAMESPACE @@ -159,6 +159,47 @@ template detail::AsmThreadStore(ptr, val); } +/// \brief Volatile thread store. +/// +/// \tparam T - Type of data to be stored. +/// \param ptr [in] - Pointer to place where to store data. +/// \param data [in] - Data to be stored. +template +ROCPRIM_DEVICE ROCPRIM_INLINE +void thread_store_volatile(T* ptr, const T& data) +{ + detail::thread_fused_copy(ptr, + &data, + [](auto& dst, const auto& src) + { + using U = std::remove_reference_t; + *static_cast(&dst) = src; + }); +} + +/// \brief Store with non-temporal hint. +/// +/// Non-temporal stores help the compiler and hardware to optimize storing +/// data which not expected to be re-used, for example by bypassing the +/// data cache. +/// +/// \tparam T - Type of data to be stored. +/// \param ptr [in] - Pointer to place where to store data. +/// \param data [in] - Data to be stored. +template +ROCPRIM_DEVICE ROCPRIM_INLINE +void thread_store_nontemporal(T* ptr, const T& data) +{ +#if __has_builtin(__builtin_nontemporal_store) + detail::thread_fused_copy(ptr, + &data, + [](auto& dst, const auto& src) + { __builtin_nontemporal_store(src, &dst); }); +#else + *ptr = data; +#endif +} + /// @} // end group thread_store diff --git a/test/rocprim/test_thread_algos.cpp b/test/rocprim/test_thread_algos.cpp index a245179c3..4c5b6d754 100644 --- a/test/rocprim/test_thread_algos.cpp +++ b/test/rocprim/test_thread_algos.cpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2024, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -160,6 +160,50 @@ TYPED_TEST(RocprimThreadOperationTests, VolatileLoad) } } +template +__global__ +void thread_load_nt_kernel(Type* volatile const device_input, Type* device_output) +{ + size_t index = blockIdx.x * blockDim.x + threadIdx.x; + device_output[index] = rocprim::thread_load_nontemporal(device_input + index); +} + +TYPED_TEST(RocprimThreadOperationTests, LoadNontemporal) +{ + using T = typename TestFixture::type; + static constexpr uint32_t block_size = 256; + static constexpr uint32_t grid_size = 128; + static constexpr uint32_t size = block_size * grid_size; + + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) + { + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); + + // Generate data + std::vector input = test_utils::get_random_data(size, 2, 200, seed_value); + std::vector output(size); + + // Calculate expected results on host + std::vector expected = input; + + // Preparing device + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(input.size()); + + thread_load_nt_kernel + <<>>(device_input.get(), device_output.get()); + HIP_CHECK(hipGetLastError()); + + // Reading results back + output = device_output.load(); + + // Verifying results + ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); + } +} + template __global__ void thread_copy_unroll_kernel(Type* device_input, Type* device_output) @@ -249,6 +293,94 @@ TYPED_TEST(RocprimThreadOperationTests, Store) } } +template +__global__ +void thread_store_volatile_kernel(Type* const device_input, Type* device_output) +{ + size_t index = blockIdx.x * blockDim.x + threadIdx.x; + rocprim::thread_store_volatile(device_output + index, device_input[index]); +} + +TYPED_TEST(RocprimThreadOperationTests, StoreVolatile) +{ + using T = typename TestFixture::type; + static constexpr uint32_t block_size = 256; + static constexpr uint32_t grid_size = 128; + static constexpr uint32_t size = block_size * grid_size; + + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) + { + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); + + // Generate data + std::vector input = test_utils::get_random_data(size, 2, 200, seed_value); + std::vector output(size); + + // Calculate expected results on host + std::vector expected = input; + + // Preparing device + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(input.size()); + + thread_store_volatile_kernel + <<>>(device_input.get(), device_output.get()); + HIP_CHECK(hipGetLastError()); + + // Reading results back + output = device_output.load(); + + // Verifying results + ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); + } +} + +template +__global__ +void thread_store_nt_kernel(Type* const device_input, Type* device_output) +{ + size_t index = blockIdx.x * blockDim.x + threadIdx.x; + rocprim::thread_store_nontemporal(device_output + index, device_input[index]); +} + +TYPED_TEST(RocprimThreadOperationTests, StoreNontemporal) +{ + using T = typename TestFixture::type; + static constexpr uint32_t block_size = 256; + static constexpr uint32_t grid_size = 128; + static constexpr uint32_t size = block_size * grid_size; + + for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) + { + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); + + // Generate data + std::vector input = test_utils::get_random_data(size, 2, 200, seed_value); + std::vector output(size); + + // Calculate expected results on host + std::vector expected = input; + + // Preparing device + test_utils::device_ptr device_input(input); + test_utils::device_ptr device_output(input.size()); + + thread_store_nt_kernel + <<>>(device_input.get(), device_output.get()); + HIP_CHECK(hipGetLastError()); + + // Reading results back + output = device_output.load(); + + // Verifying results + ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); + } +} + struct sum_op { template ROCPRIM_HOST_DEVICE From 0c36f828b007197e8ca48b6c21b7cfd74215af59 Mon Sep 17 00:00:00 2001 From: Cenxuan Tian Date: Thu, 16 Jan 2025 14:45:54 +0000 Subject: [PATCH 71/84] Resolve "Fix compile errors in "rocprim-release-tune" mode" --- benchmark/benchmark_device_binary_search.cpp | 8 ++++---- benchmark/benchmark_device_histogram.parallel.hpp | 10 +++++----- ...hmark_device_segmented_radix_sort_keys.parallel.hpp | 4 ++-- ...mark_device_segmented_radix_sort_pairs.parallel.hpp | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/benchmark/benchmark_device_binary_search.cpp b/benchmark/benchmark_device_binary_search.cpp index 76d3e233f..b5c655344 100644 --- a/benchmark/benchmark_device_binary_search.cpp +++ b/benchmark/benchmark_device_binary_search.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2019-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2019-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -240,9 +240,6 @@ int main(int argc, char* argv[]) benchmark::AddCustomContext("bytes", std::to_string(bytes)); benchmark::AddCustomContext("seed", seed_type); - using custom_float2 = custom_type; - using custom_double2 = custom_type; - // Add benchmarks std::vector benchmarks; #ifdef BENCHMARK_CONFIG_TUNING @@ -255,6 +252,9 @@ int main(int argc, char* argv[]) seed, stream); #else // BENCHMARK_CONFIG_TUNING + using custom_float2 = custom_type; + using custom_double2 = custom_type; + benchmarks = {BENCHMARK_TYPE(float), BENCHMARK_TYPE(double), BENCHMARK_TYPE(int8_t), diff --git a/benchmark/benchmark_device_histogram.parallel.hpp b/benchmark/benchmark_device_histogram.parallel.hpp index be9e8c8ea..f0151d6c4 100644 --- a/benchmark/benchmark_device_histogram.parallel.hpp +++ b/benchmark/benchmark_device_histogram.parallel.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2023-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -203,9 +203,9 @@ struct device_histogram_benchmark : public config_autotune_interface { unsigned int bins; int entropy_reduction; - level_type lower_level[ActiveChannels]; - level_type upper_level[ActiveChannels]; - unsigned int num_levels[ActiveChannels]; + level_type lower_level[ActiveChannels]{}; + level_type upper_level[ActiveChannels]{}; + unsigned int num_levels[ActiveChannels]{}; T* get_d_input(size_t full_size) { return input_cache::instance().get_or_generate( @@ -228,7 +228,7 @@ struct device_histogram_benchmark : public config_autotune_interface { for(int entropy_reduction : {0, 2, 4, 6}) { - case_data data = {.bins = bins, .entropy_reduction = entropy_reduction}; + case_data data = {bins, entropy_reduction}; // Reuse inputs for the same sample type. This autotune uses multipe inputs for all // combinations of bins and entropy, but the inputs do not depend on autotuned diff --git a/benchmark/benchmark_device_segmented_radix_sort_keys.parallel.hpp b/benchmark/benchmark_device_segmented_radix_sort_keys.parallel.hpp index 34bd6db5d..9dca3572c 100644 --- a/benchmark/benchmark_device_segmented_radix_sort_keys.parallel.hpp +++ b/benchmark/benchmark_device_segmented_radix_sort_keys.parallel.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -307,7 +307,7 @@ struct device_segmented_radix_sort_benchmark_generator UnpartitionWarpAllowed>>>()); } template - static auto __create(std::vector>& storage) -> + static auto __create(std::vector>&) -> typename std::enable_if::type {} diff --git a/benchmark/benchmark_device_segmented_radix_sort_pairs.parallel.hpp b/benchmark/benchmark_device_segmented_radix_sort_pairs.parallel.hpp index 88ae7baa0..d36145632 100644 --- a/benchmark/benchmark_device_segmented_radix_sort_pairs.parallel.hpp +++ b/benchmark/benchmark_device_segmented_radix_sort_pairs.parallel.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -335,7 +335,7 @@ struct device_segmented_radix_sort_benchmark_generator UnpartitionWarpAllowed>>>()); } template - static auto __create(std::vector>& storage) -> + static auto __create(std::vector>&) -> typename std::enable_if::type From bc6605b9119d61baf52af2d248d19bd39afa1826 Mon Sep 17 00:00:00 2001 From: Borys Petrov Date: Thu, 16 Jan 2025 16:32:33 +0000 Subject: [PATCH 72/84] Documentation to CI upload --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9abd34ecb..67fbcf6a6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -485,6 +485,10 @@ test:docs: extends: - .rules:test - .build:docs + artifacts: + paths: + - $DOCS_DIR/_build/html/ + expire_in: 2 weeks .benchmark-base: stage: benchmark From bf8985d9de9ac1415fa279f94ecfb42c5a86a862 Mon Sep 17 00:00:00 2001 From: Nick Breed Date: Fri, 17 Jan 2025 06:37:32 +0000 Subject: [PATCH 73/84] Resolve "Remove ASM from thread_load and thread_store" --- CHANGELOG.md | 6 + docs/conf.py | 6 +- .../include/rocprim/thread/thread_copy.hpp | 21 +- .../include/rocprim/thread/thread_load.hpp | 207 +++++++++------ .../include/rocprim/thread/thread_store.hpp | 122 +++++---- test/rocprim/test_thread_algos.cpp | 245 +++++------------- 6 files changed, 283 insertions(+), 324 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4becb366d..65a7cb2c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,17 @@ Full documentation for rocPRIM is available at [https://rocm.docs.amd.com/projec * Added gfx950 support. * Added missing operator `rocprim::key_value_pair::operator==`. * Added `rocprim::unrolled_copy` thread function to copy multiple items inside the thread. +* Added `rocprim::unrolled_thread_load` thread function to load multiple items inside the thread using `rocprim::thread_load`. * Added `rocprim::int128_t` and `rocprim::uint128_t` to benchmarks for improved performance evaluation on 128-bit integers. * Added `rocprim::int128_t` to autotuning supported types to improve performance for 128-bit integers. * Added `rocprim::merge_inplace` function that allows merging in-place. * Added initial value support for warp- and block-level inclusive scan. +### Changed + +* Undeprecated `rocprim::thread_load` uses dereference as default and updated other cache load modifiers. +* Undeprecated `rocprim::thread_store` uses dereference as default and updated other cache store modifiers. + ### Resolved issues * Fixed device radix sort not returning the correct required temporary storage when a double buffer contains `nullptr`. diff --git a/docs/conf.py b/docs/conf.py index 58c7445ed..877a7b36e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -35,7 +35,7 @@ project = "rocPRIM Documentation" html_title = f"rocPRIM {version_number} Documentation" author = "Advanced Micro Devices, Inc." -copyright = "Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved." +copyright = "Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved." version = version_number release = version_number @@ -54,5 +54,5 @@ external_projects = [] external_projects_current_project = "rocprim" -cpp_id_attributes = ["__global__", "__device__", "__host__", "__forceinline__", "static"] +cpp_id_attributes = ["__global__", "__device__", "__host__", "__forceinline__", "__restrict__", "static"] cpp_paren_attributes = ["__declspec"] diff --git a/rocprim/include/rocprim/thread/thread_copy.hpp b/rocprim/include/rocprim/thread/thread_copy.hpp index 317051830..5a8e4b371 100644 --- a/rocprim/include/rocprim/thread/thread_copy.hpp +++ b/rocprim/include/rocprim/thread/thread_copy.hpp @@ -96,8 +96,8 @@ struct next_type /// type that we try to load with (leave this as the default); if it doesn't apply, /// the chain defined by `next_type` is followed until one is found that applies. /// -/// \tparam Alignment - Total alignment of the memory to be copied. -/// \tparam Size - Total size of the memory to be copied. +/// \tparam Alignment Total alignment of the memory to be copied. +/// \tparam Size Total size of the memory to be copied. template struct fused_copy; @@ -172,12 +172,12 @@ void thread_fused_copy(T* __restrict__ dst, const U* __restrict__ src, F copy_op /// of the highest alignment and size possible, depending /// on the alignment and size of the input types. /// -/// \tparam T - The destination data type. -/// \tparam U - The source data type. -/// \tparam Alignment - If given, explicit alignment of the input types. Both `dst` and `src` +/// \tparam T The destination data type. +/// \tparam U The source data type. +/// \tparam Alignment If given, explicit alignment of the input types. Both `dst` and `src` /// must be aligned to this size. -/// \param dst - Target memory, where the result is written. -/// \param src - Source memory, where the data is read from. +/// \param dst Target memory, where the result is written. +/// \param src Source memory, where the data is read from. template ROCPRIM_DEVICE ROCPRIM_INLINE void thread_fused_copy(T* __restrict__ dst, const U* __restrict__ src) @@ -194,6 +194,9 @@ template ROCPRIM_DEVICE ROCPRIM_INLINE void unrolled_copy_impl(InputIteratorT src, T* dst, std::integer_sequence) { + // Unroll multiple thread loads by unpacking an integer sequence + // into a dummy array. We assign the destination values inside the + // constructor of this dummy array. int dummy[] = {(dst[Is] = src[Is], 0)...}; (void)dummy; } @@ -204,8 +207,8 @@ void unrolled_copy_impl(InputIteratorT src, T* dst, std::integer_sequence ROCPRIM_DEVICE ROCPRIM_INLINE void unrolled_copy(InputIteratorT src, T* dst) diff --git a/rocprim/include/rocprim/thread/thread_load.hpp b/rocprim/include/rocprim/thread/thread_load.hpp index 6185b61cc..e69b703dd 100644 --- a/rocprim/include/rocprim/thread/thread_load.hpp +++ b/rocprim/include/rocprim/thread/thread_load.hpp @@ -48,13 +48,14 @@ BEGIN_ROCPRIM_NAMESPACE /// \brief These enum values are used to specify caching behaviour on load enum cache_load_modifier : int { - load_default, ///< Default (no modifier) - load_ca, ///< Cache at all levels - load_cg, ///< Cache at global level - load_cs, ///< Cache streaming (likely to be accessed once) - load_cv, ///< Cache as volatile (including cached system lines) - load_ldg, ///< Cache as texture - load_volatile, ///< Volatile (any memory space) + load_default, ///< Default (no modifier) + load_ca, ///< Cache at all levels + load_cg, ///< Cache at global level + load_nontemporal, ///< Cache streaming (likely not to be accessed again after loading) + load_cv, ///< Cache as volatile (including cached system lines) + load_ldg, ///< Cache as texture + load_volatile, ///< Volatile (any memory space) + load_cs = load_nontemporal ///< Alias for load_nontemporal (will be deprecated in 7.0) }; /// @} @@ -63,8 +64,9 @@ enum cache_load_modifier : int namespace detail { -template -ROCPRIM_DEVICE __forceinline__ T AsmThreadLoad(void * ptr) +template +ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE +T asm_thread_load(void* ptr) { T retval{}; __builtin_memcpy(&retval, ptr, sizeof(T)); @@ -75,26 +77,27 @@ ROCPRIM_DEVICE __forceinline__ T AsmThreadLoad(void * ptr) // Important for syncing. Check section 9.2.2 or 7.3 in the following document // http://developer.amd.com/wordpress/media/2013/12/AMD_GCN3_Instruction_Set_Architecture_rev1.1.pdf - #define ROCPRIM_ASM_THREAD_LOAD(cache_modifier, \ - llvm_cache_modifier, \ - type, \ - interim_type, \ - asm_operator, \ - output_modifier, \ - wait_inst, \ - wait_cmd) \ - template<> \ - ROCPRIM_DEVICE __forceinline__ type AsmThreadLoad(void* ptr) \ - { \ - interim_type retval; \ - asm volatile(#asm_operator " %0, %1 " llvm_cache_modifier "\n\t" wait_inst wait_cmd \ - "(%2)" \ - : "=" #output_modifier(retval) \ - : "v"(ptr), "I"(0x00)); \ - return *bit_cast(&retval); \ + #define ROCPRIM_ASM_THREAD_LOAD(cache_modifier, \ + llvm_cache_modifier, \ + type, \ + interim_type, \ + asm_operator, \ + output_modifier, \ + wait_inst, \ + wait_cmd) \ + template<> \ + ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE type asm_thread_load(void* ptr) \ + { \ + interim_type retval; \ + asm volatile(#asm_operator " %0, %1 " llvm_cache_modifier "\n\t" wait_inst wait_cmd \ + "(%2)" \ + : "=" #output_modifier(retval) \ + : "v"(ptr), "I"(0x00)); \ + return *bit_cast(&retval); \ } -// TODO Add specialization for custom larger data types + // TODO Add specialization for custom larger data types + // clang-format off #define ROCPRIM_ASM_THREAD_LOAD_GROUP(cache_modifier, llvm_cache_modifier, wait_inst, wait_cmd) \ ROCPRIM_ASM_THREAD_LOAD(cache_modifier, llvm_cache_modifier, int8_t, int16_t, flat_load_sbyte, v, wait_inst, wait_cmd); \ ROCPRIM_ASM_THREAD_LOAD(cache_modifier, llvm_cache_modifier, int16_t, int16_t, flat_load_sshort, v, wait_inst, wait_cmd); \ @@ -104,83 +107,93 @@ ROCPRIM_DEVICE __forceinline__ T AsmThreadLoad(void * ptr) ROCPRIM_ASM_THREAD_LOAD(cache_modifier, llvm_cache_modifier, float, uint32_t, flat_load_dword, v, wait_inst, wait_cmd); \ ROCPRIM_ASM_THREAD_LOAD(cache_modifier, llvm_cache_modifier, uint64_t, uint64_t, flat_load_dwordx2, v, wait_inst, wait_cmd); \ ROCPRIM_ASM_THREAD_LOAD(cache_modifier, llvm_cache_modifier, double, uint64_t, flat_load_dwordx2, v, wait_inst, wait_cmd); + // clang-format on -#if defined(__gfx940__) || defined(__gfx941__) -ROCPRIM_ASM_THREAD_LOAD_GROUP(load_ca, "sc0", "s_waitcnt", ""); + #if defined(__gfx940__) || defined(__gfx941__) ROCPRIM_ASM_THREAD_LOAD_GROUP(load_cg, "sc1", "s_waitcnt", ""); -ROCPRIM_ASM_THREAD_LOAD_GROUP(load_cv, "sc0 sc1", "s_waitcnt", "vmcnt"); -ROCPRIM_ASM_THREAD_LOAD_GROUP(load_volatile, "sc0 sc1", "s_waitcnt", "vmcnt"); -#elif defined(__gfx942__) || defined(__gfx950__) -ROCPRIM_ASM_THREAD_LOAD_GROUP(load_ca, "sc0", "s_waitcnt", ""); + #elif defined(__gfx942__) || defined(__gfx950__) ROCPRIM_ASM_THREAD_LOAD_GROUP(load_cg, "sc0 nt", "s_waitcnt", ""); -ROCPRIM_ASM_THREAD_LOAD_GROUP(load_cv, "sc0", "s_waitcnt", "vmcnt"); -ROCPRIM_ASM_THREAD_LOAD_GROUP(load_volatile, "sc0", "s_waitcnt", "vmcnt"); -#elif defined(__gfx1200__) || defined(__gfx1201__) -ROCPRIM_ASM_THREAD_LOAD_GROUP(load_ca, "scope:SCOPE_DEV", "s_wait_loadcnt_dscnt", ""); + #elif defined(__gfx1200__) || defined(__gfx1201__) ROCPRIM_ASM_THREAD_LOAD_GROUP(load_cg, "th:TH_DEFAULT scope:SCOPE_DEV", "s_wait_loadcnt_dscnt", ""); -ROCPRIM_ASM_THREAD_LOAD_GROUP(load_cv, "th:TH_DEFAULT scope:SCOPE_DEV", "s_wait_loadcnt_dscnt", ""); -ROCPRIM_ASM_THREAD_LOAD_GROUP(load_volatile, "th:TH_DEFAULT scope:SCOPE_DEV", "s_wait_loadcnt_dscnt", ""); -#else -ROCPRIM_ASM_THREAD_LOAD_GROUP(load_ca, "glc", "s_waitcnt", ""); + #else ROCPRIM_ASM_THREAD_LOAD_GROUP(load_cg, "glc slc", "s_waitcnt", ""); -ROCPRIM_ASM_THREAD_LOAD_GROUP(load_cv, "glc", "s_waitcnt", "vmcnt"); -ROCPRIM_ASM_THREAD_LOAD_GROUP(load_volatile, "glc", "s_waitcnt", "vmcnt"); -#endif - -// TODO find correct modifiers to match these -ROCPRIM_ASM_THREAD_LOAD_GROUP(load_ldg, "", "s_waitcnt", ""); -ROCPRIM_ASM_THREAD_LOAD_GROUP(load_cs, "", "s_waitcnt", ""); + #endif #endif -} +} // namespace detail /// \addtogroup thread_load /// @{ -/// \brief Store data using the default load instruction. No support for cache modified stores yet -/// \tparam MODIFIER Value in enum for determine which type of cache store modifier to be used +/// \brief Load data using the load instruction specified by CacheLoadModifier. +/// \tparam CacheLoadModifier Value in enum for determine which type of cache store modifier to be used /// \tparam InputIteratorT Type of Output Iterator /// \param itr [in] Iterator to location where data is to be stored /// \return Data that is loaded from memory -template -[[deprecated("Use a dereference instead.")]] ROCPRIM_DEVICE ROCPRIM_INLINE - typename std::iterator_traits::value_type - thread_load(InputIteratorT itr) +template +ROCPRIM_DEVICE ROCPRIM_INLINE +typename std::iterator_traits::value_type thread_load(InputIteratorT itr) { - using T = typename std::iterator_traits::value_type; - T retval = thread_load(&(*itr)); - return *itr; + using T = typename std::iterator_traits::value_type; + return thread_load(&(*itr)); } -/// \brief Load data using the default load instruction. No support for cache modified loads yet -/// \tparam MODIFIER Value in enum for determine which type of cache store modifier to be used +/// \brief Load data using the default load instruction. +/// \tparam CacheLoadModifier Value in enum for determine which type of cache store modifier to be used /// \tparam T Type of Data to be loaded -/// \param ptr [in] - Pointer to data to be loaded +/// \tparam Alignment Explicit alignment of the source data. +/// \param ptr [in] Pointer to data to be loaded /// \return Data that is loaded from memory -template -[[deprecated("Use a dereference instead.")]] ROCPRIM_DEVICE ROCPRIM_INLINE T thread_load(T* ptr) +template +ROCPRIM_DEVICE ROCPRIM_INLINE +std::enable_if_t thread_load(T* ptr) +{ + alignas(Alignment) T result; + detail::thread_fused_copy(&result, + ptr, + [](auto& dst, const auto& src) { dst = src; }); + return result; +} + +/// \brief Global thread load. +/// +/// \tparam CacheLoadModifier Value in enum for determine which type of cache store modifier to be used +/// \tparam T Type of Data to be loaded +/// \tparam Alignment (Unused) +/// \param ptr [in] Input pointer for data that will be loaded +/// \return returns loaded value +template +ROCPRIM_DEVICE ROCPRIM_INLINE +std::enable_if_t thread_load(T* ptr) { - return detail::AsmThreadLoad(ptr); + return detail::asm_thread_load::type>(ptr); } /// \brief Volatile thread load. /// +/// \tparam CacheLoadModifier Value in enum for determine which type of cache store modifier to be used /// \tparam T Type of Data to be copied -/// \param ptr [in] - Input pointer for data that will be copied +/// \tparam Alignment Explicit alignment of the source data. +/// \param ptr [in] Input pointer for data that will be copied /// \return returns loaded value -template +template ROCPRIM_DEVICE ROCPRIM_INLINE -T thread_load_volatile(const T* ptr) +std::enable_if_t + thread_load(T* ptr) { - T result; - detail::thread_fused_copy(&result, - ptr, - [](auto& dst, const auto& src) - { - using U = std::remove_reference_t; - dst = *static_cast(&src); - }); + alignas(Alignment) T result; + detail::thread_fused_copy(&result, + ptr, + [](auto& dst, const auto& src) + { + using U = std::remove_reference_t; + dst = *static_cast(&src); + }); return result; } @@ -190,13 +203,14 @@ T thread_load_volatile(const T* ptr) /// data which is not expected to be re-used, for example by bypassing /// the data cache. /// -/// \tparam T - Type of data to be loaded. -/// \tparam Alignment - Explicit alignment of the source data. -/// \param ptr [in] - Pointer to data to be loaded. +/// \tparam CacheLoadModifier Value in enum for determine which type of cache store modifier to be used +/// \tparam T Type of data to be loaded. +/// \tparam Alignment Explicit alignment of the source data. +/// \param ptr [in] Pointer to data to be loaded. /// \return Returns loaded value. -template +template ROCPRIM_DEVICE ROCPRIM_INLINE -T thread_load_nontemporal(const T* ptr) +std::enable_if_t thread_load(T* ptr) { #if __has_builtin(__builtin_nontemporal_load) alignas(Alignment) T result; @@ -206,10 +220,41 @@ T thread_load_nontemporal(const T* ptr) { dst = __builtin_nontemporal_load(&src); }); return result; #else - return *ptr; + return thread_load(ptr); #endif } +namespace detail +{ + +template +ROCPRIM_DEVICE ROCPRIM_INLINE +void unrolled_thread_load_impl(T* src, T* dst, std::integer_sequence) +{ + // Unroll multiple thread loads by unpacking an integer sequence + // into a dummy array. We assign the destination values inside the + // constructor of this dummy array. + int dummy[] = {(dst[Is] = thread_load(src + Is), 0)...}; + (void)dummy; +} + +} // namespace detail + +/// \brief Load Count number of items from src to dst. +/// \tparam Count number of items to load +/// \tparam CacheLoadModifier the modifier used for the thread_load +/// \tparam T Type of Data to be copied to +/// \param src [in] Input iterator for data that will be loaded in +/// \param dst [out] The pointer the data will be loaded to. +template +ROCPRIM_DEVICE ROCPRIM_INLINE +void unrolled_thread_load(T* src, T* dst) +{ + detail::unrolled_thread_load_impl(src, + dst, + std::make_integer_sequence{}); +} + /// @} // end of group thread_load diff --git a/rocprim/include/rocprim/thread/thread_store.hpp b/rocprim/include/rocprim/thread/thread_store.hpp index fd2ec5ec8..b3d6fb4c5 100644 --- a/rocprim/include/rocprim/thread/thread_store.hpp +++ b/rocprim/include/rocprim/thread/thread_store.hpp @@ -45,12 +45,13 @@ BEGIN_ROCPRIM_NAMESPACE /// \brief These enum values are used to specify caching behaviour on store enum cache_store_modifier { - store_default, ///< Default (no modifier) - store_wb, ///< Cache write-back all coherent levels - store_cg, ///< Cache at global level - store_cs, ///< Cache streaming (likely to be accessed once) - store_wt, ///< Cache write-through (to system memory) - store_volatile, ///< Volatile shared (any memory space) + store_default, ///< Default (no modifier) + store_wb, ///< Cache write-back all coherent levels + store_cg, ///< Cache at global level + store_nontemporal, ///< Cache streaming (likely not to be accessed again after storing) + store_wt, ///< Cache write-through (to system memory) + store_volatile, ///< Volatile (any memory space) + store_cs = store_nontemporal ///< Alias for store_nontemporal (will be deprecated in 7.0) }; /// @} @@ -59,8 +60,9 @@ enum cache_store_modifier namespace detail { -template -ROCPRIM_DEVICE __forceinline__ void AsmThreadStore(void * ptr, T val) +template +ROCPRIM_DEVICE __forceinline__ +void asm_thread_store(void* ptr, T val) { __builtin_memcpy(ptr, &val, sizeof(T)); } @@ -81,8 +83,8 @@ ROCPRIM_DEVICE __forceinline__ void AsmThreadStore(void * ptr, T val) wait_inst, \ wait_cmd) \ template<> \ - ROCPRIM_DEVICE __forceinline__ void AsmThreadStore(void* ptr, \ - type val) \ + ROCPRIM_DEVICE __forceinline__ void asm_thread_store(void* ptr, \ + type val) \ { \ interim_type temp_val = *bit_cast(&val); \ asm volatile(#asm_operator " %0, %1 " llvm_cache_modifier "\n\t" wait_inst wait_cmd \ @@ -91,8 +93,9 @@ ROCPRIM_DEVICE __forceinline__ void AsmThreadStore(void * ptr, T val) : "v"(ptr), #output_modifier(temp_val), "I"(0x00)); \ } -// TODO fix flat_store_ubyte and flat_store_sbyte issues -// TODO Add specialization for custom larger data types + // TODO fix flat_store_ubyte and flat_store_sbyte issues + // TODO Add specialization for custom larger data types + // clang-format off #define ROCPRIM_ASM_THREAD_STORE_GROUP(cache_modifier, llvm_cache_modifier, wait_inst, wait_cmd) \ ROCPRIM_ASM_THREAD_STORE(cache_modifier, llvm_cache_modifier, int8_t, int16_t, flat_store_byte, v, wait_inst, wait_cmd); \ ROCPRIM_ASM_THREAD_STORE(cache_modifier, llvm_cache_modifier, int16_t, int16_t, flat_store_short, v, wait_inst, wait_cmd); \ @@ -102,74 +105,82 @@ ROCPRIM_DEVICE __forceinline__ void AsmThreadStore(void * ptr, T val) ROCPRIM_ASM_THREAD_STORE(cache_modifier, llvm_cache_modifier, float, uint32_t, flat_store_dword, v, wait_inst, wait_cmd); \ ROCPRIM_ASM_THREAD_STORE(cache_modifier, llvm_cache_modifier, uint64_t, uint64_t, flat_store_dwordx2, v, wait_inst, wait_cmd); \ ROCPRIM_ASM_THREAD_STORE(cache_modifier, llvm_cache_modifier, double, uint64_t, flat_store_dwordx2, v, wait_inst, wait_cmd); + // clang-format on -#if defined(__gfx940__) || defined(__gfx941__) -ROCPRIM_ASM_THREAD_STORE_GROUP(store_wb, "sc0 sc1", "s_waitcnt", ""); // TODO: gfx942 validation + #if defined(__gfx940__) || defined(__gfx941__) ROCPRIM_ASM_THREAD_STORE_GROUP(store_cg, "sc0 sc1", "s_waitcnt", ""); -ROCPRIM_ASM_THREAD_STORE_GROUP(store_wt, "sc0 sc1", "s_waitcnt", "vmcnt"); -ROCPRIM_ASM_THREAD_STORE_GROUP(store_volatile, "sc0 sc1", "s_waitcnt", "vmcnt"); -#elif defined(__gfx942__) || defined(__gfx950__) -ROCPRIM_ASM_THREAD_STORE_GROUP(store_wb, "sc0", "s_waitcnt", ""); + #elif defined(__gfx942__) || defined(__gfx950__) ROCPRIM_ASM_THREAD_STORE_GROUP(store_cg, "sc0 nt", "s_waitcnt", ""); -ROCPRIM_ASM_THREAD_STORE_GROUP(store_wt, "sc0", "s_waitcnt", "vmcnt"); -ROCPRIM_ASM_THREAD_STORE_GROUP(store_volatile, "sc0", "s_waitcnt", "vmcnt"); -#elif defined(__gfx1200__) || defined(__gfx1201__) -ROCPRIM_ASM_THREAD_STORE_GROUP(store_wb, "scope:SCOPE_DEV", "s_wait_storecnt_dscnt", ""); // TODO: gfx942 validation -ROCPRIM_ASM_THREAD_STORE_GROUP(store_cg, "th:TH_DEFAULT scope:SCOPE_DEV", "s_wait_storecnt_dscnt", ""); -ROCPRIM_ASM_THREAD_STORE_GROUP(store_wt, "scope:SCOPE_DEV", "s_wait_storecnt_dscnt", ""); -ROCPRIM_ASM_THREAD_STORE_GROUP(store_volatile, "scope:SCOPE_DEV", "s_wait_storecnt_dscnt", ""); -#else -ROCPRIM_ASM_THREAD_STORE_GROUP(store_wb, "glc", "s_waitcnt", ""); + #elif defined(__gfx1200__) || defined(__gfx1201__) +ROCPRIM_ASM_THREAD_STORE_GROUP(store_cg, + "th:TH_DEFAULT scope:SCOPE_DEV", + "s_wait_storecnt_dscnt", + ""); + #else ROCPRIM_ASM_THREAD_STORE_GROUP(store_cg, "glc slc", "s_waitcnt", ""); -ROCPRIM_ASM_THREAD_STORE_GROUP(store_wt, "glc", "s_waitcnt", "vmcnt"); -ROCPRIM_ASM_THREAD_STORE_GROUP(store_volatile, "glc", "s_waitcnt", "vmcnt"); -#endif -// TODO find correct modifiers to match these -ROCPRIM_ASM_THREAD_STORE_GROUP(store_cs, "", "s_waitcnt", ""); + #endif #endif -} +} // namespace detail /// \addtogroup thread_store /// @{ /// \brief Store data using the default load instruction. No support for cache modified stores yet -/// \tparam MODIFIER Value in enum for determine which type of cache store modifier to be used +/// \tparam CacheStoreModifier Value in enum for determine which type of cache store modifier to be used /// \tparam OutputIteratorT Type of Output Iterator /// \tparam T Type of Data to be stored /// \param itr [in] Iterator to location where data is to be stored /// \param val [in] Data to be stored -template -[[deprecated("Use a dereference instead.")]] ROCPRIM_DEVICE ROCPRIM_INLINE void - thread_store(OutputIteratorT itr, T val) +template +ROCPRIM_DEVICE ROCPRIM_INLINE +void thread_store(OutputIteratorT itr, T val) { - thread_store(&(*itr), val); + thread_store(&(*itr), val); } /// \brief Store data using the default load instruction. No support for cache modified stores yet -/// \tparam MODIFIER Value in enum for determine which type of cache store modifier to be used +/// \tparam CacheStoreModifier Value in enum for determine which type of cache store modifier to be used /// \tparam T Type of Data to be stored /// \param ptr [in] Pointer to location where data is to be stored /// \param val [in] Data to be stored -template -[[deprecated("Use a dereference instead.")]] ROCPRIM_DEVICE ROCPRIM_INLINE void thread_store(T* ptr, - T val) +template +ROCPRIM_DEVICE ROCPRIM_INLINE +std::enable_if_t + thread_store(T* ptr, T val) { - detail::AsmThreadStore(ptr, val); + detail::thread_fused_copy(ptr, &val, [](auto& dst, const auto& src) { dst = src; }); +} + +/// \brief Global cached store. +/// +/// \tparam CacheStoreModifier Value in enum for determine which type of cache store modifier to be used +/// \tparam T Type of data to be stored. +/// \param ptr [in] Pointer to place where to store data. +/// \param val [in] Data to be stored. +template +ROCPRIM_DEVICE ROCPRIM_INLINE +std::enable_if_t thread_store(T* ptr, T val) +{ + detail::asm_thread_store(ptr, val); } /// \brief Volatile thread store. /// -/// \tparam T - Type of data to be stored. -/// \param ptr [in] - Pointer to place where to store data. -/// \param data [in] - Data to be stored. -template +/// \tparam CacheStoreModifier Value in enum for determine which type of cache store modifier to be used +/// \tparam T Type of data to be stored. +/// \param ptr [in] Pointer to place where to store data. +/// \param val [in] Data to be stored. +template ROCPRIM_DEVICE ROCPRIM_INLINE -void thread_store_volatile(T* ptr, const T& data) +std::enable_if_t + thread_store(T* ptr, T val) { detail::thread_fused_copy(ptr, - &data, + &val, [](auto& dst, const auto& src) { using U = std::remove_reference_t; @@ -183,20 +194,21 @@ void thread_store_volatile(T* ptr, const T& data) /// data which not expected to be re-used, for example by bypassing the /// data cache. /// +/// \tparam CacheStoreModifier Value in enum for determine which type of cache store modifier to be used /// \tparam T - Type of data to be stored. -/// \param ptr [in] - Pointer to place where to store data. -/// \param data [in] - Data to be stored. -template +/// \param ptr [in] Pointer to place where to store data. +/// \param val [in] Data to be stored. +template ROCPRIM_DEVICE ROCPRIM_INLINE -void thread_store_nontemporal(T* ptr, const T& data) +std::enable_if_t thread_store(T* ptr, T val) { #if __has_builtin(__builtin_nontemporal_store) detail::thread_fused_copy(ptr, - &data, + &val, [](auto& dst, const auto& src) { __builtin_nontemporal_store(src, &dst); }); #else - *ptr = data; + thread_store(ptr, val); #endif } diff --git a/test/rocprim/test_thread_algos.cpp b/test/rocprim/test_thread_algos.cpp index 4c5b6d754..6788a1be5 100644 --- a/test/rocprim/test_thread_algos.cpp +++ b/test/rocprim/test_thread_algos.cpp @@ -77,9 +77,40 @@ __global__ void thread_load_kernel(Type* volatile const device_input, Type* device_output) { size_t index = blockIdx.x * blockDim.x + threadIdx.x; - ROCPRIM_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations"); - device_output[index] = rocprim::thread_load(device_input + index); - ROCPRIM_CLANG_SUPPRESS_WARNING_POP + + if(index % 8 == 0) + { + device_output[index] = rocprim::thread_load(device_input + index); + } + else if(index % 8 == 1) + { + device_output[index] = rocprim::thread_load(device_input + index); + } + else if(index % 8 == 2) + { + device_output[index] = rocprim::thread_load(device_input + index); + } + else if(index % 8 == 3) + { + device_output[index] + = rocprim::thread_load(device_input + index); + } + else if(index % 8 == 4) + { + device_output[index] = rocprim::thread_load(device_input + index); + } + else if(index % 8 == 5) + { + device_output[index] = rocprim::thread_load(device_input + index); + } + else if(index % 8 == 6) + { + device_output[index] = rocprim::thread_load(device_input + index); + } + else // index % 8 == 7 + { + device_output[index] = rocprim::thread_load(device_input + index); + } } TYPED_TEST(RocprimThreadOperationTests, Load) @@ -116,102 +147,24 @@ TYPED_TEST(RocprimThreadOperationTests, Load) } } -template +template __global__ -void thread_volatile_load_kernel(Type* volatile const device_input, Type* device_output) +void thread_copy_unroll_kernel(Type* device_input, Type* device_output) { - size_t index = blockIdx.x * blockDim.x + threadIdx.x; - device_output[index] = rocprim::thread_load_volatile(device_input + index); -} + size_t thread_id = blockIdx.x * blockDim.x + threadIdx.x; + size_t index = thread_id * ItemsPerThread; -TYPED_TEST(RocprimThreadOperationTests, VolatileLoad) -{ - using T = typename TestFixture::type; - static constexpr uint32_t block_size = 256; - static constexpr uint32_t grid_size = 128; - static constexpr uint32_t size = block_size * grid_size; - - for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) + if(thread_id % 2 == 0) { - unsigned int seed_value - = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; - SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); - - // Generate data - std::vector input = test_utils::get_random_data(size, 2, 200, seed_value); - std::vector output(size); - - // Calculate expected results on host - std::vector expected = input; - - // Preparing device - test_utils::device_ptr device_input(input); - test_utils::device_ptr device_output(input.size()); - - thread_volatile_load_kernel - <<>>(device_input.get(), device_output.get()); - HIP_CHECK(hipGetLastError()); - - // Reading results back - output = device_output.load(); - - // Verifying results - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); + rocprim::unrolled_copy(device_input + index, device_output + index); } -} - -template -__global__ -void thread_load_nt_kernel(Type* volatile const device_input, Type* device_output) -{ - size_t index = blockIdx.x * blockDim.x + threadIdx.x; - device_output[index] = rocprim::thread_load_nontemporal(device_input + index); -} - -TYPED_TEST(RocprimThreadOperationTests, LoadNontemporal) -{ - using T = typename TestFixture::type; - static constexpr uint32_t block_size = 256; - static constexpr uint32_t grid_size = 128; - static constexpr uint32_t size = block_size * grid_size; - - for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) + else { - unsigned int seed_value - = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; - SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); - - // Generate data - std::vector input = test_utils::get_random_data(size, 2, 200, seed_value); - std::vector output(size); - - // Calculate expected results on host - std::vector expected = input; - - // Preparing device - test_utils::device_ptr device_input(input); - test_utils::device_ptr device_output(input.size()); - - thread_load_nt_kernel - <<>>(device_input.get(), device_output.get()); - HIP_CHECK(hipGetLastError()); - - // Reading results back - output = device_output.load(); - - // Verifying results - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); + rocprim::unrolled_thread_load(device_input + index, + device_output + index); } } -template -__global__ -void thread_copy_unroll_kernel(Type* device_input, Type* device_output) -{ - size_t index = (blockIdx.x * blockDim.x + threadIdx.x) * ItemsPerThread; - rocprim::unrolled_copy(device_input + index, device_output + index); -} - TYPED_TEST(RocprimThreadOperationTests, CopyUnroll) { using T = typename TestFixture::type; @@ -254,95 +207,36 @@ __global__ void thread_store_kernel(Type* const device_input, Type* device_output) { size_t index = blockIdx.x * blockDim.x + threadIdx.x; - ROCPRIM_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations") - rocprim::thread_store(device_output + index, device_input[index]); - ROCPRIM_CLANG_SUPPRESS_WARNING_POP -} - -TYPED_TEST(RocprimThreadOperationTests, Store) -{ - using T = typename TestFixture::type; - static constexpr uint32_t block_size = 256; - static constexpr uint32_t grid_size = 128; - static constexpr uint32_t size = block_size * grid_size; - for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) + if(index % 7 == 0) { - unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; - SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); - - // Generate data - std::vector input = test_utils::get_random_data(size, 2, 200, seed_value); - std::vector output(size); - - // Calculate expected results on host - std::vector expected = input; - - // Preparing device - test_utils::device_ptr device_input(input); - test_utils::device_ptr device_output(input.size()); - - thread_store_kernel<<>>(device_input.get(), device_output.get()); - HIP_CHECK(hipGetLastError()); - - // Reading results back - output = device_output.load(); - - // Verifying results - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); + rocprim::thread_store(device_output + index, device_input[index]); } -} - -template -__global__ -void thread_store_volatile_kernel(Type* const device_input, Type* device_output) -{ - size_t index = blockIdx.x * blockDim.x + threadIdx.x; - rocprim::thread_store_volatile(device_output + index, device_input[index]); -} - -TYPED_TEST(RocprimThreadOperationTests, StoreVolatile) -{ - using T = typename TestFixture::type; - static constexpr uint32_t block_size = 256; - static constexpr uint32_t grid_size = 128; - static constexpr uint32_t size = block_size * grid_size; - - for(size_t seed_index = 0; seed_index < number_of_runs; seed_index++) + else if(index % 7 == 1) { - unsigned int seed_value - = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; - SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); - - // Generate data - std::vector input = test_utils::get_random_data(size, 2, 200, seed_value); - std::vector output(size); - - // Calculate expected results on host - std::vector expected = input; - - // Preparing device - test_utils::device_ptr device_input(input); - test_utils::device_ptr device_output(input.size()); - - thread_store_volatile_kernel - <<>>(device_input.get(), device_output.get()); - HIP_CHECK(hipGetLastError()); - - // Reading results back - output = device_output.load(); - - // Verifying results - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); + rocprim::thread_store(device_output + index, device_input[index]); + } + else if(index % 7 == 2) + { + rocprim::thread_store(device_output + index, device_input[index]); + } + else if(index % 7 == 3) + { + rocprim::thread_store(device_output + index, + device_input[index]); + } + else if(index % 7 == 4) + { + rocprim::thread_store(device_output + index, device_input[index]); + } + else if(index % 7 == 5) + { + rocprim::thread_store(device_output + index, device_input[index]); + } + else // index % 7 == 6 + { + rocprim::thread_store(device_output + index, device_input[index]); } -} - -template -__global__ -void thread_store_nt_kernel(Type* const device_input, Type* device_output) -{ - size_t index = blockIdx.x * blockDim.x + threadIdx.x; - rocprim::thread_store_nontemporal(device_output + index, device_input[index]); } TYPED_TEST(RocprimThreadOperationTests, StoreNontemporal) @@ -369,8 +263,7 @@ TYPED_TEST(RocprimThreadOperationTests, StoreNontemporal) test_utils::device_ptr device_input(input); test_utils::device_ptr device_output(input.size()); - thread_store_nt_kernel - <<>>(device_input.get(), device_output.get()); + thread_store_kernel<<>>(device_input.get(), device_output.get()); HIP_CHECK(hipGetLastError()); // Reading results back From 2d9620977d4600220ec7233ef837923824f9cad6 Mon Sep 17 00:00:00 2001 From: Sander Bos Date: Fri, 17 Jan 2025 08:00:27 +0000 Subject: [PATCH 74/84] Add documentation steps to developer guidelines --- docs/concepts/developer.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/concepts/developer.rst b/docs/concepts/developer.rst index 20a842b75..5eb485021 100644 --- a/docs/concepts/developer.rst +++ b/docs/concepts/developer.rst @@ -337,3 +337,22 @@ Pitfalls and common mistakes HIP code is compiled in multiple passes: one for the host and one for each targeted device architecture. As such, host code is agnostic of device architecture, and should be designed as such. Only with a ``hipStream`` can the device be inferred and can certain properties be obtained. Since device code is compiled for a specific architecture, it can contain compile-time optimizations for specific architectures. Note that AMD GPUs have a warp size of 32 or 64, and unless specialized, algorithms should work for both warp sizes. All variables with the ``__shared__`` memory space specifier should either be in a function with the ``__global__`` (``ROCPRIM_KERNEL``) execution space specifier or in a function with the ``__device__`` (``ROCPRIM_DEVICE``) execution space specifier marked with ``__forceinline__`` (``ROCPRIM_FORCE_INLINE``). The reason for this is that without forcing the inlining of the function, the compiler may choose not to optimize shared memory allocations, leading to exceeding the limit dictated by hardware. + +Documenting algorithms +====================== + +Documenting algorithms requires updating several files. + +See `Contributing to the ROCm documentation `_ for general guidelines. + +The ``docs/`` directory contains the ``.rst`` files that form this website. These ``.rst`` files use `Breathe directives `_ to display `Doxygen's special commands `_, which are used in C++ comments to document code. + +Algorithms should be documented in their appropriate ``docs/`` subdirectories, like ``block_ops/`` and ``device_ops/``, based on their scope. :ref:`dev-nth_element` for example is a :ref:`device-wide operation `, documented in ``docs/device_ops/nth_element.rst``. + +There are several pages that link to the :ref:`dev-nth_element` documentation: + +- ``docs/device_ops/index.rst``, which adds :ref:`dev-nth_element` to :ref:`dev-index`. +- ``docs/reference/ops_summary.rst``, which adds :ref:`dev-nth_element` to :ref:`ops-summary`. +- ``docs/sphinx/_toc.yml.in``, which adds :ref:`dev-nth_element` to Sphinx its `Table Of Contents `_, and ``docs/sphinx/_toc.yml`` is automatically generated from this file. + +Any newly added algorithm should also update these files. As one can easily forget to update one of them, it is recommended to `build and preview the documentation locally `_. From 033ced9df1e458883d3ba6c4a258d88375ea035a Mon Sep 17 00:00:00 2001 From: Cenxuan Tian Date: Fri, 17 Jan 2025 13:46:58 +0000 Subject: [PATCH 75/84] Resolve "Refactor lookback_scan_state API" --- .../device/detail/device_scan_common.hpp | 4 +- .../device/detail/lookback_scan_state.hpp | 222 +++++++----------- 2 files changed, 91 insertions(+), 135 deletions(-) diff --git a/rocprim/include/rocprim/device/detail/device_scan_common.hpp b/rocprim/include/rocprim/device/detail/device_scan_common.hpp index 1a9008762..4207f986a 100644 --- a/rocprim/include/rocprim/device/detail/device_scan_common.hpp +++ b/rocprim/include/rocprim/device/detail/device_scan_common.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -48,7 +48,7 @@ void access_indexed_lookback_value(LookBackScanState lookback_scan_state, if(flat_thread_id == (save_index < number_of_blocks ? save_index : 0)) { typename LookBackScanState::value_type value; - prefix_flag dummy_flag; + lookback_scan_prefix_flag dummy_flag; lookback_scan_state.get(save_index, dummy_flag, value); access_function(value); } diff --git a/rocprim/include/rocprim/device/detail/lookback_scan_state.hpp b/rocprim/include/rocprim/device/detail/lookback_scan_state.hpp index 59b7d70e1..eb3d08b70 100644 --- a/rocprim/include/rocprim/device/detail/lookback_scan_state.hpp +++ b/rocprim/include/rocprim/device/detail/lookback_scan_state.hpp @@ -61,27 +61,27 @@ BEGIN_ROCPRIM_NAMESPACE // Single pass prefix scan was implemented based on: // Merrill, D. and Garland, M. Single-pass Parallel Prefix Scan with Decoupled Look-back. -// Technical Report NVR2016-001, NVIDIA Research. Mar. 2016. +// Technical Report NVR2016-002, NVIDIA Research. Mar. 2016. namespace detail { -enum class prefix_flag : uint8_t +enum class lookback_scan_prefix_flag : uint8_t { // flag for padding, values should be discarded - INVALID = 0xFF, + invalid = 0xFF, // initialized, not result in value - EMPTY = 0, + empty = 0, // partial prefix value (from single block) - PARTIAL = 1, + partial = 1, // final prefix value - COMPLETE = 2 + complete = 2 }; template struct match_prefix_underlying_type { - using value_and_prefix = tuple; + using value_and_prefix = tuple; using type = select_type, select_type_case, @@ -190,7 +190,7 @@ struct lookback_scan_state struct prefix_type { T value; - prefix_flag flag; + lookback_scan_prefix_flag flag; }; static_assert(sizeof(prefix_underlying_type) >= sizeof(prefix_type), ""); @@ -213,17 +213,6 @@ struct lookback_scan_state return hipSuccess; } - [[deprecated("Please use the overload returns an error code, this function assumes the default" - " stream and silently ignores errors.")]] - ROCPRIM_HOST_DEVICE - static inline lookback_scan_state create(void* temp_storage, - const unsigned int number_of_blocks) - { - lookback_scan_state result; - (void)create(result, temp_storage, number_of_blocks, /*default stream*/ 0); - return result; - } - ROCPRIM_HOST_DEVICE static inline hipError_t get_storage_size(const unsigned int number_of_blocks, const hipStream_t stream, @@ -237,20 +226,10 @@ struct lookback_scan_state return error; } - [[deprecated("Please use the overload returns an error code, this function assumes the default" - " stream and silently ignores errors.")]] ROCPRIM_HOST_DEVICE - static inline size_t get_storage_size(const unsigned int number_of_blocks) - { - size_t result; - (void)get_storage_size(number_of_blocks, /*default stream*/ 0, result); - return result; - } - - ROCPRIM_HOST static inline hipError_t - get_temp_storage_layout(const unsigned int number_of_blocks, - const hipStream_t stream, - detail::temp_storage::layout& layout) + static inline hipError_t get_temp_storage_layout(const unsigned int number_of_blocks, + const hipStream_t stream, + detail::temp_storage::layout& layout) { size_t storage_size = 0; hipError_t error = get_storage_size(number_of_blocks, stream, storage_size); @@ -258,16 +237,6 @@ struct lookback_scan_state return error; } - [[deprecated("Please use the overload returns an error code, this function assumes the default" - " stream and silently ignores errors.")]] ROCPRIM_HOST static inline detail:: - temp_storage::layout - get_temp_storage_layout(const unsigned int number_of_blocks) - { - detail::temp_storage::layout result; - (void)get_temp_storage_layout(number_of_blocks, /*default stream*/ 0, result); - return result; - } - ROCPRIM_DEVICE ROCPRIM_INLINE void initialize_prefix(const unsigned int block_id, const unsigned int number_of_blocks) { @@ -276,7 +245,7 @@ struct lookback_scan_state if(block_id < number_of_blocks) { prefix_type prefix; - prefix.flag = prefix_flag::EMPTY; + prefix.flag = lookback_scan_prefix_flag::empty; prefix_underlying_type p; memcpy(&p, &prefix, sizeof(prefix_type)); prefixes[padding + block_id] = p; @@ -284,7 +253,7 @@ struct lookback_scan_state if(block_id < padding) { prefix_type prefix; - prefix.flag = prefix_flag::INVALID; + prefix.flag = lookback_scan_prefix_flag::invalid; prefix_underlying_type p; memcpy(&p, &prefix, sizeof(prefix_type)); prefixes[block_id] = p; @@ -293,16 +262,17 @@ struct lookback_scan_state ROCPRIM_DEVICE ROCPRIM_INLINE void set_partial(const unsigned int block_id, const T value) { - this->set(block_id, prefix_flag::PARTIAL, value); + this->set(block_id, lookback_scan_prefix_flag::partial, value); } ROCPRIM_DEVICE ROCPRIM_INLINE void set_complete(const unsigned int block_id, const T value) { - this->set(block_id, prefix_flag::COMPLETE, value); + this->set(block_id, lookback_scan_prefix_flag::complete, value); } // block_id must be > 0 - ROCPRIM_DEVICE ROCPRIM_INLINE void get(const unsigned int block_id, prefix_flag& flag, T& value) + ROCPRIM_DEVICE ROCPRIM_INLINE + void get(const unsigned int block_id, lookback_scan_prefix_flag& flag, T& value) { constexpr unsigned int padding = ::rocprim::device_warp_size(); @@ -313,7 +283,7 @@ struct lookback_scan_state prefix_underlying_type p = ::rocprim::detail::atomic_load(&prefixes[padding + block_id]); memcpy(&prefix, &p, sizeof(prefix_type)); - while(prefix.flag == prefix_flag::EMPTY) + while(prefix.flag == lookback_scan_prefix_flag::empty) { if ROCPRIM_IF_CONSTEXPR(UseSleep) { @@ -361,11 +331,12 @@ struct lookback_scan_state T cache[max_lookback_per_thread]; int cache_offset = 0; - prefix_flag flag; + lookback_scan_prefix_flag flag; T block_prefix; this->get(lookback_block_id, flag, block_prefix); - while(warp_all(flag != prefix_flag::COMPLETE && flag != prefix_flag::INVALID) + while(warp_all(flag != lookback_scan_prefix_flag::complete + && flag != lookback_scan_prefix_flag::invalid) && cache_offset < max_lookback_per_thread) { cache[cache_offset++] = block_prefix; @@ -379,9 +350,9 @@ struct lookback_scan_state // - We have run out of available space in the cache. In this case, wait until // any of the current lookback flags pointed to by lookback_block_id changes // to complete. - if(warp_all(flag != prefix_flag::COMPLETE)) + if(warp_all(flag != lookback_scan_prefix_flag::complete)) { - if(warp_all(flag == prefix_flag::INVALID)) + if(warp_all(flag == lookback_scan_prefix_flag::invalid)) { // All invalid, so we have to move one block back to // get back to known civilization. @@ -394,14 +365,14 @@ struct lookback_scan_state { this->get(lookback_block_id, flag, block_prefix); } - while(warp_all(flag != prefix_flag::COMPLETE)); + while(warp_all(flag != lookback_scan_prefix_flag::complete)); } // Now just sum all these values to get the prefix // Note that the values are striped across the threads. // In the first iteration, the current prefix is at the value cache for the current - // offset at the lowest warp number that has prefix_flag::COMPLETE set. - const auto bits = ballot(flag == prefix_flag::COMPLETE); + // offset at the lowest warp number that has lookback_scan_prefix_flag::complete set. + const auto bits = ballot(flag == lookback_scan_prefix_flag::complete); const auto lowest = ctz(bits); // Now sum all the values from block_prefix that are lower than the current prefix. @@ -419,8 +390,8 @@ struct lookback_scan_state } private: - ROCPRIM_DEVICE ROCPRIM_INLINE void - set(const unsigned int block_id, const prefix_flag flag, const T value) + ROCPRIM_DEVICE ROCPRIM_INLINE + void set(const unsigned int block_id, const lookback_scan_prefix_flag flag, const T value) { constexpr unsigned int padding = ::rocprim::device_warp_size(); @@ -440,16 +411,17 @@ struct lookback_scan_state { public: - using flag_underlying_type = std::underlying_type_t; + using flag_underlying_type = std::underlying_type_t; using value_type = T; static constexpr bool use_sleep = UseSleep; // temp_storage must point to allocation of get_storage_size(number_of_blocks) bytes - ROCPRIM_HOST static inline hipError_t create(lookback_scan_state& state, - void* temp_storage, - const unsigned int number_of_blocks, - const hipStream_t stream) + ROCPRIM_HOST_DEVICE + static inline hipError_t create(lookback_scan_state& state, + void* temp_storage, + const unsigned int number_of_blocks, + const hipStream_t stream) { unsigned int warp_size; hipError_t error = ::rocprim::host_warp_size(stream, warp_size); @@ -469,19 +441,10 @@ struct lookback_scan_state return error; } - [[deprecated( - "Please use the overload returns an error code, this function assumes the default" - " stream and silently ignores errors.")]] ROCPRIM_HOST static inline lookback_scan_state - create(void* temp_storage, const unsigned int number_of_blocks) - { - lookback_scan_state result; - (void)create(result, temp_storage, number_of_blocks, /*default stream*/ 0); - return result; - } - - ROCPRIM_HOST static inline hipError_t get_storage_size(const unsigned int number_of_blocks, - const hipStream_t stream, - size_t& storage_size) + ROCPRIM_HOST_DEVICE + static inline hipError_t get_storage_size(const unsigned int number_of_blocks, + const hipStream_t stream, + size_t& storage_size) { unsigned int warp_size; hipError_t error = ::rocprim::host_warp_size(stream, warp_size); @@ -489,42 +452,23 @@ struct lookback_scan_state // Always use sizeof(value_underlying_type) instead of sizeof(T) because storage is // allocated by host so it can hold both types no matter what device is used. storage_size = 2 * ::rocprim::detail::align_size(n * sizeof(value_underlying_type)); - storage_size += n * sizeof(prefix_flag); + storage_size += n * sizeof(lookback_scan_prefix_flag); return error; } - [[deprecated("Please use the overload returns an error code, this function assumes the default" - " stream and silently ignores errors.")]] ROCPRIM_HOST static inline size_t - get_storage_size(const unsigned int number_of_blocks) - { - size_t result; - (void)get_storage_size(number_of_blocks, /*default stream*/ 0, result); - return result; - } - - ROCPRIM_HOST static inline hipError_t - get_temp_storage_layout(const unsigned int number_of_blocks, - const hipStream_t stream, - detail::temp_storage::layout& layout) + ROCPRIM_HOST_DEVICE + static inline hipError_t get_temp_storage_layout(const unsigned int number_of_blocks, + const hipStream_t stream, + detail::temp_storage::layout& layout) { size_t storage_size = 0; - size_t alignment - = std::max({alignof(prefix_flag), alignof(T), alignof(value_underlying_type)}); + size_t alignment = std::max( + {alignof(lookback_scan_prefix_flag), alignof(T), alignof(value_underlying_type)}); hipError_t error = get_storage_size(number_of_blocks, stream, storage_size); layout = detail::temp_storage::layout{storage_size, alignment}; return error; } - [[deprecated("Please use the overload returns an error code, this function assumes the default" - " stream and silently ignores errors.")]] ROCPRIM_HOST static inline detail:: - temp_storage::layout - get_temp_storage_layout(const unsigned int number_of_blocks) - { - detail::temp_storage::layout result; - (void)get_temp_storage_layout(number_of_blocks, /*default stream*/ 0, result); - return result; - } - ROCPRIM_DEVICE ROCPRIM_INLINE void initialize_prefix(const unsigned int block_id, const unsigned int number_of_blocks) { @@ -532,26 +476,28 @@ struct lookback_scan_state if(block_id < number_of_blocks) { prefixes_flags[padding + block_id] - = static_cast(prefix_flag::EMPTY); + = static_cast(lookback_scan_prefix_flag::empty); } if(block_id < padding) { - prefixes_flags[block_id] = static_cast(prefix_flag::INVALID); + prefixes_flags[block_id] + = static_cast(lookback_scan_prefix_flag::invalid); } } ROCPRIM_DEVICE ROCPRIM_INLINE void set_partial(const unsigned int block_id, const T value) { - this->set(block_id, prefix_flag::PARTIAL, value); + this->set(block_id, lookback_scan_prefix_flag::partial, value); } ROCPRIM_DEVICE ROCPRIM_INLINE void set_complete(const unsigned int block_id, const T value) { - this->set(block_id, prefix_flag::COMPLETE, value); + this->set(block_id, lookback_scan_prefix_flag::complete, value); } // block_id must be > 0 - ROCPRIM_DEVICE ROCPRIM_INLINE void get(const unsigned int block_id, prefix_flag& flag, T& value) + ROCPRIM_DEVICE ROCPRIM_INLINE + void get(const unsigned int block_id, lookback_scan_prefix_flag& flag, T& value) { constexpr unsigned int padding = ::rocprim::device_warp_size(); @@ -560,7 +506,8 @@ struct lookback_scan_state rocprim::detail::atomic_fence_acquire_order_only(); const auto* values = static_cast( - flag == prefix_flag::PARTIAL ? prefixes_partial_values : prefixes_complete_values); + flag == lookback_scan_prefix_flag::partial ? prefixes_partial_values + : prefixes_complete_values); value_underlying_type v; for(unsigned int i = 0; i < value_underlying_type::words_no; ++i) { @@ -570,8 +517,9 @@ struct lookback_scan_state #else ::rocprim::detail::memory_fence_device(); - const auto* values = static_cast( - flag == prefix_flag::PARTIAL ? prefixes_partial_values : prefixes_complete_values); + const auto* values = static_cast(flag == lookback_scan_prefix_flag::partial + ? prefixes_partial_values + : prefixes_complete_values); value = values[padding + block_id]; #endif } @@ -625,9 +573,10 @@ struct lookback_scan_state int cache_offset = 0; - prefix_flag flag = this->get_flag(lookback_block_id); + lookback_scan_prefix_flag flag = this->get_flag(lookback_block_id); - while(warp_all(flag != prefix_flag::COMPLETE && flag != prefix_flag::INVALID)) + while(warp_all(flag != lookback_scan_prefix_flag::complete + && flag != lookback_scan_prefix_flag::invalid)) { ++cache_offset; lookback_block_id -= device_warp_size(); @@ -640,9 +589,9 @@ struct lookback_scan_state // - We have run out of available space in the cache. In this case, wait until // any of the current lookback flags pointed to by lookback_block_id changes // to complete. - if(warp_all(flag != prefix_flag::COMPLETE)) + if(warp_all(flag != lookback_scan_prefix_flag::complete)) { - if(warp_all(flag == prefix_flag::INVALID)) + if(warp_all(flag == lookback_scan_prefix_flag::invalid)) { // All invalid, so we have to move one block back to // get back to known civilization. @@ -655,7 +604,7 @@ struct lookback_scan_state { flag = this->get_flag(lookback_block_id); } - while(warp_all(flag != prefix_flag::COMPLETE)); + while(warp_all(flag != lookback_scan_prefix_flag::complete)); } T block_prefix; @@ -664,8 +613,8 @@ struct lookback_scan_state // Now just sum all these values to get the prefix // Note that the values are striped across the threads. // In the first iteration, the current prefix is at the value cache for the current - // offset at the lowest warp number that has prefix_flag::COMPLETE set. - const auto bits = ballot(flag == prefix_flag::COMPLETE); + // offset at the lowest warp number that has lookback_scan_prefix_flag::complete set. + const auto bits = ballot(flag == lookback_scan_prefix_flag::complete); const auto lowest = ctz(bits); // Now sum all the values from block_prefix that are lower than the current prefix. @@ -685,16 +634,17 @@ struct lookback_scan_state } private: - ROCPRIM_DEVICE ROCPRIM_INLINE prefix_flag get_flag(const unsigned int block_id) + ROCPRIM_DEVICE ROCPRIM_INLINE + lookback_scan_prefix_flag get_flag(const unsigned int block_id) { constexpr unsigned int padding = ::rocprim::device_warp_size(); const unsigned int SLEEP_MAX = 32; unsigned int times_through = 1; - prefix_flag flag = static_cast( + lookback_scan_prefix_flag flag = static_cast( ::rocprim::detail::atomic_load(&prefixes_flags[padding + block_id])); - while(flag == prefix_flag::EMPTY) + while(flag == lookback_scan_prefix_flag::empty) { if(UseSleep) { @@ -704,20 +654,21 @@ struct lookback_scan_state times_through++; } - flag = static_cast( + flag = static_cast( ::rocprim::detail::atomic_load(&prefixes_flags[padding + block_id])); } return flag; } - ROCPRIM_DEVICE ROCPRIM_INLINE void - set(const unsigned int block_id, const prefix_flag flag, const T value) + ROCPRIM_DEVICE ROCPRIM_INLINE + void set(const unsigned int block_id, const lookback_scan_prefix_flag flag, const T value) { constexpr unsigned int padding = ::rocprim::device_warp_size(); #if ROCPRIM_DETAIL_LOOKBACK_SCAN_STATE_WITHOUT_SLOW_FENCES auto* values = static_cast( - flag == prefix_flag::PARTIAL ? prefixes_partial_values : prefixes_complete_values); + flag == lookback_scan_prefix_flag::partial ? prefixes_partial_values + : prefixes_complete_values); value_underlying_type v; __builtin_memcpy(&v, &value, sizeof(value)); for(unsigned int i = 0; i < value_underlying_type::words_no; ++i) @@ -727,8 +678,9 @@ struct lookback_scan_state // Wait for all atomic stores of prefixes_*_values before signaling complete / partial state rocprim::detail::atomic_fence_release_vmem_order_only(); #else - auto* values = static_cast(flag == prefix_flag::PARTIAL ? prefixes_partial_values - : prefixes_complete_values); + auto* values = static_cast(flag == lookback_scan_prefix_flag::partial + ? prefixes_partial_values + : prefixes_complete_values); values[padding + block_id] = value; ::rocprim::detail::memory_fence_device(); #endif @@ -771,8 +723,10 @@ class lookback_scan_prefix_op ROCPRIM_DEVICE ROCPRIM_INLINE ~lookback_scan_prefix_op() = default; private: - ROCPRIM_DEVICE ROCPRIM_INLINE void - reduce_partial_prefixes(unsigned int block_id, prefix_flag& flag, T& partial_prefix) + ROCPRIM_DEVICE ROCPRIM_INLINE + void reduce_partial_prefixes(unsigned int block_id, + lookback_scan_prefix_flag& flag, + T& partial_prefix) { // Order of reduction must be reversed, because 0th thread has // prefix from the (block_id_ - 1) block, 1st thread has prefix @@ -785,17 +739,18 @@ class lookback_scan_prefix_op scan_state_.get(block_id, flag, block_prefix); auto headflag_scan_op = headflag_scan_op_type(scan_op_); - warp_reduce_prefix_type().tail_segmented_reduce(block_prefix, - partial_prefix, - (flag == prefix_flag::COMPLETE), - headflag_scan_op); + warp_reduce_prefix_type().tail_segmented_reduce( + block_prefix, + partial_prefix, + (flag == lookback_scan_prefix_flag::complete), + headflag_scan_op); } ROCPRIM_DEVICE ROCPRIM_INLINE T get_prefix() { if ROCPRIM_IF_CONSTEXPR(Determinism == lookback_scan_determinism::nondeterministic) { - prefix_flag flag; + lookback_scan_prefix_flag flag; T partial_prefix; unsigned int previous_block_id = block_id_ - ::rocprim::lane_id() - 1; @@ -805,7 +760,7 @@ class lookback_scan_prefix_op T prefix = partial_prefix; // while we don't load a complete prefix, reduce partial prefixes - while(::rocprim::detail::warp_all(flag != prefix_flag::COMPLETE)) + while(::rocprim::detail::warp_all(flag != lookback_scan_prefix_flag::complete)) { previous_block_id -= ::rocprim::device_warp_size(); reduce_partial_prefixes(previous_block_id, flag, partial_prefix); @@ -845,6 +800,7 @@ class lookback_scan_prefix_op LookbackScanState& scan_state_; }; +// This is a HOST only API // It is known that early revisions of MI100 (gfx908) hang in the wait loop of // lookback_scan_state::get() without sleeping (s_sleep). // is_sleep_scan_state_used() checks the architecture/revision of the device on host in runtime, From a7dec4b46168e829246600855e4d1e57f7707bce Mon Sep 17 00:00:00 2001 From: Anton Gorenko Date: Fri, 17 Jan 2025 16:22:50 +0000 Subject: [PATCH 76/84] Adopt symbol visibility rules in rocPRIM --- .gitlab-ci.yml | 1 + CHANGELOG.md | 1 + CMakeLists.txt | 3 +- rocprim/include/rocprim/config.hpp | 54 +++-- .../device/detail/device_batch_memcpy.hpp | 28 +-- rocprim/include/rocprim/rocprim.hpp | 2 - .../include/rocprim/rocprim_version.hpp.in | 6 +- test/rocprim/CMakeLists.txt | 100 +++++++++ test/rocprim/test_linking.cpp | 42 ++++ test/rocprim/test_linking_lib.cpp | 100 +++++++++ test/rocprim/test_linking_new_scan.hpp | 196 ++++++++++++++++++ 11 files changed, 500 insertions(+), 33 deletions(-) create mode 100644 test/rocprim/test_linking.cpp create mode 100644 test/rocprim/test_linking_lib.cpp create mode 100644 test/rocprim/test_linking_new_scan.hpp diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 67fbcf6a6..772e787a5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -207,6 +207,7 @@ build:cmake-minimum-apt: - $BUILD_DIR/test/CTestTestfile.cmake - $BUILD_DIR/test/rocprim/CTestTestfile.cmake - $BUILD_DIR/test/rocprim/test_* + - $BUILD_DIR/test/rocprim/libtest_* - $BUILD_DIR/test/test_* expire_in: 1 day diff --git a/CHANGELOG.md b/CHANGELOG.md index 65a7cb2c9..46bedbdee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ Full documentation for rocPRIM is available at [https://rocm.docs.amd.com/projec * Undeprecated `rocprim::thread_load` uses dereference as default and updated other cache load modifiers. * Undeprecated `rocprim::thread_store` uses dereference as default and updated other cache store modifiers. +* Changed symbol visibility: all kernels now have hidden visibility, inline namespace with the library version is added, e.g. `rocprim::ROCPRIM_300400_NS::...`. This allows to link multiple libraries built with different versions of rocPRIM. ### Resolved issues diff --git a/CMakeLists.txt b/CMakeLists.txt index 773188d54..4380dea26 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ # MIT License # -# Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -183,6 +183,7 @@ endif() # Setup VERSION set(VERSION_STRING "3.4.0") rocm_setup_version(VERSION ${VERSION_STRING}) +math(EXPR rocprim_VERSION_NUMBER "${rocprim_VERSION_MAJOR} * 100000 + ${rocprim_VERSION_MINOR} * 100 + ${rocprim_VERSION_PATCH}") # Print configuration summary include(cmake/Summary.cmake) diff --git a/rocprim/include/rocprim/config.hpp b/rocprim/include/rocprim/config.hpp index cc69d95e0..11caf44fb 100644 --- a/rocprim/include/rocprim/config.hpp +++ b/rocprim/include/rocprim/config.hpp @@ -21,17 +21,46 @@ #ifndef ROCPRIM_CONFIG_HPP_ #define ROCPRIM_CONFIG_HPP_ -#define BEGIN_ROCPRIM_NAMESPACE \ - namespace rocprim { - -#define END_ROCPRIM_NAMESPACE \ - } /* rocprim */ - #include -#include -#include #include +#include +#include + +#include "rocprim_version.hpp" + +// Inline namespace (e.g. ROCPRIM_300400_NS where 300400 is the rocPRIM version) is used to +// eliminate issues when shared libraries are built with different versions of rocPRIM so they may +// have symbols with the same name but different content. +// ROCPRIM_DISABLE_INLINE_NAMESPACE can be defined to disable inline namespaces (the old behavior). +// ROCPRIM_INLINE_NAMESPACE can be defined to override the standard inline namespace name. +// Additionally, all kernels have hidden visibility (see ROCPRIM_KERNEL). +#if defined(DOXYGEN_DOCUMENTATION_BUILD) || defined(ROCPRIM_DISABLE_INLINE_NAMESPACE) + #define ROCPRIM_INLINE_NAMESPACE + #define BEGIN_ROCPRIM_INLINE_NAMESPACE + #define END_ROCPRIM_INLINE_NAMESPACE +#else + #define ROCPRIM_CONCAT_(SEP, A, B) A##SEP##B + #define ROCPRIM_CONCAT(SEP, A, B) ROCPRIM_CONCAT_(SEP, A, B) + + #ifndef ROCPRIM_INLINE_NAMESPACE + #define ROCPRIM_INLINE_NAMESPACE \ + ROCPRIM_CONCAT(_, ROCPRIM, ROCPRIM_CONCAT(_, ROCPRIM_VERSION, NS)) + #endif + #define BEGIN_ROCPRIM_INLINE_NAMESPACE \ + inline namespace ROCPRIM_INLINE_NAMESPACE \ + { + #define END_ROCPRIM_INLINE_NAMESPACE } /* inline namespace */ +#endif + +#define BEGIN_ROCPRIM_NAMESPACE \ + namespace rocprim \ + { \ + BEGIN_ROCPRIM_INLINE_NAMESPACE + +#define END_ROCPRIM_NAMESPACE \ + END_ROCPRIM_INLINE_NAMESPACE \ + } /* namespace rocprim */ #if __cplusplus < 201402L #error "rocPRIM requires at least C++14" @@ -42,11 +71,6 @@ #define ROCPRIM_HOST __host__ #define ROCPRIM_HOST_DEVICE __host__ __device__ #define ROCPRIM_SHARED_MEMORY __shared__ - #ifdef _WIN32 - #define ROCPRIM_KERNEL __global__ static - #else - #define ROCPRIM_KERNEL __global__ - #endif // TODO: These parameters should be tuned for NAVI in the close future. #ifndef ROCPRIM_DEFAULT_MAX_BLOCK_SIZE #define ROCPRIM_DEFAULT_MAX_BLOCK_SIZE 256 @@ -56,16 +80,18 @@ #endif #ifndef DOXYGEN_DOCUMENTATION_BUILD + #define ROCPRIM_FORCE_INLINE __attribute__((always_inline)) #define ROCPRIM_INLINE inline + #define ROCPRIM_KERNEL __global__ __attribute__((__visibility__("hidden"))) #define ROCPRIM_LAUNCH_BOUNDS(...) __launch_bounds__(__VA_ARGS__) #else // Prefer simpler signatures to let Sphinx/Breathe parse them #define ROCPRIM_FORCE_INLINE inline #define ROCPRIM_INLINE inline + #define ROCPRIM_KERNEL __global__ // Ignore __launch_bounds__ for doxygen builds #define ROCPRIM_LAUNCH_BOUNDS(...) #endif - #define ROCPRIM_FORCE_INLINE __attribute__((always_inline)) #endif #undef ROCPRIM_TARGET_GCN3 diff --git a/rocprim/include/rocprim/device/detail/device_batch_memcpy.hpp b/rocprim/include/rocprim/device/detail/device_batch_memcpy.hpp index 902545fab..283ae9eeb 100644 --- a/rocprim/include/rocprim/device/detail/device_batch_memcpy.hpp +++ b/rocprim/include/rocprim/device/detail/device_batch_memcpy.hpp @@ -1,6 +1,6 @@ /****************************************************************************** * Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2023-2024, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2023-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -900,9 +900,10 @@ struct batch_memcpy_impl }; public: - __global__ static void init_tile_state_kernel(blev_buffer_scan_state_type buffer_scan_state, - blev_block_scan_state_type block_scan_state, - tile_offset_type num_tiles) + static ROCPRIM_KERNEL + void init_tile_state_kernel(blev_buffer_scan_state_type buffer_scan_state, + blev_block_scan_state_type block_scan_state, + tile_offset_type num_tiles) { const uint32_t block_id = rocprim::detail::block_id<0>(); const uint32_t block_size = rocprim::detail::block_size<0>(); @@ -914,12 +915,12 @@ struct batch_memcpy_impl block_scan_state.initialize_prefix(flat_thread_id, num_tiles); } - __global__ static void - non_blev_memcpy_kernel(copyable_buffers buffers, - buffer_offset_type num_buffers, - copyable_blev_buffers blev_buffers, - blev_buffer_scan_state_type blev_buffer_scan_state, - blev_block_scan_state_type blev_block_scan_state) + static ROCPRIM_KERNEL + void non_blev_memcpy_kernel(copyable_buffers buffers, + buffer_offset_type num_buffers, + copyable_blev_buffers blev_buffers, + blev_buffer_scan_state_type blev_buffer_scan_state, + blev_block_scan_state_type blev_block_scan_state) { ROCPRIM_SHARED_MEMORY typename non_blev_memcpy::storage_type temp_storage; non_blev_memcpy{}.copy(temp_storage.get(), @@ -931,9 +932,10 @@ struct batch_memcpy_impl rocprim::flat_block_id()); } - __global__ static void blev_memcpy_kernel(copyable_blev_buffers blev_buffers, - blev_buffer_scan_state_type buffer_offset_tile, - tile_offset_type last_tile_offset) + static ROCPRIM_KERNEL + void blev_memcpy_kernel(copyable_blev_buffers blev_buffers, + blev_buffer_scan_state_type buffer_offset_tile, + tile_offset_type last_tile_offset) { const auto flat_block_thread_id = ::rocprim::detail::block_thread_id<0>(); const auto flat_block_id = ::rocprim::detail::block_id<0>(); diff --git a/rocprim/include/rocprim/rocprim.hpp b/rocprim/include/rocprim/rocprim.hpp index 5fa4410dc..37f1571a5 100644 --- a/rocprim/include/rocprim/rocprim.hpp +++ b/rocprim/include/rocprim/rocprim.hpp @@ -28,8 +28,6 @@ // Meta configuration for rocPRIM #include "config.hpp" -#include "rocprim_version.hpp" - #include "intrinsics.hpp" #include "functional.hpp" #include "types.hpp" diff --git a/rocprim/include/rocprim/rocprim_version.hpp.in b/rocprim/include/rocprim/rocprim_version.hpp.in index 67c5fe85c..0a3b673c5 100644 --- a/rocprim/include/rocprim/rocprim_version.hpp.in +++ b/rocprim/include/rocprim/rocprim_version.hpp.in @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -22,7 +22,7 @@ #define ROCPRIM_VERSION_HPP_ /// \def ROCPRIM_VERSION -/// \brief ROCPRIM library version +/// \brief rocPRIM library version /// /// Version number may not be visible in the documentation. /// @@ -32,7 +32,7 @@ /// /// For example, if ROCPRIM_VERSION is 100500, then the major version is 1, /// the minor version is 5, and the patch level is 0. -#define ROCPRIM_VERSION @rocprim_VERSION_MAJOR@ * 100000 + @rocprim_VERSION_MINOR@ * 100 + @rocprim_VERSION_PATCH@ +#define ROCPRIM_VERSION @rocprim_VERSION_NUMBER@ #define ROCPRIM_VERSION_MAJOR @rocprim_VERSION_MAJOR@ #define ROCPRIM_VERSION_MINOR @rocprim_VERSION_MINOR@ diff --git a/test/rocprim/CMakeLists.txt b/test/rocprim/CMakeLists.txt index 87542b75b..baa85d829 100644 --- a/test/rocprim/CMakeLists.txt +++ b/test/rocprim/CMakeLists.txt @@ -326,3 +326,103 @@ add_rocprim_test("rocprim.warp_scan" test_warp_scan.cpp) add_rocprim_test("rocprim.warp_sort" test_warp_sort.cpp) add_rocprim_test("rocprim.warp_store" test_warp_store.cpp) add_rocprim_test("rocprim.zip_iterator" test_zip_iterator.cpp) + +if(NOT WIN32) + # Linking tests check if all external rocPRIM symbols are in the inline namespace, kernel are not + # visible and if it is possible to link two libraries that use different rocPRIM versions. + # See comments in test_linking_new_scan.hpp for more details. + + # For testing how the old behavior (without inline namespace) works set it to ON, the tests will + # fail + set(WITHOUT_INLINE_NAMESPACE OFF) + + if(WITHOUT_INLINE_NAMESPACE) + set(COMMON_DEFINITIONS "ROCPRIM_DISABLE_INLINE_NAMESPACE") + endif() + + add_library(test_linking_lib1 SHARED test_linking_lib.cpp) + target_include_directories(test_linking_lib1 SYSTEM BEFORE + PUBLIC + $ + $ + ) + target_link_libraries(test_linking_lib1 PUBLIC rocprim_hip) + set_target_properties(test_linking_lib1 + PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/test/rocprim" + COMPILE_DEFINITIONS "TEST_FUNC=test1;${COMMON_DEFINITIONS};" + ) + + add_library(test_linking_lib2 SHARED test_linking_lib.cpp) + target_include_directories(test_linking_lib2 SYSTEM BEFORE + PUBLIC + $ + $ + ) + target_link_libraries(test_linking_lib2 PUBLIC rocprim_hip) + # Pretend that we build this library with another version of rocPRIM which uses another + # implementation of scan + if(NOT WITHOUT_INLINE_NAMESPACE) + set(LIB2_NAMESPACE_DEFINITION "ROCPRIM_INLINE_NAMESPACE=ROCPRIM_300201_NS") + endif() + set_target_properties(test_linking_lib2 + PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/test/rocprim" + COMPILE_DEFINITIONS "TEST_FUNC=test2;TEST_LINKING_EMULATE_ANOTHER_VERSION;${COMMON_DEFINITIONS};${LIB2_NAMESPACE_DEFINITION};" + ) + + add_rocprim_test_internal("rocprim.linking" "test_linking.cpp;test_linking_lib.cpp" test_linking) + target_link_libraries(test_linking PUBLIC test_linking_lib1 test_linking_lib2) + set_target_properties(test_linking + PROPERTIES + COMPILE_DEFINITIONS "TEST_FUNC=test0;${COMMON_DEFINITIONS};" + ) + + # Check what symbols are exported by the libraries and what their names are + + set(NM_ARGS --demangle --defined-only --extern-only) + + add_custom_command( + TARGET test_linking_lib1 + POST_BUILD + COMMAND ${CMAKE_NM} ${NM_ARGS} $ | grep "rocprim::" + COMMENT "Printing external symbols of test_linking_lib1" + ) + add_custom_command( + TARGET test_linking_lib2 + POST_BUILD + COMMAND ${CMAKE_NM} ${NM_ARGS} $ | grep "rocprim::" + COMMENT "Printing external symbols of test_linking_lib2" + ) + add_custom_command( + TARGET test_linking + POST_BUILD + COMMAND ${CMAKE_NM} ${NM_ARGS} $ | grep "rocprim::" + COMMENT "Printing external symbols of test_linking" + ) + + # The tests fail if + # * there are symbols in the rocprim namespace but not in the inline namespace with the + # corresponding rocPRIM version; + # * any kernels are visible (usually they have "kernel" in their names). + string(REPLACE ";" " " NM_ARGS_STR "${NM_ARGS}") + add_test( + NAME "rocprim.linking_lib1_symbols" + COMMAND sh -c "${CMAKE_NM} ${NM_ARGS_STR} $ | grep -c -P \"(rocprim::(?!ROCPRIM_${rocprim_VERSION_NUMBER}_NS))|kernel\"" + ) + add_test( + NAME "rocprim.linking_lib2_symbols" + COMMAND sh -c "${CMAKE_NM} ${NM_ARGS_STR} $ | grep -c -P \"(rocprim::(?!ROCPRIM_300201_NS))|kernel\"" + ) + add_test( + NAME "rocprim.linking_symbols" + COMMAND sh -c "${CMAKE_NM} ${NM_ARGS_STR} $ | grep -c -P \"(rocprim::(?!ROCPRIM_${rocprim_VERSION_NUMBER}_NS))|kernel\"" + ) + set_tests_properties( + "rocprim.linking_lib1_symbols" + "rocprim.linking_lib2_symbols" + "rocprim.linking_symbols" + PROPERTIES + PASS_REGULAR_EXPRESSION "^0" # no such symbols must be found (grep -c returns 0) + ) +endif() diff --git a/test/rocprim/test_linking.cpp b/test/rocprim/test_linking.cpp new file mode 100644 index 000000000..6ee7d0700 --- /dev/null +++ b/test/rocprim/test_linking.cpp @@ -0,0 +1,42 @@ +// MIT License +// +// Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "../common_test_header.hpp" + +extern void test0(size_t size); +extern void test1(size_t size); +extern void test2(size_t size); + +TEST(TestLinking, Test0) +{ + test0(34); +} + +TEST(TestLinking, Test1) +{ + test1(45); +} + +TEST(TestLinking, Test2) +{ + test2(56); +} diff --git a/test/rocprim/test_linking_lib.cpp b/test/rocprim/test_linking_lib.cpp new file mode 100644 index 000000000..8dab403da --- /dev/null +++ b/test/rocprim/test_linking_lib.cpp @@ -0,0 +1,100 @@ +// MIT License +// +// Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#ifdef TEST_LINKING_EMULATE_ANOTHER_VERSION + // Pretend that another version of rocPRIM is used with a different implementation if scan + #define ROCPRIM_DEVICE_DEVICE_SCAN_HPP_ + #include "test_linking_new_scan.hpp" +#else + #include +#endif + +#include "../common_test_header.hpp" +#include "test_utils_device_ptr.hpp" +#include "test_utils_types.hpp" + +#include + +#ifndef TEST_FUNC + #define TEST_FUNC test0 +#endif + +void TEST_FUNC(size_t size) +{ + using T = int; + + const int seed_value = 123; + + // Generate data + std::vector input = test_utils::get_random_data(size, 0, 100, seed_value); + + test_utils::device_ptr d_input(input); + test_utils::device_ptr d_output(input.size()); + + std::vector expected(size); + // Calculate expected results on host +#ifdef TEST_LINKING_EMULATE_ANOTHER_VERSION + // The "new" version of the function fills the output with a constant value + std::fill(expected.begin(), expected.end(), 12345); +#else + test_utils::host_inclusive_scan(input.begin(), + input.end(), + expected.begin(), + rocprim::plus()); +#endif + + size_t temp_storage_size_bytes; + // Get size of d_temp_storage + HIP_CHECK(rocprim::inclusive_scan(nullptr, + temp_storage_size_bytes, + d_input.get(), + d_output.get(), + input.size(), + rocprim::plus())); + +#ifdef TEST_LINKING_EMULATE_ANOTHER_VERSION + // The "new" version of the function requests this particular size of temporary storage + EXPECT_EQ(temp_storage_size_bytes, 12345); +#else + EXPECT_LT(temp_storage_size_bytes, 12345); +#endif + + // allocate temporary storage + test_utils::device_ptr d_temp_storage(temp_storage_size_bytes); + + // Run + HIP_CHECK(rocprim::inclusive_scan(d_temp_storage.get(), + temp_storage_size_bytes, + d_input.get(), + d_output.get(), + input.size(), + rocprim::plus())); + + HIP_CHECK(hipGetLastError()); + HIP_CHECK(hipDeviceSynchronize()); + + // Copy output to host + const auto output = d_output.load(); + + // Check if output values are as expected + ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output, expected)); +} diff --git a/test/rocprim/test_linking_new_scan.hpp b/test/rocprim/test_linking_new_scan.hpp new file mode 100644 index 000000000..cb2745103 --- /dev/null +++ b/test/rocprim/test_linking_new_scan.hpp @@ -0,0 +1,196 @@ +// MIT License +// +// Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#ifndef TEST_LINKING_NEW_SCAN_HPP_ +#define TEST_LINKING_NEW_SCAN_HPP_ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +// This file contains simplified versions of functions and kernels from +// rocprim/device/device_scan.hpp. The idea is to emulate a situation when two libraries use +// different versions of rocPRIM. Their implementations are intentionally different: the kernel +// fills the output with a constant value, and the function requests a different size of temporary +// storage. +// The functions/kernels here MUST have exactly the same signatures as in the original file. +// Then, if the inline namespace is used, the linker is able to distinguish between two +// implementations of one function. Otherwise, one of the tests in test_linking.cpp MAY fail +// because the linker uses an incorrect implementation of the function (as they share one name). + +BEGIN_ROCPRIM_NAMESPACE + +namespace detail +{ + +template +ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void single_scan_kernel_impl(InputIterator input, + const size_t input_size, + AccType initial_value, + OutputIterator output, + BinaryFunction scan_op) +{ + (void)input; + (void)initial_value; + (void)scan_op; + + static constexpr scan_config_params params = device_params(); + + constexpr unsigned int block_size = params.kernel_config.block_size; + constexpr unsigned int items_per_thread = params.kernel_config.items_per_thread; + + AccType values[items_per_thread]; + for(unsigned int i = 0; i < items_per_thread; ++i) + { + values[i] = 12345; + } + block_store_direct_striped(flat_block_thread_id(), output, values, input_size); +} + +template +ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(device_params().kernel_config.block_size) void + single_scan_kernel(InputIterator input, + const size_t size, + const InitValueType initial_value, + OutputIterator output, + BinaryFunction scan_op) +{ + single_scan_kernel_impl(input, + size, + static_cast(get_input_value(initial_value)), + output, + scan_op); +} + +template +inline auto scan_impl(void* temporary_storage, + size_t& storage_size, + InputIterator input, + OutputIterator output, + const InitValueType initial_value, + const size_t size, + BinaryFunction scan_op, + const hipStream_t stream, + bool debug_synchronous) +{ + (void)debug_synchronous; + + using config = wrapped_scan_config; + + detail::target_arch target_arch; + hipError_t result = host_target_arch(stream, target_arch); + if(result != hipSuccess) + { + return result; + } + const scan_config_params params = dispatch_target_arch(target_arch); + + const unsigned int block_size = params.kernel_config.block_size; + const unsigned int items_per_thread = params.kernel_config.items_per_thread; + const auto items_per_block = block_size * items_per_thread; + + if(temporary_storage == nullptr) + { + storage_size = 12345; + return hipSuccess; + } + + const unsigned int number_of_blocks = (size + items_per_block - 1) / items_per_block; + if(number_of_blocks > 1) + { + return hipErrorInvalidValue; + } + + single_scan_kernel + <<>>(input, size, initial_value, output, scan_op); + return hipGetLastError(); +} + +} // namespace detail + +template::value_type>, + class AccType = typename std::iterator_traits::value_type> +inline hipError_t inclusive_scan(void* temporary_storage, + size_t& storage_size, + InputIterator input, + OutputIterator output, + const size_t size, + BinaryFunction scan_op = BinaryFunction(), + const hipStream_t stream = 0, + bool debug_synchronous = false) +{ + return detail::scan_impl(temporary_storage, + storage_size, + input, + output, + AccType{}, + size, + scan_op, + stream, + debug_synchronous); +} + +END_ROCPRIM_NAMESPACE + +#endif // TEST_LINKING_NEW_SCAN_HPP_ From b38f541f1d56ab4bcc6a6e586086a081dff96a17 Mon Sep 17 00:00:00 2001 From: Robin Voetter Date: Mon, 20 Jan 2025 12:41:16 +0100 Subject: [PATCH 77/84] remove nm debug messages from cmake --- test/rocprim/CMakeLists.txt | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/test/rocprim/CMakeLists.txt b/test/rocprim/CMakeLists.txt index baa85d829..2394a5ca1 100644 --- a/test/rocprim/CMakeLists.txt +++ b/test/rocprim/CMakeLists.txt @@ -379,28 +379,11 @@ if(NOT WIN32) ) # Check what symbols are exported by the libraries and what their names are + # nm --demangle --defined-only --extern-only is used to print + # the symbols that are exported from a library. set(NM_ARGS --demangle --defined-only --extern-only) - add_custom_command( - TARGET test_linking_lib1 - POST_BUILD - COMMAND ${CMAKE_NM} ${NM_ARGS} $ | grep "rocprim::" - COMMENT "Printing external symbols of test_linking_lib1" - ) - add_custom_command( - TARGET test_linking_lib2 - POST_BUILD - COMMAND ${CMAKE_NM} ${NM_ARGS} $ | grep "rocprim::" - COMMENT "Printing external symbols of test_linking_lib2" - ) - add_custom_command( - TARGET test_linking - POST_BUILD - COMMAND ${CMAKE_NM} ${NM_ARGS} $ | grep "rocprim::" - COMMENT "Printing external symbols of test_linking" - ) - # The tests fail if # * there are symbols in the rocprim namespace but not in the inline namespace with the # corresponding rocPRIM version; From 44d19afb4470915738744a5c8f1d1e4b9dc37854 Mon Sep 17 00:00:00 2001 From: Sander Bos Date: Mon, 20 Jan 2025 08:16:17 +0000 Subject: [PATCH 78/84] Remove extra dashes in the documentation its \param and \tparam --- .../block/block_adjacent_difference.hpp | 246 +++++++------- .../rocprim/block/block_discontinuity.hpp | 138 ++++---- .../include/rocprim/block/block_exchange.hpp | 196 +++++------ .../include/rocprim/block/block_histogram.hpp | 44 +-- rocprim/include/rocprim/block/block_load.hpp | 68 ++-- .../include/rocprim/block/block_load_func.hpp | 158 ++++----- .../rocprim/block/block_radix_rank.hpp | 134 ++++---- .../rocprim/block/block_radix_sort.hpp | 140 ++++---- .../include/rocprim/block/block_reduce.hpp | 72 ++-- rocprim/include/rocprim/block/block_scan.hpp | 318 +++++++++--------- .../include/rocprim/block/block_shuffle.hpp | 142 ++++---- rocprim/include/rocprim/block/block_sort.hpp | 88 ++--- rocprim/include/rocprim/block/block_store.hpp | 42 +-- .../rocprim/block/block_store_func.hpp | 100 +++--- .../include/rocprim/detail/temp_storage.hpp | 46 +-- .../include/rocprim/device/config_types.hpp | 14 +- .../device/detail/device_config_helper.hpp | 110 +++--- .../detail/device_run_length_encode.hpp | 14 +- .../rocprim/device/device_binary_search.hpp | 92 ++--- .../include/rocprim/device/device_copy.hpp | 4 +- .../rocprim/device/device_copy_config.hpp | 16 +- .../rocprim/device/device_histogram.hpp | 250 +++++++------- .../include/rocprim/device/device_memcpy.hpp | 4 +- .../rocprim/device/device_memcpy_config.hpp | 16 +- .../include/rocprim/device/device_merge.hpp | 70 ++-- .../rocprim/device/device_merge_sort.hpp | 54 +-- .../device/device_merge_sort_config.hpp | 16 +- .../device/device_partial_sort_config.hpp | 8 +- .../rocprim/device/device_partition.hpp | 162 ++++----- .../device/device_radix_sort_config.hpp | 10 +- .../include/rocprim/device/device_reduce.hpp | 54 +-- .../rocprim/device/device_reduce_by_key.hpp | 40 +-- .../device/device_run_length_encode.hpp | 56 +-- .../device_run_length_encode_config.hpp | 6 +- .../include/rocprim/device/device_scan.hpp | 58 ++-- .../rocprim/device/device_scan_by_key.hpp | 74 ++-- .../device/device_segmented_radix_sort.hpp | 266 +++++++-------- .../device_segmented_radix_sort_config.hpp | 6 +- .../device/device_segmented_reduce.hpp | 34 +- .../rocprim/device/device_segmented_scan.hpp | 126 +++---- .../include/rocprim/device/device_select.hpp | 154 ++++----- .../rocprim/device/device_transform.hpp | 46 +-- rocprim/include/rocprim/intrinsics/thread.hpp | 4 +- rocprim/include/rocprim/intrinsics/warp.hpp | 4 +- .../rocprim/intrinsics/warp_shuffle.hpp | 46 +-- .../rocprim/iterator/arg_index_iterator.hpp | 14 +- .../rocprim/iterator/constant_iterator.hpp | 14 +- .../rocprim/iterator/counting_iterator.hpp | 12 +- .../rocprim/iterator/discard_iterator.hpp | 6 +- .../rocprim/iterator/reverse_iterator.hpp | 8 +- .../iterator/texture_cache_iterator.hpp | 12 +- .../rocprim/iterator/transform_iterator.hpp | 16 +- .../include/rocprim/iterator/zip_iterator.hpp | 6 +- rocprim/include/rocprim/type_traits.hpp | 6 +- .../rocprim/types/integer_sequence.hpp | 10 +- rocprim/include/rocprim/types/tuple.hpp | 30 +- .../rocprim/warp/detail/warp_sort_stable.hpp | 4 +- .../include/rocprim/warp/warp_exchange.hpp | 46 +-- rocprim/include/rocprim/warp/warp_load.hpp | 34 +- rocprim/include/rocprim/warp/warp_reduce.hpp | 58 ++-- rocprim/include/rocprim/warp/warp_scan.hpp | 114 +++---- rocprim/include/rocprim/warp/warp_sort.hpp | 66 ++-- rocprim/include/rocprim/warp/warp_store.hpp | 24 +- 63 files changed, 2113 insertions(+), 2113 deletions(-) diff --git a/rocprim/include/rocprim/block/block_adjacent_difference.hpp b/rocprim/include/rocprim/block/block_adjacent_difference.hpp index 9591fcf4f..f8c2154de 100644 --- a/rocprim/include/rocprim/block/block_adjacent_difference.hpp +++ b/rocprim/include/rocprim/block/block_adjacent_difference.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. -* Modifications Copyright (c) 2022-2024, Advanced Micro Devices, Inc. All rights reserved. +* Modifications Copyright (c) 2022-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -47,8 +47,8 @@ BEGIN_ROCPRIM_NAMESPACE /// methods for applying binary functions for pairs of consecutive items partition across a thread /// block. /// -/// \tparam T - the input type. -/// \tparam BlockSize - the number of threads in a block. +/// \tparam T the input type. +/// \tparam BlockSize the number of threads in a block. /// /// \par Overview /// * There are two types of flags: @@ -126,19 +126,19 @@ class block_adjacent_difference /// flagged. /// \deprecated The flags API of block_adjacent_difference is deprecated, /// use subtract_left() or block_discontinuity::flag_heads() instead. - /// \tparam ItemsPerThread - [inferred] the number of items to be processed by + /// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread. - /// \tparam Flag - [inferred] the flag type. - /// \tparam FlagOp - [inferred] type of binary function used for flagging. + /// \tparam Flag [inferred] the flag type. + /// \tparam FlagOp [inferred] type of binary function used for flagging. /// - /// \param [out] head_flags - array that contains the head flags. - /// \param [in] input - array that data is loaded from. - /// \param [in] flag_op - binary operation function object that will be used for flagging. + /// \param [out] head_flags array that contains the head flags. + /// \param [in] input array that data is loaded from. + /// \param [in] flag_op binary operation function object that will be used for flagging. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b); or bool (const T& a, const T& b, unsigned int b_index);. /// The signature does not need to have const &, but function object /// must not modify the objects passed to it. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reuse /// Synchronization barrier should be placed before \p storage is reused @@ -203,21 +203,21 @@ class block_adjacent_difference /// \deprecated The flags API of block_adjacent_difference is deprecated, /// use subtract_left() or block_discontinuity::flag_heads() instead. /// - /// \tparam ItemsPerThread - [inferred] the number of items to be processed by + /// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread. - /// \tparam Flag - [inferred] the flag type. - /// \tparam FlagOp - [inferred] type of binary function used for flagging. + /// \tparam Flag [inferred] the flag type. + /// \tparam FlagOp [inferred] type of binary function used for flagging. /// - /// \param [out] head_flags - array that contains the head flags. - /// \param [in] tile_predecessor_item - first tile item from thread to be compared + /// \param [out] head_flags array that contains the head flags. + /// \param [in] tile_predecessor_item first tile item from thread to be compared /// against. - /// \param [in] input - array that data is loaded from. - /// \param [in] flag_op - binary operation function object that will be used for flagging. + /// \param [in] input array that data is loaded from. + /// \param [in] flag_op binary operation function object that will be used for flagging. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b); or bool (const T& a, const T& b, unsigned int b_index);. /// The signature does not need to have const &, but function object /// must not modify the objects passed to it. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reuse /// Synchronization barrier should be placed before \p storage is reused @@ -291,19 +291,19 @@ class block_adjacent_difference /// \deprecated The flags API of block_adjacent_difference is deprecated, /// use subtract_right() or block_discontinuity::flag_tails() instead. /// - /// \tparam ItemsPerThread - [inferred] the number of items to be processed by + /// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread. - /// \tparam Flag - [inferred] the flag type. - /// \tparam FlagOp - [inferred] type of binary function used for flagging. + /// \tparam Flag [inferred] the flag type. + /// \tparam FlagOp [inferred] type of binary function used for flagging. /// - /// \param [out] tail_flags - array that contains the tail flags. - /// \param [in] input - array that data is loaded from. - /// \param [in] flag_op - binary operation function object that will be used for flagging. + /// \param [out] tail_flags array that contains the tail flags. + /// \param [in] input array that data is loaded from. + /// \param [in] flag_op binary operation function object that will be used for flagging. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b); or bool (const T& a, const T& b, unsigned int b_index);. /// The signature does not need to have const &, but function object /// must not modify the objects passed to it. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reuse /// Synchronization barrier should be placed before \p storage is reused @@ -369,21 +369,21 @@ class block_adjacent_difference /// \deprecated The flags API of block_adjacent_difference is deprecated, /// use subtract_right() or block_discontinuity::flag_tails() instead. /// - /// \tparam ItemsPerThread - [inferred] the number of items to be processed by + /// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread. - /// \tparam Flag - [inferred] the flag type. - /// \tparam FlagOp - [inferred] type of binary function used for flagging. + /// \tparam Flag [inferred] the flag type. + /// \tparam FlagOp [inferred] type of binary function used for flagging. /// - /// \param [out] tail_flags - array that contains the tail flags. - /// \param [in] tile_successor_item - last tile item from thread to be compared + /// \param [out] tail_flags array that contains the tail flags. + /// \param [in] tile_successor_item last tile item from thread to be compared /// against. - /// \param [in] input - array that data is loaded from. - /// \param [in] flag_op - binary operation function object that will be used for flagging. + /// \param [in] input array that data is loaded from. + /// \param [in] flag_op binary operation function object that will be used for flagging. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b); or bool (const T& a, const T& b, unsigned int b_index);. /// The signature does not need to have const &, but function object /// must not modify the objects passed to it. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reuse /// Synchronization barrier should be placed before \p storage is reused @@ -454,20 +454,20 @@ class block_adjacent_difference /// \brief Tags both \p head_flags and\p tail_flags that indicate discontinuities /// between items partitioned across the thread block. /// - /// \tparam ItemsPerThread - [inferred] the number of items to be processed by + /// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread. - /// \tparam Flag - [inferred] the flag type. - /// \tparam FlagOp - [inferred] type of binary function used for flagging. + /// \tparam Flag [inferred] the flag type. + /// \tparam FlagOp [inferred] type of binary function used for flagging. /// - /// \param [out] head_flags - array that contains the head flags. - /// \param [out] tail_flags - array that contains the tail flags. - /// \param [in] input - array that data is loaded from. - /// \param [in] flag_op - binary operation function object that will be used for flagging. + /// \param [out] head_flags array that contains the head flags. + /// \param [out] tail_flags array that contains the tail flags. + /// \param [in] input array that data is loaded from. + /// \param [in] flag_op binary operation function object that will be used for flagging. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b); or bool (const T& a, const T& b, unsigned int b_index);. /// The signature does not need to have const &, but function object /// must not modify the objects passed to it. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reuse /// Synchronization barrier should be placed before \p storage is reused @@ -550,22 +550,22 @@ class block_adjacent_difference /// \deprecated The flags API of block_adjacent_difference is deprecated, /// use block_discontinuity::flag_heads_and_tails() instead. /// - /// \tparam ItemsPerThread - [inferred] the number of items to be processed by + /// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread. - /// \tparam Flag - [inferred] the flag type. - /// \tparam FlagOp - [inferred] type of binary function used for flagging. + /// \tparam Flag [inferred] the flag type. + /// \tparam FlagOp [inferred] type of binary function used for flagging. /// - /// \param [out] head_flags - array that contains the head flags. - /// \param [out] tail_flags - array that contains the tail flags. - /// \param [in] tile_successor_item - last tile item from thread to be compared + /// \param [out] head_flags array that contains the head flags. + /// \param [out] tail_flags array that contains the tail flags. + /// \param [in] tile_successor_item last tile item from thread to be compared /// against. - /// \param [in] input - array that data is loaded from. - /// \param [in] flag_op - binary operation function object that will be used for flagging. + /// \param [in] input array that data is loaded from. + /// \param [in] flag_op binary operation function object that will be used for flagging. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b); or bool (const T& a, const T& b, unsigned int b_index);. /// The signature does not need to have const &, but function object /// must not modify the objects passed to it. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reuse /// Synchronization barrier should be placed before \p storage is reused @@ -656,22 +656,22 @@ class block_adjacent_difference /// \deprecated The flags API of block_adjacent_difference is deprecated, /// use block_discontinuity::flag_heads_and_tails() instead. /// - /// \tparam ItemsPerThread - [inferred] the number of items to be processed by + /// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread. - /// \tparam Flag - [inferred] the flag type. - /// \tparam FlagOp - [inferred] type of binary function used for flagging. + /// \tparam Flag [inferred] the flag type. + /// \tparam FlagOp [inferred] type of binary function used for flagging. /// - /// \param [out] head_flags - array that contains the head flags. - /// \param [in] tile_predecessor_item - first tile item from thread to be compared + /// \param [out] head_flags array that contains the head flags. + /// \param [in] tile_predecessor_item first tile item from thread to be compared /// against. - /// \param [out] tail_flags - array that contains the tail flags. - /// \param [in] input - array that data is loaded from. - /// \param [in] flag_op - binary operation function object that will be used for flagging. + /// \param [out] tail_flags array that contains the tail flags. + /// \param [in] input array that data is loaded from. + /// \param [in] flag_op binary operation function object that will be used for flagging. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b); or bool (const T& a, const T& b, unsigned int b_index);. /// The signature does not need to have const &, but function object /// must not modify the objects passed to it. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reuse /// Synchronization barrier should be placed before \p storage is reused @@ -763,24 +763,24 @@ class block_adjacent_difference /// \deprecated The flags API of block_adjacent_difference is deprecated, /// use block_discontinuity::flag_heads_and_tails() instead. /// - /// \tparam ItemsPerThread - [inferred] the number of items to be processed by + /// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread. - /// \tparam Flag - [inferred] the flag type. - /// \tparam FlagOp - [inferred] type of binary function used for flagging. + /// \tparam Flag [inferred] the flag type. + /// \tparam FlagOp [inferred] type of binary function used for flagging. /// - /// \param [out] head_flags - array that contains the head flags. - /// \param [in] tile_predecessor_item - first tile item from thread to be compared + /// \param [out] head_flags array that contains the head flags. + /// \param [in] tile_predecessor_item first tile item from thread to be compared /// against. - /// \param [out] tail_flags - array that contains the tail flags. - /// \param [in] tile_successor_item - last tile item from thread to be compared + /// \param [out] tail_flags array that contains the tail flags. + /// \param [in] tile_successor_item last tile item from thread to be compared /// against. - /// \param [in] input - array that data is loaded from. - /// \param [in] flag_op - binary operation function object that will be used for flagging. + /// \param [in] input array that data is loaded from. + /// \param [in] flag_op binary operation function object that will be used for flagging. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b); or bool (const T& a, const T& b, unsigned int b_index);. /// The signature does not need to have const &, but function object /// must not modify the objects passed to it. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reuse /// Synchronization barrier should be placed before \p storage is reused @@ -883,12 +883,12 @@ class block_adjacent_difference /// output[i] = op(input[i], input[i-1]); /// \endcode /// - /// \tparam Output - [inferred] the type of output, must be assignable from the result of `op` - /// \tparam ItemsPerThread - [inferred] the number of items processed by each thread - /// \tparam BinaryFunction - [inferred] the type of the function to apply - /// \param [in] input - array that data is loaded from partitioned across the threads in the block - /// \param [out] output - array where the result of function application will be written to - /// \param [in] op - binary function applied to the items. + /// \tparam Output [inferred] the type of output, must be assignable from the result of `op` + /// \tparam ItemsPerThread [inferred] the number of items processed by each thread + /// \tparam BinaryFunction [inferred] the type of the function to apply + /// \param [in] input array that data is loaded from partitioned across the threads in the block + /// \param [out] output array where the result of function application will be written to + /// \param [in] op binary function applied to the items. /// The signature of the function should be equivalent to the following: /// `bool f(const T &a, const T &b)` The signature does not need to have /// `const &` but the function object must not modify the objects passed to it. @@ -921,18 +921,18 @@ class block_adjacent_difference /// output[i] = op(input[i], input[i-1]); /// \endcode /// - /// \tparam Output - [inferred] the type of output, must be assignable from the result of `op` - /// \tparam ItemsPerThread - [inferred] the number of items processed by each thread - /// \tparam BinaryFunction - [inferred] the type of the function to apply - /// \param [in] input - array that data is loaded from partitioned across the threads in the block - /// \param [out] output - array where the result of function application will be written to - /// \param [in] op - binary function applied to the items. + /// \tparam Output [inferred] the type of output, must be assignable from the result of `op` + /// \tparam ItemsPerThread [inferred] the number of items processed by each thread + /// \tparam BinaryFunction [inferred] the type of the function to apply + /// \param [in] input array that data is loaded from partitioned across the threads in the block + /// \param [out] output array where the result of function application will be written to + /// \param [in] op binary function applied to the items. /// The signature of the function should be equivalent to the following: /// `bool f(const T &a, const T &b)` The signature does not need to have /// `const &` but the function object must not modify the objects passed to it. - /// \param [in] tile_predecessor - the item before the tile, will be used as the input + /// \param [in] tile_predecessor the item before the tile, will be used as the input /// of the first application of `op` - /// \param storage - reference to a temporary storage object of type #storage_type + /// \param storage reference to a temporary storage object of type #storage_type /// \par Storage reuse /// Synchronization barrier should be placed before `storage` is reused /// or repurposed: `__syncthreads()` or \link syncthreads() rocprim::syncthreads() \endlink. @@ -962,18 +962,18 @@ class block_adjacent_difference /// output[i] = input[i] /// \endcode /// - /// \tparam Output - [inferred] the type of output, must be assignable from the result of `op` - /// \tparam ItemsPerThread - [inferred] the number of items processed by each thread - /// \tparam BinaryFunction - [inferred] the type of the function to apply - /// \param [in] input - array that data is loaded from partitioned across the threads in the block - /// \param [out] output - array where the result of function application will be written to - /// \param [in] op - binary function applied to the items. + /// \tparam Output [inferred] the type of output, must be assignable from the result of `op` + /// \tparam ItemsPerThread [inferred] the number of items processed by each thread + /// \tparam BinaryFunction [inferred] the type of the function to apply + /// \param [in] input array that data is loaded from partitioned across the threads in the block + /// \param [out] output array where the result of function application will be written to + /// \param [in] op binary function applied to the items. /// The signature of the function should be equivalent to the following: /// `bool f(const T &a, const T &b)` The signature does not need to have /// `const &` but the function object must not modify the objects passed to it. - /// \param [in] valid_items - number of items in the block which are considered "valid" and will + /// \param [in] valid_items number of items in the block which are considered "valid" and will /// be used. Must be less or equal to `BlockSize` * `ItemsPerThread` - /// \param storage - reference to a temporary storage object of type #storage_type + /// \param storage reference to a temporary storage object of type #storage_type /// \par Storage reuse /// Synchronization barrier should be placed before `storage` is reused /// or repurposed: `__syncthreads()` or \link syncthreads() rocprim::syncthreads() \endlink. @@ -997,20 +997,20 @@ class block_adjacent_difference /// predecessor. /// /// This combines subtract_left_partial() with a tile predecessor. - /// \tparam Output - [inferred] the type of output, must be assignable from the result of `op` - /// \tparam ItemsPerThread - [inferred] the number of items processed by each thread - /// \tparam BinaryFunction - [inferred] the type of the function to apply - /// \param [in] input - array that data is loaded from partitioned across the threads in the block - /// \param [out] output - array where the result of function application will be written to - /// \param [in] op - binary function applied to the items. + /// \tparam Output [inferred] the type of output, must be assignable from the result of `op` + /// \tparam ItemsPerThread [inferred] the number of items processed by each thread + /// \tparam BinaryFunction [inferred] the type of the function to apply + /// \param [in] input array that data is loaded from partitioned across the threads in the block + /// \param [out] output array where the result of function application will be written to + /// \param [in] op binary function applied to the items. /// The signature of the function should be equivalent to the following: /// `bool f(const T &a, const T &b)` The signature does not need to have /// `const &` but the function object must not modify the objects passed to it. - /// \param [in] tile_predecessor - the item before the tile, will be used as the input + /// \param [in] tile_predecessor the item before the tile, will be used as the input /// of the first application of `op` - /// \param [in] valid_items - number of items in the block which are considered "valid" and will + /// \param [in] valid_items number of items in the block which are considered "valid" and will /// be used. Must be less or equal to `BlockSize` * `ItemsPerThread` - /// \param storage - reference to a temporary storage object of type #storage_type + /// \param storage reference to a temporary storage object of type #storage_type /// \par Storage reuse /// Synchronization barrier should be placed before `storage` is reused /// or repurposed: `__syncthreads()` or \link syncthreads() rocprim::syncthreads() \endlink. @@ -1040,16 +1040,16 @@ class block_adjacent_difference /// output[i] = op(input[i], input[i+1]); /// \endcode /// - /// \tparam Output - [inferred] the type of output, must be assignable from the result of `op` - /// \tparam ItemsPerThread - [inferred] the number of items processed by each thread - /// \tparam BinaryFunction - [inferred] the type of the function to apply - /// \param [in] input - array that data is loaded from partitioned across the threads in the block - /// \param [out] output - array where the result of function application will be written to - /// \param [in] op - binary function applied to the items. + /// \tparam Output [inferred] the type of output, must be assignable from the result of `op` + /// \tparam ItemsPerThread [inferred] the number of items processed by each thread + /// \tparam BinaryFunction [inferred] the type of the function to apply + /// \param [in] input array that data is loaded from partitioned across the threads in the block + /// \param [out] output array where the result of function application will be written to + /// \param [in] op binary function applied to the items. /// The signature of the function should be equivalent to the following: /// `bool f(const T &a, const T &b)` The signature does not need to have /// `const &` but the function object must not modify the objects passed to it. - /// \param storage - reference to a temporary storage object of type #storage_type + /// \param storage reference to a temporary storage object of type #storage_type /// \par Storage reuse /// Synchronization barrier should be placed before `storage` is reused /// or repurposed: `__syncthreads()` or \link syncthreads() rocprim::syncthreads() \endlink. @@ -1079,18 +1079,18 @@ class block_adjacent_difference /// op(input[block_size * ItemsPerThread - 1], tile_successor) /// \endcode /// - /// \tparam Output - [inferred] the type of output, must be assignable from the result of `op` - /// \tparam ItemsPerThread - [inferred] the number of items processed by each thread - /// \tparam BinaryFunction - [inferred] the type of the function to apply - /// \param [in] input - array that data is loaded from partitioned across the threads in the block - /// \param [out] output - array where the result of function application will be written to - /// \param [in] op - binary function applied to the items. + /// \tparam Output [inferred] the type of output, must be assignable from the result of `op` + /// \tparam ItemsPerThread [inferred] the number of items processed by each thread + /// \tparam BinaryFunction [inferred] the type of the function to apply + /// \param [in] input array that data is loaded from partitioned across the threads in the block + /// \param [out] output array where the result of function application will be written to + /// \param [in] op binary function applied to the items. /// The signature of the function should be equivalent to the following: /// `bool f(const T &a, const T &b)` The signature does not need to have /// `const &` but the function object must not modify the objects passed to it. - /// \param [in] tile_successor - the item after the tile, will be used as the input + /// \param [in] tile_successor the item after the tile, will be used as the input /// of the last application of `op` - /// \param storage - reference to a temporary storage object of type #storage_type + /// \param storage reference to a temporary storage object of type #storage_type /// \par Storage reuse /// Synchronization barrier should be placed before `storage` is reused /// or repurposed: `__syncthreads()` or \link syncthreads() rocprim::syncthreads() \endlink. @@ -1119,18 +1119,18 @@ class block_adjacent_difference /// output[i] = input[i] /// \endcode /// - /// \tparam Output - [inferred] the type of output, must be assignable from the result of `op` - /// \tparam ItemsPerThread - [inferred] the number of items processed by each thread - /// \tparam BinaryFunction - [inferred] the type of the function to apply - /// \param [in] input - array that data is loaded from partitioned across the threads in the block - /// \param [out] output - array where the result of function application will be written to - /// \param [in] op - binary function applied to the items. + /// \tparam Output [inferred] the type of output, must be assignable from the result of `op` + /// \tparam ItemsPerThread [inferred] the number of items processed by each thread + /// \tparam BinaryFunction [inferred] the type of the function to apply + /// \param [in] input array that data is loaded from partitioned across the threads in the block + /// \param [out] output array where the result of function application will be written to + /// \param [in] op binary function applied to the items. /// The signature of the function should be equivalent to the following: /// `bool f(const T &a, const T &b)` The signature does not need to have /// `const &` but the function object must not modify the objects passed to it. - /// \param [in] valid_items - number of items in the block which are considered "valid" and will + /// \param [in] valid_items number of items in the block which are considered "valid" and will /// be used. Must be less or equal to `BlockSize` * `ItemsPerThread` - /// \param storage - reference to a temporary storage object of type #storage_type + /// \param storage reference to a temporary storage object of type #storage_type /// \par Storage reuse /// Synchronization barrier should be placed before `storage` is reused /// or repurposed: `__syncthreads()` or \link syncthreads() rocprim::syncthreads() \endlink. diff --git a/rocprim/include/rocprim/block/block_discontinuity.hpp b/rocprim/include/rocprim/block/block_discontinuity.hpp index dbb777908..cba82cdd5 100644 --- a/rocprim/include/rocprim/block/block_discontinuity.hpp +++ b/rocprim/include/rocprim/block/block_discontinuity.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -38,8 +38,8 @@ BEGIN_ROCPRIM_NAMESPACE /// methods for flagging items that are discontinued within an ordered set of items across /// threads in a block. /// -/// \tparam T - the input type. -/// \tparam BlockSize - the number of threads in a block. +/// \tparam T the input type. +/// \tparam BlockSize the number of threads in a block. /// /// \par Overview /// * There are two types of flags: @@ -116,19 +116,19 @@ class block_discontinuity /// across the thread block, where the first item has no reference and is always /// flagged. /// - /// \tparam ItemsPerThread - [inferred] the number of items to be processed by + /// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread. - /// \tparam Flag - [inferred] the flag type. - /// \tparam FlagOp - [inferred] type of binary function used for flagging. + /// \tparam Flag [inferred] the flag type. + /// \tparam FlagOp [inferred] type of binary function used for flagging. /// - /// \param [out] head_flags - array that contains the head flags. - /// \param [in] input - array that data is loaded from. - /// \param [in] flag_op - binary operation function object that will be used for flagging. + /// \param [out] head_flags array that contains the head flags. + /// \param [in] input array that data is loaded from. + /// \param [in] flag_op binary operation function object that will be used for flagging. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b); or bool (const T& a, const T& b, unsigned int b_index);. /// The signature does not need to have const &, but function object /// must not modify the objects passed to it. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reusage /// Synchronization barrier should be placed before \p storage is reused @@ -185,21 +185,21 @@ class block_discontinuity /// across the thread block, where the first item of the first thread is compared against /// a \p tile_predecessor_item. /// - /// \tparam ItemsPerThread - [inferred] the number of items to be processed by + /// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread. - /// \tparam Flag - [inferred] the flag type. - /// \tparam FlagOp - [inferred] type of binary function used for flagging. + /// \tparam Flag [inferred] the flag type. + /// \tparam FlagOp [inferred] type of binary function used for flagging. /// - /// \param [out] head_flags - array that contains the head flags. - /// \param [in] tile_predecessor_item - first tile item from thread to be compared + /// \param [out] head_flags array that contains the head flags. + /// \param [in] tile_predecessor_item first tile item from thread to be compared /// against. - /// \param [in] input - array that data is loaded from. - /// \param [in] flag_op - binary operation function object that will be used for flagging. + /// \param [in] input array that data is loaded from. + /// \param [in] flag_op binary operation function object that will be used for flagging. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b); or bool (const T& a, const T& b, unsigned int b_index);. /// The signature does not need to have const &, but function object /// must not modify the objects passed to it. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reusage /// Synchronization barrier should be placed before \p storage is reused @@ -264,19 +264,19 @@ class block_discontinuity /// across the thread block, where the last item has no reference and is always /// flagged. /// - /// \tparam ItemsPerThread - [inferred] the number of items to be processed by + /// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread. - /// \tparam Flag - [inferred] the flag type. - /// \tparam FlagOp - [inferred] type of binary function used for flagging. + /// \tparam Flag [inferred] the flag type. + /// \tparam FlagOp [inferred] type of binary function used for flagging. /// - /// \param [out] tail_flags - array that contains the tail flags. - /// \param [in] input - array that data is loaded from. - /// \param [in] flag_op - binary operation function object that will be used for flagging. + /// \param [out] tail_flags array that contains the tail flags. + /// \param [in] input array that data is loaded from. + /// \param [in] flag_op binary operation function object that will be used for flagging. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b); or bool (const T& a, const T& b, unsigned int b_index);. /// The signature does not need to have const &, but function object /// must not modify the objects passed to it. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reusage /// Synchronization barrier should be placed before \p storage is reused @@ -333,21 +333,21 @@ class block_discontinuity /// across the thread block, where the last item of the last thread is compared against /// a \p tile_successor_item. /// - /// \tparam ItemsPerThread - [inferred] the number of items to be processed by + /// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread. - /// \tparam Flag - [inferred] the flag type. - /// \tparam FlagOp - [inferred] type of binary function used for flagging. + /// \tparam Flag [inferred] the flag type. + /// \tparam FlagOp [inferred] type of binary function used for flagging. /// - /// \param [out] tail_flags - array that contains the tail flags. - /// \param [in] tile_successor_item - last tile item from thread to be compared + /// \param [out] tail_flags array that contains the tail flags. + /// \param [in] tile_successor_item last tile item from thread to be compared /// against. - /// \param [in] input - array that data is loaded from. - /// \param [in] flag_op - binary operation function object that will be used for flagging. + /// \param [in] input array that data is loaded from. + /// \param [in] flag_op binary operation function object that will be used for flagging. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b); or bool (const T& a, const T& b, unsigned int b_index);. /// The signature does not need to have const &, but function object /// must not modify the objects passed to it. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reusage /// Synchronization barrier should be placed before \p storage is reused @@ -411,20 +411,20 @@ class block_discontinuity /// \brief Tags both \p head_flags and\p tail_flags that indicate discontinuities /// between items partitioned across the thread block. /// - /// \tparam ItemsPerThread - [inferred] the number of items to be processed by + /// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread. - /// \tparam Flag - [inferred] the flag type. - /// \tparam FlagOp - [inferred] type of binary function used for flagging. + /// \tparam Flag [inferred] the flag type. + /// \tparam FlagOp [inferred] type of binary function used for flagging. /// - /// \param [out] head_flags - array that contains the head flags. - /// \param [out] tail_flags - array that contains the tail flags. - /// \param [in] input - array that data is loaded from. - /// \param [in] flag_op - binary operation function object that will be used for flagging. + /// \param [out] head_flags array that contains the head flags. + /// \param [out] tail_flags array that contains the tail flags. + /// \param [in] input array that data is loaded from. + /// \param [in] flag_op binary operation function object that will be used for flagging. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b); or bool (const T& a, const T& b, unsigned int b_index);. /// The signature does not need to have const &, but function object /// must not modify the objects passed to it. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reusage /// Synchronization barrier should be placed before \p storage is reused @@ -498,22 +498,22 @@ class block_discontinuity /// between items partitioned across the thread block, where the last item of the /// last thread is compared against a \p tile_successor_item. /// - /// \tparam ItemsPerThread - [inferred] the number of items to be processed by + /// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread. - /// \tparam Flag - [inferred] the flag type. - /// \tparam FlagOp - [inferred] type of binary function used for flagging. + /// \tparam Flag [inferred] the flag type. + /// \tparam FlagOp [inferred] type of binary function used for flagging. /// - /// \param [out] head_flags - array that contains the head flags. - /// \param [out] tail_flags - array that contains the tail flags. - /// \param [in] tile_successor_item - last tile item from thread to be compared + /// \param [out] head_flags array that contains the head flags. + /// \param [out] tail_flags array that contains the tail flags. + /// \param [in] tile_successor_item last tile item from thread to be compared /// against. - /// \param [in] input - array that data is loaded from. - /// \param [in] flag_op - binary operation function object that will be used for flagging. + /// \param [in] input array that data is loaded from. + /// \param [in] flag_op binary operation function object that will be used for flagging. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b); or bool (const T& a, const T& b, unsigned int b_index);. /// The signature does not need to have const &, but function object /// must not modify the objects passed to it. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reusage /// Synchronization barrier should be placed before \p storage is reused @@ -595,22 +595,22 @@ class block_discontinuity /// between items partitioned across the thread block, where the first item of the /// first thread is compared against a \p tile_predecessor_item. /// - /// \tparam ItemsPerThread - [inferred] the number of items to be processed by + /// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread. - /// \tparam Flag - [inferred] the flag type. - /// \tparam FlagOp - [inferred] type of binary function used for flagging. + /// \tparam Flag [inferred] the flag type. + /// \tparam FlagOp [inferred] type of binary function used for flagging. /// - /// \param [out] head_flags - array that contains the head flags. - /// \param [in] tile_predecessor_item - first tile item from thread to be compared + /// \param [out] head_flags array that contains the head flags. + /// \param [in] tile_predecessor_item first tile item from thread to be compared /// against. - /// \param [out] tail_flags - array that contains the tail flags. - /// \param [in] input - array that data is loaded from. - /// \param [in] flag_op - binary operation function object that will be used for flagging. + /// \param [out] tail_flags array that contains the tail flags. + /// \param [in] input array that data is loaded from. + /// \param [in] flag_op binary operation function object that will be used for flagging. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b); or bool (const T& a, const T& b, unsigned int b_index);. /// The signature does not need to have const &, but function object /// must not modify the objects passed to it. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reusage /// Synchronization barrier should be placed before \p storage is reused @@ -693,24 +693,24 @@ class block_discontinuity /// the first and last thread is compared against a \p tile_predecessor_item and /// a \p tile_successor_item. /// - /// \tparam ItemsPerThread - [inferred] the number of items to be processed by + /// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread. - /// \tparam Flag - [inferred] the flag type. - /// \tparam FlagOp - [inferred] type of binary function used for flagging. + /// \tparam Flag [inferred] the flag type. + /// \tparam FlagOp [inferred] type of binary function used for flagging. /// - /// \param [out] head_flags - array that contains the head flags. - /// \param [in] tile_predecessor_item - first tile item from thread to be compared + /// \param [out] head_flags array that contains the head flags. + /// \param [in] tile_predecessor_item first tile item from thread to be compared /// against. - /// \param [out] tail_flags - array that contains the tail flags. - /// \param [in] tile_successor_item - last tile item from thread to be compared + /// \param [out] tail_flags array that contains the tail flags. + /// \param [in] tile_successor_item last tile item from thread to be compared /// against. - /// \param [in] input - array that data is loaded from. - /// \param [in] flag_op - binary operation function object that will be used for flagging. + /// \param [in] input array that data is loaded from. + /// \param [in] flag_op binary operation function object that will be used for flagging. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b); or bool (const T& a, const T& b, unsigned int b_index);. /// The signature does not need to have const &, but function object /// must not modify the objects passed to it. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reusage /// Synchronization barrier should be placed before \p storage is reused diff --git a/rocprim/include/rocprim/block/block_exchange.hpp b/rocprim/include/rocprim/block/block_exchange.hpp index 0962c77b0..47ef2277a 100644 --- a/rocprim/include/rocprim/block/block_exchange.hpp +++ b/rocprim/include/rocprim/block/block_exchange.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -40,10 +40,10 @@ BEGIN_ROCPRIM_NAMESPACE /// \brief The \p block_exchange class is a block level parallel primitive which provides /// methods for rearranging items partitioned across threads in a block. /// -/// \tparam T - the input type. -/// \tparam BlockSize - the number of threads in a block. -/// \tparam ItemsPerThread - the number of items contributed by each thread. -/// \tparam PaddingHint - a hint that decides when to use padding. May not always be applicable. +/// \tparam T the input type. +/// \tparam BlockSize the number of threads in a block. +/// \tparam ItemsPerThread the number of items contributed by each thread. +/// \tparam PaddingHint a hint that decides when to use padding. May not always be applicable. /// /// \par Overview /// * The \p block_exchange class supports the following rearrangement methods: @@ -148,10 +148,10 @@ class block_exchange /// \brief Transposes a blocked arrangement of items to a striped arrangement /// across the thread block. /// - /// \tparam U - [inferred] the output type. + /// \tparam U [inferred] the output type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. template ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void blocked_to_striped(const T (&input)[ItemsPerThread], @@ -164,11 +164,11 @@ class block_exchange /// \brief Transposes a blocked arrangement of items to a striped arrangement /// across the thread block, using temporary storage. /// - /// \tparam U - [inferred] the output type. + /// \tparam U [inferred] the output type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reusage /// Synchronization barrier should be placed before \p storage is reused @@ -215,10 +215,10 @@ class block_exchange /// \brief Transposes a striped arrangement of items to a blocked arrangement /// across the thread block. /// - /// \tparam U - [inferred] the output type. + /// \tparam U [inferred] the output type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. template ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void striped_to_blocked(const T (&input)[ItemsPerThread], @@ -231,11 +231,11 @@ class block_exchange /// \brief Transposes a striped arrangement of items to a blocked arrangement /// across the thread block, using temporary storage. /// - /// \tparam U - [inferred] the output type. + /// \tparam U [inferred] the output type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reusage /// Synchronization barrier should be placed before \p storage is reused @@ -282,10 +282,10 @@ class block_exchange /// \brief Transposes a blocked arrangement of items to a warp-striped arrangement /// across the thread block. /// - /// \tparam U - [inferred] the output type. + /// \tparam U [inferred] the output type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. template ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void blocked_to_warp_striped(const T (&input)[ItemsPerThread], @@ -298,11 +298,11 @@ class block_exchange /// \brief Transposes a blocked arrangement of items to a warp-striped arrangement /// across the thread block, using temporary storage. /// - /// \tparam U - [inferred] the output type. + /// \tparam U [inferred] the output type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reusage /// Synchronization barrier should be placed before \p storage is reused @@ -353,10 +353,10 @@ class block_exchange /// \brief Transposes a warp-striped arrangement of items to a blocked arrangement /// across the thread block. /// - /// \tparam U - [inferred] the output type. + /// \tparam U [inferred] the output type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. template ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void warp_striped_to_blocked(const T (&input)[ItemsPerThread], @@ -369,11 +369,11 @@ class block_exchange /// \brief Transposes a warp-striped arrangement of items to a blocked arrangement /// across the thread block, using temporary storage. /// - /// \tparam U - [inferred] the output type. + /// \tparam U [inferred] the output type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reusage /// Synchronization barrier should be placed before \p storage is reused @@ -424,12 +424,12 @@ class block_exchange /// \brief Scatters items to a blocked arrangement based on their ranks /// across the thread block. /// - /// \tparam U - [inferred] the output type. - /// \tparam Offset - [inferred] the rank type. + /// \tparam U [inferred] the output type. + /// \tparam Offset [inferred] the rank type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. - /// \param [out] ranks - array that has rank of data. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. + /// \param [out] ranks array that has rank of data. template ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void scatter_to_blocked(const T (&input)[ItemsPerThread], @@ -443,12 +443,12 @@ class block_exchange /// \brief Gathers items from a striped arrangement based on their ranks /// across the thread block. /// - /// \tparam U - [inferred] the output type. - /// \tparam Offset - [inferred] the rank type. + /// \tparam U [inferred] the output type. + /// \tparam Offset [inferred] the rank type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. - /// \param [out] ranks - array that has rank of data. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. + /// \param [out] ranks array that has rank of data. template ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void gather_from_striped(const T (&input)[ItemsPerThread], @@ -462,13 +462,13 @@ class block_exchange /// \brief Scatters items to a blocked arrangement based on their ranks /// across the thread block, using temporary storage. /// - /// \tparam U - [inferred] the output type. - /// \tparam Offset - [inferred] the rank type. + /// \tparam U [inferred] the output type. + /// \tparam Offset [inferred] the rank type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. - /// \param [out] ranks - array that has rank of data. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. + /// \param [out] ranks array that has rank of data. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reusage /// Synchronization barrier should be placed before \p storage is reused @@ -518,13 +518,13 @@ class block_exchange /// \brief Gathers items from a striped arrangement based on their ranks /// across the thread block, using temporary storage. /// - /// \tparam U - [inferred] the output type. - /// \tparam Offset - [inferred] the rank type. + /// \tparam U [inferred] the output type. + /// \tparam Offset [inferred] the rank type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. - /// \param [out] ranks - array that has rank of data. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. + /// \param [out] ranks array that has rank of data. + /// \param [in] storage reference to a temporary storage object of type storage_type. template ROCPRIM_DEVICE ROCPRIM_INLINE void gather_from_striped(const T (&input)[ItemsPerThread], @@ -552,12 +552,12 @@ class block_exchange /// \brief Scatters items to a striped arrangement based on their ranks /// across the thread block. /// - /// \tparam U - [inferred] the output type. - /// \tparam Offset - [inferred] the rank type. + /// \tparam U [inferred] the output type. + /// \tparam Offset [inferred] the rank type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. - /// \param [out] ranks - array that has rank of data. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. + /// \param [out] ranks array that has rank of data. template ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void scatter_to_striped(const T (&input)[ItemsPerThread], @@ -571,13 +571,13 @@ class block_exchange /// \brief Scatters items to a striped arrangement based on their ranks /// across the thread block, using temporary storage. /// - /// \tparam U - [inferred] the output type. - /// \tparam Offset - [inferred] the rank type. + /// \tparam U [inferred] the output type. + /// \tparam Offset [inferred] the rank type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. - /// \param [out] ranks - array that has rank of data. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. + /// \param [out] ranks array that has rank of data. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reusage /// Synchronization barrier should be placed before \p storage is reused @@ -627,13 +627,13 @@ class block_exchange /// \brief Scatters items to a *warp* striped arrangement based on their ranks /// across the thread block, using temporary storage. /// - /// \tparam U - [inferred] the output type. - /// \tparam Offset - [inferred] the rank type. + /// \tparam U [inferred] the output type. + /// \tparam Offset [inferred] the rank type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. - /// \param [out] ranks - array that has rank of data. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. + /// \param [out] ranks array that has rank of data. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reusage /// Synchronization barrier should be placed before \p storage is reused @@ -697,12 +697,12 @@ class block_exchange /// \par Overview /// * Items with rank -1 are not scattered. /// - /// \tparam U - [inferred] the output type. - /// \tparam Offset - [inferred] the rank type. + /// \tparam U [inferred] the output type. + /// \tparam Offset [inferred] the rank type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. - /// \param [in] ranks - array that has rank of data. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. + /// \param [in] ranks array that has rank of data. template ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void scatter_to_striped_guarded(const T (&input)[ItemsPerThread], @@ -719,13 +719,13 @@ class block_exchange /// \par Overview /// * Items with rank -1 are not scattered. /// - /// \tparam U - [inferred] the output type. - /// \tparam Offset - [inferred] the rank type. + /// \tparam U [inferred] the output type. + /// \tparam Offset [inferred] the rank type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. - /// \param [in] ranks - array that has rank of data. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. + /// \param [in] ranks array that has rank of data. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reusage /// Synchronization barrier should be placed before \p storage is reused @@ -778,14 +778,14 @@ class block_exchange /// \brief Scatters items to a striped arrangement based on their ranks /// across the thread block, with a flag to denote validity. /// - /// \tparam U - [inferred] the output type. - /// \tparam Offset - [inferred] the rank type. - /// \tparam ValidFlag - [inferred] the validity flag type. + /// \tparam U [inferred] the output type. + /// \tparam Offset [inferred] the rank type. + /// \tparam ValidFlag [inferred] the validity flag type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. - /// \param [in] ranks - array that has rank of data. - /// \param [in] is_valid - array that has flags to denote validity. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. + /// \param [in] ranks array that has rank of data. + /// \param [in] is_valid array that has flags to denote validity. template ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void scatter_to_striped_flagged(const T (&input)[ItemsPerThread], @@ -801,15 +801,15 @@ class block_exchange /// across the thread block, with a flag to denote validity, using temporary /// storage. /// - /// \tparam U - [inferred] the output type. - /// \tparam Offset - [inferred] the rank type. - /// \tparam ValidFlag - [inferred] the validity flag type. + /// \tparam U [inferred] the output type. + /// \tparam Offset [inferred] the rank type. + /// \tparam ValidFlag [inferred] the validity flag type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. - /// \param [in] ranks - array that has rank of data. - /// \param [in] is_valid - array that has flags to denote validity. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. + /// \param [in] ranks array that has rank of data. + /// \param [in] is_valid array that has flags to denote validity. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reusage /// Synchronization barrier should be placed before \p storage is reused diff --git a/rocprim/include/rocprim/block/block_histogram.hpp b/rocprim/include/rocprim/block/block_histogram.hpp index f74c35d5b..52f3515de 100644 --- a/rocprim/include/rocprim/block/block_histogram.hpp +++ b/rocprim/include/rocprim/block/block_histogram.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -86,11 +86,11 @@ struct select_block_histogram_impl /// \brief The block_histogram class is a block level parallel primitive which provides methods /// for constructing block-wide histograms from items partitioned across threads in a block. /// -/// \tparam T - the input/output type. -/// \tparam BlockSize - the number of threads in a block. -/// \tparam ItemsPerThread - the number of items to be processed by each thread. -/// \tparam Bins - the number of bins within the histogram. -/// \tparam Algorithm - selected histogram algorithm, block_histogram_algorithm::default_algorithm by default. +/// \tparam T the input/output type. +/// \tparam BlockSize the number of threads in a block. +/// \tparam ItemsPerThread the number of items to be processed by each thread. +/// \tparam Bins the number of bins within the histogram. +/// \tparam Algorithm selected histogram algorithm, block_histogram_algorithm::default_algorithm by default. /// /// \par Overview /// * block_histogram has two alternative implementations: \p block_histogram_algorithm::using_atomic @@ -152,9 +152,9 @@ class block_histogram /// \brief Initialize histogram counters to zero. /// - /// \tparam Counter - [inferred] counter type of histogram. + /// \tparam Counter [inferred] counter type of histogram. /// - /// \param [out] hist - histogram bin count. + /// \param [out] hist histogram bin count. template ROCPRIM_DEVICE ROCPRIM_INLINE void init_histogram(Counter hist[Bins]) @@ -175,11 +175,11 @@ class block_histogram /// \brief Update an existing block-wide histogram. Each thread composites an array of /// input elements. /// - /// \tparam Counter - [inferred] counter type of histogram. + /// \tparam Counter [inferred] counter type of histogram. /// - /// \param [in] input - reference to an array containing thread input values. The function expects each value to satisfy 0 <= input[i] < BINS. - /// \param [out] hist - histogram bin count. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] input reference to an array containing thread input values. The function expects each value to satisfy 0 <= input[i] < BINS. + /// \param [out] hist histogram bin count. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reusage /// Synchronization barrier should be placed before \p storage is reused @@ -235,10 +235,10 @@ class block_histogram /// * This overload does not accept storage argument. Required shared memory is /// allocated by the method itself. /// - /// \tparam Counter - [inferred] counter type of histogram. + /// \tparam Counter [inferred] counter type of histogram. /// - /// \param [in] input - reference to an array containing thread input values. The function expects each value to satisfy 0 <= input[i] < BINS. - /// \param [out] hist - histogram bin count. + /// \param [in] input reference to an array containing thread input values. The function expects each value to satisfy 0 <= input[i] < BINS. + /// \param [out] hist histogram bin count. template ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void composite(T (&input)[ItemsPerThread], @@ -250,11 +250,11 @@ class block_histogram /// \brief Construct a new block-wide histogram. Each thread contributes an array of /// input elements. /// - /// \tparam Counter - [inferred] counter type of histogram. + /// \tparam Counter [inferred] counter type of histogram. /// - /// \param [in] input - reference to an array containing thread input values. The function expects each value to satisfy 0 <= input[i] < BINS. - /// \param [out] hist - histogram bin count. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] input reference to an array containing thread input values. The function expects each value to satisfy 0 <= input[i] < BINS. + /// \param [out] hist histogram bin count. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reusage /// Synchronization barrier should be placed before \p storage is reused @@ -305,10 +305,10 @@ class block_histogram /// * This overload does not accept storage argument. Required shared memory is /// allocated by the method itself. /// - /// \tparam Counter - [inferred] counter type of histogram. + /// \tparam Counter [inferred] counter type of histogram. /// - /// \param [in] input - reference to an array containing thread input values. The function expects each value to satisfy 0 <= input[i] < BINS. - /// \param [out] hist - histogram bin count. + /// \param [in] input reference to an array containing thread input values. The function expects each value to satisfy 0 <= input[i] < BINS. + /// \param [out] hist histogram bin count. template ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void histogram(T (&input)[ItemsPerThread], diff --git a/rocprim/include/rocprim/block/block_load.hpp b/rocprim/include/rocprim/block/block_load.hpp index cc0bec2ab..ab46ab730 100644 --- a/rocprim/include/rocprim/block/block_load.hpp +++ b/rocprim/include/rocprim/block/block_load.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -92,11 +92,11 @@ enum class block_load_method /// for loading data from continuous memory into a blocked arrangement of items across the thread /// block. /// -/// \tparam T - the input/output type. -/// \tparam BlockSize - the number of threads in a block. -/// \tparam ItemsPerThread - the number of items to be processed by +/// \tparam T the input/output type. +/// \tparam BlockSize the number of threads in a block. +/// \tparam ItemsPerThread the number of items to be processed by /// each thread. -/// \tparam Method - the method to load data. +/// \tparam Method the method to load data. /// /// \par Overview /// * The \p block_load class has a number of different methods to load data: @@ -153,11 +153,11 @@ class block_load /// \brief Loads data from continuous memory into an arrangement of items across the /// thread block. /// - /// \tparam InputIterator - [inferred] an iterator type for input (can be a simple + /// \tparam InputIterator [inferred] an iterator type for input (can be a simple /// pointer. /// - /// \param [in] block_input - the input iterator from the thread block to load from. - /// \param [out] items - array that data is loaded to. + /// \param [in] block_input the input iterator from the thread block to load from. + /// \param [out] items array that data is loaded to. /// /// \par Overview /// * The type \p T must be such that an object of type \p InputIterator @@ -178,12 +178,12 @@ class block_load /// \brief Loads data from continuous memory into an arrangement of items across the /// thread block, which is guarded by range \p valid. /// - /// \tparam InputIterator - [inferred] an iterator type for input (can be a simple + /// \tparam InputIterator [inferred] an iterator type for input (can be a simple /// pointer. /// - /// \param [in] block_input - the input iterator from the thread block to load from. - /// \param [out] items - array that data is loaded to. - /// \param [in] valid - maximum range of valid numbers to load. + /// \param [in] block_input the input iterator from the thread block to load from. + /// \param [out] items array that data is loaded to. + /// \param [in] valid maximum range of valid numbers to load. /// /// \par Overview /// * The type \p T must be such that an object of type \p InputIterator @@ -206,14 +206,14 @@ class block_load /// thread block, which is guarded by range with a fall-back value for out-of-bound /// elements. /// - /// \tparam InputIterator - [inferred] an iterator type for input (can be a simple + /// \tparam InputIterator [inferred] an iterator type for input (can be a simple /// pointer. - /// \tparam Default - [inferred] The data type of the default value. + /// \tparam Default [inferred] The data type of the default value. /// - /// \param [in] block_input - the input iterator from the thread block to load from. - /// \param [out] items - array that data is loaded to. - /// \param [in] valid - maximum range of valid numbers to load. - /// \param [in] out_of_bounds - default value assigned to out-of-bound items. + /// \param [in] block_input the input iterator from the thread block to load from. + /// \param [out] items array that data is loaded to. + /// \param [in] valid maximum range of valid numbers to load. + /// \param [in] out_of_bounds default value assigned to out-of-bound items. /// /// \par Overview /// * The type \p T must be such that an object of type \p InputIterator @@ -240,12 +240,12 @@ class block_load /// \brief Loads data from continuous memory into an arrangement of items across the /// thread block, using temporary storage. /// - /// \tparam InputIterator - [inferred] an iterator type for input (can be a simple + /// \tparam InputIterator [inferred] an iterator type for input (can be a simple /// pointer. /// - /// \param [in] block_input - the input iterator from the thread block to load from. - /// \param [out] items - array that data is loaded to. - /// \param [in] storage - temporary storage for inputs. + /// \param [in] block_input the input iterator from the thread block to load from. + /// \param [out] items array that data is loaded to. + /// \param [in] storage temporary storage for inputs. /// /// \par Overview /// * The type \p T must be such that an object of type \p InputIterator @@ -284,13 +284,13 @@ class block_load /// \brief Loads data from continuous memory into an arrangement of items across the /// thread block, which is guarded by range \p valid, using temporary storage. /// - /// \tparam InputIterator - [inferred] an iterator type for input (can be a simple + /// \tparam InputIterator [inferred] an iterator type for input (can be a simple /// pointer /// - /// \param [in] block_input - the input iterator from the thread block to load from. - /// \param [out] items - array that data is loaded to. - /// \param [in] valid - maximum range of valid numbers to load. - /// \param [in] storage - temporary storage for inputs. + /// \param [in] block_input the input iterator from the thread block to load from. + /// \param [out] items array that data is loaded to. + /// \param [in] valid maximum range of valid numbers to load. + /// \param [in] storage temporary storage for inputs. /// /// \par Overview /// * The type \p T must be such that an object of type \p InputIterator @@ -331,15 +331,15 @@ class block_load /// thread block, which is guarded by range with a fall-back value for out-of-bound /// elements, using temporary storage. /// - /// \tparam InputIterator - [inferred] an iterator type for input (can be a simple + /// \tparam InputIterator [inferred] an iterator type for input (can be a simple /// pointer. - /// \tparam Default - [inferred] The data type of the default value. + /// \tparam Default [inferred] The data type of the default value. /// - /// \param [in] block_input - the input iterator from the thread block to load from. - /// \param [out] items - array that data is loaded to. - /// \param [in] valid - maximum range of valid numbers to load. - /// \param [in] out_of_bounds - default value assigned to out-of-bound items. - /// \param [in] storage - temporary storage for inputs. + /// \param [in] block_input the input iterator from the thread block to load from. + /// \param [out] items array that data is loaded to. + /// \param [in] valid maximum range of valid numbers to load. + /// \param [in] out_of_bounds default value assigned to out-of-bound items. + /// \param [in] storage temporary storage for inputs. /// /// \par Overview /// * The type \p T must be such that an object of type \p InputIterator diff --git a/rocprim/include/rocprim/block/block_load_func.hpp b/rocprim/include/rocprim/block/block_load_func.hpp index 4e578a16f..5360dd61e 100644 --- a/rocprim/include/rocprim/block/block_load_func.hpp +++ b/rocprim/include/rocprim/block/block_load_func.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -40,15 +40,15 @@ BEGIN_ROCPRIM_NAMESPACE /// across a thread block. Each thread uses a \p flat_id to load a range of /// \p ItemsPerThread into \p items. /// -/// \tparam InputIterator - [inferred] an iterator type for input (can be a simple +/// \tparam InputIterator [inferred] an iterator type for input (can be a simple /// pointer -/// \tparam T - [inferred] the data type -/// \tparam ItemsPerThread - [inferred] the number of items to be processed by +/// \tparam T [inferred] the data type +/// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread /// -/// \param flat_id - a local flat 1D thread id in a block (tile) for the calling thread -/// \param block_input - the input iterator from the thread block to load from -/// \param items - array that data is loaded to +/// \param flat_id a local flat 1D thread id in a block (tile) for the calling thread +/// \param block_input the input iterator from the thread block to load from +/// \param items array that data is loaded to template< class InputIterator, class T, @@ -75,16 +75,16 @@ void block_load_direct_blocked(unsigned int flat_id, /// across a thread block. Each thread uses a \p flat_id to load a range of /// \p ItemsPerThread into \p items. /// -/// \tparam InputIterator - [inferred] an iterator type for input (can be a simple +/// \tparam InputIterator [inferred] an iterator type for input (can be a simple /// pointer -/// \tparam T - [inferred] the data type -/// \tparam ItemsPerThread - [inferred] the number of items to be processed by +/// \tparam T [inferred] the data type +/// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread /// -/// \param flat_id - a local flat 1D thread id in a block (tile) for the calling thread -/// \param block_input - the input iterator from the thread block to load from -/// \param items - array that data is loaded to -/// \param valid - maximum range of valid numbers to load +/// \param flat_id a local flat 1D thread id in a block (tile) for the calling thread +/// \param block_input the input iterator from the thread block to load from +/// \param items array that data is loaded to +/// \param valid maximum range of valid numbers to load template< class InputIterator, class T, @@ -116,18 +116,18 @@ void block_load_direct_blocked(unsigned int flat_id, /// across a thread block. Each thread uses a \p flat_id to load a range of /// \p ItemsPerThread into \p items. /// -/// \tparam InputIterator - [inferred] an iterator type for input (can be a simple +/// \tparam InputIterator [inferred] an iterator type for input (can be a simple /// pointer -/// \tparam T - [inferred] the data type -/// \tparam ItemsPerThread - [inferred] the number of items to be processed by +/// \tparam T [inferred] the data type +/// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread -/// \tparam Default - [inferred] The data type of the default value +/// \tparam Default [inferred] The data type of the default value /// -/// \param flat_id - a local flat 1D thread id in a block (tile) for the calling thread -/// \param block_input - the input iterator from the thread block to load from -/// \param items - array that data is loaded to -/// \param valid - maximum range of valid numbers to load -/// \param out_of_bounds - default value assigned to out-of-bound items +/// \param flat_id a local flat 1D thread id in a block (tile) for the calling thread +/// \param block_input the input iterator from the thread block to load from +/// \param items array that data is loaded to +/// \param valid maximum range of valid numbers to load +/// \param out_of_bounds default value assigned to out-of-bound items template< class InputIterator, class T, @@ -165,16 +165,16 @@ void block_load_direct_blocked(unsigned int flat_id, /// * The datatype \p T is not a primitive or a HIP vector type (e.g. int2, /// int4, etc. /// -/// \tparam T - [inferred] the input data type -/// \tparam U - [inferred] the output data type -/// \tparam ItemsPerThread - [inferred] the number of items to be processed by +/// \tparam T [inferred] the input data type +/// \tparam U [inferred] the output data type +/// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread /// /// The type \p T must be such that it can be implicitly converted to \p U. /// -/// \param flat_id - a local flat 1D thread id in a block (tile) for the calling thread -/// \param block_input - the input iterator from the thread block to load from -/// \param items - array that data is loaded to +/// \param flat_id a local flat 1D thread id in a block (tile) for the calling thread +/// \param block_input the input iterator from the thread block to load from +/// \param items array that data is loaded to template< class T, class U, @@ -227,16 +227,16 @@ block_load_direct_blocked_vectorized(unsigned int flat_id, /// across a thread block. Each thread uses a \p flat_id to load a range of /// \p ItemsPerThread into \p items. /// -/// \tparam BlockSize - the number of threads in a block -/// \tparam InputIterator - [inferred] an iterator type for input (can be a simple +/// \tparam BlockSize the number of threads in a block +/// \tparam InputIterator [inferred] an iterator type for input (can be a simple /// pointer -/// \tparam T - [inferred] the data type -/// \tparam ItemsPerThread - [inferred] the number of items to be processed by +/// \tparam T [inferred] the data type +/// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread /// -/// \param flat_id - a local flat 1D thread id in a block (tile) for the calling thread -/// \param block_input - the input iterator from the thread block to load from -/// \param items - array that data is loaded to +/// \param flat_id a local flat 1D thread id in a block (tile) for the calling thread +/// \param block_input the input iterator from the thread block to load from +/// \param items array that data is loaded to template< unsigned int BlockSize, class InputIterator, @@ -263,17 +263,17 @@ void block_load_direct_striped(unsigned int flat_id, /// across a thread block. Each thread uses a \p flat_id to load a range of /// \p ItemsPerThread into \p items. /// -/// \tparam BlockSize - the number of threads in a block -/// \tparam InputIterator - [inferred] an iterator type for input (can be a simple +/// \tparam BlockSize the number of threads in a block +/// \tparam InputIterator [inferred] an iterator type for input (can be a simple /// pointer -/// \tparam T - [inferred] the data type -/// \tparam ItemsPerThread - [inferred] the number of items to be processed by +/// \tparam T [inferred] the data type +/// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread /// -/// \param flat_id - a local flat 1D thread id in a block (tile) for the calling thread -/// \param block_input - the input iterator from the thread block to load from -/// \param items - array that data is loaded to -/// \param valid - maximum range of valid numbers to load +/// \param flat_id a local flat 1D thread id in a block (tile) for the calling thread +/// \param block_input the input iterator from the thread block to load from +/// \param items array that data is loaded to +/// \param valid maximum range of valid numbers to load template< unsigned int BlockSize, class InputIterator, @@ -306,19 +306,19 @@ void block_load_direct_striped(unsigned int flat_id, /// across a thread block. Each thread uses a \p flat_id to load a range of /// \p ItemsPerThread into \p items. /// -/// \tparam BlockSize - the number of threads in a block -/// \tparam InputIterator - [inferred] an iterator type for input (can be a simple +/// \tparam BlockSize the number of threads in a block +/// \tparam InputIterator [inferred] an iterator type for input (can be a simple /// pointer -/// \tparam T - [inferred] the data type -/// \tparam ItemsPerThread - [inferred] the number of items to be processed by +/// \tparam T [inferred] the data type +/// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread -/// \tparam Default - [inferred] The data type of the default value +/// \tparam Default [inferred] The data type of the default value /// -/// \param flat_id - a local flat 1D thread id in a block (tile) for the calling thread -/// \param block_input - the input iterator from the thread block to load from -/// \param items - array that data is loaded to -/// \param valid - maximum range of valid numbers to load -/// \param out_of_bounds - default value assigned to out-of-bound items +/// \param flat_id a local flat 1D thread id in a block (tile) for the calling thread +/// \param block_input the input iterator from the thread block to load from +/// \param items array that data is loaded to +/// \param valid maximum range of valid numbers to load +/// \param out_of_bounds default value assigned to out-of-bound items template< unsigned int BlockSize, class InputIterator, @@ -357,16 +357,16 @@ void block_load_direct_striped(unsigned int flat_id, /// * Using \p WarpSize smaller than hardware warpsize could result in lower /// performance. /// -/// \tparam WarpSize - [optional] the number of threads in a warp -/// \tparam InputIterator - [inferred] an iterator type for input (can be a simple +/// \tparam WarpSize [optional] the number of threads in a warp +/// \tparam InputIterator [inferred] an iterator type for input (can be a simple /// pointer -/// \tparam T - [inferred] the data type -/// \tparam ItemsPerThread - [inferred] the number of items to be processed by +/// \tparam T [inferred] the data type +/// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread /// -/// \param flat_id - a local flat 1D thread id in a block (tile) for the calling thread -/// \param block_input - the input iterator from the thread block to load from -/// \param items - array that data is loaded to +/// \param flat_id a local flat 1D thread id in a block (tile) for the calling thread +/// \param block_input the input iterator from the thread block to load from +/// \param items array that data is loaded to template< unsigned int WarpSize = device_warp_size(), class InputIterator, @@ -408,17 +408,17 @@ void block_load_direct_warp_striped(unsigned int flat_id, /// * Using \p WarpSize smaller than hardware warpsize could result in lower /// performance. /// -/// \tparam WarpSize - [optional] the number of threads in a warp -/// \tparam InputIterator - [inferred] an iterator type for input (can be a simple +/// \tparam WarpSize [optional] the number of threads in a warp +/// \tparam InputIterator [inferred] an iterator type for input (can be a simple /// pointer -/// \tparam T - [inferred] the data type -/// \tparam ItemsPerThread - [inferred] the number of items to be processed by +/// \tparam T [inferred] the data type +/// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread /// -/// \param flat_id - a local flat 1D thread id in a block (tile) for the calling thread -/// \param block_input - the input iterator from the thread block to load from -/// \param items - array that data is loaded to -/// \param valid - maximum range of valid numbers to load +/// \param flat_id a local flat 1D thread id in a block (tile) for the calling thread +/// \param block_input the input iterator from the thread block to load from +/// \param items array that data is loaded to +/// \param valid maximum range of valid numbers to load template< unsigned int WarpSize = device_warp_size(), class InputIterator, @@ -466,19 +466,19 @@ void block_load_direct_warp_striped(unsigned int flat_id, /// * Using \p WarpSize smaller than hardware warpsize could result in lower /// performance. /// -/// \tparam WarpSize - [optional] the number of threads in a warp -/// \tparam InputIterator - [inferred] an iterator type for input (can be a simple +/// \tparam WarpSize [optional] the number of threads in a warp +/// \tparam InputIterator [inferred] an iterator type for input (can be a simple /// pointer -/// \tparam T - [inferred] the data type -/// \tparam ItemsPerThread - [inferred] the number of items to be processed by +/// \tparam T [inferred] the data type +/// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread -/// \tparam Default - [inferred] The data type of the default value +/// \tparam Default [inferred] The data type of the default value /// -/// \param flat_id - a local flat 1D thread id in a block (tile) for the calling thread -/// \param block_input - the input iterator from the thread block to load from -/// \param items - array that data is loaded to -/// \param valid - maximum range of valid numbers to load -/// \param out_of_bounds - default value assigned to out-of-bound items +/// \param flat_id a local flat 1D thread id in a block (tile) for the calling thread +/// \param block_input the input iterator from the thread block to load from +/// \param items array that data is loaded to +/// \param valid maximum range of valid numbers to load +/// \param out_of_bounds default value assigned to out-of-bound items template< unsigned int WarpSize = device_warp_size(), class InputIterator, diff --git a/rocprim/include/rocprim/block/block_radix_rank.hpp b/rocprim/include/rocprim/block/block_radix_rank.hpp index 8229fe355..82ca16d6f 100644 --- a/rocprim/include/rocprim/block/block_radix_rank.hpp +++ b/rocprim/include/rocprim/block/block_radix_rank.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -94,13 +94,13 @@ struct select_block_radix_rank_impl /// associates each item with the index it would gain if the keys were sorted into an array, /// according to a radix comparison. Ranking is performed in a stable manner. /// -/// \tparam BlockSizeX - the number of threads in a block's x dimension. -/// \tparam RadixBits - the maximum number of radix digit bits that comparisons are performed by. -/// \tparam MemoizeOuterScan - whether to cache digit counters in local memory. This omits loading +/// \tparam BlockSizeX the number of threads in a block's x dimension. +/// \tparam RadixBits the maximum number of radix digit bits that comparisons are performed by. +/// \tparam MemoizeOuterScan whether to cache digit counters in local memory. This omits loading /// the same values from shared memory twice, at the expense of more register usage. -/// \tparam BlockSizeY - the number of threads in a block's y dimension, defaults to 1. -/// \tparam BlockSizeZ - the number of threads in a block's z dimension, defaults to 1. -/// \tparam PaddingHint - a hint that decides when to use padding. May not always be applicable. +/// \tparam BlockSizeY the number of threads in a block's y dimension, defaults to 1. +/// \tparam BlockSizeZ the number of threads in a block's z dimension, defaults to 1. +/// \tparam PaddingHint a hint that decides when to use padding. May not always be applicable. /// /// \par Overview /// * Key type must be an arithmetic type (that is, an integral type or a floating point type). @@ -169,14 +169,14 @@ class block_radix_rank /// \brief Perform ascending radix rank over keys partitioned across threads in a block. /// - /// \tparam Key - the key type. - /// \tparam ItemsPerThread - the number of items contributed by each thread in the block. - /// \param [in] keys - reference to an array of keys provided by a thread. - /// \param [out] ranks - reference to an array where the final ranks are written to. - /// \param [in] storage - reference to a temporary storage object of type \p storage_type. - /// \param [in] begin_bit - index of the first (least significant) bit used in key comparison. + /// \tparam Key the key type. + /// \tparam ItemsPerThread the number of items contributed by each thread in the block. + /// \param [in] keys reference to an array of keys provided by a thread. + /// \param [out] ranks reference to an array where the final ranks are written to. + /// \param [in] storage reference to a temporary storage object of type \p storage_type. + /// \param [in] begin_bit index of the first (least significant) bit used in key comparison. /// Must be in range [0; 8 * sizeof(Key)). - /// \param [in] pass_bits - [optional] the number of bits used in key comparison. Must be in + /// \param [in] pass_bits [optional] the number of bits used in key comparison. Must be in /// the range (0; RadixBits]. Default value: RadixBits. /// /// \par Storage reusage @@ -224,13 +224,13 @@ class block_radix_rank /// * This overload does not accept storage argument. Required shared memory is allocated /// by the method itself. /// - /// \tparam Key - the key type. - /// \tparam ItemsPerThread - the number of items contributed by each thread in the block. - /// \param [in] keys - reference to an array of keys provided by a thread. - /// \param [out] ranks - reference to an array where the final ranks are written to. - /// \param [in] begin_bit - index of the first (least significant) bit used in key comparison. + /// \tparam Key the key type. + /// \tparam ItemsPerThread the number of items contributed by each thread in the block. + /// \param [in] keys reference to an array of keys provided by a thread. + /// \param [out] ranks reference to an array where the final ranks are written to. + /// \param [in] begin_bit index of the first (least significant) bit used in key comparison. /// Must be in range [0; 8 * sizeof(Key)). - /// \param [in] pass_bits - [optional] the number of bits used in key comparison. Must be in + /// \param [in] pass_bits [optional] the number of bits used in key comparison. Must be in /// the range (0; RadixBits]. Default value: RadixBits. template ROCPRIM_DEVICE void rank_keys(const Key (&keys)[ItemsPerThread], @@ -245,13 +245,13 @@ class block_radix_rank /// \brief Perform ascending radix rank over bit keys partitioned across threads in a block. /// This overload accepts a callback used to extract the radix digit from a key. /// - /// \tparam ItemsPerThread - the number of items contributed by each thread in the block. - /// \param [in] keys - reference to an array of keys provided by a thread. - /// \param [out] ranks - reference to an array where the final ranks are written to. - /// \param [in] storage - reference to a temporary storage object of type \p storage_type. - /// \param [in] begin_bit - index of the first (least significant) bit used in key comparison. + /// \tparam ItemsPerThread the number of items contributed by each thread in the block. + /// \param [in] keys reference to an array of keys provided by a thread. + /// \param [out] ranks reference to an array where the final ranks are written to. + /// \param [in] storage reference to a temporary storage object of type \p storage_type. + /// \param [in] begin_bit index of the first (least significant) bit used in key comparison. /// Must be in range [0; 8 * sizeof(Key)). - /// \param [in] pass_bits - [optional] the number of bits used in key comparison. Must be in + /// \param [in] pass_bits [optional] the number of bits used in key comparison. Must be in /// the range (0; RadixBits]. Default value: RadixBits. /// /// \par Storage reusage @@ -299,13 +299,13 @@ class block_radix_rank /// * This overload does not accept storage argument. Required shared memory is allocated /// by the method itself. /// - /// \tparam Key - the key type. - /// \tparam ItemsPerThread - the number of items contributed by each thread in the block. - /// \param [in] keys - reference to an array of keys provided by a thread. - /// \param [out] ranks - reference to an array where the final ranks are written to. - /// \param [in] begin_bit - index of the first (least significant) bit used in key comparison. + /// \tparam Key the key type. + /// \tparam ItemsPerThread the number of items contributed by each thread in the block. + /// \param [in] keys reference to an array of keys provided by a thread. + /// \param [out] ranks reference to an array where the final ranks are written to. + /// \param [in] begin_bit index of the first (least significant) bit used in key comparison. /// Must be in range [0; 8 * sizeof(Key)). - /// \param [in] pass_bits - [optional] the number of bits used in key comparison. Must be in + /// \param [in] pass_bits [optional] the number of bits used in key comparison. Must be in /// the range (0; RadixBits]. Default value: RadixBits. template ROCPRIM_DEVICE void rank_keys_desc(const Key (&keys)[ItemsPerThread], @@ -320,14 +320,14 @@ class block_radix_rank /// \brief Perform ascending radix rank over bit keys partitioned across threads in a block. /// This overload accepts a callback used to extract the radix digit from a key. /// - /// \tparam Key - the key type. - /// \tparam ItemsPerThread - the number of items contributed by each thread in the block. - /// \tparam DigitExtractor - type of the unary function object used to extract a digit from + /// \tparam Key the key type. + /// \tparam ItemsPerThread the number of items contributed by each thread in the block. + /// \tparam DigitExtractor type of the unary function object used to extract a digit from /// a key. - /// \param [in] keys - reference to an array of keys provided by a thread. - /// \param [out] ranks - reference to an array where the final ranks are written to. - /// \param [in] storage - reference to a temporary storage object of type \p storage_type. - /// \param [in] digit_extractor - function object used to convert a key to a digit. + /// \param [in] keys reference to an array of keys provided by a thread. + /// \param [out] ranks reference to an array where the final ranks are written to. + /// \param [in] storage reference to a temporary storage object of type \p storage_type. + /// \param [in] digit_extractor function object used to convert a key to a digit. /// The signature of the \p digit_extractor should be equivalent to the following: /// unsigned int f(const Key &key);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -382,13 +382,13 @@ class block_radix_rank /// * This overload does not accept storage argument. Required shared memory is allocated /// by the method itself. /// - /// \tparam Key - the key type. - /// \tparam ItemsPerThread - the number of items contributed by each thread in the block. - /// \tparam DigitExtractor - type of the unary function object used to extract a digit from + /// \tparam Key the key type. + /// \tparam ItemsPerThread the number of items contributed by each thread in the block. + /// \tparam DigitExtractor type of the unary function object used to extract a digit from /// a key. - /// \param [in] keys - reference to an array of keys provided by a thread. - /// \param [out] ranks - reference to an array where the final ranks are written to. - /// \param [in] digit_extractor - function object used to convert a key to a digit. + /// \param [in] keys reference to an array of keys provided by a thread. + /// \param [out] ranks reference to an array where the final ranks are written to. + /// \param [in] digit_extractor function object used to convert a key to a digit. /// The signature of the \p digit_extractor should be equivalent to the following: /// unsigned int f(const Key &key);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -406,14 +406,14 @@ class block_radix_rank /// \brief Perform descending radix rank over bit keys partitioned across threads in a block. /// This overload accepts a callback used to extract the radix digit from a key. /// - /// \tparam Key - the key type. - /// \tparam ItemsPerThread - the number of items contributed by each thread in the block. - /// \tparam DigitExtractor - type of the unary function object used to extract a digit from + /// \tparam Key the key type. + /// \tparam ItemsPerThread the number of items contributed by each thread in the block. + /// \tparam DigitExtractor type of the unary function object used to extract a digit from /// a key. - /// \param [in] keys - reference to an array of keys provided by a thread. - /// \param [out] ranks - reference to an array where the final ranks are written to. - /// \param [in] storage - reference to a temporary storage object of type \p storage_type. - /// \param [in] digit_extractor - function object used to convert a key to a digit. + /// \param [in] keys reference to an array of keys provided by a thread. + /// \param [out] ranks reference to an array where the final ranks are written to. + /// \param [in] storage reference to a temporary storage object of type \p storage_type. + /// \param [in] digit_extractor function object used to convert a key to a digit. /// The signature of the \p digit_extractor should be equivalent to the following: /// unsigned int f(const Key &key);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -468,13 +468,13 @@ class block_radix_rank /// * This overload does not accept storage argument. Required shared memory is allocated /// by the method itself. /// - /// \tparam Key - the key type. - /// \tparam ItemsPerThread - the number of items contributed by each thread in the block. - /// \tparam DigitExtractor - type of the unary function object used to extract a digit from + /// \tparam Key the key type. + /// \tparam ItemsPerThread the number of items contributed by each thread in the block. + /// \tparam DigitExtractor type of the unary function object used to extract a digit from /// a key. - /// \param [in] keys - reference to an array of keys provided by a thread. - /// \param [out] ranks - reference to an array where the final ranks are written to. - /// \param [in] digit_extractor - function object used to convert a key to a digit. + /// \param [in] keys reference to an array of keys provided by a thread. + /// \param [out] ranks reference to an array where the final ranks are written to. + /// \param [in] digit_extractor function object used to convert a key to a digit. /// The signature of the \p digit_extractor should be equivalent to the following: /// unsinged int f(const Key &key);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -493,23 +493,23 @@ class block_radix_rank /// This overload accepts a callback used to extract the radix digit from a key, and provides /// the counts of each digit and a prefix scan thereof in a blocked arrangement. /// - /// \tparam Key - the key type. - /// \tparam ItemsPerThread - the number of items contributed by each thread in the block. - /// \tparam DigitExtractor - type of the unary function object used to extract a digit from + /// \tparam Key the key type. + /// \tparam ItemsPerThread the number of items contributed by each thread in the block. + /// \tparam DigitExtractor type of the unary function object used to extract a digit from /// a key. - /// \param [in] keys - reference to an array of keys provided by a thread. Keys are expected in + /// \param [in] keys reference to an array of keys provided by a thread. Keys are expected in /// warp-striped arrangement. - /// \param [out] ranks - reference to an array where the final ranks are written to. Ranks are + /// \param [out] ranks reference to an array where the final ranks are written to. Ranks are /// provided in warp-striped arrangement. - /// \param [in] storage - reference to a temporary storage object of type \p storage_type. - /// \param [in] digit_extractor - function object used to convert a key to a digit. + /// \param [in] storage reference to a temporary storage object of type \p storage_type. + /// \param [in] digit_extractor function object used to convert a key to a digit. /// The signature of the \p digit_extractor should be equivalent to the following: /// unsigned int f(const Key &key);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. /// This function will be used during ranking to extract the digit that indicates /// the key's value. Values return by this function object must be in range [0; 1 << RadixBits). - /// \param [in] prefix - An exclusive prefix scan of the counts per digit. - /// \param [in] counts - The number of keys with a particular digit in the input, per digit. + /// \param [in] prefix An exclusive prefix scan of the counts per digit. + /// \param [in] counts The number of keys with a particular digit in the input, per digit. /// /// \par Storage reusage /// A synchronization barrier should be placed before \p storage is reused diff --git a/rocprim/include/rocprim/block/block_radix_sort.hpp b/rocprim/include/rocprim/block/block_radix_sort.hpp index c84ff1a95..e6af87777 100644 --- a/rocprim/include/rocprim/block/block_radix_sort.hpp +++ b/rocprim/include/rocprim/block/block_radix_sort.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -44,12 +44,12 @@ BEGIN_ROCPRIM_NAMESPACE /// methods for sorting of items (keys or key-value pairs) partitioned across threads in a block /// using radix sort algorithm. /// -/// \tparam Key - the key type. -/// \tparam BlockSize - the number of threads in a block. -/// \tparam ItemsPerThread - the number of items contributed by each thread. -/// \tparam Value - the value type. Default type empty_type indicates +/// \tparam Key the key type. +/// \tparam BlockSize the number of threads in a block. +/// \tparam ItemsPerThread the number of items contributed by each thread. +/// \tparam Value the value type. Default type empty_type indicates /// a keys-only sort. -/// \tparam RadixBitsPerPass - amount of bits to sort per pass. The Default is 4. +/// \tparam RadixBitsPerPass amount of bits to sort per pass. The Default is 4. /// \tparam RadixRankAlgorithm the rank algorithm used. /// /// \par Overview @@ -164,11 +164,11 @@ class block_radix_sort /// /// \tparam Decomposer The type of the decomposer argument. Defaults to the identity decomposer. /// - /// \param [in, out] keys - reference to an array of keys provided by a thread. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] begin_bit - [optional] index of the first (least significant) bit used in + /// \param [in, out] keys reference to an array of keys provided by a thread. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. - /// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in + /// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). /// \param [in] decomposer [optional] If `Key` is not an arithmetic type (integral, floating point), @@ -224,10 +224,10 @@ class block_radix_sort /// * This overload does not accept storage argument. Required shared memory is /// allocated by the method itself. /// - /// \param [in, out] keys - reference to an array of keys provided by a thread. - /// \param [in] begin_bit - [optional] index of the first (least significant) bit used in + /// \param [in, out] keys reference to an array of keys provided by a thread. + /// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. - /// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in + /// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). /// \param [in] decomposer [optional] If `Key` is not an arithmetic type (integral, floating point), @@ -247,11 +247,11 @@ class block_radix_sort /// /// \tparam Decomposer The type of the decomposer argument. Defaults to the identity decomposer. /// - /// \param [in, out] keys - reference to an array of keys provided by a thread. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] begin_bit - [optional] index of the first (least significant) bit used in + /// \param [in, out] keys reference to an array of keys provided by a thread. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. - /// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in + /// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). /// \param [in] decomposer [optional] If `Key` is not an arithmetic type (integral, floating point), @@ -309,10 +309,10 @@ class block_radix_sort /// /// \tparam Decomposer The type of the decomposer argument. Defaults to the identity decomposer. /// - /// \param [in, out] keys - reference to an array of keys provided by a thread. - /// \param [in] begin_bit - [optional] index of the first (least significant) bit used in + /// \param [in, out] keys reference to an array of keys provided by a thread. + /// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. - /// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in + /// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). /// \param [in] decomposer [optional] If `Key` is not an arithmetic type (integral, floating point), @@ -335,12 +335,12 @@ class block_radix_sort /// /// \tparam Decomposer The type of the decomposer argument. Defaults to the identity decomposer. /// - /// \param [in, out] keys - reference to an array of keys provided by a thread. - /// \param [in, out] values - reference to an array of values provided by a thread. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] begin_bit - [optional] index of the first (least significant) bit used in + /// \param [in, out] keys reference to an array of keys provided by a thread. + /// \param [in, out] values reference to an array of values provided by a thread. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. - /// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in + /// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). /// \param [in] decomposer [optional] If `Key` is not an arithmetic type (integral, floating point), @@ -405,11 +405,11 @@ class block_radix_sort /// /// \tparam Decomposer The type of the decomposer argument. Defaults to the identity decomposer. /// - /// \param [in, out] keys - reference to an array of keys provided by a thread. - /// \param [in, out] values - reference to an array of values provided by a thread. - /// \param [in] begin_bit - [optional] index of the first (least significant) bit used in + /// \param [in, out] keys reference to an array of keys provided by a thread. + /// \param [in, out] values reference to an array of values provided by a thread. + /// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. - /// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in + /// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). /// \param [in] decomposer [optional] If `Key` is not an arithmetic type (integral, floating point), @@ -434,12 +434,12 @@ class block_radix_sort /// /// \tparam Decomposer The type of the decomposer argument. Defaults to the identity decomposer. /// - /// \param [in, out] keys - reference to an array of keys provided by a thread. - /// \param [in, out] values - reference to an array of values provided by a thread. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] begin_bit - [optional] index of the first (least significant) bit used in + /// \param [in, out] keys reference to an array of keys provided by a thread. + /// \param [in, out] values reference to an array of values provided by a thread. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. - /// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in + /// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). /// \param [in] decomposer [optional] If `Key` is not an arithmetic type (integral, floating point), @@ -504,11 +504,11 @@ class block_radix_sort /// /// \tparam Decomposer The type of the decomposer argument. Defaults to the identity decomposer. /// - /// \param [in, out] keys - reference to an array of keys provided by a thread. - /// \param [in, out] values - reference to an array of values provided by a thread. - /// \param [in] begin_bit - [optional] index of the first (least significant) bit used in + /// \param [in, out] keys reference to an array of keys provided by a thread. + /// \param [in, out] values reference to an array of values provided by a thread. + /// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. - /// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in + /// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). /// \param [in] decomposer [optional] If `Key` is not an arithmetic type (integral, floating point), @@ -531,11 +531,11 @@ class block_radix_sort /// /// \tparam Decomposer The type of the decomposer argument. Defaults to the identity decomposer. /// - /// \param [in, out] keys - reference to an array of keys provided by a thread. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] begin_bit - [optional] index of the first (least significant) bit used in + /// \param [in, out] keys reference to an array of keys provided by a thread. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. - /// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in + /// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). /// \param [in] decomposer [optional] If `Key` is not an arithmetic type (integral, floating point), @@ -594,10 +594,10 @@ class block_radix_sort /// /// \tparam Decomposer The type of the decomposer argument. Defaults to the identity decomposer. /// - /// \param [in, out] keys - reference to an array of keys provided by a thread. - /// \param [in] begin_bit - [optional] index of the first (least significant) bit used in + /// \param [in, out] keys reference to an array of keys provided by a thread. + /// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. - /// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in + /// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). /// \param [in] decomposer [optional] If `Key` is not an arithmetic type (integral, floating point), @@ -618,11 +618,11 @@ class block_radix_sort /// /// \tparam Decomposer The type of the decomposer argument. Defaults to the identity decomposer. /// - /// \param [in, out] keys - reference to an array of keys provided by a thread. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] begin_bit - [optional] index of the first (least significant) bit used in + /// \param [in, out] keys reference to an array of keys provided by a thread. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. - /// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in + /// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). /// \param [in] decomposer [optional] If `Key` is not an arithmetic type (integral, floating point), @@ -681,10 +681,10 @@ class block_radix_sort /// /// \tparam Decomposer The type of the decomposer argument. Defaults to the identity decomposer. /// - /// \param [in, out] keys - reference to an array of keys provided by a thread. - /// \param [in] begin_bit - [optional] index of the first (least significant) bit used in + /// \param [in, out] keys reference to an array of keys provided by a thread. + /// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. - /// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in + /// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). /// \param [in] decomposer [optional] If `Key` is not an arithmetic type (integral, floating point), @@ -708,12 +708,12 @@ class block_radix_sort /// /// \tparam Decomposer The type of the decomposer argument. Defaults to the identity decomposer. /// - /// \param [in, out] keys - reference to an array of keys provided by a thread. - /// \param [in, out] values - reference to an array of values provided by a thread. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] begin_bit - [optional] index of the first (least significant) bit used in + /// \param [in, out] keys reference to an array of keys provided by a thread. + /// \param [in, out] values reference to an array of values provided by a thread. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. - /// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in + /// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). /// \param [in] decomposer [optional] If `Key` is not an arithmetic type (integral, floating point), @@ -776,11 +776,11 @@ class block_radix_sort /// /// \tparam Decomposer The type of the decomposer argument. Defaults to the identity decomposer. /// - /// \param [in, out] keys - reference to an array of keys provided by a thread. - /// \param [in, out] values - reference to an array of values provided by a thread. - /// \param [in] begin_bit - [optional] index of the first (least significant) bit used in + /// \param [in, out] keys reference to an array of keys provided by a thread. + /// \param [in, out] values reference to an array of values provided by a thread. + /// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. - /// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in + /// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). /// \param [in] decomposer [optional] If `Key` is not an arithmetic type (integral, floating point), @@ -805,12 +805,12 @@ class block_radix_sort /// /// \tparam Decomposer The type of the decomposer argument. Defaults to the identity decomposer. /// - /// \param [in, out] keys - reference to an array of keys provided by a thread. - /// \param [in, out] values - reference to an array of values provided by a thread. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] begin_bit - [optional] index of the first (least significant) bit used in + /// \param [in, out] keys reference to an array of keys provided by a thread. + /// \param [in, out] values reference to an array of values provided by a thread. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. - /// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in + /// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). /// \param [in] decomposer [optional] If `Key` is not an arithmetic type (integral, floating point), @@ -874,11 +874,11 @@ class block_radix_sort /// /// \tparam Decomposer The type of the decomposer argument. Defaults to the identity decomposer. /// - /// \param [in, out] keys - reference to an array of keys provided by a thread. - /// \param [in, out] values - reference to an array of values provided by a thread. - /// \param [in] begin_bit - [optional] index of the first (least significant) bit used in + /// \param [in, out] keys reference to an array of keys provided by a thread. + /// \param [in, out] values reference to an array of values provided by a thread. + /// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. - /// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in + /// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). /// \param [in] decomposer [optional] If `Key` is not an arithmetic type (integral, floating point), diff --git a/rocprim/include/rocprim/block/block_reduce.hpp b/rocprim/include/rocprim/block/block_reduce.hpp index f72966267..f3168904f 100644 --- a/rocprim/include/rocprim/block/block_reduce.hpp +++ b/rocprim/include/rocprim/block/block_reduce.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -86,9 +86,9 @@ struct select_block_reduce_impl. /// - /// \param [in] input - thread input value. - /// \param [out] output - reference to a thread output value. May be aliased with \p input. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] reduce_op - binary operation function object that will be used for reduce. + /// \param [in] input thread input value. + /// \param [out] output reference to a thread output value. May be aliased with \p input. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] reduce_op binary operation function object that will be used for reduce. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -215,12 +215,12 @@ class block_reduce /// * This overload does not accept storage argument. Required shared memory is /// allocated by the method itself. /// - /// \tparam BinaryFunction - type of binary function used for reduce. Default type + /// \tparam BinaryFunction type of binary function used for reduce. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] output - reference to a thread output value. May be aliased with \p input. - /// \param [in] reduce_op - binary operation function object that will be used for reduce. + /// \param [in] input thread input value. + /// \param [out] output reference to a thread output value. May be aliased with \p input. + /// \param [in] reduce_op binary operation function object that will be used for reduce. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -235,14 +235,14 @@ class block_reduce /// \brief Performs reduction across threads in a block. /// - /// \tparam ItemsPerThread - number of items in the \p input array. - /// \tparam BinaryFunction - type of binary function used for reduce. Default type + /// \tparam ItemsPerThread number of items in the \p input array. + /// \tparam BinaryFunction type of binary function used for reduce. Default type /// is rocprim::plus. /// - /// \param [in] input - reference to an array containing thread input values. - /// \param [out] output - reference to a thread output array. May be aliased with \p input. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] reduce_op - binary operation function object that will be used for reduce. + /// \param [in] input reference to an array containing thread input values. + /// \param [out] output reference to a thread output array. May be aliased with \p input. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] reduce_op binary operation function object that will be used for reduce. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -299,13 +299,13 @@ class block_reduce /// * This overload does not accept storage argument. Required shared memory is /// allocated by the method itself. /// - /// \tparam ItemsPerThread - number of items in the \p input array. - /// \tparam BinaryFunction - type of binary function used for reduce. Default type + /// \tparam ItemsPerThread number of items in the \p input array. + /// \tparam BinaryFunction type of binary function used for reduce. Default type /// is rocprim::plus. /// - /// \param [in] input - reference to an array containing thread input values. - /// \param [out] output - reference to a thread output array. May be aliased with \p input. - /// \param [in] reduce_op - binary operation function object that will be used for reduce. + /// \param [in] input reference to an array containing thread input values. + /// \param [out] output reference to a thread output array. May be aliased with \p input. + /// \param [in] reduce_op binary operation function object that will be used for reduce. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -323,14 +323,14 @@ class block_reduce /// \brief Performs reduction across threads in a block. /// - /// \tparam BinaryFunction - type of binary function used for reduce. Default type + /// \tparam BinaryFunction type of binary function used for reduce. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] output - reference to a thread output value. May be aliased with \p input. - /// \param [in] valid_items - number of items that will be reduced in the block. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] reduce_op - binary operation function object that will be used for reduce. + /// \param [in] input thread input value. + /// \param [out] output reference to a thread output value. May be aliased with \p input. + /// \param [in] valid_items number of items that will be reduced in the block. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] reduce_op binary operation function object that will be used for reduce. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -384,14 +384,14 @@ class block_reduce /// * This overload does not accept storage argument. Required shared memory is /// allocated by the method itself. /// - /// \tparam ItemsPerThread - number of items in the \p input array. - /// \tparam BinaryFunction - type of binary function used for reduce. Default type + /// \tparam ItemsPerThread number of items in the \p input array. + /// \tparam BinaryFunction type of binary function used for reduce. Default type /// is rocprim::plus. /// - /// \param [in] input - reference to an array containing thread input values. - /// \param [out] output - reference to a thread output array. May be aliased with \p input. - /// \param [in] valid_items - number of items that will be reduced in the block. - /// \param [in] reduce_op - binary operation function object that will be used for reduce. + /// \param [in] input reference to an array containing thread input values. + /// \param [out] output reference to a thread output array. May be aliased with \p input. + /// \param [in] valid_items number of items that will be reduced in the block. + /// \param [in] reduce_op binary operation function object that will be used for reduce. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. diff --git a/rocprim/include/rocprim/block/block_scan.hpp b/rocprim/include/rocprim/block/block_scan.hpp index a7958fc75..f679a489c 100644 --- a/rocprim/include/rocprim/block/block_scan.hpp +++ b/rocprim/include/rocprim/block/block_scan.hpp @@ -82,11 +82,11 @@ struct select_block_scan_impl /// for performing inclusive and exclusive scan operations of items partitioned across /// threads in a block. /// -/// \tparam T - the input/output type. -/// \tparam BlockSizeX - the number of threads in a block's x dimension. -/// \tparam Algorithm - selected scan algorithm, block_scan_algorithm::default_algorithm by default. -/// \tparam BlockSizeY - the number of threads in a block's y dimension, defaults to 1. -/// \tparam BlockSizeZ - the number of threads in a block's z dimension, defaults to 1. +/// \tparam T the input/output type. +/// \tparam BlockSizeX the number of threads in a block's x dimension. +/// \tparam Algorithm selected scan algorithm, block_scan_algorithm::default_algorithm by default. +/// \tparam BlockSizeY the number of threads in a block's y dimension, defaults to 1. +/// \tparam BlockSizeZ the number of threads in a block's z dimension, defaults to 1. /// /// \par Overview /// * Supports non-commutative scan operators. However, a scan operator should be @@ -150,13 +150,13 @@ class block_scan /// \brief Performs inclusive scan across threads in a block. /// - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] output - reference to a thread output value. May be aliased with \p input. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] input thread input value. + /// \param [out] output reference to a thread output value. May be aliased with \p input. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -210,12 +210,12 @@ class block_scan /// * This overload does not accept storage argument. Required shared memory is /// allocated by the method itself. /// - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] output - reference to a thread output value. May be aliased with \p input. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] input thread input value. + /// \param [out] output reference to a thread output value. May be aliased with \p input. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -230,14 +230,14 @@ class block_scan /// \brief Performs inclusive scan and reduction across threads in a block. /// - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] output - reference to a thread output value. May be aliased with \p input. - /// \param [out] reduction - result of reducing of all \p input values in a block. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] input thread input value. + /// \param [out] output reference to a thread output value. May be aliased with \p input. + /// \param [out] reduction result of reducing of all \p input values in a block. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -295,13 +295,13 @@ class block_scan /// * This overload does not accept storage argument. Required shared memory is /// allocated by the method itself. /// - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] output - reference to a thread output value. May be aliased with \p input. - /// \param [out] reduction - result of reducing of all \p input values in a block. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] input thread input value. + /// \param [out] output reference to a thread output value. May be aliased with \p input. + /// \param [out] reduction result of reducing of all \p input values in a block. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -318,22 +318,22 @@ class block_scan /// \brief Performs inclusive scan across threads in a block, and uses /// \p prefix_callback_op to generate prefix value for the whole block. /// - /// \tparam PrefixCallback - type of the unary function object used for generating + /// \tparam PrefixCallback type of the unary function object used for generating /// block-wide prefix value for the scan operation. - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] output - reference to a thread output value. May be aliased with \p input. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in,out] prefix_callback_op - function object for generating block prefix value. + /// \param [in] input thread input value. + /// \param [out] output reference to a thread output value. May be aliased with \p input. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in,out] prefix_callback_op function object for generating block prefix value. /// The signature of the \p prefix_callback_op should be equivalent to the following: /// T f(const T &block_reduction);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. /// The object will be called by the first warp of the block with block reduction of /// \p input values as input argument. The result of the first thread will be used as the /// block-wide prefix. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -407,14 +407,14 @@ class block_scan /// \brief Performs inclusive scan across threads in a block. /// - /// \tparam ItemsPerThread - number of items in the \p input array. - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam ItemsPerThread number of items in the \p input array. + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - reference to an array containing thread input values. - /// \param [out] output - reference to a thread output array. May be aliased with \p input. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] input reference to an array containing thread input values. + /// \param [out] output reference to a thread output array. May be aliased with \p input. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -478,13 +478,13 @@ class block_scan /// * This overload does not accept storage argument. Required shared memory is /// allocated by the method itself. /// - /// \tparam ItemsPerThread - number of items in the \p input array. - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam ItemsPerThread number of items in the \p input array. + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - reference to an array containing thread input values. - /// \param [out] output - reference to a thread output array. May be aliased with \p input. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] input reference to an array containing thread input values. + /// \param [out] output reference to a thread output array. May be aliased with \p input. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -509,15 +509,15 @@ class block_scan /// \brief Performs seeded inclusive scan across threads in a block. /// - /// \tparam ItemsPerThread - number of items in the \p input array. - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam ItemsPerThread number of items in the \p input array. + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - reference to an array containing thread input values. - /// \param [in] init - initial value to seed the inclusive scan. - /// \param [out] output - reference to a thread output array. May be aliased with \p input. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] input reference to an array containing thread input values. + /// \param [in] init initial value to seed the inclusive scan. + /// \param [out] output reference to a thread output array. May be aliased with \p input. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -574,14 +574,14 @@ class block_scan /// * This overload does not accept storage argument. Required shared memory is /// allocated by the method itself. /// - /// \tparam ItemsPerThread - number of items in the \p input array. - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam ItemsPerThread number of items in the \p input array. + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - reference to an array containing thread input values. - /// \param [in] init - initial value to seed the inclusive scan. - /// \param [out] output - reference to a thread output array. May be aliased with \p input. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] input reference to an array containing thread input values. + /// \param [in] init initial value to seed the inclusive scan. + /// \param [out] output reference to a thread output array. May be aliased with \p input. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -600,15 +600,15 @@ class block_scan /// \brief Performs inclusive scan and reduction across threads in a block. /// - /// \tparam ItemsPerThread - number of items in the \p input array. - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam ItemsPerThread number of items in the \p input array. + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - reference to an array containing thread input values. - /// \param [out] output - reference to a thread output array. May be aliased with \p input. - /// \param [out] reduction - result of reducing of all \p input values in a block. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] input reference to an array containing thread input values. + /// \param [out] output reference to a thread output array. May be aliased with \p input. + /// \param [out] reduction result of reducing of all \p input values in a block. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -675,14 +675,14 @@ class block_scan /// * This overload does not accept storage argument. Required shared memory is /// allocated by the method itself. /// - /// \tparam ItemsPerThread - number of items in the \p input array. - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam ItemsPerThread number of items in the \p input array. + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - reference to an array containing thread input values. - /// \param [out] output - reference to a thread output array. May be aliased with \p input. - /// \param [out] reduction - result of reducing of all \p input values in a block. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] input reference to an array containing thread input values. + /// \param [out] output reference to a thread output array. May be aliased with \p input. + /// \param [out] reduction result of reducing of all \p input values in a block. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -708,16 +708,16 @@ class block_scan /// \brief Performs seeded inclusive scan and reduction across threads in a block. /// - /// \tparam ItemsPerThread - number of items in the \p input array. - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam ItemsPerThread number of items in the \p input array. + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - reference to an array containing thread input values. - /// \param [in] init - initial value to seed the inclusive scan. - /// \param [out] output - reference to a thread output array. May be aliased with \p input. - /// \param [out] reduction - result of reducing of all \p input values in a block. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] input reference to an array containing thread input values. + /// \param [in] init initial value to seed the inclusive scan. + /// \param [out] output reference to a thread output array. May be aliased with \p input. + /// \param [out] reduction result of reducing of all \p input values in a block. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -778,15 +778,15 @@ class block_scan /// * This overload does not accept storage argument. Required shared memory is /// allocated by the method itself. /// - /// \tparam ItemsPerThread - number of items in the \p input array. - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam ItemsPerThread number of items in the \p input array. + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - reference to an array containing thread input values. - /// \param [in] init - initial value to seed the inclusive scan. - /// \param [out] output - reference to a thread output array. May be aliased with \p input. - /// \param [out] reduction - result of reducing of all \p input values in a block. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] input reference to an array containing thread input values. + /// \param [in] init initial value to seed the inclusive scan. + /// \param [out] output reference to a thread output array. May be aliased with \p input. + /// \param [out] reduction result of reducing of all \p input values in a block. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -807,23 +807,23 @@ class block_scan /// \brief Performs inclusive scan across threads in a block, and uses /// \p prefix_callback_op to generate prefix value for the whole block. /// - /// \tparam ItemsPerThread - number of items in the \p input array. - /// \tparam PrefixCallback - type of the unary function object used for generating + /// \tparam ItemsPerThread number of items in the \p input array. + /// \tparam PrefixCallback type of the unary function object used for generating /// block-wide prefix value for the scan operation. - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - reference to an array containing thread input values. - /// \param [out] output - reference to a thread output array. May be aliased with \p input. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in,out] prefix_callback_op - function object for generating block prefix value. + /// \param [in] input reference to an array containing thread input values. + /// \param [out] output reference to a thread output array. May be aliased with \p input. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in,out] prefix_callback_op function object for generating block prefix value. /// The signature of the \p prefix_callback_op should be equivalent to the following: /// T f(const T &block_reduction);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. /// The object will be called by the first warp of the block with block reduction of /// \p input values as input argument. The result of the first thread will be used as the /// block-wide prefix. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -905,15 +905,15 @@ class block_scan /// \brief Performs exclusive scan across threads in a block. /// - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] output - reference to a thread output value. May be aliased with \p input. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] init - initial value used to start the exclusive scan. Should be the same + /// \param [in] input thread input value. + /// \param [out] output reference to a thread output value. May be aliased with \p input. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] init initial value used to start the exclusive scan. Should be the same /// for all threads in a block. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -970,14 +970,14 @@ class block_scan /// * This overload does not accept storage argument. Required shared memory is /// allocated by the method itself. /// - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] output - reference to a thread output value. May be aliased with \p input. - /// \param [in] init - initial value used to start the exclusive scan. Should be the same + /// \param [in] input thread input value. + /// \param [out] output reference to a thread output value. May be aliased with \p input. + /// \param [in] init initial value used to start the exclusive scan. Should be the same /// for all threads in a block. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -993,16 +993,16 @@ class block_scan /// \brief Performs exclusive scan and reduction across threads in a block. /// - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] output - reference to a thread output value. May be aliased with \p input. - /// \param [in] init - initial value used to start the exclusive scan. Should be the same + /// \param [in] input thread input value. + /// \param [out] output reference to a thread output value. May be aliased with \p input. + /// \param [in] init initial value used to start the exclusive scan. Should be the same /// for all threads in a block. - /// \param [out] reduction - result of reducing of all \p input values in a block. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [out] reduction result of reducing of all \p input values in a block. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -1063,15 +1063,15 @@ class block_scan /// * This overload does not accept storage argument. Required shared memory is /// allocated by the method itself. /// - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] output - reference to a thread output value. May be aliased with \p input. - /// \param [in] init - initial value used to start the exclusive scan. Should be the same + /// \param [in] input thread input value. + /// \param [out] output reference to a thread output value. May be aliased with \p input. + /// \param [in] init initial value used to start the exclusive scan. Should be the same /// for all threads in a block. - /// \param [out] reduction - result of reducing of all \p input values in a block. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [out] reduction result of reducing of all \p input values in a block. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -1089,22 +1089,22 @@ class block_scan /// \brief Performs exclusive scan across threads in a block, and uses /// \p prefix_callback_op to generate prefix value for the whole block. /// - /// \tparam PrefixCallback - type of the unary function object used for generating + /// \tparam PrefixCallback type of the unary function object used for generating /// block-wide prefix value for the scan operation. - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] output - reference to a thread output value. May be aliased with \p input. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in,out] prefix_callback_op - function object for generating block prefix value. + /// \param [in] input thread input value. + /// \param [out] output reference to a thread output value. May be aliased with \p input. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in,out] prefix_callback_op function object for generating block prefix value. /// The signature of the \p prefix_callback_op should be equivalent to the following: /// T f(const T &block_reduction);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. /// The object will be called by the first warp of the block with block reduction of /// \p input values as input argument. The result of the first thread will be used as the /// block-wide prefix. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -1178,16 +1178,16 @@ class block_scan /// \brief Performs exclusive scan across threads in a block. /// - /// \tparam ItemsPerThread - number of items in the \p input array. - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam ItemsPerThread number of items in the \p input array. + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - reference to an array containing thread input values. - /// \param [out] output - reference to a thread output array. May be aliased with \p input. - /// \param [in] init - initial value used to start the exclusive scan. Should be the same + /// \param [in] input reference to an array containing thread input values. + /// \param [out] output reference to a thread output array. May be aliased with \p input. + /// \param [in] init initial value used to start the exclusive scan. Should be the same /// for all threads in a block. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -1254,15 +1254,15 @@ class block_scan /// * This overload does not accept storage argument. Required shared memory is /// allocated by the method itself. /// - /// \tparam ItemsPerThread - number of items in the \p input array. - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam ItemsPerThread number of items in the \p input array. + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - reference to an array containing thread input values. - /// \param [out] output - reference to a thread output array. May be aliased with \p input. - /// \param [in] init - initial value used to start the exclusive scan. Should be the same + /// \param [in] input reference to an array containing thread input values. + /// \param [out] output reference to a thread output array. May be aliased with \p input. + /// \param [in] init initial value used to start the exclusive scan. Should be the same /// for all threads in a block. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -1288,17 +1288,17 @@ class block_scan /// \brief Performs exclusive scan and reduction across threads in a block. /// - /// \tparam ItemsPerThread - number of items in the \p input array. - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam ItemsPerThread number of items in the \p input array. + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - reference to an array containing thread input values. - /// \param [out] output - reference to a thread output array. May be aliased with \p input. - /// \param [in] init - initial value used to start the exclusive scan. Should be the same + /// \param [in] input reference to an array containing thread input values. + /// \param [out] output reference to a thread output array. May be aliased with \p input. + /// \param [in] init initial value used to start the exclusive scan. Should be the same /// for all threads in a block. - /// \param [out] reduction - result of reducing of all \p input values in a block. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [out] reduction result of reducing of all \p input values in a block. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -1369,16 +1369,16 @@ class block_scan /// * This overload does not accept storage argument. Required shared memory is /// allocated by the method itself. /// - /// \tparam ItemsPerThread - number of items in the \p input array. - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam ItemsPerThread number of items in the \p input array. + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - reference to an array containing thread input values. - /// \param [out] output - reference to a thread output array. May be aliased with \p input. - /// \param [in] init - initial value used to start the exclusive scan. Should be the same + /// \param [in] input reference to an array containing thread input values. + /// \param [out] output reference to a thread output array. May be aliased with \p input. + /// \param [in] init initial value used to start the exclusive scan. Should be the same /// for all threads in a block. - /// \param [out] reduction - result of reducing of all \p input values in a block. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [out] reduction result of reducing of all \p input values in a block. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -1406,23 +1406,23 @@ class block_scan /// \brief Performs exclusive scan across threads in a block, and uses /// \p prefix_callback_op to generate prefix value for the whole block. /// - /// \tparam ItemsPerThread - number of items in the \p input array. - /// \tparam PrefixCallback - type of the unary function object used for generating + /// \tparam ItemsPerThread number of items in the \p input array. + /// \tparam PrefixCallback type of the unary function object used for generating /// block-wide prefix value for the scan operation. - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - reference to an array containing thread input values. - /// \param [out] output - reference to a thread output array. May be aliased with \p input. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in,out] prefix_callback_op - function object for generating block prefix value. + /// \param [in] input reference to an array containing thread input values. + /// \param [out] output reference to a thread output array. May be aliased with \p input. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in,out] prefix_callback_op function object for generating block prefix value. /// The signature of the \p prefix_callback_op should be equivalent to the following: /// T f(const T &block_reduction);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. /// The object will be called by the first warp of the block with block reduction of /// \p input values as input argument. The result of the first thread will be used as the /// block-wide prefix. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. diff --git a/rocprim/include/rocprim/block/block_shuffle.hpp b/rocprim/include/rocprim/block/block_shuffle.hpp index 8d7cd7373..0db3ea34f 100644 --- a/rocprim/include/rocprim/block/block_shuffle.hpp +++ b/rocprim/include/rocprim/block/block_shuffle.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2024, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -50,10 +50,10 @@ BEGIN_ROCPRIM_NAMESPACE /// \brief The block_shuffle class is a block level parallel primitive which provides methods /// for shuffling data partitioned across a block /// -/// \tparam T - the input/output type. -/// \tparam BlockSizeX - the number of threads in a block's x dimension, it has no defaults value. -/// \tparam BlockSizeY - the number of threads in a block's y dimension, defaults to 1. -/// \tparam BlockSizeZ - the number of threads in a block's z dimension, defaults to 1. +/// \tparam T the input/output type. +/// \tparam BlockSizeX the number of threads in a block's x dimension, it has no defaults value. +/// \tparam BlockSizeY the number of threads in a block's y dimension, defaults to 1. +/// \tparam BlockSizeZ the number of threads in a block's z dimension, defaults to 1. /// /// \par Overview /// It is commonplace for blocks of threads to rearrange data items between @@ -114,9 +114,9 @@ class block_shuffle /// \par A thread with threadIdx i receives data from a thread with threadIdx (i - distance), where distance may be a negative value. /// \par Any shuffle operation with invalid input or output threadIds are not carried out, i.e. threadId < 0 || threadId >= BlockSize. /// - /// \param [in] input - input data to be shuffled to another thread. - /// \param [out] output - reference to a output value, that receives data from another thread - /// \param [in] distance - The input threadId + distance = output threadId. + /// \param [in] input input data to be shuffled to another thread. + /// \param [out] output reference to a output value, that receives data from another thread + /// \param [in] distance The input threadId + distance = output threadId. /// /// \par Example. /// \code{.cpp} @@ -147,10 +147,10 @@ class block_shuffle /// \par A thread with threadIdx i receives data from a thread with threadIdx (i - distance), where distance may be a negative value. /// \par Any shuffle operation with invalid input or output threadIds are not carried out, i.e. threadId < 0 || threadId >= BlockSize. /// - /// \param [in] flat_id - flat thread ID obtained from rocprim::flat_block_thread_id - /// \param [in] input - input data to be shuffled to another thread. - /// \param [out] output - reference to a output value, that receives data from another thread - /// \param [in] distance - The input threadId + distance = output threadId. + /// \param [in] flat_id flat thread ID obtained from rocprim::flat_block_thread_id + /// \param [in] input input data to be shuffled to another thread. + /// \param [out] output reference to a output value, that receives data from another thread + /// \param [in] distance The input threadId + distance = output threadId. ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void offset(const size_t& flat_id, T input, T& output, int distance) { @@ -163,11 +163,11 @@ class block_shuffle /// \par A thread with threadIdx i receives data from a thread with threadIdx (i - distance), where distance may be a negative value. /// \par Any shuffle operation with invalid input or output threadIds are not carried out, i.e. threadId < 0 || threadId >= BlockSize. /// - /// \param [in] flat_id - flat thread ID obtained from rocprim::flat_block_thread_id - /// \param [in] input - input data to be shuffled to another thread. - /// \param [out] output - reference to a output value, that receives data from another thread - /// \param [in] distance - The input threadId + distance = output threadId. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] flat_id flat thread ID obtained from rocprim::flat_block_thread_id + /// \param [in] input input data to be shuffled to another thread. + /// \param [out] output reference to a output value, that receives data from another thread + /// \param [in] distance The input threadId + distance = output threadId. + /// \param [in] storage reference to a temporary storage object of type storage_type. ROCPRIM_DEVICE ROCPRIM_INLINE void offset(const size_t& flat_id, T input, T& output, int distance, storage_type& storage) { @@ -189,9 +189,9 @@ class block_shuffle /// \par A thread with threadIdx i receives data from a thread with threadIdx (i - distance) % BlockSize, where distance may be a negative value. /// \par Data is rotated around the block, using (input_threadId + distance) modulous BlockSize to ensure valid threadIds. /// - /// \param [in] input - input data to be shuffled to another thread. - /// \param [out] output - reference to a output value, that receives data from another thread - /// \param [in] distance - The input threadId + distance = output threadId. Distance magnitude should be <= BlockSize. + /// \param [in] input input data to be shuffled to another thread. + /// \param [out] output reference to a output value, that receives data from another thread + /// \param [in] distance The input threadId + distance = output threadId. Distance magnitude should be <= BlockSize. /// /// \par Example. /// \code{.cpp} @@ -222,10 +222,10 @@ class block_shuffle /// \par A thread with threadIdx i receives data from a thread with threadIdx (i - distance) % BlockSize, where distance may be a negative value. /// \par Data is rotated around the block, using (input_threadId + distance) modulous BlockSize to ensure valid threadIds. /// - /// \param [in] flat_id - flat thread ID obtained from rocprim::flat_block_thread_id - /// \param [in] input - input data to be shuffled to another thread. - /// \param [out] output - reference to a output value, that receives data from another thread - /// \param [in] distance - The input threadId + distance = output threadId. + /// \param [in] flat_id flat thread ID obtained from rocprim::flat_block_thread_id + /// \param [in] input input data to be shuffled to another thread. + /// \param [out] output reference to a output value, that receives data from another thread + /// \param [in] distance The input threadId + distance = output threadId. ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void rotate(const size_t& flat_id, T input, T& output, int distance) { @@ -238,11 +238,11 @@ class block_shuffle /// \par A thread with threadIdx i receives data from a thread with threadIdx (i - distance) % BlockSize, where distance may be a negative value. /// \par Data is rotated around the block, using (input_threadId + distance) modulous BlockSize to ensure valid threadIds. /// - /// \param [in] flat_id - flat thread ID obtained from rocprim::flat_block_thread_id - /// \param [in] input - input data to be shuffled to another thread. - /// \param [out] output - reference to a output value, that receives data from another thread - /// \param [in] distance - The input threadId + distance = output threadId. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] flat_id flat thread ID obtained from rocprim::flat_block_thread_id + /// \param [in] input input data to be shuffled to another thread. + /// \param [out] output reference to a output value, that receives data from another thread + /// \param [in] distance The input threadId + distance = output threadId. + /// \param [in] storage reference to a temporary storage object of type storage_type. ROCPRIM_DEVICE ROCPRIM_INLINE void rotate(const size_t& flat_id, T input, T& output, int distance, storage_type& storage) { @@ -267,8 +267,8 @@ class block_shuffle /// \brief The thread block rotates a blocked arrange of input items, /// shifting it up by one item /// - /// \param [in] input - The calling thread's input items - /// \param [out] prev - The corresponding predecessor items (may be aliased to \p input). + /// \param [in] input The calling thread's input items + /// \param [out] prev The corresponding predecessor items (may be aliased to \p input). /// The item \p prev[0] is not updated for thread0. /// /// \par Example. @@ -299,9 +299,9 @@ class block_shuffle /// \brief The thread block rotates a blocked arrange of input items, /// shifting it up by one item /// - /// \param [in] flat_id - flat thread ID obtained from rocprim::flat_block_thread_id - /// \param [in] input - The calling thread's input items - /// \param [out] prev - The corresponding predecessor items (may be aliased to \p input). + /// \param [in] flat_id flat thread ID obtained from rocprim::flat_block_thread_id + /// \param [in] input The calling thread's input items + /// \param [out] prev The corresponding predecessor items (may be aliased to \p input). /// The item \p prev[0] is not updated for thread0. template ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void @@ -314,10 +314,10 @@ class block_shuffle /// \brief The thread block rotates a blocked arrange of input items, /// shifting it up by one item, using temporary storage. /// - /// \param [in] flat_id - flat thread ID obtained from rocprim::flat_block_thread_id - /// \param [in] input - The calling thread's input items - /// \param [out] prev - The corresponding predecessor items (may be aliased to \p input). - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] flat_id flat thread ID obtained from rocprim::flat_block_thread_id + /// \param [in] input The calling thread's input items + /// \param [out] prev The corresponding predecessor items (may be aliased to \p input). + /// \param [in] storage reference to a temporary storage object of type storage_type. /// The item \p prev[0] is not updated for thread0. template ROCPRIM_DEVICE ROCPRIM_INLINE void up(const size_t& flat_id, @@ -345,10 +345,10 @@ class block_shuffle /// \brief The thread block rotates a blocked arrange of input items, /// shifting it up by one item /// - /// \param [in] input - The calling thread's input items - /// \param [out] prev - The corresponding predecessor items (may be aliased to \p input). + /// \param [in] input The calling thread's input items + /// \param [out] prev The corresponding predecessor items (may be aliased to \p input). /// The item \p prev[0] is not updated for thread0. - /// \param [out] block_suffix - The item \p input[ItemsPerThread-1] from + /// \param [out] block_suffix The item \p input[ItemsPerThread-1] from /// threadBlockSize-1, provided to all threads template ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void @@ -363,11 +363,11 @@ class block_shuffle /// \brief The thread block rotates a blocked arrange of input items, /// shifting it up by one item /// - /// \param [in] flat_id - flat thread ID obtained from rocprim::flat_block_thread_id - /// \param [in] input - The calling thread's input items - /// \param [out] prev - The corresponding predecessor items (may be aliased to \p input). + /// \param [in] flat_id flat thread ID obtained from rocprim::flat_block_thread_id + /// \param [in] input The calling thread's input items + /// \param [out] prev The corresponding predecessor items (may be aliased to \p input). /// The item \p prev[0] is not updated for thread0. - /// \param [out] block_suffix - The item \p input[ItemsPerThread-1] from + /// \param [out] block_suffix The item \p input[ItemsPerThread-1] from /// threadBlockSize-1, provided to all threads template ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void up(const size_t& flat_id, @@ -382,13 +382,13 @@ class block_shuffle /// \brief The thread block rotates a blocked arrange of input items, /// shifting it up by one item, using temporary storage. /// - /// \param [in] flat_id - flat thread ID obtained from rocprim::flat_block_thread_id - /// \param [in] input - The calling thread's input items - /// \param [out] prev - The corresponding predecessor items (may be aliased to \p input). + /// \param [in] flat_id flat thread ID obtained from rocprim::flat_block_thread_id + /// \param [in] input The calling thread's input items + /// \param [out] prev The corresponding predecessor items (may be aliased to \p input). /// The item \p prev[0] is not updated for thread0. - /// \param [out] block_suffix - The item \p input[ItemsPerThread-1] from + /// \param [out] block_suffix The item \p input[ItemsPerThread-1] from /// threadBlockSize-1, provided to all threads - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] storage reference to a temporary storage object of type storage_type. template ROCPRIM_DEVICE ROCPRIM_INLINE void up(const size_t& flat_id, T (&input)[ItemsPerThread], @@ -405,8 +405,8 @@ class block_shuffle /// \brief The thread block rotates a blocked arrange of input items, /// shifting it down by one item /// - /// \param [in] input - The calling thread's input items - /// \param [out] next - The corresponding successor items (may be aliased to \p input). + /// \param [in] input The calling thread's input items + /// \param [out] next The corresponding successor items (may be aliased to \p input). /// The item \p prev[0] is not updated for threadBlockSize - 1. /// /// \par Example. @@ -437,9 +437,9 @@ class block_shuffle /// \brief The thread block rotates a blocked arrange of input items, /// shifting it down by one item /// - /// \param [in] flat_id - flat thread ID obtained from rocprim::flat_block_thread_id - /// \param [in] input - The calling thread's input items - /// \param [out] next - The corresponding successor items (may be aliased to \p input). + /// \param [in] flat_id flat thread ID obtained from rocprim::flat_block_thread_id + /// \param [in] input The calling thread's input items + /// \param [out] next The corresponding successor items (may be aliased to \p input). /// The item \p prev[0] is not updated for threadBlockSize - 1. template ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void @@ -452,11 +452,11 @@ class block_shuffle /// \brief The thread block rotates a blocked arrange of input items, /// shifting it down by one item, using temporary storage. /// - /// \param [in] flat_id - flat thread ID obtained from rocprim::flat_block_thread_id - /// \param [in] input - The calling thread's input items - /// \param [out] next - The corresponding successor items (may be aliased to \p input). + /// \param [in] flat_id flat thread ID obtained from rocprim::flat_block_thread_id + /// \param [in] input The calling thread's input items + /// \param [out] next The corresponding successor items (may be aliased to \p input). /// The item \p prev[0] is not updated for threadBlockSize - 1. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] storage reference to a temporary storage object of type storage_type. template ROCPRIM_DEVICE ROCPRIM_INLINE void down(const size_t& flat_id, T (&input)[ItemsPerThread], @@ -483,10 +483,10 @@ class block_shuffle /// \brief The thread block rotates a blocked arrange of input items, /// shifting it down by one item /// - /// \param [in] input - The calling thread's input items - /// \param [out] next - The corresponding successor items (may be aliased to \p input). + /// \param [in] input The calling thread's input items + /// \param [out] next The corresponding successor items (may be aliased to \p input). /// The item \p prev[0] is not updated for threadBlockSize - 1. - /// \param [out] block_prefix - The item \p input[0] from thread0, provided to all threads + /// \param [out] block_prefix The item \p input[0] from thread0, provided to all threads template ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void down(T (&input)[ItemsPerThread], T (&next)[ItemsPerThread], T& block_prefix) @@ -500,11 +500,11 @@ class block_shuffle /// \brief The thread block rotates a blocked arrange of input items, /// shifting it down by one item /// - /// \param [in] flat_id - flat thread ID obtained from rocprim::flat_block_thread_id - /// \param [in] input - The calling thread's input items - /// \param [out] next - The corresponding successor items (may be aliased to \p input). + /// \param [in] flat_id flat thread ID obtained from rocprim::flat_block_thread_id + /// \param [in] input The calling thread's input items + /// \param [out] next The corresponding successor items (may be aliased to \p input). /// The item \p prev[0] is not updated for threadBlockSize - 1. - /// \param [out] block_prefix - The item \p input[0] from thread0, provided to all threads + /// \param [out] block_prefix The item \p input[0] from thread0, provided to all threads template ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE void down(const size_t& flat_id, T (&input)[ItemsPerThread], @@ -518,12 +518,12 @@ class block_shuffle /// \brief The thread block rotates a blocked arrange of input items, /// shifting it down by one item, using temporary storage. /// - /// \param [in] flat_id - flat thread ID obtained from rocprim::flat_block_thread_id - /// \param [in] input - The calling thread's input items - /// \param [out] next - The corresponding successor items (may be aliased to \p input). + /// \param [in] flat_id flat thread ID obtained from rocprim::flat_block_thread_id + /// \param [in] input The calling thread's input items + /// \param [out] next The corresponding successor items (may be aliased to \p input). /// The item \p prev[0] is not updated for threadBlockSize - 1. - /// \param [out] block_prefix - The item \p input[0] from thread0, provided to all threads - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [out] block_prefix The item \p input[0] from thread0, provided to all threads + /// \param [in] storage reference to a temporary storage object of type storage_type. template ROCPRIM_DEVICE ROCPRIM_INLINE void down(const size_t& flat_id, T (&input)[ItemsPerThread], diff --git a/rocprim/include/rocprim/block/block_sort.hpp b/rocprim/include/rocprim/block/block_sort.hpp index 9b16dd2cf..04f3fd1d0 100644 --- a/rocprim/include/rocprim/block/block_sort.hpp +++ b/rocprim/include/rocprim/block/block_sort.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -127,13 +127,13 @@ struct select_block_sort_impl /// methods sorting items (keys or key-value pairs) partitioned across threads in a block /// using comparison-based sort algorithm. /// -/// \tparam Key - the key type. -/// \tparam BlockSize - the number of threads in a block. -/// \tparam ItemsPerThread - number of items processed by each thread. +/// \tparam Key the key type. +/// \tparam BlockSize the number of threads in a block. +/// \tparam ItemsPerThread number of items processed by each thread. /// The total range will be BlockSize * ItemsPerThread long -/// \tparam Value - the value type. Default type empty_type indicates +/// \tparam Value the value type. Default type empty_type indicates /// a keys-only sort. -/// \tparam Algorithm - selected sort algorithm. The available algorithms and default choice are +/// \tparam Algorithm selected sort algorithm. The available algorithms and default choice are /// documented in \ref block_sort_algorithm. /// /// \par Overview @@ -194,11 +194,11 @@ class block_sort /// \brief Block sort for any data type. /// - /// \tparam BinaryFunction - type of binary function used for sort. Default type + /// \tparam BinaryFunction type of binary function used for sort. Default type /// is rocprim::less. /// - /// \param [in, out] thread_key - reference to a key provided by a thread. - /// \param [in] compare_function - comparison function object which returns true if the + /// \param [in, out] thread_key reference to a key provided by a thread. + /// \param [in] compare_function comparison function object which returns true if the /// first argument is is ordered before the second. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b);. The signature does not need to have @@ -223,12 +223,12 @@ class block_sort /// \brief Block sort for any data type. /// - /// \tparam BinaryFunction - type of binary function used for sort. Default type + /// \tparam BinaryFunction type of binary function used for sort. Default type /// is rocprim::less. /// - /// \param [in, out] thread_key - reference to a key provided by a thread. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] compare_function - comparison function object which returns true if the + /// \param [in, out] thread_key reference to a key provided by a thread. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] compare_function comparison function object which returns true if the /// first argument is is ordered before the second. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b);. The signature does not need to have @@ -283,12 +283,12 @@ class block_sort /// \brief Block sort by key for any data type. /// - /// \tparam BinaryFunction - type of binary function used for sort. Default type + /// \tparam BinaryFunction type of binary function used for sort. Default type /// is rocprim::less. /// - /// \param [in, out] thread_key - reference to a key provided by a thread. - /// \param [in, out] thread_value - reference to a value provided by a thread. - /// \param [in] compare_function - comparison function object which returns true if the + /// \param [in, out] thread_key reference to a key provided by a thread. + /// \param [in, out] thread_value reference to a value provided by a thread. + /// \param [in] compare_function comparison function object which returns true if the /// first argument is is ordered before the second. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b);. The signature does not need to have @@ -315,13 +315,13 @@ class block_sort /// \brief Block sort by key for any data type. /// - /// \tparam BinaryFunction - type of binary function used for sort. Default type + /// \tparam BinaryFunction type of binary function used for sort. Default type /// is rocprim::less. /// - /// \param [in, out] thread_key - reference to a key provided by a thread. - /// \param [in, out] thread_value - reference to a value provided by a thread. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] compare_function - comparison function object which returns true if the + /// \param [in, out] thread_key reference to a key provided by a thread. + /// \param [in, out] thread_value reference to a value provided by a thread. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] compare_function comparison function object which returns true if the /// first argument is is ordered before the second. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b);. The signature does not need to have @@ -380,13 +380,13 @@ class block_sort /// \brief Block sort for any data type. This function sorts up to \p size elements blocked /// across threads. /// - /// \tparam BinaryFunction - type of binary function used for sort. Default type + /// \tparam BinaryFunction type of binary function used for sort. Default type /// is rocprim::less. /// - /// \param [in, out] thread_key - reference to a key provided by a thread. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] size - custom size of block to be sorted. - /// \param [in] compare_function - comparison function object which returns true if the + /// \param [in, out] thread_key reference to a key provided by a thread. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] size custom size of block to be sorted. + /// \param [in] compare_function comparison function object which returns true if the /// first argument is is ordered before the second. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b);. The signature does not need to have @@ -404,13 +404,13 @@ class block_sort /// \brief Block sort for any data type. This function sorts up to \p size elements blocked /// across threads. /// - /// \tparam BinaryFunction - type of binary function used for sort. Default type + /// \tparam BinaryFunction type of binary function used for sort. Default type /// is rocprim::less. /// - /// \param [in, out] thread_keys - reference to keys provided by a thread. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] size - custom size of block to be sorted. - /// \param [in] compare_function - comparison function object which returns true if the + /// \param [in, out] thread_keys reference to keys provided by a thread. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] size custom size of block to be sorted. + /// \param [in] compare_function comparison function object which returns true if the /// first argument is is ordered before the second. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b);. The signature does not need to have @@ -428,14 +428,14 @@ class block_sort /// \brief Block sort by key for any data type. This function sorts up to \p size elements /// blocked across threads. /// - /// \tparam BinaryFunction - type of binary function used for sort. Default type + /// \tparam BinaryFunction type of binary function used for sort. Default type /// is rocprim::less. /// - /// \param [in, out] thread_key - reference to a key provided by a thread. - /// \param [in, out] thread_value - reference to a value provided by a thread. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] size - custom size of block to be sorted. - /// \param [in] compare_function - comparison function object which returns true if the + /// \param [in, out] thread_key reference to a key provided by a thread. + /// \param [in, out] thread_value reference to a value provided by a thread. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] size custom size of block to be sorted. + /// \param [in] compare_function comparison function object which returns true if the /// first argument is is ordered before the second. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b);. The signature does not need to have @@ -453,14 +453,14 @@ class block_sort /// \brief Block sort by key for any data type. This function sorts up to \p size elements /// blocked across threads. /// - /// \tparam BinaryFunction - type of binary function used for sort. Default type + /// \tparam BinaryFunction type of binary function used for sort. Default type /// is rocprim::less. /// - /// \param [in, out] thread_keys - reference to keys provided by a thread. - /// \param [in, out] thread_values - reference to values provided by a thread. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] size - custom size of block to be sorted. - /// \param [in] compare_function - comparison function object which returns true if the + /// \param [in, out] thread_keys reference to keys provided by a thread. + /// \param [in, out] thread_values reference to values provided by a thread. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] size custom size of block to be sorted. + /// \param [in] compare_function comparison function object which returns true if the /// first argument is is ordered before the second. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b);. The signature does not need to have diff --git a/rocprim/include/rocprim/block/block_store.hpp b/rocprim/include/rocprim/block/block_store.hpp index cbd7ed739..905cb0c01 100644 --- a/rocprim/include/rocprim/block/block_store.hpp +++ b/rocprim/include/rocprim/block/block_store.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -93,11 +93,11 @@ enum class block_store_method /// \brief The \p block_store class is a block level parallel primitive which provides methods /// for storing an arrangement of items into a blocked/striped arrangement on continous memory. /// -/// \tparam T - the output/output type. -/// \tparam BlockSize - the number of threads in a block. -/// \tparam ItemsPerThread - the number of items to be processed by +/// \tparam T the output/output type. +/// \tparam BlockSize the number of threads in a block. +/// \tparam ItemsPerThread the number of items to be processed by /// each thread. -/// \tparam Method - the method to store data. +/// \tparam Method the method to store data. /// /// \par Overview /// * The \p block_store class has a number of different methods to store data: @@ -154,11 +154,11 @@ class block_store /// \brief Stores an arrangement of items from across the thread block into an /// arrangement on continuous memory. /// - /// \tparam OutputIterator - [inferred] an iterator type for output (can be a simple + /// \tparam OutputIterator [inferred] an iterator type for output (can be a simple /// pointer. /// - /// \param [out] block_output - the output iterator from the thread block to store to. - /// \param [in] items - array that data is read from. + /// \param [out] block_output the output iterator from the thread block to store to. + /// \param [in] items array that data is read from. /// /// \par Overview /// * The type \p T must be such that an object of type \p InputIterator @@ -175,12 +175,12 @@ class block_store /// \brief Stores an arrangement of items from across the thread block into an /// arrangement on continuous memory, which is guarded by range \p valid. /// - /// \tparam OutputIterator - [inferred] an iterator type for output (can be a simple + /// \tparam OutputIterator [inferred] an iterator type for output (can be a simple /// pointer. /// - /// \param [out] block_output - the output iterator from the thread block to store to. - /// \param [in] items - array that data is read from. - /// \param [in] valid - maximum range of valid numbers to read. + /// \param [out] block_output the output iterator from the thread block to store to. + /// \param [in] items array that data is read from. + /// \param [in] valid maximum range of valid numbers to read. /// /// \par Overview /// * The type \p T must be such that an object of type \p InputIterator @@ -198,12 +198,12 @@ class block_store /// \brief Stores an arrangement of items from across the thread block into an /// arrangement on continuous memory, using temporary storage. /// - /// \tparam OutputIterator - [inferred] an iterator type for output (can be a simple + /// \tparam OutputIterator [inferred] an iterator type for output (can be a simple /// pointer. /// - /// \param [out] block_output - the output iterator from the thread block to store to. - /// \param [in] items - array that data is read from. - /// \param [in] storage - temporary storage for outputs. + /// \param [out] block_output the output iterator from the thread block to store to. + /// \param [in] items array that data is read from. + /// \param [in] storage temporary storage for outputs. /// /// \par Overview /// * The type \p T must be such that an object of type \p InputIterator @@ -239,13 +239,13 @@ class block_store /// arrangement on continuous memory, which is guarded by range \p valid, /// using temporary storage /// - /// \tparam OutputIterator - [inferred] an iterator type for output (can be a simple + /// \tparam OutputIterator [inferred] an iterator type for output (can be a simple /// pointer. /// - /// \param [out] block_output - the output iterator from the thread block to store to. - /// \param [in] items - array that data is read from. - /// \param [in] valid - maximum range of valid numbers to read. - /// \param [in] storage - temporary storage for outputs. + /// \param [out] block_output the output iterator from the thread block to store to. + /// \param [in] items array that data is read from. + /// \param [in] valid maximum range of valid numbers to read. + /// \param [in] storage temporary storage for outputs. /// /// \par Overview /// * The type \p T must be such that an object of type \p InputIterator diff --git a/rocprim/include/rocprim/block/block_store_func.hpp b/rocprim/include/rocprim/block/block_store_func.hpp index ba6abc495..225bd1671 100644 --- a/rocprim/include/rocprim/block/block_store_func.hpp +++ b/rocprim/include/rocprim/block/block_store_func.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -40,15 +40,15 @@ BEGIN_ROCPRIM_NAMESPACE /// across a thread block. Each thread uses a \p flat_id to store a range of /// \p ItemsPerThread \p items to the thread block. /// -/// \tparam OutputIterator - [inferred] an iterator type for input (can be a simple +/// \tparam OutputIterator [inferred] an iterator type for input (can be a simple /// pointer -/// \tparam T - [inferred] the data type -/// \tparam ItemsPerThread - [inferred] the number of items to be processed by +/// \tparam T [inferred] the data type +/// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread /// -/// \param flat_id - a local flat 1D thread id in a block (tile) for the calling thread -/// \param block_output - the input iterator from the thread block to store to -/// \param items - array that data is stored to thread block +/// \param flat_id a local flat 1D thread id in a block (tile) for the calling thread +/// \param block_output the input iterator from the thread block to store to +/// \param items array that data is stored to thread block template< class OutputIterator, class T, @@ -79,16 +79,16 @@ void block_store_direct_blocked(unsigned int flat_id, /// across a thread block. Each thread uses a \p flat_id to store a range of /// \p ItemsPerThread \p items to the thread block. /// -/// \tparam OutputIterator - [inferred] an iterator type for input (can be a simple +/// \tparam OutputIterator [inferred] an iterator type for input (can be a simple /// pointer -/// \tparam T - [inferred] the data type -/// \tparam ItemsPerThread - [inferred] the number of items to be processed by +/// \tparam T [inferred] the data type +/// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread /// -/// \param flat_id - a local flat 1D thread id in a block (tile) for the calling thread -/// \param block_output - the input iterator from the thread block to store to -/// \param items - array that data is stored to thread block -/// \param valid - maximum range of valid numbers to store +/// \param flat_id a local flat 1D thread id in a block (tile) for the calling thread +/// \param block_output the input iterator from the thread block to store to +/// \param items array that data is stored to thread block +/// \param valid maximum range of valid numbers to store template< class OutputIterator, class T, @@ -131,16 +131,16 @@ void block_store_direct_blocked(unsigned int flat_id, /// * The datatype \p T is not a primitive or a HIP vector type (e.g. int2, /// int4, etc. /// -/// \tparam T - [inferred] the output data type -/// \tparam U - [inferred] the input data type -/// \tparam ItemsPerThread - [inferred] the number of items to be processed by +/// \tparam T [inferred] the output data type +/// \tparam U [inferred] the input data type +/// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread /// /// The type \p U must be such that it can be implicitly converted to \p T. /// -/// \param flat_id - a local flat 1D thread id in a block (tile) for the calling thread -/// \param block_output - the input iterator from the thread block to load from -/// \param items - array that data is loaded to +/// \param flat_id a local flat 1D thread id in a block (tile) for the calling thread +/// \param block_output the input iterator from the thread block to load from +/// \param items array that data is loaded to template< class T, class U, @@ -192,16 +192,16 @@ block_store_direct_blocked_vectorized(unsigned int flat_id, /// across a thread block. Each thread uses a \p flat_id to store a range of /// \p ItemsPerThread \p items to the thread block. /// -/// \tparam BlockSize - the number of threads in a block -/// \tparam OutputIterator - [inferred] an iterator type for input (can be a simple +/// \tparam BlockSize the number of threads in a block +/// \tparam OutputIterator [inferred] an iterator type for input (can be a simple /// pointer -/// \tparam T - [inferred] the data type -/// \tparam ItemsPerThread - [inferred] the number of items to be processed by +/// \tparam T [inferred] the data type +/// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread /// -/// \param flat_id - a local flat 1D thread id in a block (tile) for the calling thread -/// \param block_output - the input iterator from the thread block to store to -/// \param items - array that data is stored to thread block +/// \param flat_id a local flat 1D thread id in a block (tile) for the calling thread +/// \param block_output the input iterator from the thread block to store to +/// \param items array that data is stored to thread block template< unsigned int BlockSize, class OutputIterator, @@ -232,17 +232,17 @@ void block_store_direct_striped(unsigned int flat_id, /// across a thread block. Each thread uses a \p flat_id to store a range of /// \p ItemsPerThread \p items to the thread block. /// -/// \tparam BlockSize - the number of threads in a block -/// \tparam OutputIterator - [inferred] an iterator type for input (can be a simple +/// \tparam BlockSize the number of threads in a block +/// \tparam OutputIterator [inferred] an iterator type for input (can be a simple /// pointer -/// \tparam T - [inferred] the data type -/// \tparam ItemsPerThread - [inferred] the number of items to be processed by +/// \tparam T [inferred] the data type +/// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread /// -/// \param flat_id - a local flat 1D thread id in a block (tile) for the calling thread -/// \param block_output - the input iterator from the thread block to store to -/// \param items - array that data is stored to thread block -/// \param valid - maximum range of valid numbers to store +/// \param flat_id a local flat 1D thread id in a block (tile) for the calling thread +/// \param block_output the input iterator from the thread block to store to +/// \param items array that data is stored to thread block +/// \param valid maximum range of valid numbers to store template< unsigned int BlockSize, class OutputIterator, @@ -286,16 +286,16 @@ void block_store_direct_striped(unsigned int flat_id, /// * Using \p WarpSize smaller than hardware warpsize could result in lower /// performance. /// -/// \tparam WarpSize - [optional] the number of threads in a warp -/// \tparam OutputIterator - [inferred] an iterator type for input (can be a simple +/// \tparam WarpSize [optional] the number of threads in a warp +/// \tparam OutputIterator [inferred] an iterator type for input (can be a simple /// pointer -/// \tparam T - [inferred] the data type -/// \tparam ItemsPerThread - [inferred] the number of items to be processed by +/// \tparam T [inferred] the data type +/// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread /// -/// \param flat_id - a local flat 1D thread id in a block (tile) for the calling thread -/// \param block_output - the input iterator from the thread block to store to -/// \param items - array that data is stored to thread block +/// \param flat_id a local flat 1D thread id in a block (tile) for the calling thread +/// \param block_output the input iterator from the thread block to store to +/// \param items array that data is stored to thread block template< unsigned int WarpSize = device_warp_size(), class OutputIterator, @@ -341,17 +341,17 @@ void block_store_direct_warp_striped(unsigned int flat_id, /// * Using \p WarpSize smaller than hardware warpsize could result in lower /// performance. /// -/// \tparam WarpSize - [optional] the number of threads in a warp -/// \tparam OutputIterator - [inferred] an iterator type for input (can be a simple +/// \tparam WarpSize [optional] the number of threads in a warp +/// \tparam OutputIterator [inferred] an iterator type for input (can be a simple /// pointer -/// \tparam T - [inferred] the data type -/// \tparam ItemsPerThread - [inferred] the number of items to be processed by +/// \tparam T [inferred] the data type +/// \tparam ItemsPerThread [inferred] the number of items to be processed by /// each thread /// -/// \param flat_id - a local flat 1D thread id in a block (tile) for the calling thread -/// \param block_output - the input iterator from the thread block to store to -/// \param items - array that data is stored to thread block -/// \param valid - maximum range of valid numbers to store +/// \param flat_id a local flat 1D thread id in a block (tile) for the calling thread +/// \param block_output the input iterator from the thread block to store to +/// \param items array that data is stored to thread block +/// \param valid maximum range of valid numbers to store template< unsigned int WarpSize = device_warp_size(), class OutputIterator, diff --git a/rocprim/include/rocprim/detail/temp_storage.hpp b/rocprim/include/rocprim/detail/temp_storage.hpp index d1c27b747..797e5c711 100644 --- a/rocprim/include/rocprim/detail/temp_storage.hpp +++ b/rocprim/include/rocprim/detail/temp_storage.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -55,7 +55,7 @@ struct layout /// \brief This structure describes a single required partition of temporary global memory, as well /// as where to store the allocated pointer. -/// \tparam T - The base type to allocate temporary memory for. +/// \tparam T The base type to allocate temporary memory for. template struct simple_partition { @@ -73,7 +73,7 @@ struct simple_partition /// \brief Assigns the final storage for this partition. `storage` is assumed to have the required /// alignment and size as described by the layout returned by `get_layout()`. - /// \param storage - Base pointer to the storage to be used for this partition. + /// \param storage Base pointer to the storage to be used for this partition. ROCPRIM_HOST_DEVICE void set_storage(void* const storage) { @@ -82,9 +82,9 @@ struct simple_partition }; /// \brief Construct a simple `simple_partition` with a particular layout. -/// \tparam T - The base type to allocate temporary memory for -/// \param dest - Pointer to where to store the final allocated pointer -/// \param storage_layout - The required layout that the memory allocated to `*dest` should have. +/// \tparam T The base type to allocate temporary memory for +/// \param dest Pointer to where to store the final allocated pointer +/// \param storage_layout The required layout that the memory allocated to `*dest` should have. template ROCPRIM_HOST_DEVICE simple_partition make_partition(T** dest, layout storage_layout) @@ -93,10 +93,10 @@ simple_partition make_partition(T** dest, layout storage_layout) } /// \brief Construct a simple `simple_partition` from a size and an alignment that forms the layout. -/// \tparam T - The base type to allocate temporary memory for -/// \param dest - Pointer to where to store the final allocated pointer -/// \param size - The required size that the memory allocated to `*dest` should have. -/// \param alignment - The required alignment that the memory allocated to `*dest` should have. +/// \tparam T The base type to allocate temporary memory for +/// \param dest Pointer to where to store the final allocated pointer +/// \param size The required size that the memory allocated to `*dest` should have. +/// \param alignment The required alignment that the memory allocated to `*dest` should have. template ROCPRIM_HOST_DEVICE simple_partition make_partition(T** dest, size_t size, size_t alignment = default_alignment) @@ -106,9 +106,9 @@ simple_partition make_partition(T** dest, size_t size, size_t alignment = def /// \brief Construct a `simple_partition` for a type, given a total number of _elements_ that the allocated /// temporary memory should consist of. The natural alignment for `T` is used. -/// \tparam T - The base type to allocate temporary memory for -/// \param dest - Pointer to where to store the final allocated pointer -/// \param elements - The number of elements of `T` that the memory allocated to `dest` should consist of. +/// \tparam T The base type to allocate temporary memory for +/// \param dest Pointer to where to store the final allocated pointer +/// \param elements The number of elements of `T` that the memory allocated to `dest` should consist of. template ROCPRIM_HOST_DEVICE simple_partition ptr_aligned_array(T** dest, size_t elements) @@ -119,7 +119,7 @@ simple_partition ptr_aligned_array(T** dest, size_t elements) /// \brief A partition that represents a linear sequence of sub-partitions. This structure can be used to /// allocate multiple sub-partitions, each of which are sequentially allocated in order, and packed /// such that the only padding between memory of different sub-partitions is due to required alignment. -/// \tparam Ts - The sub-partitions to allocate temporary memory for. Each should have the following member functions: +/// \tparam Ts The sub-partitions to allocate temporary memory for. Each should have the following member functions: /// `layout get_layout()` - Compute the required storage layout for the partition. /// `void set_storage(void* const storage)` - Update the internal destination pointer or the destination pointers /// of sub-partitions with the given pointer. `storage` has at least the required size and alignment as described @@ -159,7 +159,7 @@ struct linear_partition /// \brief Assigns the final storage for this partition. `storage` is assumed to have the required /// alignment and size as described by the layout returned by `get_layout()`. - /// \param storage - Base pointer to the storage to be used for this partition. + /// \param storage Base pointer to the storage to be used for this partition. ROCPRIM_HOST_DEVICE void set_storage(void* const storage) { @@ -180,7 +180,7 @@ struct linear_partition }; /// \brief Construct a `linear_partition` from sub-partitions. -/// \tparam Ts - The sub-partitions to allocate temporary memory for. +/// \tparam Ts The sub-partitions to allocate temporary memory for. /// \see linear_partition template ROCPRIM_HOST_DEVICE @@ -191,7 +191,7 @@ linear_partition make_linear_partition(Ts... ts) /// \brief A partition that represents a union of sub-partitions of temporary memories which are not used /// at the same time, and for which the allocated temporary memory can be shared. -/// \tparam Ts - The sub-partitions to allocate temporary memory for. Each should have the following member functions: +/// \tparam Ts The sub-partitions to allocate temporary memory for. Each should have the following member functions: /// `layout get_layout()` - Compute the required storage layout for the partition. /// `void set_storage(void* const storage)` - Update the internal destination pointer or the destination pointers /// of sub-partitions with the given pointer. `storage` has at least the required size and alignment as described @@ -228,7 +228,7 @@ struct union_partition /// \brief Assigns the final storage for this partition. `storage` is assumed to have the required /// alignment and size as described by the layout returned by `get_layout()`. - /// \param storage - Base pointer to the storage to be used for this partition. + /// \param storage Base pointer to the storage to be used for this partition. ROCPRIM_HOST_DEVICE void set_storage(void* const storage) { @@ -238,7 +238,7 @@ struct union_partition }; /// \brief Construct a `union_partition` from sub-partitions. -/// \tparam Ts - The sub-partitions to allocate temporary memory for. +/// \tparam Ts The sub-partitions to allocate temporary memory for. /// \see union_partition template ROCPRIM_HOST_DEVICE @@ -264,15 +264,15 @@ union_partition make_union_partition(Ts... ts) /// memory, its alignment is not factored into the total required memory, and its destination pointer will be set to /// `nullptr`. /// -/// \tparam TempStoragePartition - The root partition to allocate temporary memory for. It should have the following +/// \tparam TempStoragePartition The root partition to allocate temporary memory for. It should have the following /// member functions: /// `layout get_layout()` - Compute the required storage layout for the partition. /// `void set_storage(void* const storage)` - Update the internal destination pointer or the destination pointers of /// sub-partitions with the given pointer. `storage` has at least the required size and alignment as described by /// the result of `get_layout()`. -/// \param temporary_storage - The base pointer to the allocated temporary memory. May be `nullptr`. -/// \param storage_size [in,out] - The size of `temporary_storage`. -/// \param partition [in,out] - The root partition to allocate temporary memory to. +/// \param temporary_storage The base pointer to the allocated temporary memory. May be `nullptr`. +/// \param storage_size [in,out] The size of `temporary_storage`. +/// \param partition [in,out] The root partition to allocate temporary memory to. template ROCPRIM_HOST_DEVICE hipError_t diff --git a/rocprim/include/rocprim/device/config_types.hpp b/rocprim/include/rocprim/device/config_types.hpp index ee3c75290..d9f37d19b 100644 --- a/rocprim/include/rocprim/device/config_types.hpp +++ b/rocprim/include/rocprim/device/config_types.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -74,8 +74,8 @@ struct kernel_config_params /// \brief Configuration of particular kernels launched by device-level operation /// -/// \tparam BlockSize - number of threads in a block. -/// \tparam ItemsPerThread - number of items processed by each thread. +/// \tparam BlockSize number of threads in a block. +/// \tparam ItemsPerThread number of items processed by each thread. template @@ -359,8 +359,8 @@ inline hipError_t host_target_arch(const hipStream_t stream, target_arch& arch) /// \brief Returns a number of threads in a hardware warp for the actual device. /// At host side this constant is available at runtime only. -/// \param device_id - the device that should be queried. -/// \param warp_size - out parameter for the warp size. +/// \param device_id the device that should be queried. +/// \param warp_size out parameter for the warp size. /// \return hipError_t any error that might occur. /// /// It is constant for a device. @@ -379,8 +379,8 @@ ROCPRIM_HOST inline hipError_t host_warp_size(const int device_id, unsigned int& /// \brief Returns the number of threads in a hardware warp for the device associated with the stream. /// At host side this constant is available at runtime only. -/// \param stream - the stream, whose device should be queried. -/// \param warp_size - out parameter for the warp size. +/// \param stream the stream, whose device should be queried. +/// \param warp_size out parameter for the warp size. /// \return hipError_t any error that might occur. /// /// It is constant for a device. diff --git a/rocprim/include/rocprim/device/detail/device_config_helper.hpp b/rocprim/include/rocprim/device/detail/device_config_helper.hpp index 4cdfc19a5..d175c6609 100644 --- a/rocprim/include/rocprim/device/detail/device_config_helper.hpp +++ b/rocprim/include/rocprim/device/detail/device_config_helper.hpp @@ -184,10 +184,10 @@ struct radix_sort_onesweep_config_params /// \brief Configuration of subalgorithm Onesweep. /// -/// \tparam HistogramConfig - configuration of histogram kernel. -/// \tparam SortConfig - configuration of sort kernel. -/// \tparam RadixBits - number of bits per iteration. -/// \tparam RadixRankAlgorithm - algorithm used for radix rank. +/// \tparam HistogramConfig configuration of histogram kernel. +/// \tparam SortConfig configuration of sort kernel. +/// \tparam RadixBits number of bits per iteration. +/// \tparam RadixRankAlgorithm algorithm used for radix rank. template, class SortConfig = kernel_config<256, 12>, unsigned int RadixBits = 4, @@ -240,10 +240,10 @@ struct reduce_config_params /// \brief Configuration of device-level reduce primitives. /// -/// \tparam BlockSize - number of threads in a block. -/// \tparam ItemsPerThread - number of items processed by each thread. -/// \tparam BlockReduceMethod - algorithm for block reduce. -/// \tparam SizeLimit - limit on the number of items reduced by a single launch +/// \tparam BlockSize number of threads in a block. +/// \tparam ItemsPerThread number of items processed by each thread. +/// \tparam BlockReduceMethod algorithm for block reduce. +/// \tparam SizeLimit limit on the number of items reduced by a single launch template struct default_segmented_radix_sort_config_base { @@ -764,12 +764,12 @@ struct histogram_config_params /// \brief Configuration of device-level histogram operation. /// -/// \tparam HistogramConfig - configuration of histogram kernel. Must be \p kernel_config. -/// \tparam MaxGridSize - maximum number of blocks to launch. -/// \tparam SharedImplMaxBins - maximum total number of bins for all active channels +/// \tparam HistogramConfig configuration of histogram kernel. Must be \p kernel_config. +/// \tparam MaxGridSize maximum number of blocks to launch. +/// \tparam SharedImplMaxBins maximum total number of bins for all active channels /// for the shared memory histogram implementation (samples -> shared memory bins -> global memory bins), /// when exceeded the global memory implementation is used (samples -> global memory bins). -/// \tparam SharedImplHistograms - number of histograms in the shared memory to reduce bank conflicts +/// \tparam SharedImplHistograms number of histograms in the shared memory to reduce bank conflicts /// for atomic operations with narrow sample distributions. Sweetspot for 9xx and 10xx is 3. template /// in the semi-open interval `[lower_output[i], upper_output[i])` of `haystack,` in case of /// being present at all. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `lower_bound_config`. -/// \tparam HaystackIterator - [inferred] Random-access iterator type of the search range. Must meet +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `lower_bound_config`. +/// \tparam HaystackIterator [inferred] Random-access iterator type of the search range. Must meet /// the requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam NeedlesIterator - [inferred] Random-access iterator type of the input range. Must meet +/// \tparam NeedlesIterator [inferred] Random-access iterator type of the input range. Must meet /// the requirements of a C++ InputIterator concept. It can be a simple pointer type. Elements of /// the type pointed by it must be comparable to elements of the type pointed by HaystackIterator /// as either operand of `compare_op.` -/// \tparam OutputIterator - [inferred] Random-access iterator type of the output range. Must meet +/// \tparam OutputIterator [inferred] Random-access iterator type of the output range. Must meet /// the requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam CompareFunction - [inferred] Type of binary function that accepts two arguments of the +/// \tparam CompareFunction [inferred] Type of binary function that accepts two arguments of the /// types pointed by `HaystackIterator` and `NeedlesIterator,` and returns a value convertible /// to bool. Default type is `::rocprim::less<>.` -/// \param [in] temporary_storage - Pointer to a device-accessible temporary storage. -/// \param [in,out] storage_size - Reference to the size (in bytes) of `temporary_storage.` -/// \param [in] haystack - Iterator to the first element in the search range. Elements of this +/// \param [in] temporary_storage Pointer to a device-accessible temporary storage. +/// \param [in,out] storage_size Reference to the size (in bytes) of `temporary_storage.` +/// \param [in] haystack Iterator to the first element in the search range. Elements of this /// range must be sorted. -/// \param [in] needles - Iterator to the first element in the range of values to search for on +/// \param [in] needles Iterator to the first element in the range of values to search for on /// `haystack.` -/// \param [out] output - Iterator to the first element in the output range. -/// \param [in] haystack_size - Number of elements in the search range `haystack.` -/// \param [in] needles_size - Number of elements in the input range `needles.` -/// \param [in] compare_op - Binary operation function object that is used to compare values. The +/// \param [out] output Iterator to the first element in the output range. +/// \param [in] haystack_size Number of elements in the search range `haystack.` +/// \param [in] needles_size Number of elements in the input range `needles.` +/// \param [in] compare_op Binary operation function object that is used to compare values. The /// signature of the function should be equivalent to the following: /// `bool f(const T &a, const U &b);`. It does not need to have `const &`, but the /// function object must not modify the objects passed to it. Default is `CompareFunction().` -/// \param [in] stream - [optional] HIP stream object. Default is `0` (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel launch is +/// \param [in] stream [optional] HIP stream object. Default is `0` (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel launch is /// forced in order to check for errors. /// \return `hipSuccess` (0) after a successful search; otherwise a HIP runtime error of /// type `hipError_t`. @@ -251,33 +251,33 @@ hipError_t lower_bound(void * temporary_storage, /// in the semi-open interval `[lower_output[i], upper_output[i])` of `haystack,` in case of /// being present at all. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `upper_bound_config`. -/// \tparam HaystackIterator - [inferred] Random-access iterator type of the search range. Must meet +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `upper_bound_config`. +/// \tparam HaystackIterator [inferred] Random-access iterator type of the search range. Must meet /// the requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam NeedlesIterator - [inferred] Random-access iterator type of the input range. Must meet +/// \tparam NeedlesIterator [inferred] Random-access iterator type of the input range. Must meet /// the requirements of a C++ InputIterator concept. It can be a simple pointer type. Elements of /// the type pointed by it must be comparable to elements of the type pointed by HaystackIterator /// as either operand of `compare_op.` -/// \tparam OutputIterator - [inferred] Random-access iterator type of the output range. Must meet +/// \tparam OutputIterator [inferred] Random-access iterator type of the output range. Must meet /// the requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam CompareFunction - [inferred] Type of binary function that accepts two arguments of the +/// \tparam CompareFunction [inferred] Type of binary function that accepts two arguments of the /// types pointed by `HaystackIterator` and `NeedlesIterator,` and returns a value convertible /// to bool. Default type is `::rocprim::less<>.` -/// \param [in] temporary_storage - Pointer to a device-accessible temporary storage. -/// \param [in,out] storage_size - Reference to the size (in bytes) of `temporary_storage.` -/// \param [in] haystack - Iterator to the first element in the search range. Elements of this +/// \param [in] temporary_storage Pointer to a device-accessible temporary storage. +/// \param [in,out] storage_size Reference to the size (in bytes) of `temporary_storage.` +/// \param [in] haystack Iterator to the first element in the search range. Elements of this /// range must be sorted. -/// \param [in] needles - Iterator to the first element in the range of values to search for on +/// \param [in] needles Iterator to the first element in the range of values to search for on /// `haystack.` -/// \param [out] output - Iterator to the first element in the output range. -/// \param [in] haystack_size - Number of elements in the search range `haystack.` -/// \param [in] needles_size - Number of elements in the input range `needles.` -/// \param [in] compare_op - Binary operation function object that is used to compare values. The +/// \param [out] output Iterator to the first element in the output range. +/// \param [in] haystack_size Number of elements in the search range `haystack.` +/// \param [in] needles_size Number of elements in the input range `needles.` +/// \param [in] compare_op Binary operation function object that is used to compare values. The /// signature of the function should be equivalent to the following: /// `bool f(const T &a, const U &b);`. It does not need to have `const &`, but the /// function object must not modify the objects passed to it. Default is `CompareFunction().` -/// \param [in] stream - [optional] HIP stream object. Default is `0` (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel launch is +/// \param [in] stream [optional] HIP stream object. Default is `0` (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel launch is /// forced in order to check for errors. /// \return `hipSuccess` (`0)` after a successful search; otherwise a HIP runtime error of /// type `hipError_t.` @@ -382,33 +382,33 @@ hipError_t upper_bound(void * temporary_storage, /// * When a null pointer is passed as `temporary_storage`, the required allocation size (in bytes) /// is written to `storage_size` and the function returns without performing the search operation. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `binary_search_config`. -/// \tparam HaystackIterator - [inferred] Random-access iterator type of the search range. Must meet +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `binary_search_config`. +/// \tparam HaystackIterator [inferred] Random-access iterator type of the search range. Must meet /// the requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam NeedlesIterator - [inferred] Random-access iterator type of the input range. Must meet +/// \tparam NeedlesIterator [inferred] Random-access iterator type of the input range. Must meet /// the requirements of a C++ InputIterator concept. It can be a simple pointer type. Elements of /// the type pointed by it must be comparable to elements of the type pointed by `HaystackIterator` /// as either operand of `compare_op`. -/// \tparam OutputIterator - [inferred] Random-access iterator type of the output range. Must meet +/// \tparam OutputIterator [inferred] Random-access iterator type of the output range. Must meet /// the requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam CompareFunction - [inferred] Type of binary function that accepts two arguments of the +/// \tparam CompareFunction [inferred] Type of binary function that accepts two arguments of the /// types pointed by `HaystackIterator` and `NeedlesIterator`, and returns a value convertible to /// bool. Default type is `::rocprim::less<>`. -/// \param [in] temporary_storage - Pointer to a device-accessible temporary storage. -/// \param [in,out] storage_size - Reference to the size (in bytes) of `temporary_storage`. -/// \param [in] haystack - Iterator to the first element in the search range. Elements of this +/// \param [in] temporary_storage Pointer to a device-accessible temporary storage. +/// \param [in,out] storage_size Reference to the size (in bytes) of `temporary_storage`. +/// \param [in] haystack Iterator to the first element in the search range. Elements of this /// range must be sorted. -/// \param [in] needles - Iterator to the first element in the range of values to search for on +/// \param [in] needles Iterator to the first element in the range of values to search for on /// `haystack`. -/// \param [out] output - Iterator to the first element in the output range of boolean values. -/// \param [in] haystack_size - Number of elements in the search range `haystack`. -/// \param [in] needles_size - Number of elements in the input range `needles`. -/// \param [in] compare_op - Binary operation function object that is used to compare values. The +/// \param [out] output Iterator to the first element in the output range of boolean values. +/// \param [in] haystack_size Number of elements in the search range `haystack`. +/// \param [in] needles_size Number of elements in the input range `needles`. +/// \param [in] compare_op Binary operation function object that is used to compare values. The /// signature of the function should be equivalent to the following: /// `bool f(const T &a, const U &b);`. It does not need to have `const &`, but the /// function object must not modify the objects passed to it. Default is `CompareFunction()`. -/// \param [in] stream - [optional] HIP stream object. Default is `0` (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel launch is +/// \param [in] stream [optional] HIP stream object. Default is `0` (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel launch is /// forced in order to check for errors. /// \return `hipSuccess` (`0`) after a successful search; otherwise a HIP runtime error of /// type `hipError_t`. diff --git a/rocprim/include/rocprim/device/device_copy.hpp b/rocprim/include/rocprim/device/device_copy.hpp index 1ddae74ef..57c526907 100644 --- a/rocprim/include/rocprim/device/device_copy.hpp +++ b/rocprim/include/rocprim/device/device_copy.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -48,7 +48,7 @@ BEGIN_ROCPRIM_NAMESPACE /// \param [in] sizes iterator of range sizes to copy. /// \param [in] num_copies number of ranges to copy /// \param [in] stream [optional] HIP stream object to enqueue the copy on. Default is `hipStreamDefault`. -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. The default value is `false`. /// /// Performs multiple device to device copies as a single batched operation. diff --git a/rocprim/include/rocprim/device/device_copy_config.hpp b/rocprim/include/rocprim/device/device_copy_config.hpp index 275956e87..ce9312eaa 100644 --- a/rocprim/include/rocprim/device/device_copy_config.hpp +++ b/rocprim/include/rocprim/device/device_copy_config.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -32,13 +32,13 @@ BEGIN_ROCPRIM_NAMESPACE /// \brief /// -/// \tparam NonBlevBlockSize - number of threads per block for thread- and warp-level copy. -/// \tparam NonBlevBuffersPerThreaed - number of buffers processed per thread. -/// \tparam TlevBytesPerThread - number of bytes per thread for thread-level copy. -/// \tparam BlevBlockSize - number of thread per block for block-level copy. -/// \tparam BlevBytesPerThread - number of bytes per thread for block-level copy. -/// \tparam WlevSizeThreshold - minimum size to use warp-level copy instead of thread-level. -/// \tparam BlevSizeThreshold - minimum size to use block-level copy instead of warp-level. +/// \tparam NonBlevBlockSize number of threads per block for thread- and warp-level copy. +/// \tparam NonBlevBuffersPerThreaed number of buffers processed per thread. +/// \tparam TlevBytesPerThread number of bytes per thread for thread-level copy. +/// \tparam BlevBlockSize number of thread per block for block-level copy. +/// \tparam BlevBytesPerThread number of bytes per thread for block-level copy. +/// \tparam WlevSizeThreshold minimum size to use warp-level copy instead of thread-level. +/// \tparam BlevSizeThreshold minimum size to use block-level copy instead of warp-level. templatebool f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. /// The default value is \p BinaryFunction(). -/// \param [in] stream - [optional] HIP stream object. Default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. Default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. Default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful sort; otherwise a HIP runtime error of @@ -325,39 +325,39 @@ hipError_t merge(void * temporary_storage, /// if \p temporary_storage in a null pointer. /// * Accepts custom compare_functions for merging across the device. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `merge_config`. -/// \tparam KeysInputIterator1 - random-access iterator type of the first keys input range. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `merge_config`. +/// \tparam KeysInputIterator1 random-access iterator type of the first keys input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam KeysInputIterator2 - random-access iterator type of the second keys input range. Must meet the +/// \tparam KeysInputIterator2 random-access iterator type of the second keys input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam KeysOutputIterator - random-access iterator type of the keys output range. Must meet the +/// \tparam KeysOutputIterator random-access iterator type of the keys output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam ValuesInputIterator1 - random-access iterator type of the first values input range. Must meet the +/// \tparam ValuesInputIterator1 random-access iterator type of the first values input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam ValuesInputIterator2 - random-access iterator type of the second values input range. Must meet the +/// \tparam ValuesInputIterator2 random-access iterator type of the second values input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam ValuesOutputIterator - random-access iterator type of the values output range. Must meet the +/// \tparam ValuesOutputIterator random-access iterator type of the values output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the sort operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] keys_input1 - iterator to the first key in the first range to merge. -/// \param [in] keys_input2 - iterator to the first key in the second range to merge. -/// \param [out] keys_output - iterator to the first key in the output range. -/// \param [in] values_input1 - iterator to the first value in the first range to merge. -/// \param [in] values_input2 - iterator to the first value in the second range to merge. -/// \param [out] values_output - iterator to the first value in the output range. -/// \param [in] input1_size - number of element in the first input range. -/// \param [in] input2_size - number of element in the second input range. -/// \param [in] compare_function - binary operation function object that will be used for key comparison. +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] keys_input1 iterator to the first key in the first range to merge. +/// \param [in] keys_input2 iterator to the first key in the second range to merge. +/// \param [out] keys_output iterator to the first key in the output range. +/// \param [in] values_input1 iterator to the first value in the first range to merge. +/// \param [in] values_input2 iterator to the first value in the second range to merge. +/// \param [out] values_output iterator to the first value in the output range. +/// \param [in] input1_size number of element in the first input range. +/// \param [in] input2_size number of element in the second input range. +/// \param [in] compare_function binary operation function object that will be used for key comparison. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. /// The default value is \p BinaryFunction(). -/// \param [in] stream - [optional] HIP stream object. Default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. Default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. Default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful sort; otherwise a HIP runtime error of diff --git a/rocprim/include/rocprim/device/device_merge_sort.hpp b/rocprim/include/rocprim/device/device_merge_sort.hpp index 035d93adc..c1631954f 100644 --- a/rocprim/include/rocprim/device/device_merge_sort.hpp +++ b/rocprim/include/rocprim/device/device_merge_sort.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -639,26 +639,26 @@ inline hipError_t merge_sort_impl( /// * op(a, b) and op(b, a) are both false, /// then it is \b guaranteed that \p a will precede \p b as well in the output (ordered) keys. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `merge_sort_config`. -/// \tparam KeysInputIterator - random-access iterator type of the input range. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `merge_sort_config`. +/// \tparam KeysInputIterator random-access iterator type of the input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam KeysOutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam KeysOutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the sort operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] keys_input - pointer to the first element in the range to sort. -/// \param [out] keys_output - pointer to the first element in the output range. -/// \param [in] size - number of element in the input range. -/// \param [in] compare_function - binary operation function object that will be used for comparison. +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] keys_input pointer to the first element in the range to sort. +/// \param [out] keys_output pointer to the first element in the output range. +/// \param [in] size number of element in the input range. +/// \param [in] compare_function binary operation function object that will be used for comparison. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. /// The default value is \p BinaryFunction(). -/// \param [in] stream - [optional] HIP stream object. Default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. Default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. Default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful sort; otherwise a HIP runtime error of @@ -738,32 +738,32 @@ hipError_t merge_sort(void * temporary_storage, /// * op(a, b) and op(b, a) are both false, /// then it is \b guaranteed that \p a will precede \p b as well in the output (ordered) keys. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `merge_sort_config`. -/// \tparam KeysInputIterator - random-access iterator type of the input range. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `merge_sort_config`. +/// \tparam KeysInputIterator random-access iterator type of the input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam KeysOutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam KeysOutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam ValuesInputIterator - random-access iterator type of the input range. Must meet the +/// \tparam ValuesInputIterator random-access iterator type of the input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam ValuesOutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam ValuesOutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the sort operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] keys_input - pointer to the first element in the range to sort. -/// \param [out] keys_output - pointer to the first element in the output range. -/// \param [in] values_input - pointer to the first element in the range to sort. -/// \param [out] values_output - pointer to the first element in the output range. -/// \param [in] size - number of element in the input range. -/// \param [in] compare_function - binary operation function object that will be used for comparison. +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] keys_input pointer to the first element in the range to sort. +/// \param [out] keys_output pointer to the first element in the output range. +/// \param [in] values_input pointer to the first element in the range to sort. +/// \param [out] values_output pointer to the first element in the output range. +/// \param [in] size number of element in the input range. +/// \param [in] compare_function binary operation function object that will be used for comparison. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. /// The default value is \p BinaryFunction(). -/// \param [in] stream - [optional] HIP stream object. Default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. Default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. Default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful sort; otherwise a HIP runtime error of diff --git a/rocprim/include/rocprim/device/device_merge_sort_config.hpp b/rocprim/include/rocprim/device/device_merge_sort_config.hpp index 5da8e0ab0..4883c4a08 100644 --- a/rocprim/include/rocprim/device/device_merge_sort_config.hpp +++ b/rocprim/include/rocprim/device/device_merge_sort_config.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -45,16 +45,16 @@ struct merge_sort_config_params /// \brief Configuration of device-level merge primitives. /// -/// \tparam SortBlockSize - block size in the block-sort step -/// \tparam SortItemsPerThread - ItemsPerThread in the block-sort step -/// \tparam MergeOddevenBlockSize - block size in the block merge step using oddeven impl +/// \tparam SortBlockSize block size in the block-sort step +/// \tparam SortItemsPerThread ItemsPerThread in the block-sort step +/// \tparam MergeOddevenBlockSize block size in the block merge step using oddeven impl /// (used when input_size < MinInputSizeMergepath) -/// \tparam MergeMergepathPartitionBlockSize - block size of the partition kernel in the block merge +/// \tparam MergeMergepathPartitionBlockSize block size of the partition kernel in the block merge /// step using mergepath impl -/// \tparam MergeMergepathBlockSize - block size in the block merge step using mergepath impl -/// \tparam MergeMergepathItemsPerThread - ItemsPerThread in the block merge step using +/// \tparam MergeMergepathBlockSize block size in the block merge step using mergepath impl +/// \tparam MergeMergepathItemsPerThread ItemsPerThread in the block merge step using /// mergepath impl -/// \tparam MinInputSizeMergepath - breakpoint of input-size to use mergepath impl for +/// \tparam MinInputSizeMergepath breakpoint of input-size to use mergepath impl for /// block merge step templatebool f(const T &a);. The signature does not need to have /// const &, but function object must not modify the object passed to it. -/// \param [in] stream - [optional] HIP stream object. The default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. The default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. The default value is \p false. /// /// \par Example @@ -898,43 +898,43 @@ hipError_t partition(void * temporary_storage, /// minus the number of elements written to \p output_first_part minus the number of elements written /// to \p output_second_part. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `select_config`. -/// \tparam InputIterator - random-access iterator type of the input range. It can be +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `select_config`. +/// \tparam InputIterator random-access iterator type of the input range. It can be /// a simple pointer type. -/// \tparam FirstOutputIterator - random-access iterator type of the first output range. It can be +/// \tparam FirstOutputIterator random-access iterator type of the first output range. It can be /// a simple pointer type. -/// \tparam SecondOutputIterator - random-access iterator type of the second output range. It can be +/// \tparam SecondOutputIterator random-access iterator type of the second output range. It can be /// a simple pointer type. -/// \tparam UnselectedOutputIterator - random-access iterator type of the unselected output range. +/// \tparam UnselectedOutputIterator random-access iterator type of the unselected output range. /// It can be a simple pointer type. -/// \tparam SelectedCountOutputIterator - random-access iterator type of the selected_count_output +/// \tparam SelectedCountOutputIterator random-access iterator type of the selected_count_output /// value. It can be a simple pointer type. -/// \tparam FirstUnaryPredicate - type of the first unary selection predicate. -/// \tparam SecondUnaryPredicate - type of the second unary selection predicate. +/// \tparam FirstUnaryPredicate type of the first unary selection predicate. +/// \tparam SecondUnaryPredicate type of the second unary selection predicate. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the select operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] input - iterator to the first element in the range to select values from. -/// \param [out] output_first_part - iterator to the first element in the first output range. -/// \param [out] output_second_part - iterator to the first element in the second output range. -/// \param [out] output_unselected - iterator to the first element in the unselected output range. -/// \param [out] selected_count_output - iterator to the total number of selected values in +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] input iterator to the first element in the range to select values from. +/// \param [out] output_first_part iterator to the first element in the first output range. +/// \param [out] output_second_part iterator to the first element in the second output range. +/// \param [out] output_unselected iterator to the first element in the unselected output range. +/// \param [out] selected_count_output iterator to the total number of selected values in /// \p output_first_part and \p output_second_part respectively. -/// \param [in] size - number of elements in the input range. -/// \param [in] select_first_part_op - unary function object which returns \p true if the element +/// \param [in] size number of elements in the input range. +/// \param [in] select_first_part_op unary function object which returns \p true if the element /// should be in \p output_first_part range /// The signature of the function should be equivalent to the following: /// bool f(const T &a);. The signature does not need to have /// const &, but function object must not modify the object passed to it. -/// \param [in] select_second_part_op - unary function object which returns \p true if the element +/// \param [in] select_second_part_op unary function object which returns \p true if the element /// should be in \p output_second_part range (given that \p select_first_part_op returned \p false) /// The signature of the function should be equivalent to the following: /// bool f(const T &a);. The signature does not need to have /// const &, but function object must not modify the object passed to it. -/// \param [in] stream - [optional] HIP stream object. The default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. The default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. The default value is \p false. /// /// \par Example diff --git a/rocprim/include/rocprim/device/device_radix_sort_config.hpp b/rocprim/include/rocprim/device/device_radix_sort_config.hpp index 7bbdba6b1..52ac82766 100644 --- a/rocprim/include/rocprim/device/device_radix_sort_config.hpp +++ b/rocprim/include/rocprim/device/device_radix_sort_config.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -35,14 +35,14 @@ BEGIN_ROCPRIM_NAMESPACE /// One of three algorithms is used: single sort (launches only a single block), /// merge sort, or Onesweep. /// -/// \tparam SortSingleConfig - Configuration for the single kernel subalgorithm. +/// \tparam SortSingleConfig Configuration for the single kernel subalgorithm. /// must be \p kernel_config or \p default_config. -/// \tparam MergeSortConfig - Configuration for the merge sort subalgorithm. +/// \tparam MergeSortConfig Configuration for the merge sort subalgorithm. /// must be \p merge_sort_config or \p default_config. If \p merge_sort_config, the sorted /// items per block must be a power of two. -/// \tparam OnesweepConfig - Configuration for the Onesweep subalgorithm. +/// \tparam OnesweepConfig Configuration for the Onesweep subalgorithm. /// must be \p radix_sort_onesweep_config or \p default_config. -/// \tparam MergeSortLimit - The largest number of items for which the merge sort algorithm will be +/// \tparam MergeSortLimit The largest number of items for which the merge sort algorithm will be /// used. Note that below this limit, a different algorithm may be used. templaterocprim::transform_iterator, see the example below. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `reduce_config`. -/// \tparam InputIterator - random-access iterator type of the input range. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `reduce_config`. +/// \tparam InputIterator random-access iterator type of the input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam OutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam OutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam InitValueType - type of the initial value. -/// \tparam BinaryFunction - type of binary function used for reduction. Default type +/// \tparam InitValueType type of the initial value. +/// \tparam BinaryFunction type of binary function used for reduction. Default type /// is \p rocprim::plus, where \p T is a \p value_type of \p InputIterator. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the reduction operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] input - iterator to the first element in the range to reduce. -/// \param [out] output - iterator to the first element in the output range. It can be +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] input iterator to the first element in the range to reduce. +/// \param [out] output iterator to the first element in the output range. It can be /// same as \p input. -/// \param [in] initial_value - initial value to start the reduction. -/// \param [in] size - number of element in the input range. -/// \param [in] reduce_op - binary operation function object that will be used for reduction. +/// \param [in] initial_value initial value to start the reduction. +/// \param [in] size number of element in the input range. +/// \param [in] reduce_op binary operation function object that will be used for reduction. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. /// The default value is \p BinaryFunction(). -/// \param [in] stream - [optional] HIP stream object. The default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. The default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. The default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful reduction; otherwise a HIP runtime error of @@ -456,29 +456,29 @@ inline hipError_t reduce(void* temporary_storage, /// * By default, the input type is used for accumulation. A custom type /// can be specified using rocprim::transform_iterator, see the example below. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `reduce_config`. -/// \tparam InputIterator - random-access iterator type of the input range. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `reduce_config`. +/// \tparam InputIterator random-access iterator type of the input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam OutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam OutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam BinaryFunction - type of binary function used for reduction. Default type +/// \tparam BinaryFunction type of binary function used for reduction. Default type /// is \p rocprim::plus, where \p T is a \p value_type of \p InputIterator. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the reduction operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] input - iterator to the first element in the range to reduce. -/// \param [out] output - iterator to the first element in the output range. It can be +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] input iterator to the first element in the range to reduce. +/// \param [out] output iterator to the first element in the output range. It can be /// same as \p input. -/// \param [in] size - number of element in the input range. -/// \param [in] reduce_op - binary operation function object that will be used for reduction. +/// \param [in] size number of element in the input range. +/// \param [in] reduce_op binary operation function object that will be used for reduction. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. /// Default is BinaryFunction(). -/// \param [in] stream - [optional] HIP stream object. Default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. Default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. Default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful reduction; otherwise a HIP runtime error of diff --git a/rocprim/include/rocprim/device/device_reduce_by_key.hpp b/rocprim/include/rocprim/device/device_reduce_by_key.hpp index c8607afec..aabdaba8d 100644 --- a/rocprim/include/rocprim/device/device_reduce_by_key.hpp +++ b/rocprim/include/rocprim/device/device_reduce_by_key.hpp @@ -397,46 +397,46 @@ hipError_t reduce_by_key_impl(void* temporary_storage, /// * Ranges specified by \p unique_output and \p aggregates_output must have at least /// *unique_count_output (i.e. the number of unique keys) elements. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `reduce_by_key_config`. -/// \tparam KeysInputIterator - random-access iterator type of the input range. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `reduce_by_key_config`. +/// \tparam KeysInputIterator random-access iterator type of the input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam ValuesInputIterator - random-access iterator type of the input range. Must meet the +/// \tparam ValuesInputIterator random-access iterator type of the input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam UniqueOutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam UniqueOutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam AggregatesOutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam AggregatesOutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam UniqueCountOutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam UniqueCountOutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam BinaryFunction - type of binary function used for reduction. Default type +/// \tparam BinaryFunction type of binary function used for reduction. Default type /// is \p rocprim::plus, where \p T is a \p value_type of \p ValuesInputIterator. -/// \tparam KeyCompareFunction - type of binary function used to determine keys equality. Default type +/// \tparam KeyCompareFunction type of binary function used to determine keys equality. Default type /// is \p rocprim::equal_to, where \p T is a \p value_type of \p KeysInputIterator. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the reduction operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] keys_input - iterator to the first element in the range of keys. -/// \param [in] values_input - iterator to the first element in the range of values to reduce. -/// \param [in] size - number of element in the input range. -/// \param [out] unique_output - iterator to the first element in the output range of unique keys. -/// \param [out] aggregates_output - iterator to the first element in the output range of reductions. -/// \param [out] unique_count_output - iterator to total number of groups. -/// \param [in] reduce_op - binary operation function object that will be used for reduction. +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] keys_input iterator to the first element in the range of keys. +/// \param [in] values_input iterator to the first element in the range of values to reduce. +/// \param [in] size number of element in the input range. +/// \param [out] unique_output iterator to the first element in the output range of unique keys. +/// \param [out] aggregates_output iterator to the first element in the output range of reductions. +/// \param [out] unique_count_output iterator to total number of groups. +/// \param [in] reduce_op binary operation function object that will be used for reduction. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it and must not have /// any side effects since the function may be called on uninitalized data. /// Default is BinaryFunction(). -/// \param [in] key_compare_op - binary operation function object that will be used to determine key equality. +/// \param [in] key_compare_op binary operation function object that will be used to determine key equality. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it and must not have /// any side effects since the function may be called on uninitalized data. /// Default is KeyCompareFunction(). -/// \param [in] stream - [optional] HIP stream object. Default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. Default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. Default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful reduction; otherwise a HIP runtime error of diff --git a/rocprim/include/rocprim/device/device_run_length_encode.hpp b/rocprim/include/rocprim/device/device_run_length_encode.hpp index 4d55cf8a8..88e05138e 100644 --- a/rocprim/include/rocprim/device/device_run_length_encode.hpp +++ b/rocprim/include/rocprim/device/device_run_length_encode.hpp @@ -277,27 +277,27 @@ hipError_t run_length_encode_non_trivial_runs_impl(void* tempo /// * Ranges specified by \p unique_output and \p counts_output must have at least /// *runs_count_output (i.e. the number of runs) elements. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `run_length_encode_config`. -/// \tparam InputIterator - random-access iterator type of the input range. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `run_length_encode_config`. +/// \tparam InputIterator random-access iterator type of the input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam UniqueOutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam UniqueOutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam CountsOutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam CountsOutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam RunsCountOutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam RunsCountOutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] input - iterator to the first element in the range of values. -/// \param [in] size - number of element in the input range. -/// \param [out] unique_output - iterator to the first element in the output range of unique values. -/// \param [out] counts_output - iterator to the first element in the output range of lenghts. -/// \param [out] runs_count_output - iterator to total number of runs. -/// \param [in] stream - [optional] HIP stream object. Default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] input iterator to the first element in the range of values. +/// \param [in] size number of element in the input range. +/// \param [out] unique_output iterator to the first element in the output range of unique values. +/// \param [out] counts_output iterator to the first element in the output range of lenghts. +/// \param [out] runs_count_output iterator to total number of runs. +/// \param [in] stream [optional] HIP stream object. Default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. Default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful operation; otherwise a HIP runtime error of @@ -390,27 +390,27 @@ inline hipError_t run_length_encode(void* temporary_storage, /// * Ranges specified by \p offsets_output and \p counts_output must have at least /// *runs_count_output (i.e. the number of non-trivial runs) elements. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `run_length_encode_config`. -/// \tparam InputIterator - random-access iterator type of the input range. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `run_length_encode_config`. +/// \tparam InputIterator random-access iterator type of the input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam OffsetsOutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam OffsetsOutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam CountsOutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam CountsOutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam RunsCountOutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam RunsCountOutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] input - iterator to the first element in the range of values. -/// \param [in] size - number of element in the input range. -/// \param [out] offsets_output - iterator to the first element in the output range of offsets. -/// \param [out] counts_output - iterator to the first element in the output range of lenghts. -/// \param [out] runs_count_output - iterator to total number of runs. -/// \param [in] stream - [optional] HIP stream object. Default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] input iterator to the first element in the range of values. +/// \param [in] size number of element in the input range. +/// \param [out] offsets_output iterator to the first element in the output range of offsets. +/// \param [out] counts_output iterator to the first element in the output range of lenghts. +/// \param [out] runs_count_output iterator to total number of runs. +/// \param [in] stream [optional] HIP stream object. Default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. Default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful operation; otherwise a HIP runtime error of diff --git a/rocprim/include/rocprim/device/device_run_length_encode_config.hpp b/rocprim/include/rocprim/device/device_run_length_encode_config.hpp index 53b7ee8ff..708e89be3 100644 --- a/rocprim/include/rocprim/device/device_run_length_encode_config.hpp +++ b/rocprim/include/rocprim/device/device_run_length_encode_config.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -39,9 +39,9 @@ BEGIN_ROCPRIM_NAMESPACE /// \brief Configuration of device-level run-length encoding operation. /// -/// \tparam ReduceByKeyConfig - configuration of device-level reduce-by-key operation. +/// \tparam ReduceByKeyConfig configuration of device-level reduce-by-key operation. /// Must be \p reduce_by_key_config or \p default_config. -/// \tparam SelectConfig - configuration of device-level select operation. +/// \tparam SelectConfig configuration of device-level select operation. /// Must be \p select_config or \p default_config. template struct run_length_encode_config diff --git a/rocprim/include/rocprim/device/device_scan.hpp b/rocprim/include/rocprim/device/device_scan.hpp index 29c46e8fb..18bce9dce 100644 --- a/rocprim/include/rocprim/device/device_scan.hpp +++ b/rocprim/include/rocprim/device/device_scan.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -396,31 +396,31 @@ inline auto scan_impl(void* temporary_storage, /// * By default, the input type is used for accumulation. A custom type /// can be specified using the \p AccType type parameter, see the example below. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `scan_config`. -/// \tparam InputIterator - random-access iterator type of the input range. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `scan_config`. +/// \tparam InputIterator random-access iterator type of the input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam OutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam OutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam BinaryFunction - type of binary function used for scan. Default type +/// \tparam BinaryFunction type of binary function used for scan. Default type /// is \p rocprim::plus, where \p T is a \p value_type of \p InputIterator. -/// \tparam AccType - accumulator type used to propagate the scanned values. Default type +/// \tparam AccType accumulator type used to propagate the scanned values. Default type /// is value type of the input iterator. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the scan operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] input - iterator to the first element in the range to scan. -/// \param [out] output - iterator to the first element in the output range. It can be +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] input iterator to the first element in the range to scan. +/// \param [out] output iterator to the first element in the output range. It can be /// same as \p input. -/// \param [in] size - number of element in the input range. -/// \param [in] scan_op - binary operation function object that will be used for scan. +/// \param [in] size number of element in the input range. +/// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. /// Default is BinaryFunction(). -/// \param [in] stream - [optional] HIP stream object. Default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. Default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. Default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful scan; otherwise a HIP runtime error of @@ -582,34 +582,34 @@ inline hipError_t deterministic_inclusive_scan(void* temporary_stora /// if \p temporary_storage in a null pointer. /// * Ranges specified by \p input and \p output must have at least \p size elements. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `scan_config`. -/// \tparam InputIterator - random-access iterator type of the input range. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `scan_config`. +/// \tparam InputIterator random-access iterator type of the input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam OutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam OutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam InitValueType - type of the initial value. -/// \tparam BinaryFunction - type of binary function used for scan. Default type +/// \tparam InitValueType type of the initial value. +/// \tparam BinaryFunction type of binary function used for scan. Default type /// is \p rocprim::plus, where \p T is a \p value_type of \p InputIterator. -/// \tparam AccType - accumulator type used to propagate the scanned values. Default type +/// \tparam AccType accumulator type used to propagate the scanned values. Default type /// is 'InitValueType', unless it's 'rocprim::future_value'. Then it will be the wrapped input type. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the scan operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] input - iterator to the first element in the range to scan. -/// \param [out] output - iterator to the first element in the output range. It can be +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] input iterator to the first element in the range to scan. +/// \param [out] output iterator to the first element in the output range. It can be /// same as \p input. -/// \param [in] initial_value - initial value to start the scan. +/// \param [in] initial_value initial value to start the scan. /// A rocpim::future_value may be passed to use a value that will be later computed. -/// \param [in] size - number of element in the input range. -/// \param [in] scan_op - binary operation function object that will be used for scan. +/// \param [in] size number of element in the input range. +/// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. /// The default value is \p BinaryFunction(). -/// \param [in] stream - [optional] HIP stream object. The default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. The default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. The default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful scan; otherwise a HIP runtime error of diff --git a/rocprim/include/rocprim/device/device_scan_by_key.hpp b/rocprim/include/rocprim/device/device_scan_by_key.hpp index a0f762675..93ca4bc93 100644 --- a/rocprim/include/rocprim/device/device_scan_by_key.hpp +++ b/rocprim/include/rocprim/device/device_scan_by_key.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -313,41 +313,41 @@ inline hipError_t scan_by_key_impl(void* const temporary_storage, /// * Ranges specified by \p keys_input, \p values_input, and \p values_output must have /// at least \p size elements. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `scan_by_key_config`. -/// \tparam KeysInputIterator - random-access iterator type of the input range. It can be +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `scan_by_key_config`. +/// \tparam KeysInputIterator random-access iterator type of the input range. It can be /// a simple pointer type. -/// \tparam ValuesInputIterator - random-access iterator type of the input range. It can be +/// \tparam ValuesInputIterator random-access iterator type of the input range. It can be /// a simple pointer type. -/// \tparam ValuesOutputIterator - random-access iterator type of the output range. It can be +/// \tparam ValuesOutputIterator random-access iterator type of the output range. It can be /// a simple pointer type. -/// \tparam BinaryFunction - type of binary function used for scan. Default type +/// \tparam BinaryFunction type of binary function used for scan. Default type /// is \p rocprim::plus, where \p T is a \p value_type of \p InputIterator. -/// \tparam KeyCompareFunction - type of binary function used to determine keys equality. Default type +/// \tparam KeyCompareFunction type of binary function used to determine keys equality. Default type /// is \p rocprim::equal_to, where \p T is a \p value_type of \p KeysInputIterator. -/// \tparam AccType - accumulator type used to propagate the scanned values. Default type +/// \tparam AccType accumulator type used to propagate the scanned values. Default type /// is value type of the input iterator. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the scan operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] keys_input - iterator to the first element in the range of keys. -/// \param [in] values_input - iterator to the first element in the range of values to scan. -/// \param [out] values_output - iterator to the first element in the output value range. -/// \param [in] size - number of element in the input range. -/// \param [in] scan_op - binary operation function object that will be used for scanning +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] keys_input iterator to the first element in the range of keys. +/// \param [in] values_input iterator to the first element in the range of values to scan. +/// \param [out] values_output iterator to the first element in the output value range. +/// \param [in] size number of element in the input range. +/// \param [in] scan_op binary operation function object that will be used for scanning /// input values. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. /// Default is BinaryFunction(). -/// \param [in] key_compare_op - binary operation function object that will be used to determine keys equality. +/// \param [in] key_compare_op binary operation function object that will be used to determine keys equality. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. /// Default is KeyCompareFunction(). -/// \param [in] stream - [optional] HIP stream object. Default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. Default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. Default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful scan; otherwise a HIP runtime error of @@ -505,44 +505,44 @@ inline hipError_t deterministic_inclusive_scan_by_key(void* const /// * Ranges specified by \p keys_input, \p values_input, and \p values_output must have /// at least \p size elements. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `scan_by_key_config`. -/// \tparam KeysInputIterator - random-access iterator type of the input range. It can be +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `scan_by_key_config`. +/// \tparam KeysInputIterator random-access iterator type of the input range. It can be /// a simple pointer type. -/// \tparam ValuesInputIterator - random-access iterator type of the input range. It can be +/// \tparam ValuesInputIterator random-access iterator type of the input range. It can be /// a simple pointer type. -/// \tparam ValuesOutputIterator - random-access iterator type of the output range. It can be +/// \tparam ValuesOutputIterator random-access iterator type of the output range. It can be /// a simple pointer type. -/// \tparam InitValueType - type of the initial value. -/// \tparam BinaryFunction - type of binary function used for scan. Default type +/// \tparam InitValueType type of the initial value. +/// \tparam BinaryFunction type of binary function used for scan. Default type /// is \p rocprim::plus, where \p T is a \p value_type of \p InputIterator. -/// \tparam KeyCompareFunction - type of binary function used to determine keys equality. Default type +/// \tparam KeyCompareFunction type of binary function used to determine keys equality. Default type /// is \p rocprim::equal_to, where \p T is a \p value_type of \p KeysInputIterator. -/// \tparam AccType - accumulator type used to propagate the scanned values. Default type +/// \tparam AccType accumulator type used to propagate the scanned values. Default type /// is 'InitValueType', unless it's 'rocprim::future_value'. Then it will be the wrapped input type. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the scan operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] keys_input - iterator to the first element in the range of keys. -/// \param [in] values_input - iterator to the first element in the range of values to scan. -/// \param [out] values_output - iterator to the first element in the output value range. -/// \param [in] initial_value - initial value to start the scan. +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] keys_input iterator to the first element in the range of keys. +/// \param [in] values_input iterator to the first element in the range of values to scan. +/// \param [out] values_output iterator to the first element in the output value range. +/// \param [in] initial_value initial value to start the scan. /// A rocpim::future_value may be passed to use a value that will be later computed. -/// \param [in] size - number of element in the input range. -/// \param [in] scan_op - binary operation function object that will be used for scanning +/// \param [in] size number of element in the input range. +/// \param [in] scan_op binary operation function object that will be used for scanning /// input values. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. /// Default is BinaryFunction(). -/// \param [in] key_compare_op - binary operation function object that will be used to determine keys equality. +/// \param [in] key_compare_op binary operation function object that will be used to determine keys equality. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. /// Default is KeyCompareFunction(). -/// \param [in] stream - [optional] HIP stream object. Default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. Default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. Default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful scan; otherwise a HIP runtime error of diff --git a/rocprim/include/rocprim/device/device_segmented_radix_sort.hpp b/rocprim/include/rocprim/device/device_segmented_radix_sort.hpp index fd63149e4..4b385e5c5 100644 --- a/rocprim/include/rocprim/device/device_segmented_radix_sort.hpp +++ b/rocprim/include/rocprim/device/device_segmented_radix_sort.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -625,32 +625,32 @@ hipError_t segmented_radix_sort_impl(void * temporary_storage, /// * op(a, b) and op(b, a) are both false, /// then it is \b guaranteed that \p a will precede \p b as well in the output (ordered) keys. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `segmented_radix_sort_config`. -/// \tparam KeysInputIterator - random-access iterator type of the input range. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `segmented_radix_sort_config`. +/// \tparam KeysInputIterator random-access iterator type of the input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam KeysOutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam KeysOutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam OffsetIterator - random-access iterator type of segment offsets. Must meet the +/// \tparam OffsetIterator random-access iterator type of segment offsets. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the sort operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] keys_input - pointer to the first element in the range to sort. -/// \param [out] keys_output - pointer to the first element in the output range. -/// \param [in] size - number of element in the input range. -/// \param [in] segments - number of segments in the input range. -/// \param [in] begin_offsets - iterator to the first element in the range of beginning offsets. -/// \param [in] end_offsets - iterator to the first element in the range of ending offsets. -/// \param [in] begin_bit - [optional] index of the first (least significant) bit used in +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] keys_input pointer to the first element in the range to sort. +/// \param [out] keys_output pointer to the first element in the output range. +/// \param [in] size number of element in the input range. +/// \param [in] segments number of segments in the input range. +/// \param [in] begin_offsets iterator to the first element in the range of beginning offsets. +/// \param [in] end_offsets iterator to the first element in the range of ending offsets. +/// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. /// Non-default value not supported for floating-point key-types. -/// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in +/// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). Non-default value not supported for floating-point key-types. -/// \param [in] stream - [optional] HIP stream object. Default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. Default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. Default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful sort; otherwise a HIP runtime error of @@ -753,32 +753,32 @@ hipError_t segmented_radix_sort_keys(void * temporary_storage, /// * op(a, b) and op(b, a) are both false, /// then it is \b guaranteed that \p a will precede \p b as well in the output (ordered) keys. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `segmented_radix_sort_config`. -/// \tparam KeysInputIterator - random-access iterator type of the input range. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `segmented_radix_sort_config`. +/// \tparam KeysInputIterator random-access iterator type of the input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam KeysOutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam KeysOutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam OffsetIterator - random-access iterator type of segment offsets. Must meet the +/// \tparam OffsetIterator random-access iterator type of segment offsets. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the sort operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] keys_input - pointer to the first element in the range to sort. -/// \param [out] keys_output - pointer to the first element in the output range. -/// \param [in] size - number of element in the input range. -/// \param [in] segments - number of segments in the input range. -/// \param [in] begin_offsets - iterator to the first element in the range of beginning offsets. -/// \param [in] end_offsets - iterator to the first element in the range of ending offsets. -/// \param [in] begin_bit - [optional] index of the first (least significant) bit used in +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] keys_input pointer to the first element in the range to sort. +/// \param [out] keys_output pointer to the first element in the output range. +/// \param [in] size number of element in the input range. +/// \param [in] segments number of segments in the input range. +/// \param [in] begin_offsets iterator to the first element in the range of beginning offsets. +/// \param [in] end_offsets iterator to the first element in the range of ending offsets. +/// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. /// Non-default value not supported for floating-point key-types. -/// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in +/// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). Non-default value not supported for floating-point key-types. -/// \param [in] stream - [optional] HIP stream object. Default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. Default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. Default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful sort; otherwise a HIP runtime error of @@ -882,38 +882,38 @@ hipError_t segmented_radix_sort_keys_desc(void * temporary_storage, /// * op(a, b) and op(b, a) are both false, /// then it is \b guaranteed that \p a will precede \p b as well in the output (ordered) keys. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `segmented_radix_sort_config`. -/// \tparam KeysInputIterator - random-access iterator type of the input range. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `segmented_radix_sort_config`. +/// \tparam KeysInputIterator random-access iterator type of the input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam KeysOutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam KeysOutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam ValuesInputIterator - random-access iterator type of the input range. Must meet the +/// \tparam ValuesInputIterator random-access iterator type of the input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam ValuesOutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam ValuesOutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam OffsetIterator - random-access iterator type of segment offsets. Must meet the +/// \tparam OffsetIterator random-access iterator type of segment offsets. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the sort operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] keys_input - pointer to the first element in the range to sort. -/// \param [out] keys_output - pointer to the first element in the output range. -/// \param [in] values_input - pointer to the first element in the range to sort. -/// \param [out] values_output - pointer to the first element in the output range. -/// \param [in] size - number of element in the input range. -/// \param [in] segments - number of segments in the input range. -/// \param [in] begin_offsets - iterator to the first element in the range of beginning offsets. -/// \param [in] end_offsets - iterator to the first element in the range of ending offsets. -/// \param [in] begin_bit - [optional] index of the first (least significant) bit used in +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] keys_input pointer to the first element in the range to sort. +/// \param [out] keys_output pointer to the first element in the output range. +/// \param [in] values_input pointer to the first element in the range to sort. +/// \param [out] values_output pointer to the first element in the output range. +/// \param [in] size number of element in the input range. +/// \param [in] segments number of segments in the input range. +/// \param [in] begin_offsets iterator to the first element in the range of beginning offsets. +/// \param [in] end_offsets iterator to the first element in the range of ending offsets. +/// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. /// Non-default value not supported for floating-point key-types. -/// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in +/// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). Non-default value not supported for floating-point key-types. -/// \param [in] stream - [optional] HIP stream object. Default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. Default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. Default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful sort; otherwise a HIP runtime error of @@ -1030,38 +1030,38 @@ hipError_t segmented_radix_sort_pairs(void * temporary_storage, /// then it is \b guaranteed that \p a will precede \p b as well in the output (ordered) keys. /// (ordered) keys. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `segmented_radix_sort_config`. -/// \tparam KeysInputIterator - random-access iterator type of the input range. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `segmented_radix_sort_config`. +/// \tparam KeysInputIterator random-access iterator type of the input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam KeysOutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam KeysOutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam ValuesInputIterator - random-access iterator type of the input range. Must meet the +/// \tparam ValuesInputIterator random-access iterator type of the input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam ValuesOutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam ValuesOutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam OffsetIterator - random-access iterator type of segment offsets. Must meet the +/// \tparam OffsetIterator random-access iterator type of segment offsets. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the sort operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] keys_input - pointer to the first element in the range to sort. -/// \param [out] keys_output - pointer to the first element in the output range. -/// \param [in] values_input - pointer to the first element in the range to sort. -/// \param [out] values_output - pointer to the first element in the output range. -/// \param [in] size - number of element in the input range. -/// \param [in] segments - number of segments in the input range. -/// \param [in] begin_offsets - iterator to the first element in the range of beginning offsets. -/// \param [in] end_offsets - iterator to the first element in the range of ending offsets. -/// \param [in] begin_bit - [optional] index of the first (least significant) bit used in +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] keys_input pointer to the first element in the range to sort. +/// \param [out] keys_output pointer to the first element in the output range. +/// \param [in] values_input pointer to the first element in the range to sort. +/// \param [out] values_output pointer to the first element in the output range. +/// \param [in] size number of element in the input range. +/// \param [in] segments number of segments in the input range. +/// \param [in] begin_offsets iterator to the first element in the range of beginning offsets. +/// \param [in] end_offsets iterator to the first element in the range of ending offsets. +/// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. /// Non-default value not supported for floating-point key-types. -/// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in +/// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). Non-default value not supported for floating-point key-types. -/// \param [in] stream - [optional] HIP stream object. Default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. Default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. Default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful sort; otherwise a HIP runtime error of @@ -1177,29 +1177,29 @@ hipError_t segmented_radix_sort_pairs_desc(void * temporary_storage, /// * op(a, b) and op(b, a) are both false, /// then it is \b guaranteed that \p a will precede \p b as well in the output (ordered) keys. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `segmented_radix_sort_config`. -/// \tparam Key - key type. Must be an integral type or a floating-point type. -/// \tparam OffsetIterator - random-access iterator type of segment offsets. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `segmented_radix_sort_config`. +/// \tparam Key key type. Must be an integral type or a floating-point type. +/// \tparam OffsetIterator random-access iterator type of segment offsets. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the sort operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in,out] keys - reference to the double-buffer of keys, its \p current() +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in,out] keys reference to the double-buffer of keys, its \p current() /// contains the input range and will be updated to point to the output range. -/// \param [in] size - number of element in the input range. -/// \param [in] segments - number of segments in the input range. -/// \param [in] begin_offsets - iterator to the first element in the range of beginning offsets. -/// \param [in] end_offsets - iterator to the first element in the range of ending offsets. -/// \param [in] begin_bit - [optional] index of the first (least significant) bit used in +/// \param [in] size number of element in the input range. +/// \param [in] segments number of segments in the input range. +/// \param [in] begin_offsets iterator to the first element in the range of beginning offsets. +/// \param [in] end_offsets iterator to the first element in the range of ending offsets. +/// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. /// Non-default value not supported for floating-point key-types. -/// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in +/// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). Non-default value not supported for floating-point key-types. -/// \param [in] stream - [optional] HIP stream object. Default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. Default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. Default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful sort; otherwise a HIP runtime error of @@ -1311,29 +1311,29 @@ hipError_t segmented_radix_sort_keys(void * temporary_storage, /// * op(a, b) and op(b, a) are both false, /// then it is \b guaranteed that \p a will precede \p b as well in the output (ordered) keys. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `segmented_radix_sort_config`. -/// \tparam Key - key type. Must be an integral type or a floating-point type. -/// \tparam OffsetIterator - random-access iterator type of segment offsets. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `segmented_radix_sort_config`. +/// \tparam Key key type. Must be an integral type or a floating-point type. +/// \tparam OffsetIterator random-access iterator type of segment offsets. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the sort operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in,out] keys - reference to the double-buffer of keys, its \p current() +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in,out] keys reference to the double-buffer of keys, its \p current() /// contains the input range and will be updated to point to the output range. -/// \param [in] size - number of element in the input range. -/// \param [in] segments - number of segments in the input range. -/// \param [in] begin_offsets - iterator to the first element in the range of beginning offsets. -/// \param [in] end_offsets - iterator to the first element in the range of ending offsets. -/// \param [in] begin_bit - [optional] index of the first (least significant) bit used in +/// \param [in] size number of element in the input range. +/// \param [in] segments number of segments in the input range. +/// \param [in] begin_offsets iterator to the first element in the range of beginning offsets. +/// \param [in] end_offsets iterator to the first element in the range of ending offsets. +/// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. /// Non-default value not supported for floating-point key-types. -/// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in +/// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). Non-default value not supported for floating-point key-types. -/// \param [in] stream - [optional] HIP stream object. Default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. Default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. Default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful sort; otherwise a HIP runtime error of @@ -1446,32 +1446,32 @@ hipError_t segmented_radix_sort_keys_desc(void * temporary_storage, /// then it is \b guaranteed that \p a will precede \p b as well in the output (ordered) keys. /// (ordered) keys. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `segmented_radix_sort_config`. -/// \tparam Key - key type. Must be an integral type or a floating-point type. -/// \tparam Value - value type. -/// \tparam OffsetIterator - random-access iterator type of segment offsets. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `segmented_radix_sort_config`. +/// \tparam Key key type. Must be an integral type or a floating-point type. +/// \tparam Value value type. +/// \tparam OffsetIterator random-access iterator type of segment offsets. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the sort operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in,out] keys - reference to the double-buffer of keys, its \p current() +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in,out] keys reference to the double-buffer of keys, its \p current() /// contains the input range and will be updated to point to the output range. -/// \param [in,out] values - reference to the double-buffer of values, its \p current() +/// \param [in,out] values reference to the double-buffer of values, its \p current() /// contains the input range and will be updated to point to the output range. -/// \param [in] size - number of element in the input range. -/// \param [in] segments - number of segments in the input range. -/// \param [in] begin_offsets - iterator to the first element in the range of beginning offsets. -/// \param [in] end_offsets - iterator to the first element in the range of ending offsets. -/// \param [in] begin_bit - [optional] index of the first (least significant) bit used in +/// \param [in] size number of element in the input range. +/// \param [in] segments number of segments in the input range. +/// \param [in] begin_offsets iterator to the first element in the range of beginning offsets. +/// \param [in] end_offsets iterator to the first element in the range of ending offsets. +/// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. /// Non-default value not supported for floating-point key-types. -/// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in +/// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). Non-default value not supported for floating-point key-types. -/// \param [in] stream - [optional] HIP stream object. Default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. Default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. Default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful sort; otherwise a HIP runtime error of @@ -1596,32 +1596,32 @@ hipError_t segmented_radix_sort_pairs(void * temporary_storage, /// then it is \b guaranteed that \p a will precede \p b as well in the output (ordered) keys. /// (ordered) keys. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `segmented_radix_sort_config`. -/// \tparam Key - key type. Must be an integral type or a floating-point type. -/// \tparam Value - value type. -/// \tparam OffsetIterator - random-access iterator type of segment offsets. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `segmented_radix_sort_config`. +/// \tparam Key key type. Must be an integral type or a floating-point type. +/// \tparam Value value type. +/// \tparam OffsetIterator random-access iterator type of segment offsets. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the sort operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in,out] keys - reference to the double-buffer of keys, its \p current() +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in,out] keys reference to the double-buffer of keys, its \p current() /// contains the input range and will be updated to point to the output range. -/// \param [in,out] values - reference to the double-buffer of values, its \p current() +/// \param [in,out] values reference to the double-buffer of values, its \p current() /// contains the input range and will be updated to point to the output range. -/// \param [in] size - number of element in the input range. -/// \param [in] segments - number of segments in the input range. -/// \param [in] begin_offsets - iterator to the first element in the range of beginning offsets. -/// \param [in] end_offsets - iterator to the first element in the range of ending offsets. -/// \param [in] begin_bit - [optional] index of the first (least significant) bit used in +/// \param [in] size number of element in the input range. +/// \param [in] segments number of segments in the input range. +/// \param [in] begin_offsets iterator to the first element in the range of beginning offsets. +/// \param [in] end_offsets iterator to the first element in the range of ending offsets. +/// \param [in] begin_bit [optional] index of the first (least significant) bit used in /// key comparison. Must be in range [0; 8 * sizeof(Key)). Default value: \p 0. /// Non-default value not supported for floating-point key-types. -/// \param [in] end_bit - [optional] past-the-end index (most significant) bit used in +/// \param [in] end_bit [optional] past-the-end index (most significant) bit used in /// key comparison. Must be in range (begin_bit; 8 * sizeof(Key)]. Default /// value: \p 8 * sizeof(Key). Non-default value not supported for floating-point key-types. -/// \param [in] stream - [optional] HIP stream object. Default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. Default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. Default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful sort; otherwise a HIP runtime error of diff --git a/rocprim/include/rocprim/device/device_segmented_radix_sort_config.hpp b/rocprim/include/rocprim/device/device_segmented_radix_sort_config.hpp index 62def5dff..f4a1c8de9 100644 --- a/rocprim/include/rocprim/device/device_segmented_radix_sort_config.hpp +++ b/rocprim/include/rocprim/device/device_segmented_radix_sort_config.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -39,8 +39,8 @@ BEGIN_ROCPRIM_NAMESPACE /// \brief Selects the appropriate \p WarpSortConfig based on the size of the key type. /// -/// \tparam Key - the type of the sorted keys. -/// \tparam MediumWarpSize - the logical warp size of the medium segment processing kernel. +/// \tparam Key the type of the sorted keys. +/// \tparam MediumWarpSize the logical warp size of the medium segment processing kernel. template using select_warp_sort_config_t = std::conditional_tsegments + 1 elements: offsets for \p begin_offsets and /// offsets + 1 for \p end_offsets. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `reduce_config`. -/// \tparam InputIterator - random-access iterator type of the input range. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `reduce_config`. +/// \tparam InputIterator random-access iterator type of the input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam OutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam OutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam OffsetIterator - random-access iterator type of segment offsets. Must meet the +/// \tparam OffsetIterator random-access iterator type of segment offsets. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam BinaryFunction - type of binary function used for reduction. Default type +/// \tparam BinaryFunction type of binary function used for reduction. Default type /// is \p rocprim::plus, where \p T is a \p value_type of \p InputIterator. -/// \tparam InitValueType - type of the initial value. +/// \tparam InitValueType type of the initial value. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the reduction operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] input - iterator to the first element in the range to reduce. -/// \param [out] output - iterator to the first element in the output range. -/// \param [in] segments - number of segments in the input range. -/// \param [in] begin_offsets - iterator to the first element in the range of beginning offsets. -/// \param [in] end_offsets - iterator to the first element in the range of ending offsets. -/// \param [in] initial_value - initial value to start the reduction. -/// \param [in] reduce_op - binary operation function object that will be used for reduction. +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] input iterator to the first element in the range to reduce. +/// \param [out] output iterator to the first element in the output range. +/// \param [in] segments number of segments in the input range. +/// \param [in] begin_offsets iterator to the first element in the range of beginning offsets. +/// \param [in] end_offsets iterator to the first element in the range of ending offsets. +/// \param [in] initial_value initial value to start the reduction. +/// \param [in] reduce_op binary operation function object that will be used for reduction. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. /// The default value is \p BinaryFunction(). -/// \param [in] stream - [optional] HIP stream object. The default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. The default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. The default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful reduction; otherwise a HIP runtime error of diff --git a/rocprim/include/rocprim/device/device_segmented_scan.hpp b/rocprim/include/rocprim/device/device_segmented_scan.hpp index 28d43d5a5..74d37a769 100644 --- a/rocprim/include/rocprim/device/device_segmented_scan.hpp +++ b/rocprim/include/rocprim/device/device_segmented_scan.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -173,32 +173,32 @@ hipError_t segmented_scan_impl(void * temporary_storage, /// segments + 1 elements: offsets for \p begin_offsets and /// offsets + 1 for \p end_offsets. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `scan_config`. -/// \tparam InputIterator - random-access iterator type of the input range. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `scan_config`. +/// \tparam InputIterator random-access iterator type of the input range. Must meet the /// requirements of a C++ RandomAccessIterator concept. It can be a simple pointer type. -/// \tparam OutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam OutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ RandomAccessIterator concept. It can be a simple pointer type. -/// \tparam OffsetIterator - random-access iterator type of segment offsets. Must meet the +/// \tparam OffsetIterator random-access iterator type of segment offsets. Must meet the /// requirements of a C++ RandomAccessIterator concept. It can be a simple pointer type. -/// \tparam BinaryFunction - type of binary function used for scan operation. Default type +/// \tparam BinaryFunction type of binary function used for scan operation. Default type /// is \p rocprim::plus, where \p T is a \p value_type of \p InputIterator. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the scan operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] input - iterator to the first element in the range to scan. -/// \param [out] output - iterator to the first element in the output range. -/// \param [in] segments - number of segments in the input range. -/// \param [in] begin_offsets - iterator to the first element in the range of beginning offsets. -/// \param [in] end_offsets - iterator to the first element in the range of ending offsets. -/// \param [in] scan_op - binary operation function object that will be used for scan. +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] input iterator to the first element in the range to scan. +/// \param [out] output iterator to the first element in the output range. +/// \param [in] segments number of segments in the input range. +/// \param [in] begin_offsets iterator to the first element in the range of beginning offsets. +/// \param [in] end_offsets iterator to the first element in the range of ending offsets. +/// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. /// The default value is \p BinaryFunction(). -/// \param [in] stream - [optional] HIP stream object. The default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. The default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. The default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful scan; otherwise a HIP runtime error of @@ -287,34 +287,34 @@ hipError_t segmented_inclusive_scan(void * temporary_storage, /// segments + 1 elements: offsets for \p begin_offsets and /// offsets + 1 for \p end_offsets. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `scan_config`. -/// \tparam InputIterator - random-access iterator type of the input range. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `scan_config`. +/// \tparam InputIterator random-access iterator type of the input range. Must meet the /// requirements of a C++ RandomAccessIterator concept. It can be a simple pointer type. -/// \tparam OutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam OutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ RandomAccessIterator concept. It can be a simple pointer type. -/// \tparam OffsetIterator - random-access iterator type of segment offsets. Must meet the +/// \tparam OffsetIterator random-access iterator type of segment offsets. Must meet the /// requirements of a C++ RandomAccessIterator concept. It can be a simple pointer type. -/// \tparam InitValueType - type of the initial value. -/// \tparam BinaryFunction - type of binary function used for scan operation. Default type +/// \tparam InitValueType type of the initial value. +/// \tparam BinaryFunction type of binary function used for scan operation. Default type /// is \p rocprim::plus, where \p T is a \p value_type of \p InputIterator. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the scan operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] input - iterator to the first element in the range to scan. -/// \param [out] output - iterator to the first element in the output range. -/// \param [in] segments - number of segments in the input range. -/// \param [in] begin_offsets - iterator to the first element in the range of beginning offsets. -/// \param [in] end_offsets - iterator to the first element in the range of ending offsets. -/// \param [in] initial_value - initial value to start the scan. -/// \param [in] scan_op - binary operation function object that will be used for scan. +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] input iterator to the first element in the range to scan. +/// \param [out] output iterator to the first element in the output range. +/// \param [in] segments number of segments in the input range. +/// \param [in] begin_offsets iterator to the first element in the range of beginning offsets. +/// \param [in] end_offsets iterator to the first element in the range of ending offsets. +/// \param [in] initial_value initial value to start the scan. +/// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. /// The default value is \p BinaryFunction(). -/// \param [in] stream - [optional] HIP stream object. The default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. The default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. The default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful scan; otherwise a HIP runtime error of @@ -404,32 +404,32 @@ hipError_t segmented_exclusive_scan(void * temporary_storage, /// * Ranges specified by \p input, \p output, and \p flags must have at least \p size elements. /// * \p value_type of \p HeadFlagIterator iterator should be convertible to \p bool type. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `scan_config`. -/// \tparam InputIterator - random-access iterator type of the input range. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `scan_config`. +/// \tparam InputIterator random-access iterator type of the input range. Must meet the /// requirements of a C++ RandomAccessIterator concept. It can be a simple pointer type. -/// \tparam OutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam OutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ RandomAccessIterator concept. It can be a simple pointer type. -/// \tparam HeadFlagIterator - random-access iterator type of flags. Must meet the +/// \tparam HeadFlagIterator random-access iterator type of flags. Must meet the /// requirements of a C++ RandomAccessIterator concept. It can be a simple pointer type. -/// \tparam BinaryFunction - type of binary function used for scan operation. Default type +/// \tparam BinaryFunction type of binary function used for scan operation. Default type /// is \p rocprim::plus, where \p T is a \p value_type of \p InputIterator. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the scan operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] input - iterator to the first element in the range to scan. -/// \param [out] output - iterator to the first element in the output range. -/// \param [in] head_flags - iterator to the first element in the range of head flags marking +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] input iterator to the first element in the range to scan. +/// \param [out] output iterator to the first element in the output range. +/// \param [in] head_flags iterator to the first element in the range of head flags marking /// beginnings of each segment in the input range. -/// \param [in] size - number of element in the input range. -/// \param [in] scan_op - binary operation function object that will be used for scan. +/// \param [in] size number of element in the input range. +/// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. /// The default value is \p BinaryFunction(). -/// \param [in] stream - [optional] HIP stream object. The default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. The default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. The default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful scan; otherwise a HIP runtime error of @@ -516,34 +516,34 @@ hipError_t segmented_inclusive_scan(void * temporary_storage, /// * Ranges specified by \p input, \p output, and \p flags must have at least \p size elements. /// * \p value_type of \p HeadFlagIterator iterator should be convertible to \p bool type. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `scan_config`. -/// \tparam InputIterator - random-access iterator type of the input range. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `scan_config`. +/// \tparam InputIterator random-access iterator type of the input range. Must meet the /// requirements of a C++ RandomAccessIterator concept. It can be a simple pointer type. -/// \tparam OutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam OutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ RandomAccessIterator concept. It can be a simple pointer type. -/// \tparam HeadFlagIterator - random-access iterator type of flags. Must meet the +/// \tparam HeadFlagIterator random-access iterator type of flags. Must meet the /// requirements of a C++ RandomAccessIterator concept. It can be a simple pointer type. -/// \tparam InitValueType - type of the initial value. -/// \tparam BinaryFunction - type of binary function used for scan operation. Default type +/// \tparam InitValueType type of the initial value. +/// \tparam BinaryFunction type of binary function used for scan operation. Default type /// is \p rocprim::plus, where \p T is a \p value_type of \p InputIterator. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the scan operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] input - iterator to the first element in the range to scan. -/// \param [out] output - iterator to the first element in the output range. -/// \param [in] head_flags - iterator to the first element in the range of head flags marking +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] input iterator to the first element in the range to scan. +/// \param [out] output iterator to the first element in the output range. +/// \param [in] head_flags iterator to the first element in the range of head flags marking /// beginnings of each segment in the input range. -/// \param [in] initial_value - initial value to start the scan. -/// \param [in] size - number of element in the input range. -/// \param [in] scan_op - binary operation function object that will be used for scan. +/// \param [in] initial_value initial value to start the scan. +/// \param [in] size number of element in the input range. +/// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. /// The default value is \p BinaryFunction(). -/// \param [in] stream - [optional] HIP stream object. The default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. The default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. The default value is \p false. /// /// \returns \p hipSuccess (\p 0) after successful scan; otherwise a HIP runtime error of diff --git a/rocprim/include/rocprim/device/device_select.hpp b/rocprim/include/rocprim/device/device_select.hpp index 3f8267521..a7d65d1bf 100644 --- a/rocprim/include/rocprim/device/device_select.hpp +++ b/rocprim/include/rocprim/device/device_select.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -58,27 +58,27 @@ namespace detail /// * Range specified by \p selected_count_output must have at least 1 element. /// * Values of \p flag range should be implicitly convertible to `bool` type. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `select_config`. -/// \tparam InputIterator - random-access iterator type of the input range. It can be +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `select_config`. +/// \tparam InputIterator random-access iterator type of the input range. It can be /// a simple pointer type. -/// \tparam FlagIterator - random-access iterator type of the flag range. It can be +/// \tparam FlagIterator random-access iterator type of the flag range. It can be /// a simple pointer type. -/// \tparam OutputIterator - random-access iterator type of the output range. It can be +/// \tparam OutputIterator random-access iterator type of the output range. It can be /// a simple pointer type. -/// \tparam SelectedCountOutputIterator - random-access iterator type of the selected_count_output +/// \tparam SelectedCountOutputIterator random-access iterator type of the selected_count_output /// value. It can be a simple pointer type. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the select operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] input - iterator to the first element in the range to select values from. -/// \param [in] flags - iterator to the selection flag corresponding to the first element from \p input range. -/// \param [out] output - iterator to the first element in the output range. -/// \param [out] selected_count_output - iterator to the total number of selected values (length of \p output). -/// \param [in] size - number of element in the input range. -/// \param [in] stream - [optional] HIP stream object. The default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] input iterator to the first element in the range to select values from. +/// \param [in] flags iterator to the selection flag corresponding to the first element from \p input range. +/// \param [out] output iterator to the first element in the output range. +/// \param [out] selected_count_output iterator to the total number of selected values (length of \p output). +/// \param [in] size number of element in the input range. +/// \param [in] stream [optional] HIP stream object. The default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. The default value is \p false. /// /// \par Example @@ -181,24 +181,24 @@ hipError_t select(void * temporary_storage, /// values can be copied into it. /// * Range specified by \p selected_count_output must have at least 1 element. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `select_config`. -/// \tparam InputIterator - random-access iterator type of the input range. It can be +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `select_config`. +/// \tparam InputIterator random-access iterator type of the input range. It can be /// a simple pointer type. -/// \tparam OutputIterator - random-access iterator type of the output range. It can be +/// \tparam OutputIterator random-access iterator type of the output range. It can be /// a simple pointer type. -/// \tparam SelectedCountOutputIterator - random-access iterator type of the selected_count_output +/// \tparam SelectedCountOutputIterator random-access iterator type of the selected_count_output /// value. It can be a simple pointer type. -/// \tparam UnaryPredicate - type of a unary selection predicate. +/// \tparam UnaryPredicate type of a unary selection predicate. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the select operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] input - iterator to the first element in the range to select values from. -/// \param [out] output - iterator to the first element in the output range. -/// \param [out] selected_count_output - iterator to the total number of selected values (length of \p output). -/// \param [in] size - number of element in the input range. -/// \param [in] predicate - unary function object that will be used for selecting values. +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] input iterator to the first element in the range to select values from. +/// \param [out] output iterator to the first element in the output range. +/// \param [out] selected_count_output iterator to the total number of selected values (length of \p output). +/// \param [in] size number of element in the input range. +/// \param [in] predicate unary function object that will be used for selecting values. /// The predicate must meet the C++ named requirement \p BinaryPredicate : /// - The result of applying the predicate must be convertible to bool /// - The predicate must accept const object arguments, with the same behavior regardless of @@ -206,8 +206,8 @@ hipError_t select(void * temporary_storage, /// In practice, the signature of the function should be equivalent to the following: /// bool f(const T &a);. The signature does not need to have /// const &, but the function object must not modify the object passed to it. -/// \param [in] stream - [optional] HIP stream object. The default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. The default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. The default value is \p false. /// /// \par Example @@ -316,27 +316,27 @@ hipError_t select(void * temporary_storage, /// values can be copied into it. /// * Range specified by \p selected_count_output must have at least 1 element. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `select_config`. -/// \tparam InputIterator - random-access iterator type of the input range. It can be +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `select_config`. +/// \tparam InputIterator random-access iterator type of the input range. It can be /// a simple pointer type. -/// \tparam FlagIterator - random-access iterator type of the flag range. It can be +/// \tparam FlagIterator random-access iterator type of the flag range. It can be /// a simple pointer type. -/// \tparam OutputIterator - random-access iterator type of the output range. It can be +/// \tparam OutputIterator random-access iterator type of the output range. It can be /// a simple pointer type. -/// \tparam SelectedCountOutputIterator - random-access iterator type of the selected_count_output +/// \tparam SelectedCountOutputIterator random-access iterator type of the selected_count_output /// value. It can be a simple pointer type. -/// \tparam UnaryPredicate - type of a unary selection predicate. +/// \tparam UnaryPredicate type of a unary selection predicate. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the select operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] input - iterator to the first element in the range to select values from. -/// \param [in] flags - iterator to the selection flag corresponding to the first element from \p input range. -/// \param [out] output - iterator to the first element in the output range. -/// \param [out] selected_count_output - iterator to the total number of selected values (length of \p output). -/// \param [in] size - number of element in the input range. -/// \param [in] predicate - unary function object that will be used for selecting flags. +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] input iterator to the first element in the range to select values from. +/// \param [in] flags iterator to the selection flag corresponding to the first element from \p input range. +/// \param [out] output iterator to the first element in the output range. +/// \param [out] selected_count_output iterator to the total number of selected values (length of \p output). +/// \param [in] size number of element in the input range. +/// \param [in] predicate unary function object that will be used for selecting flags. /// The predicate must meet the C++ named requirement \p BinaryPredicate: /// - The result of applying the predicate must be convertible to bool. /// - The predicate must accept const object arguments, with the same behavior regardless of @@ -344,8 +344,8 @@ hipError_t select(void * temporary_storage, /// In practice, the signature of the function should be equivalent to the following: /// bool f(const T &a);. The signature does not need to have /// const &, but the function object must not modify the object passed to it. -/// \param [in] stream - [optional] HIP stream object. The default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. The default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. The default value is \p false. /// /// \par Example @@ -451,29 +451,29 @@ inline hipError_t select(void* temporary_storage, /// * By default InputIterator::value_type's equality operator is used to check /// if elements are equivalent. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `select_config`. -/// \tparam InputIterator - random-access iterator type of the input range. It can be +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `select_config`. +/// \tparam InputIterator random-access iterator type of the input range. It can be /// a simple pointer type. -/// \tparam OutputIterator - random-access iterator type of the output range. It can be +/// \tparam OutputIterator random-access iterator type of the output range. It can be /// a simple pointer type. -/// \tparam UniqueCountOutputIterator - random-access iterator type of the unique_count_output +/// \tparam UniqueCountOutputIterator random-access iterator type of the unique_count_output /// value used to return number of unique values. It can be a simple pointer type. -/// \tparam EqualityOp - type of an binary operator used to compare values for equality. +/// \tparam EqualityOp type of an binary operator used to compare values for equality. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the unique operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] input - iterator to the first element in the range to select values from. -/// \param [out] output - iterator to the first element in the output range. -/// \param [out] unique_count_output - iterator to the total number of selected values (length of \p output). -/// \param [in] size - number of element in the input range. -/// \param [in] equality_op - [optional] binary function object used to compare input values for equality. +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] input iterator to the first element in the range to select values from. +/// \param [out] output iterator to the first element in the output range. +/// \param [out] unique_count_output iterator to the total number of selected values (length of \p output). +/// \param [in] size number of element in the input range. +/// \param [in] equality_op [optional] binary function object used to compare input values for equality. /// The signature of the function should be equivalent to the following: /// bool equal_to(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the object passed to it. -/// \param [in] stream - [optional] HIP stream object. The default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. The default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. The default value is \p false. /// /// \par Example @@ -578,35 +578,35 @@ hipError_t unique(void * temporary_storage, /// * By default InputIterator::value_type's equality operator is used to check /// if elements are equivalent. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `select_config`. -/// \tparam KeyIterator - random-access iterator type of the input key range. It can be +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `select_config`. +/// \tparam KeyIterator random-access iterator type of the input key range. It can be /// a simple pointer type. -/// \tparam ValueIterator - random-access iterator type of the input value range. It can be +/// \tparam ValueIterator random-access iterator type of the input value range. It can be /// a simple pointer type. -/// \tparam OutputKeyIterator - random-access iterator type of the output key range. It can be +/// \tparam OutputKeyIterator random-access iterator type of the output key range. It can be /// a simple pointer type. -/// \tparam OutputValueIterator - random-access iterator type of the output value range. It can be +/// \tparam OutputValueIterator random-access iterator type of the output value range. It can be /// a simple pointer type. -/// \tparam UniqueCountOutputIterator - random-access iterator type of the unique_count_output +/// \tparam UniqueCountOutputIterator random-access iterator type of the unique_count_output /// value used to return number of unique keys and values. It can be a simple pointer type. -/// \tparam EqualityOp - type of an binary operator used to compare keys for equality. +/// \tparam EqualityOp type of an binary operator used to compare keys for equality. /// -/// \param [in] temporary_storage - pointer to a device-accessible temporary storage. When +/// \param [in] temporary_storage pointer to a device-accessible temporary storage. When /// a null pointer is passed, the required allocation size (in bytes) is written to /// \p storage_size and function returns without performing the unique operation. -/// \param [in,out] storage_size - reference to a size (in bytes) of \p temporary_storage. -/// \param [in] keys_input - iterator to the first element in the range to select keys from. -/// \param [in] values_input - iterator to the first element in the range of values corresponding to keys -/// \param [out] keys_output - iterator to the first element in the output key range. -/// \param [out] values_output - iterator to the first element in the output value range. -/// \param [out] unique_count_output - iterator to the total number of selected values (length of \p output). -/// \param [in] size - number of element in the input range. -/// \param [in] equality_op - [optional] binary function object used to compare input values for equality. +/// \param [in,out] storage_size reference to a size (in bytes) of \p temporary_storage. +/// \param [in] keys_input iterator to the first element in the range to select keys from. +/// \param [in] values_input iterator to the first element in the range of values corresponding to keys +/// \param [out] keys_output iterator to the first element in the output key range. +/// \param [out] values_output iterator to the first element in the output value range. +/// \param [out] unique_count_output iterator to the total number of selected values (length of \p output). +/// \param [in] size number of element in the input range. +/// \param [in] equality_op [optional] binary function object used to compare input values for equality. /// The signature of the function should be equivalent to the following: /// bool equal_to(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the object passed to it. -/// \param [in] stream - [optional] HIP stream object. The default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. The default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. The default value is \p false. template U f(const T &a);. The signature does not need to have /// const &, but function object must not modify the object passed to it. -/// \param [in] stream - [optional] HIP stream object. The default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. The default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced in order to check for errors. The default value is \p false. /// /// \par Example @@ -192,25 +192,25 @@ inline hipError_t transform(InputIterator input, /// \par Overview /// * Ranges specified by \p input1, \p input2, and \p output must have at least \p size elements. /// -/// \tparam Config - [optional] Configuration of the primitive, must be `default_config` or `transform_config`. -/// \tparam InputIterator1 - random-access iterator type of the input range. Must meet the +/// \tparam Config [optional] Configuration of the primitive, must be `default_config` or `transform_config`. +/// \tparam InputIterator1 random-access iterator type of the input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam InputIterator2 - random-access iterator type of the input range. Must meet the +/// \tparam InputIterator2 random-access iterator type of the input range. Must meet the /// requirements of a C++ InputIterator concept. It can be a simple pointer type. -/// \tparam OutputIterator - random-access iterator type of the output range. Must meet the +/// \tparam OutputIterator random-access iterator type of the output range. Must meet the /// requirements of a C++ OutputIterator concept. It can be a simple pointer type. -/// \tparam BinaryFunction - type of binary function used for transform. +/// \tparam BinaryFunction type of binary function used for transform. /// -/// \param [in] input1 - iterator to the first element in the 1st range to transform. -/// \param [in] input2 - iterator to the first element in the 2nd range to transform. -/// \param [out] output - iterator to the first element in the output range. -/// \param [in] size - number of element in the input range. -/// \param [in] transform_op - binary operation function object that will be used for transform. +/// \param [in] input1 iterator to the first element in the 1st range to transform. +/// \param [in] input2 iterator to the first element in the 2nd range to transform. +/// \param [out] output iterator to the first element in the output range. +/// \param [in] size number of element in the input range. +/// \param [in] transform_op binary operation function object that will be used for transform. /// The signature of the function should be equivalent to the following: /// U f(const T1& a, const T2& b);. The signature does not need to have /// const &, but function object must not modify the object passed to it. -/// \param [in] stream - [optional] HIP stream object. The default is \p 0 (default stream). -/// \param [in] debug_synchronous - [optional] If true, synchronization after every kernel +/// \param [in] stream [optional] HIP stream object. The default is \p 0 (default stream). +/// \param [in] debug_synchronous [optional] If true, synchronization after every kernel /// launch is forced. Default value is \p false. /// /// \par Example diff --git a/rocprim/include/rocprim/intrinsics/thread.hpp b/rocprim/include/rocprim/intrinsics/thread.hpp index a97cb021a..0a1b3c030 100644 --- a/rocprim/include/rocprim/intrinsics/thread.hpp +++ b/rocprim/include/rocprim/intrinsics/thread.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -132,7 +132,7 @@ unsigned int warp_id() } /// \brief Returns warp id in a block (tile), given the flat (linear, 1D) thread identifier in a multidimensional tile (block). -/// \param flat_id - the flat id that should be used to compute the warp id. +/// \param flat_id the flat id that should be used to compute the warp id. ROCPRIM_DEVICE ROCPRIM_INLINE unsigned int warp_id(unsigned int flat_id) { diff --git a/rocprim/include/rocprim/intrinsics/warp.hpp b/rocprim/include/rocprim/intrinsics/warp.hpp index a3d47ac35..afa15511b 100644 --- a/rocprim/include/rocprim/intrinsics/warp.hpp +++ b/rocprim/include/rocprim/intrinsics/warp.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -35,7 +35,7 @@ BEGIN_ROCPRIM_NAMESPACE /// whose i-th bit is set if and only if \p predicate is true /// for the i-th thread of the warp and the i-th thread is active. /// -/// \param predicate - input to be evaluated for all active lanes +/// \param predicate input to be evaluated for all active lanes ROCPRIM_DEVICE ROCPRIM_INLINE lane_mask_type ballot(int predicate) { diff --git a/rocprim/include/rocprim/intrinsics/warp_shuffle.hpp b/rocprim/include/rocprim/intrinsics/warp_shuffle.hpp index 7f0a86c8b..a34d9eb00 100644 --- a/rocprim/include/rocprim/intrinsics/warp_shuffle.hpp +++ b/rocprim/include/rocprim/intrinsics/warp_shuffle.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -100,7 +100,7 @@ T warp_move_dpp(const T& input) /// Each thread in warp obtains \p input from src_lane-th thread /// in warp, where src_lane is current lane with a mask applied. /// -/// \param input - input to pass to other threads +/// \param input input to pass to other threads template ROCPRIM_DEVICE ROCPRIM_INLINE T warp_swizzle(const T& input) @@ -127,9 +127,9 @@ T warp_swizzle(const T& input) /// Note: The optional \p width parameter must be a power of 2; results are /// undefined if it is not a power of 2, or it is greater than device_warp_size(). /// -/// \param input - input to pass to other threads -/// \param src_lane - warp if of a thread whose \p input should be returned -/// \param width - logical warp width +/// \param input input to pass to other threads +/// \param src_lane warp if of a thread whose \p input should be returned +/// \param width logical warp width template ROCPRIM_DEVICE ROCPRIM_INLINE T warp_shuffle(const T& input, const int src_lane, const int width = device_warp_size()) @@ -152,9 +152,9 @@ T warp_shuffle(const T& input, const int src_lane, const int width = device_warp /// Note: The optional \p width parameter must be a power of 2; results are /// undefined if it is not a power of 2, or it is greater than device_warp_size(). /// -/// \param input - input to pass to other threads -/// \param delta - offset for calculating source lane id -/// \param width - logical warp width +/// \param input input to pass to other threads +/// \param delta offset for calculating source lane id +/// \param width logical warp width template ROCPRIM_DEVICE ROCPRIM_INLINE T warp_shuffle_up(const T& input, const unsigned int delta, const int width = device_warp_size()) @@ -177,9 +177,9 @@ T warp_shuffle_up(const T& input, const unsigned int delta, const int width = de /// Note: The optional \p width parameter must be a power of 2; results are /// undefined if it is not a power of 2, or it is greater than device_warp_size(). /// -/// \param input - input to pass to other threads -/// \param delta - offset for calculating source lane id -/// \param width - logical warp width +/// \param input input to pass to other threads +/// \param delta offset for calculating source lane id +/// \param width logical warp width template ROCPRIM_DEVICE ROCPRIM_INLINE T warp_shuffle_down(const T& input, const unsigned int delta, const int width = device_warp_size()) @@ -201,9 +201,9 @@ T warp_shuffle_down(const T& input, const unsigned int delta, const int width = /// Note: The optional \p width parameter must be a power of 2; results are /// undefined if it is not a power of 2, or it is greater than device_warp_size(). /// -/// \param input - input to pass to other threads -/// \param lane_mask - mask used for calculating source lane id -/// \param width - logical warp width +/// \param input input to pass to other threads +/// \param lane_mask mask used for calculating source lane id +/// \param width logical warp width template ROCPRIM_DEVICE ROCPRIM_INLINE T warp_shuffle_xor(const T& input, const int lane_mask, const int width = device_warp_size()) @@ -229,9 +229,9 @@ namespace detail /// Note: The optional \p width parameter must be a power of 2; results are /// undefined if it is not a power of 2, or it is greater than device_warp_size(). /// -/// \param v - input to pass to other threads -/// \param mask - mask used for calculating source lane id -/// \param width - logical warp width +/// \param v input to pass to other threads +/// \param mask mask used for calculating source lane id +/// \param width logical warp width template ROCPRIM_DEVICE ROCPRIM_INLINE V warp_swizzle_shuffle(V& v, const int mask, @@ -262,9 +262,9 @@ ROCPRIM_DEVICE ROCPRIM_INLINE V warp_swizzle_shuffle(V& v, /// Note: The optional \p width parameter must be a power of 2; results are /// undefined if it is not a power of 2, or it is greater than device_warp_size(). /// -/// \param input - input to pass to other threads -/// \param dst_lane - the destination lane to which the value from this thread is written. -/// \param width - logical warp width +/// \param input input to pass to other threads +/// \param dst_lane the destination lane to which the value from this thread is written. +/// \param width logical warp width template ROCPRIM_DEVICE ROCPRIM_INLINE T warp_permute(const T& input, const int dst_lane, @@ -298,7 +298,7 @@ ROCPRIM_DEVICE ROCPRIM_INLINE T warp_permute(const T& input, /// /// \remark This operation is significantly faster than \p warp_shuffle. /// -/// \param input - the value to broadcast +/// \param input the value to broadcast template ROCPRIM_DEVICE ROCPRIM_INLINE T warp_readfirstlane(const T& input) { @@ -316,8 +316,8 @@ ROCPRIM_DEVICE ROCPRIM_INLINE T warp_readfirstlane(const T& input) /// /// \remark This operation is significantly faster than \p warp_shuffle. /// -/// \param input - the value to broadcast -/// \param src_lane - the lane whose value to broadcast to other threads in the warp +/// \param input the value to broadcast +/// \param src_lane the lane whose value to broadcast to other threads in the warp template ROCPRIM_DEVICE ROCPRIM_INLINE T warp_readlane(const T& input, const int src_lane) { diff --git a/rocprim/include/rocprim/iterator/arg_index_iterator.hpp b/rocprim/include/rocprim/iterator/arg_index_iterator.hpp index 5b96abb00..3adc89265 100644 --- a/rocprim/include/rocprim/iterator/arg_index_iterator.hpp +++ b/rocprim/include/rocprim/iterator/arg_index_iterator.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -43,11 +43,11 @@ BEGIN_ROCPRIM_NAMESPACE /// type, which includes value from the underlying range and its index in that range. /// * \p std::iterator_traits::value_type should be convertible to \p InputValueType. /// -/// \tparam InputIterator - type of the underlying random-access input iterator. Must be +/// \tparam InputIterator type of the underlying random-access input iterator. Must be /// a random-access iterator. -/// \tparam Difference - type used for identify distance between iterators and as the index type +/// \tparam Difference type used for identify distance between iterators and as the index type /// in the output pair type (see \p value_type). -/// \tparam InputValueType - value type used in the output pair type (see \p value_type). +/// \tparam InputValueType value type used in the output pair type (see \p value_type). template< class InputIterator, class Difference = std::ptrdiff_t, @@ -239,11 +239,11 @@ operator+(typename arg_index_iterator /// the underlying iterator and \p offset as the position (index) of \p iterator /// in the input range. /// -/// \tparam InputIterator - type of the underlying random-access input iterator. Must be +/// \tparam InputIterator type of the underlying random-access input iterator. Must be /// a random-access iterator. -/// \tparam Difference - type used for identify distance between iterators and as the index type +/// \tparam Difference type used for identify distance between iterators and as the index type /// in the output pair type (see \p value_type in arg_index_iterator). -/// \tparam InputValueType - value type used in the output pair type (see \p value_type +/// \tparam InputValueType value type used in the output pair type (see \p value_type /// in arg_index_iterator). /// /// \param iterator input iterator pointing to the input range. diff --git a/rocprim/include/rocprim/iterator/constant_iterator.hpp b/rocprim/include/rocprim/iterator/constant_iterator.hpp index 468a56512..d26d824d1 100644 --- a/rocprim/include/rocprim/iterator/constant_iterator.hpp +++ b/rocprim/include/rocprim/iterator/constant_iterator.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -42,8 +42,8 @@ BEGIN_ROCPRIM_NAMESPACE /// * Using it for simulating a range filled with a sequence of same values saves /// memory capacity and bandwidth. /// -/// \tparam ValueType - type of value that can be obtained by dereferencing the iterator. -/// \tparam Difference - a type used for identify distance between iterators +/// \tparam ValueType type of value that can be obtained by dereferencing the iterator. +/// \tparam Difference a type used for identify distance between iterators template< class ValueType, class Difference = std::ptrdiff_t @@ -239,11 +239,11 @@ operator+(typename constant_iterator::difference_type dis /// make_constant_iterator creates a constant_iterator with its initial value /// set to \p value. /// -/// \tparam ValueType - type of value that can be obtained by dereferencing created iterator. -/// \tparam Difference - a type used for identify distance between constant_iterator iterators. +/// \tparam ValueType type of value that can be obtained by dereferencing created iterator. +/// \tparam Difference a type used for identify distance between constant_iterator iterators. /// -/// \param value - initial value for constant_iterator. -/// \param index - optional index for constant_iterator. +/// \param value initial value for constant_iterator. +/// \param index optional index for constant_iterator. template< class ValueType, class Difference = std::ptrdiff_t diff --git a/rocprim/include/rocprim/iterator/counting_iterator.hpp b/rocprim/include/rocprim/iterator/counting_iterator.hpp index e8e02f57f..a89ed491e 100644 --- a/rocprim/include/rocprim/iterator/counting_iterator.hpp +++ b/rocprim/include/rocprim/iterator/counting_iterator.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -42,8 +42,8 @@ BEGIN_ROCPRIM_NAMESPACE /// * Using it for simulating a range filled with a sequence of consecutive values saves /// memory capacity and bandwidth. /// -/// \tparam Incrementable - type of value that can be obtained by dereferencing the iterator. -/// \tparam Difference - a type used for identify distance between iterators +/// \tparam Incrementable type of value that can be obtained by dereferencing the iterator. +/// \tparam Difference a type used for identify distance between iterators template< class Incrementable, class Difference = std::ptrdiff_t @@ -248,10 +248,10 @@ operator+(typename counting_iterator::difference_type /// make_counting_iterator creates a counting_iterator with its initial value /// set to \p value. /// -/// \tparam Incrementable - type of value that can be obtained by dereferencing created iterator. -/// \tparam Difference - a type used for identify distance between counting_iterator iterators. +/// \tparam Incrementable type of value that can be obtained by dereferencing created iterator. +/// \tparam Difference a type used for identify distance between counting_iterator iterators. /// -/// \param value - initial value for counting_iterator. +/// \param value initial value for counting_iterator. template< class Incrementable, class Difference = std::ptrdiff_t diff --git a/rocprim/include/rocprim/iterator/discard_iterator.hpp b/rocprim/include/rocprim/iterator/discard_iterator.hpp index 0cfae3407..dab938d27 100644 --- a/rocprim/include/rocprim/iterator/discard_iterator.hpp +++ b/rocprim/include/rocprim/iterator/discard_iterator.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -78,7 +78,7 @@ class discard_iterator /// \brief Creates a new discard_iterator. /// - /// \param index - optional index of discard iterator (default = 0). + /// \param index optional index of discard iterator (default = 0). ROCPRIM_HOST_DEVICE inline discard_iterator(size_t index = 0) : index_(index) @@ -226,7 +226,7 @@ operator+(typename discard_iterator::difference_type distance, /// make_discard_iterator creates a discard_iterator using optional /// index parameter \p index. /// -/// \param index - optional index of discard iterator (default = 0). +/// \param index optional index of discard iterator (default = 0). /// \return A new discard_iterator object. ROCPRIM_HOST_DEVICE inline discard_iterator diff --git a/rocprim/include/rocprim/iterator/reverse_iterator.hpp b/rocprim/include/rocprim/iterator/reverse_iterator.hpp index 5ae685514..beaaaf796 100644 --- a/rocprim/include/rocprim/iterator/reverse_iterator.hpp +++ b/rocprim/include/rocprim/iterator/reverse_iterator.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -41,7 +41,7 @@ BEGIN_ROCPRIM_NAMESPACE /// * And the decrement operators on the reverse iterators are mapped to increments on the wrapped iterator. /// * Use it to iterate over the elements of a container in reverse. /// -/// \tparam SourceIterator - type of the wrapped iterator. +/// \tparam SourceIterator type of the wrapped iterator. template class reverse_iterator; @@ -241,9 +241,9 @@ constexpr reverse_iterator /// make_reverse_iterator creates a \p reverse_iterator wrapping \p source_iterator. /// -/// \tparam SourceIterator - type of \p source_iterator. +/// \tparam SourceIterator type of \p source_iterator. /// -/// \param source_iterator - the iterator to wrap in the created \p reverse_iterator. +/// \param source_iterator the iterator to wrap in the created \p reverse_iterator. /// \return A \p reverse_iterator that wraps \p source_iterator. template ROCPRIM_HOST_DEVICE diff --git a/rocprim/include/rocprim/iterator/texture_cache_iterator.hpp b/rocprim/include/rocprim/iterator/texture_cache_iterator.hpp index 63e0cfbc6..438e35a22 100644 --- a/rocprim/include/rocprim/iterator/texture_cache_iterator.hpp +++ b/rocprim/include/rocprim/iterator/texture_cache_iterator.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -124,8 +124,8 @@ struct match_texture_type /// * This iterator is not functional on gfx94x architectures, as native texture fetch functions /// are not supported in gfx94x. /// -/// \tparam T - type of value that can be obtained by dereferencing the iterator. -/// \tparam Difference - a type used for identify distance between iterators. +/// \tparam T type of value that can be obtained by dereferencing the iterator. +/// \tparam Difference a type used for identify distance between iterators. template< class T, class Difference = std::ptrdiff_t @@ -158,9 +158,9 @@ class texture_cache_iterator /// /// \tparam Texture data pointer type /// - /// \param ptr - pointer to the texture data on the device - /// \param bytes - size of the texture data (in bytes) - /// \param texture_offset - an offset from ptr to load texture data from + /// \param ptr pointer to the texture data on the device + /// \param bytes size of the texture data (in bytes) + /// \param texture_offset an offset from ptr to load texture data from /// (Defaults to 0) template inline diff --git a/rocprim/include/rocprim/iterator/transform_iterator.hpp b/rocprim/include/rocprim/iterator/transform_iterator.hpp index 4f04eda36..6f321bfcc 100644 --- a/rocprim/include/rocprim/iterator/transform_iterator.hpp +++ b/rocprim/include/rocprim/iterator/transform_iterator.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -42,10 +42,10 @@ BEGIN_ROCPRIM_NAMESPACE /// * Using it for simulating a range filled with results of applying functor of type /// \p UnaryFunction to another range saves memory capacity and/or bandwidth. /// -/// \tparam InputIterator - type of the underlying random-access input iterator. Must be +/// \tparam InputIterator type of the underlying random-access input iterator. Must be /// a random-access iterator. -/// \tparam UnaryFunction - type of the transform functor. -/// \tparam ValueType - type of value that can be obtained by dereferencing the iterator. +/// \tparam UnaryFunction type of the transform functor. +/// \tparam ValueType type of value that can be obtained by dereferencing the iterator. /// By default it is the return type of \p UnaryFunction. template:: /// make_transform_iterator creates a transform_iterator using \p iterator as /// the underlying iterator and \p transform as the unary function. /// -/// \tparam InputIterator - type of the underlying random-access input iterator. -/// \tparam UnaryFunction - type of the transform functor. +/// \tparam InputIterator type of the underlying random-access input iterator. +/// \tparam UnaryFunction type of the transform functor. /// -/// \param iterator - input iterator. -/// \param transform - transform functor to use in created transform_iterator. +/// \param iterator input iterator. +/// \param transform transform functor to use in created transform_iterator. /// \return A new transform_iterator object which transforms the range pointed /// by \p iterator using \p transform functor. template< diff --git a/rocprim/include/rocprim/iterator/zip_iterator.hpp b/rocprim/include/rocprim/iterator/zip_iterator.hpp index 63acd5eb2..f0a4cfd09 100644 --- a/rocprim/include/rocprim/iterator/zip_iterator.hpp +++ b/rocprim/include/rocprim/iterator/zip_iterator.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -307,9 +307,9 @@ operator+(typename zip_iterator::difference_type distance, /// make_zip_iterator creates a zip_iterator using \p iterator_tuple as /// the underlying tuple of iterator. /// -/// \tparam IteratorTuple - iterator tuple type +/// \tparam IteratorTuple iterator tuple type /// -/// \param iterator_tuple - tuple of iterators to use +/// \param iterator_tuple tuple of iterators to use /// \return A new zip_iterator object template ROCPRIM_HOST_DEVICE inline diff --git a/rocprim/include/rocprim/type_traits.hpp b/rocprim/include/rocprim/type_traits.hpp index 517afd98f..98c8d7b3b 100644 --- a/rocprim/include/rocprim/type_traits.hpp +++ b/rocprim/include/rocprim/type_traits.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -111,8 +111,8 @@ struct numeric_limits : std::numeric_limits #endif // DOXYGEN_SHOULD_SKIP_THIS /// \brief Used to retrieve a type that can be treated as unsigned version of the template parameter. -/// \tparam T - The signed type to find an unsigned equivalent for. -/// \tparam size - the desired size (in bytes) of the unsigned type +/// \tparam T The signed type to find an unsigned equivalent for. +/// \tparam size the desired size (in bytes) of the unsigned type template struct get_unsigned_bits_type { diff --git a/rocprim/include/rocprim/types/integer_sequence.hpp b/rocprim/include/rocprim/types/integer_sequence.hpp index 08296a119..dc69003aa 100644 --- a/rocprim/include/rocprim/types/integer_sequence.hpp +++ b/rocprim/include/rocprim/types/integer_sequence.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2020 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -83,19 +83,19 @@ struct make_integer_sequence_impl } // end detail namespace /// \brief Generates a compile time integer sequence. -/// \tparam T - the integer type to use -/// \tparam N - the number of values to generate +/// \tparam T the integer type to use +/// \tparam N the number of values to generate template using make_integer_sequence = typename detail::make_integer_sequence_impl::type; /// \brief Generates a compile time index sequence using type size_t. -/// \tparam N - the number of values to generate +/// \tparam N the number of values to generate template using make_index_sequence = make_integer_sequence; /// \brief Generates a compile time integer sequence whose length is equal /// to the length of the supplied template parameter pack. -/// \tparam T - template parameter pack +/// \tparam T template parameter pack template using index_sequence_for = make_index_sequence; #endif diff --git a/rocprim/include/rocprim/types/tuple.hpp b/rocprim/include/rocprim/types/tuple.hpp index e9424fc92..2558871c4 100644 --- a/rocprim/include/rocprim/types/tuple.hpp +++ b/rocprim/include/rocprim/types/tuple.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -797,8 +797,8 @@ struct tuple_less_than<0> /// \brief Equal to operator for tuples. /// -/// \tparam TTypes... - the element types of \p lhs tuple. -/// \tparam UTypes... - the element types of \p rhs tuple. +/// \tparam TTypes... the element types of \p lhs tuple. +/// \tparam UTypes... the element types of \p rhs tuple. /// /// Compares every element of the tuple lhs with the corresponding element /// of the tuple rhs, and returns \p true if all are equal. @@ -823,8 +823,8 @@ bool operator==(const tuple& lhs, const tuple& rhs) /// \brief Not equal to operator for tuples. /// -/// \tparam TTypes... - the element types of \p lhs tuple. -/// \tparam UTypes... - the element types of \p rhs tuple. +/// \tparam TTypes... the element types of \p lhs tuple. +/// \tparam UTypes... the element types of \p rhs tuple. /// /// Compares every element of the tuple lhs with the corresponding element /// of the tuple rhs, and returns \p true if at least one of such pairs is @@ -842,8 +842,8 @@ bool operator!=(const tuple& lhs, const tuple& rhs) /// \brief Less than operator for tuples. /// -/// \tparam TTypes... - the element types of \p lhs tuple. -/// \tparam UTypes... - the element types of \p rhs tuple. +/// \tparam TTypes... the element types of \p lhs tuple. +/// \tparam UTypes... the element types of \p rhs tuple. /// /// Compares lhs and rhs lexicographically. /// @@ -868,8 +868,8 @@ bool operator<(const tuple& lhs, const tuple& rhs) /// \brief Greater than operator for tuples. /// -/// \tparam TTypes... - the element types of \p lhs tuple. -/// \tparam UTypes... - the element types of \p rhs tuple. +/// \tparam TTypes... the element types of \p lhs tuple. +/// \tparam UTypes... the element types of \p rhs tuple. /// /// Compares lhs and rhs lexicographically. /// @@ -885,8 +885,8 @@ bool operator>(const tuple& lhs, const tuple& rhs) /// \brief Less than or equal to operator for tuples. /// -/// \tparam TTypes... - the element types of \p lhs tuple. -/// \tparam UTypes... - the element types of \p rhs tuple. +/// \tparam TTypes... the element types of \p lhs tuple. +/// \tparam UTypes... the element types of \p rhs tuple. /// /// Compares lhs and rhs lexicographically. /// @@ -902,8 +902,8 @@ bool operator<=(const tuple& lhs, const tuple& rhs) /// \brief Greater than or equal to operator for tuples. /// -/// \tparam TTypes... - the element types of \p lhs tuple. -/// \tparam UTypes... - the element types of \p rhs tuple. +/// \tparam TTypes... the element types of \p lhs tuple. +/// \tparam UTypes... the element types of \p rhs tuple. /// /// Compares lhs and rhs lexicographically. /// @@ -1010,7 +1010,7 @@ tuple...> make_tuple(Types&&... args) noexcep /// unless \p std::decay::type results in \p std::reference_wrapper for some type U, /// in which case the deduced type is U&. /// -/// \param args - zero or more arguments to create tuple from +/// \param args zero or more arguments to create tuple from /// /// \see std::tuple template @@ -1061,7 +1061,7 @@ const ignore_type ignore; /// \brief Creates a tuple of lvalue references to its arguments \p args or instances /// of \ref rocprim::ignore. /// -/// \param args - zero or more input lvalue references used to create tuple +/// \param args zero or more input lvalue references used to create tuple /// /// \see std::tie template diff --git a/rocprim/include/rocprim/warp/detail/warp_sort_stable.hpp b/rocprim/include/rocprim/warp/detail/warp_sort_stable.hpp index 726fbcfe9..1915279a2 100644 --- a/rocprim/include/rocprim/warp/detail/warp_sort_stable.hpp +++ b/rocprim/include/rocprim/warp/detail/warp_sort_stable.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2023-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -364,7 +364,7 @@ class warp_sort_stable /// Permuting the input by the output gives the merged ranges: /// | 1 3 6 7 | 0 2 4 5 | /// - /// \param m - The size of each subsequence to merge. The output consists of indices + /// \param m The size of each subsequence to merge. The output consists of indices /// for sorted ranges of 2 * m elements. template ROCPRIM_DEVICE ROCPRIM_INLINE int merge_rank(const unsigned int m, diff --git a/rocprim/include/rocprim/warp/warp_exchange.hpp b/rocprim/include/rocprim/warp/warp_exchange.hpp index c8e2dbd31..4fa92e592 100644 --- a/rocprim/include/rocprim/warp/warp_exchange.hpp +++ b/rocprim/include/rocprim/warp/warp_exchange.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -43,9 +43,9 @@ BEGIN_ROCPRIM_NAMESPACE /// \brief The \p warp_exchange class is a warp level parallel primitive which provides /// methods for rearranging items partitioned across threads in a warp. /// -/// \tparam T - the input type. -/// \tparam ItemsPerThread - the number of items contributed by each thread. -/// \tparam WarpSize - the number of threads in a warp. +/// \tparam T the input type. +/// \tparam ItemsPerThread the number of items contributed by each thread. +/// \tparam WarpSize the number of threads in a warp. /// /// \par Overview /// * The \p warp_exchange class supports the following rearrangement methods: @@ -746,11 +746,11 @@ class warp_exchange /// \brief Transposes a blocked arrangement of items to a striped arrangement /// across the warp, using temporary storage. /// - /// \tparam U - [inferred] the output type. + /// \tparam U [inferred] the output type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reusage /// Synchronization barrier should be placed before \p storage is reused @@ -809,10 +809,10 @@ class warp_exchange /// 3. ItemsPerThread is divisible by WarpSize /// 4. WarpSize is divisible by ItemsPerThread /// - /// \tparam U - [inferred] the output type. + /// \tparam U [inferred] the output type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. /// /// \par Example. /// \code{.cpp} @@ -848,11 +848,11 @@ class warp_exchange /// \brief Transposes a striped arrangement of items to a blocked arrangement /// across the warp, using temporary storage. /// - /// \tparam U - [inferred] the output type. + /// \tparam U [inferred] the output type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reusage /// Synchronization barrier should be placed before \p storage is reused @@ -912,10 +912,10 @@ class warp_exchange /// 3. ItemsPerThread is divisible by WarpSize /// 4. WarpSize is divisible by ItemsPerThread /// - /// \tparam U - [inferred] the output type. + /// \tparam U [inferred] the output type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. /// /// \par Example. /// \code{.cpp} @@ -951,12 +951,12 @@ class warp_exchange /// \brief Orders \p input values according to ranks using temporary storage, /// then writes the values to \p output in a striped manner. /// No values in \p ranks should exists that exceed \p WarpSize*ItemsPerThread-1 . - /// \tparam U - [inferred] the output type. + /// \tparam U [inferred] the output type. /// - /// \param [in] input - array that data is loaded from. - /// \param [out] output - array that data is loaded to. - /// \param [in] ranks - array containing the positions. - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] input array that data is loaded from. + /// \param [out] output array that data is loaded to. + /// \param [in] ranks array containing the positions. + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reusage /// Synchronization barrier should be placed before \p storage is reused diff --git a/rocprim/include/rocprim/warp/warp_load.hpp b/rocprim/include/rocprim/warp/warp_load.hpp index 8bae2b566..c34e1c029 100644 --- a/rocprim/include/rocprim/warp/warp_load.hpp +++ b/rocprim/include/rocprim/warp/warp_load.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -77,12 +77,12 @@ enum class warp_load_method /// \brief The \p warp_load class is a warp level parallel primitive which provides methods /// for loading data from continuous memory into a blocked arrangement of items across a warp. /// -/// \tparam T - the input/output type. -/// \tparam ItemsPerThread - the number of items to be processed by +/// \tparam T the input/output type. +/// \tparam ItemsPerThread the number of items to be processed by /// each thread. -/// \tparam WarpSize - the number of threads in the warp. It must be a divisor of the +/// \tparam WarpSize the number of threads in the warp. It must be a divisor of the /// kernel block size. -/// \tparam Method - the method to load data. +/// \tparam Method the method to load data. /// /// \par Overview /// * The \p warp_load class has a number of different methods to load data: @@ -148,11 +148,11 @@ class warp_load /// \brief Loads data from continuous memory into an arrangement of items across the /// warp. /// - /// \tparam InputIterator - [inferred] an iterator type for input (can be a simple + /// \tparam InputIterator [inferred] an iterator type for input (can be a simple /// pointer. /// - /// \param [in] input - the input iterator to load from. - /// \param [out] items - array that data is loaded to. + /// \param [in] input the input iterator to load from. + /// \param [out] items array that data is loaded to. /// \param [in] - temporary storage for inputs. /// /// \par Overview @@ -175,12 +175,12 @@ class warp_load /// \brief Loads data from continuous memory into an arrangement of items across the /// warp. /// - /// \tparam InputIterator - [inferred] an iterator type for input (can be a simple + /// \tparam InputIterator [inferred] an iterator type for input (can be a simple /// pointer. /// - /// \param [in] input - the input iterator to load from. - /// \param [out] items - array that data is loaded to. - /// \param [in] valid - maximum range of valid numbers to load. + /// \param [in] input the input iterator to load from. + /// \param [out] items array that data is loaded to. + /// \param [in] valid maximum range of valid numbers to load. /// \param [in] - temporary storage for inputs. /// /// \par Overview @@ -204,13 +204,13 @@ class warp_load /// \brief Loads data from continuous memory into an arrangement of items across the /// warp. /// - /// \tparam InputIterator - [inferred] an iterator type for input (can be a simple + /// \tparam InputIterator [inferred] an iterator type for input (can be a simple /// pointer. /// - /// \param [in] input - the input iterator to load from. - /// \param [out] items - array that data is loaded to. - /// \param [in] valid - maximum range of valid numbers to load. - /// \param [in] out_of_bounds - default value assigned to out-of-bound items. + /// \param [in] input the input iterator to load from. + /// \param [out] items array that data is loaded to. + /// \param [in] valid maximum range of valid numbers to load. + /// \param [in] out_of_bounds default value assigned to out-of-bound items. /// \param [in] - temporary storage for inputs. /// /// \par Overview diff --git a/rocprim/include/rocprim/warp/warp_reduce.hpp b/rocprim/include/rocprim/warp/warp_reduce.hpp index 197d166a2..eb6db5635 100644 --- a/rocprim/include/rocprim/warp/warp_reduce.hpp +++ b/rocprim/include/rocprim/warp/warp_reduce.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -59,11 +59,11 @@ struct select_warp_reduce_impl /// for performing reduction operations on items partitioned across threads in a hardware /// warp. /// -/// \tparam T - the input/output type. -/// \tparam WarpSize - the size of logical warp size, which can be equal to or less than +/// \tparam T the input/output type. +/// \tparam WarpSize the size of logical warp size, which can be equal to or less than /// the size of hardware warp (see rocprim::device_warp_size()). Reduce operations are performed /// separately within groups determined by WarpSize. -/// \tparam UseAllReduce - input parameter to determine whether to broadcast final reduction +/// \tparam UseAllReduce input parameter to determine whether to broadcast final reduction /// value to all threads (default is false). /// /// \par Overview @@ -135,13 +135,13 @@ class warp_reduce /// \brief Performs reduction across threads in a logical warp. /// - /// \tparam BinaryFunction - type of binary function used for reduce. Default type + /// \tparam BinaryFunction type of binary function used for reduce. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] output - reference to a thread output value. May be aliased with \p input. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] reduce_op - binary operation function object that will be used for reduce. + /// \param [in] input thread input value. + /// \param [out] output reference to a thread output value. May be aliased with \p input. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] reduce_op binary operation function object that will be used for reduce. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -203,14 +203,14 @@ class warp_reduce /// \brief Performs reduction across threads in a logical warp. /// - /// \tparam BinaryFunction - type of binary function used for reduce. Default type + /// \tparam BinaryFunction type of binary function used for reduce. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] output - reference to a thread output value. May be aliased with \p input. - /// \param [in] valid_items - number of items that will be reduced in the warp. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] reduce_op - binary operation function object that will be used for reduce. + /// \param [in] input thread input value. + /// \param [out] output reference to a thread output value. May be aliased with \p input. + /// \param [in] valid_items number of items that will be reduced in the warp. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] reduce_op binary operation function object that will be used for reduce. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -274,15 +274,15 @@ class warp_reduce /// \brief Performs head-segmented reduction across threads in a logical warp. /// - /// \tparam Flag - type of head flags. Must be contextually convertible to \p bool. - /// \tparam BinaryFunction - type of binary function used for reduce. Default type + /// \tparam Flag type of head flags. Must be contextually convertible to \p bool. + /// \tparam BinaryFunction type of binary function used for reduce. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] output - reference to a thread output value. May be aliased with \p input. - /// \param [in] flag - thread head flag, \p true flags mark beginnings of segments. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] reduce_op - binary operation function object that will be used for reduce. + /// \param [in] input thread input value. + /// \param [out] output reference to a thread output value. May be aliased with \p input. + /// \param [in] flag thread head flag, \p true flags mark beginnings of segments. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] reduce_op binary operation function object that will be used for reduce. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -322,15 +322,15 @@ class warp_reduce /// \brief Performs tail-segmented reduction across threads in a logical warp. /// - /// \tparam Flag - type of tail flags. Must be contextually convertible to \p bool. - /// \tparam BinaryFunction - type of binary function used for reduce. Default type + /// \tparam Flag type of tail flags. Must be contextually convertible to \p bool. + /// \tparam BinaryFunction type of binary function used for reduce. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] output - reference to a thread output value. May be aliased with \p input. - /// \param [in] flag - thread tail flag, \p true flags mark ends of segments. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] reduce_op - binary operation function object that will be used for reduce. + /// \param [in] input thread input value. + /// \param [out] output reference to a thread output value. May be aliased with \p input. + /// \param [in] flag thread tail flag, \p true flags mark ends of segments. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] reduce_op binary operation function object that will be used for reduce. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. diff --git a/rocprim/include/rocprim/warp/warp_scan.hpp b/rocprim/include/rocprim/warp/warp_scan.hpp index e692ddd05..675c63032 100644 --- a/rocprim/include/rocprim/warp/warp_scan.hpp +++ b/rocprim/include/rocprim/warp/warp_scan.hpp @@ -59,8 +59,8 @@ struct select_warp_scan_impl /// for performing inclusive and exclusive scan operations of items partitioned across /// threads in a hardware warp. /// -/// \tparam T - the input/output type. -/// \tparam WarpSize - the size of logical warp size, which can be equal to or less than +/// \tparam T the input/output type. +/// \tparam WarpSize the size of logical warp size, which can be equal to or less than /// the size of hardware warp (see rocprim::device_warp_size()). Scan operations are performed /// separately within groups determined by WarpSize. /// @@ -132,13 +132,13 @@ class warp_scan /// \brief Performs inclusive scan across threads in a logical warp. /// - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] output - reference to a thread output value. May be aliased with \p input. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] input thread input value. + /// \param [out] output reference to a thread output value. May be aliased with \p input. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -204,14 +204,14 @@ class warp_scan /// \brief Performs seeded inclusive scan across threads in a logical warp. /// - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] output - reference to a thread output value. May be aliased with \p input. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] init - initial value to seed the inclusive scan. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] input thread input value. + /// \param [out] output reference to a thread output value. May be aliased with \p input. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] init initial value to seed the inclusive scan. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -280,14 +280,14 @@ class warp_scan /// \brief Performs inclusive scan and reduction across threads in a logical warp. /// - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] output - reference to a thread output value. May be aliased with \p input. - /// \param [out] reduction - result of reducing of all \p input values in logical warp. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] input thread input value. + /// \param [out] output reference to a thread output value. May be aliased with \p input. + /// \param [out] reduction result of reducing of all \p input values in logical warp. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -354,15 +354,15 @@ class warp_scan /// \brief Performs seeded inclusive scan and reduction across threads in a logical warp. /// - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] output - reference to a thread output value. May be aliased with \p input. - /// \param [out] reduction - result of reducing of all \p input values in logical warp. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] init - initial value to seed the inclusive scan. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] input thread input value. + /// \param [out] output reference to a thread output value. May be aliased with \p input. + /// \param [out] reduction result of reducing of all \p input values in logical warp. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] init initial value to seed the inclusive scan. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -433,15 +433,15 @@ class warp_scan /// \brief Performs exclusive scan across threads in a logical warp. /// - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] output - reference to a thread output value. May be aliased with \p input. - /// \param [in] init - initial value used to start the exclusive scan. Should be the same + /// \param [in] input thread input value. + /// \param [out] output reference to a thread output value. May be aliased with \p input. + /// \param [in] init initial value used to start the exclusive scan. Should be the same /// for all threads in a logical warp. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -510,17 +510,17 @@ class warp_scan /// \brief Performs exclusive scan and reduction across threads in a logical warp. /// - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] output - reference to a thread output value. May be aliased with \p input. - /// \param [in] init - initial value used to start the exclusive scan. Should be the same + /// \param [in] input thread input value. + /// \param [out] output reference to a thread output value. May be aliased with \p input. + /// \param [in] init initial value used to start the exclusive scan. Should be the same /// for all threads in a logical warp. - /// \param [out] reduction - result of reducing of all \p input values in logical warp. + /// \param [out] reduction result of reducing of all \p input values in logical warp. /// \p init value is not included in the reduction. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -668,16 +668,16 @@ class warp_scan /// \brief Performs inclusive and exclusive scan operations across threads /// in a logical warp. /// - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] inclusive_output - reference to a thread inclusive-scan output value. - /// \param [out] exclusive_output - reference to a thread exclusive-scan output value. - /// \param [in] init - initial value used to start the exclusive scan. Should be the same + /// \param [in] input thread input value. + /// \param [out] inclusive_output reference to a thread inclusive-scan output value. + /// \param [out] exclusive_output reference to a thread exclusive-scan output value. + /// \param [in] init initial value used to start the exclusive scan. Should be the same /// for all threads in a logical warp. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -752,18 +752,18 @@ class warp_scan /// \brief Performs inclusive and exclusive scan operations, and reduction across /// threads in a logical warp. /// - /// \tparam BinaryFunction - type of binary function used for scan. Default type + /// \tparam BinaryFunction type of binary function used for scan. Default type /// is rocprim::plus. /// - /// \param [in] input - thread input value. - /// \param [out] inclusive_output - reference to a thread inclusive-scan output value. - /// \param [out] exclusive_output - reference to a thread exclusive-scan output value. - /// \param [in] init - initial value used to start the exclusive scan. Should be the same + /// \param [in] input thread input value. + /// \param [out] inclusive_output reference to a thread inclusive-scan output value. + /// \param [out] exclusive_output reference to a thread exclusive-scan output value. + /// \param [in] init initial value used to start the exclusive scan. Should be the same /// for all threads in a logical warp. - /// \param [out] reduction - result of reducing of all \p input values in logical warp. + /// \param [out] reduction result of reducing of all \p input values in logical warp. /// \p init value is not included in the reduction. - /// \param [in] storage - reference to a temporary storage object of type storage_type. - /// \param [in] scan_op - binary operation function object that will be used for scan. + /// \param [in] storage reference to a temporary storage object of type storage_type. + /// \param [in] scan_op binary operation function object that will be used for scan. /// The signature of the function should be equivalent to the following: /// T f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -839,9 +839,9 @@ class warp_scan /// \brief Broadcasts value from one thread to all threads in logical warp. /// - /// \param [in] input - value to broadcast. - /// \param [in] src_lane - id of the thread whose value should be broadcasted - /// \param [in] storage - reference to a temporary storage object of type storage_type. + /// \param [in] input value to broadcast. + /// \param [in] src_lane id of the thread whose value should be broadcasted + /// \param [in] storage reference to a temporary storage object of type storage_type. /// /// \par Storage reusage /// Synchronization barrier should be placed before \p storage is reused diff --git a/rocprim/include/rocprim/warp/warp_sort.hpp b/rocprim/include/rocprim/warp/warp_sort.hpp index 3ed2736a0..4b1f6ef52 100644 --- a/rocprim/include/rocprim/warp/warp_sort.hpp +++ b/rocprim/include/rocprim/warp/warp_sort.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -126,11 +126,11 @@ class warp_sort : detail::warp_sort_shuffle /// \brief Warp sort for any data type. /// - /// \tparam BinaryFunction - type of binary function used for sort. Default type + /// \tparam BinaryFunction type of binary function used for sort. Default type /// is rocprim::less. /// - /// \param thread_key - input/output to pass to other threads - /// \param compare_function - binary operation function object that will be used for sort. + /// \param thread_key input/output to pass to other threads + /// \param compare_function binary operation function object that will be used for sort. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -157,11 +157,11 @@ class warp_sort : detail::warp_sort_shuffle /// \brief Warp sort for any data type. /// - /// \tparam BinaryFunction - type of binary function used for sort. Default type + /// \tparam BinaryFunction type of binary function used for sort. Default type /// is rocprim::less. /// - /// \param thread_keys - input/output keys to pass to other threads - /// \param compare_function - binary operation function object that will be used for sort. + /// \param thread_keys input/output keys to pass to other threads + /// \param compare_function binary operation function object that will be used for sort. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -195,12 +195,12 @@ class warp_sort : detail::warp_sort_shuffle /// \brief Warp sort for any data type using temporary storage. /// - /// \tparam BinaryFunction - type of binary function used for sort. Default type + /// \tparam BinaryFunction type of binary function used for sort. Default type /// is rocprim::less. /// - /// \param thread_key - input/output to pass to other threads - /// \param storage - temporary storage for inputs - /// \param compare_function - binary operation function object that will be used for sort. + /// \param thread_key input/output to pass to other threads + /// \param storage temporary storage for inputs + /// \param compare_function binary operation function object that will be used for sort. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -248,12 +248,12 @@ class warp_sort : detail::warp_sort_shuffle /// \brief Warp sort for any data type using temporary storage. /// - /// \tparam BinaryFunction - type of binary function used for sort. Default type + /// \tparam BinaryFunction type of binary function used for sort. Default type /// is rocprim::less. /// - /// \param thread_keys - input/output keys to pass to other threads - /// \param storage - temporary storage for inputs - /// \param compare_function - binary operation function object that will be used for sort. + /// \param thread_keys input/output keys to pass to other threads + /// \param storage temporary storage for inputs + /// \param compare_function binary operation function object that will be used for sort. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -308,12 +308,12 @@ class warp_sort : detail::warp_sort_shuffle /// \brief Warp sort by key for any data type. /// - /// \tparam BinaryFunction - type of binary function used for sort. Default type + /// \tparam BinaryFunction type of binary function used for sort. Default type /// is rocprim::less. /// - /// \param thread_key - input/output key to pass to other threads - /// \param thread_value - input/output value to pass to other threads - /// \param compare_function - binary operation function object that will be used for sort. + /// \param thread_key input/output key to pass to other threads + /// \param thread_value input/output value to pass to other threads + /// \param compare_function binary operation function object that will be used for sort. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -344,12 +344,12 @@ class warp_sort : detail::warp_sort_shuffle /// \brief Warp sort by key for any data type. /// - /// \tparam BinaryFunction - type of binary function used for sort. Default type + /// \tparam BinaryFunction type of binary function used for sort. Default type /// is rocprim::less. /// - /// \param thread_keys - input/output keys to pass to other threads - /// \param thread_values - input/outputs values to pass to other threads - /// \param compare_function - binary operation function object that will be used for sort. + /// \param thread_keys input/output keys to pass to other threads + /// \param thread_values input/outputs values to pass to other threads + /// \param compare_function binary operation function object that will be used for sort. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -388,13 +388,13 @@ class warp_sort : detail::warp_sort_shuffle /// \brief Warp sort by key for any data type using temporary storage. /// - /// \tparam BinaryFunction - type of binary function used for sort. Default type + /// \tparam BinaryFunction type of binary function used for sort. Default type /// is rocprim::less. /// - /// \param thread_key - input/output key to pass to other threads - /// \param thread_value - input/output value to pass to other threads - /// \param storage - temporary storage for inputs - /// \param compare_function - binary operation function object that will be used for sort. + /// \param thread_key input/output key to pass to other threads + /// \param thread_value input/output value to pass to other threads + /// \param storage temporary storage for inputs + /// \param compare_function binary operation function object that will be used for sort. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. @@ -443,13 +443,13 @@ class warp_sort : detail::warp_sort_shuffle /// \brief Warp sort by key for any data type using temporary storage. /// - /// \tparam BinaryFunction - type of binary function used for sort. Default type + /// \tparam BinaryFunction type of binary function used for sort. Default type /// is rocprim::less. /// - /// \param thread_keys - input/output keys to pass to other threads - /// \param thread_values - input/output values to pass to other threads - /// \param storage - temporary storage for inputs - /// \param compare_function - binary operation function object that will be used for sort. + /// \param thread_keys input/output keys to pass to other threads + /// \param thread_values input/output values to pass to other threads + /// \param storage temporary storage for inputs + /// \param compare_function binary operation function object that will be used for sort. /// The signature of the function should be equivalent to the following: /// bool f(const T &a, const T &b);. The signature does not need to have /// const &, but function object must not modify the objects passed to it. diff --git a/rocprim/include/rocprim/warp/warp_store.hpp b/rocprim/include/rocprim/warp/warp_store.hpp index d1a3b0daa..e1b41bc34 100644 --- a/rocprim/include/rocprim/warp/warp_store.hpp +++ b/rocprim/include/rocprim/warp/warp_store.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -79,12 +79,12 @@ enum class warp_store_method /// \brief The \p warp_store class is a warp level parallel primitive which provides methods /// for storing an arrangement of items into a blocked/striped arrangement on continous memory. /// -/// \tparam T - the output/output type. -/// \tparam ItemsPerThread - the number of items to be processed by +/// \tparam T the output/output type. +/// \tparam ItemsPerThread the number of items to be processed by /// each thread. -/// \tparam WarpSize - the number of threads in a warp. It must be a divisor of the +/// \tparam WarpSize the number of threads in a warp. It must be a divisor of the /// kernel block size. -/// \tparam Method - the method to store data. +/// \tparam Method the method to store data. /// /// \par Overview /// * The \p warp_store class has a number of different methods to store data: @@ -150,11 +150,11 @@ class warp_store /// \brief Stores an arrangement of items from across the warp into an /// arrangement on continuous memory. /// - /// \tparam OutputIterator - [inferred] an iterator type for output (can be a simple + /// \tparam OutputIterator [inferred] an iterator type for output (can be a simple /// pointer. /// - /// \param [out] output - the output iterator to store to. - /// \param [in] items - array that data is read from. + /// \param [out] output the output iterator to store to. + /// \param [in] items array that data is read from. /// /// \par Overview /// * The type \p T must be such that an object of type \p OutputIterator @@ -181,12 +181,12 @@ class warp_store /// arrangement on continuous memory, which is guarded by range \p valid, /// using temporary storage /// - /// \tparam OutputIterator - [inferred] an iterator type for output (can be a simple + /// \tparam OutputIterator [inferred] an iterator type for output (can be a simple /// pointer. /// - /// \param [out] output - the output iterator to store to. - /// \param [in] items - array that data is read from. - /// \param [in] valid - maximum range of valid numbers to read. + /// \param [out] output the output iterator to store to. + /// \param [in] items array that data is read from. + /// \param [in] valid maximum range of valid numbers to read. /// /// \par Overview /// * The type \p T must be such that an object of type \p OutputIterator From 0b2f462b3a92d0f0db8b68e53351998aab6f826c Mon Sep 17 00:00:00 2001 From: Robin Voetter Date: Fri, 24 Jan 2025 12:37:35 +0100 Subject: [PATCH 79/84] update changelog --- CHANGELOG.md | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46bedbdee..46c063333 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,25 +7,29 @@ Full documentation for rocPRIM is available at [https://rocm.docs.amd.com/projec ### Added * Added gfx950 support. -* Added missing operator `rocprim::key_value_pair::operator==`. -* Added `rocprim::unrolled_copy` thread function to copy multiple items inside the thread. -* Added `rocprim::unrolled_thread_load` thread function to load multiple items inside the thread using `rocprim::thread_load`. +* Added `rocprim::key_value_pair::operator==`. +* Added the `rocprim::unrolled_copy` thread function to copy multiple items inside a thread. +* Added the `rocprim::unrolled_thread_load` function to load multiple items inside a thread using `rocprim::thread_load`. * Added `rocprim::int128_t` and `rocprim::uint128_t` to benchmarks for improved performance evaluation on 128-bit integers. -* Added `rocprim::int128_t` to autotuning supported types to improve performance for 128-bit integers. -* Added `rocprim::merge_inplace` function that allows merging in-place. +* Added `rocprim::int128_t` to the supported autotuning types to improve performance for 128-bit integers. +* Added the `rocprim::merge_inplace` function for merging in-place. * Added initial value support for warp- and block-level inclusive scan. ### Changed -* Undeprecated `rocprim::thread_load` uses dereference as default and updated other cache load modifiers. -* Undeprecated `rocprim::thread_store` uses dereference as default and updated other cache store modifiers. -* Changed symbol visibility: all kernels now have hidden visibility, inline namespace with the library version is added, e.g. `rocprim::ROCPRIM_300400_NS::...`. This allows to link multiple libraries built with different versions of rocPRIM. +* A new version of `rocprim::thread_load` and `rocprim::thread_store` replace the deprecated `rocprim::thread_load` and `rocprim::thread_store` functions. The versions avoid inline assembly where possible, and don't hinder the optimizer as much as a result. +* Renamed `rocprim::load_cs` to `rocprim::load_nontemporal` and `rocprim::store_cs` to `rocprim::store_nontemporal` to express the intent of these load and store methods better. +* All kernels now have hidden symbol visibility. All symbols now have inline namespaces that include the library version, for example, `rocprim::ROCPRIM_300400_NS::symbol` instead of `rocPRIM::symbol`, letting the user link multiple libraries built with different versions of rocPRIM. ### Resolved issues * Fixed device radix sort not returning the correct required temporary storage when a double buffer contains `nullptr`. * Fixed constness of equality operators (`==` and `!=`) in `rocprim::key_value_pair`. +### Deprecations + +* `rocprim::load_cs` and `rocprim::store_cs` are deprecated. Use `rocprim::load_nontemporal` and `rocprim::store_nontemporal` now. + ## rocPRIM 3.4.0 for ROCm 6.4.0 ### Added @@ -45,8 +49,8 @@ Full documentation for rocPRIM is available at [https://rocm.docs.amd.com/projec * Added the parallel `search` and `find_end` device functions similar to `std::search` and `std::find_end`, these functions search for the first and last occurrence of the sequence respectively. * Added a parallel device-level function, `rocprim::search_n`, similar to the C++ Standard Library `std::search_n` algorithm. * Added new constructors and a `base` function, and added `constexpr` specifier to all functions in `rocprim::reverse_iterator` to improve parity with the C++17 `std::reverse_iterator`. -* Added hipGraph support to device run-length-encode for non trivial runs (`rocprim::run_length_encode_non_trivial_runs`). -* Added configuration autotuning to device run-length-encode for non trivial runs (`rocprim::run_length_encode_non_trivial_runs`) for improved performance on selected architectures. +* Added hipGraph support to device run-length-encode for nontrivial runs (`rocprim::run_length_encode_non_trivial_runs`). +* Added configuration autotuning to device run-length-encode for nontrivial runs (`rocprim::run_length_encode_non_trivial_runs`) for improved performance on selected architectures. * Added configuration autotuning to device run-length-encode for trivial runs (`rocprim::run_length_encode`) for improved performance on selected architectures. * Added a new type traits interface to enable users to provide additional type trait information to rocPRIM, facilitating better compatibility with custom types. From bc8ae44fe7b06425e653e09a915e5554fcd1953b Mon Sep 17 00:00:00 2001 From: Nara Prasetya Date: Fri, 24 Jan 2025 12:34:47 +0000 Subject: [PATCH 80/84] fix(test_device_merge_inplace.cpp): prevent merge inplace tests to tests to grow too big --- test/rocprim/test_device_merge_inplace.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/rocprim/test_device_merge_inplace.cpp b/test/rocprim/test_device_merge_inplace.cpp index 1da4373d1..a381c4e05 100644 --- a/test/rocprim/test_device_merge_inplace.cpp +++ b/test/rocprim/test_device_merge_inplace.cpp @@ -336,8 +336,14 @@ TYPED_TEST(DeviceMergeInplaceTests, MergeInplace) std::vector h_data(size_total); size_t total_bytes = sizeof(value_type) * size_total; - size_t storage_size = 0; + // Limit the total size to slightly saner numbers. + if(total_bytes >= 1LL << 35 /* 32 GiB */) + { + continue; + } + + size_t storage_size = 0; HIP_CHECK(rocprim::merge_inplace(nullptr, storage_size, static_cast(nullptr), From 6ba76f6ba198d090a57d8c47e02d7b1240d25f4b Mon Sep 17 00:00:00 2001 From: Anton Gorenko Date: Fri, 24 Jan 2025 19:02:42 +0500 Subject: [PATCH 81/84] Fix test_linking: install the test's libraries too --- test/rocprim/CMakeLists.txt | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/test/rocprim/CMakeLists.txt b/test/rocprim/CMakeLists.txt index 2394a5ca1..d867160b1 100644 --- a/test/rocprim/CMakeLists.txt +++ b/test/rocprim/CMakeLists.txt @@ -1,6 +1,6 @@ # MIT License # -# Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -341,12 +341,15 @@ if(NOT WIN32) endif() add_library(test_linking_lib1 SHARED test_linking_lib.cpp) + if(ROCPRIM_INSTALL) + rocm_install(TARGETS test_linking_lib1 COMPONENT tests) + endif() target_include_directories(test_linking_lib1 SYSTEM BEFORE - PUBLIC + PRIVATE $ $ ) - target_link_libraries(test_linking_lib1 PUBLIC rocprim_hip) + target_link_libraries(test_linking_lib1 PRIVATE rocprim_hip) set_target_properties(test_linking_lib1 PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/test/rocprim" @@ -354,12 +357,15 @@ if(NOT WIN32) ) add_library(test_linking_lib2 SHARED test_linking_lib.cpp) + if(ROCPRIM_INSTALL) + rocm_install(TARGETS test_linking_lib2 COMPONENT tests) + endif() target_include_directories(test_linking_lib2 SYSTEM BEFORE - PUBLIC + PRIVATE $ $ ) - target_link_libraries(test_linking_lib2 PUBLIC rocprim_hip) + target_link_libraries(test_linking_lib2 PRIVATE rocprim_hip) # Pretend that we build this library with another version of rocPRIM which uses another # implementation of scan if(NOT WITHOUT_INLINE_NAMESPACE) From 65c355d43c8c6d6c27fa3406fab034368d467fa8 Mon Sep 17 00:00:00 2001 From: Nara Prasetya Date: Wed, 12 Feb 2025 09:18:23 +0000 Subject: [PATCH 82/84] chore(rmake.py): bump copyright --- rmake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rmake.py b/rmake.py index 75397b0c2..dd9cc3993 100644 --- a/rmake.py +++ b/rmake.py @@ -1,5 +1,5 @@ #!/usr/bin/python3 -""" Copyright (c) 2021-2024 Advanced Micro Devices, Inc. All rights reserved. +""" Copyright (c) 2021-2025 Advanced Micro Devices, Inc. All rights reserved. Manage build and installation""" import re From c1e9924e893ccd35bdfdfc649acdff451c0cdac7 Mon Sep 17 00:00:00 2001 From: Nara Prasetya Date: Wed, 12 Feb 2025 09:19:23 +0000 Subject: [PATCH 83/84] fix(example_type_traits_interface.cpp): explicit return from main --- example/example_type_traits_interface.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/example/example_type_traits_interface.cpp b/example/example_type_traits_interface.cpp index d07cbf9a1..095928c91 100644 --- a/example/example_type_traits_interface.cpp +++ b/example/example_type_traits_interface.cpp @@ -122,4 +122,6 @@ int main() // other types some_algo(); some_algo(); + + return 0; } From 72e1ff74cbf5e1b5803d59ba48ad88b045d23b9f Mon Sep 17 00:00:00 2001 From: Anton Gorenko Date: Tue, 25 Feb 2025 13:51:11 +0500 Subject: [PATCH 84/84] Fix unintentionally exported shared libs of test_linking rocm_install adds all libraries to ${PROJECT_NAME_LOWER}-targets (i.e. rocprim-targets.cmake) unless a custom export file is specified. --- test/rocprim/CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/rocprim/CMakeLists.txt b/test/rocprim/CMakeLists.txt index d867160b1..b56c06184 100644 --- a/test/rocprim/CMakeLists.txt +++ b/test/rocprim/CMakeLists.txt @@ -342,7 +342,8 @@ if(NOT WIN32) add_library(test_linking_lib1 SHARED test_linking_lib.cpp) if(ROCPRIM_INSTALL) - rocm_install(TARGETS test_linking_lib1 COMPONENT tests) + # Use a dummy export file to avoid adding the library to rocprim-targets.cmake + rocm_install(TARGETS test_linking_lib1 COMPONENT tests EXPORT rocprim-tests) endif() target_include_directories(test_linking_lib1 SYSTEM BEFORE PRIVATE @@ -358,7 +359,8 @@ if(NOT WIN32) add_library(test_linking_lib2 SHARED test_linking_lib.cpp) if(ROCPRIM_INSTALL) - rocm_install(TARGETS test_linking_lib2 COMPONENT tests) + # Use a dummy export file to avoid adding the library to rocprim-targets.cmake + rocm_install(TARGETS test_linking_lib2 COMPONENT tests EXPORT rocprim-tests) endif() target_include_directories(test_linking_lib2 SYSTEM BEFORE PRIVATE