diff --git a/projects/hipccl/hipccl3/libhipcxx b/projects/hipccl/hipccl3/libhipcxx index 87af709ba434..5cf4deb633fb 160000 --- a/projects/hipccl/hipccl3/libhipcxx +++ b/projects/hipccl/hipccl3/libhipcxx @@ -1 +1 @@ -Subproject commit 87af709ba4349548a2bfaee73907635d889736dd +Subproject commit 5cf4deb633fbefd3868c37d2a623b4abd8822f35 diff --git a/projects/hipccl/hipccl3/rocprim/rocprim/include/rocprim/device/device_reduce_by_key.hpp b/projects/hipccl/hipccl3/rocprim/rocprim/include/rocprim/device/device_reduce_by_key.hpp index d43f41894ab2..39a362e91511 100644 --- a/projects/hipccl/hipccl3/rocprim/rocprim/include/rocprim/device/device_reduce_by_key.hpp +++ b/projects/hipccl/hipccl3/rocprim/rocprim/include/rocprim/device/device_reduce_by_key.hpp @@ -101,6 +101,7 @@ ROCPRIM_KERNEL ROCPRIM_LAUNCH_BOUNDS(ROCPRIM_DEFAULT_MAX_BLOCK_SIZE) void template; - bool use_atomic_block_id; ROCPRIM_RETURN_ON_ERROR(check_if_using_atomic_block_id(stream, use_atomic_block_id)); const auto use_atomic_block_id_variant @@ -142,7 +141,7 @@ hipError_t reduce_by_key_impl_wrapped_config(void* temporary const auto params = get_config(Config{}, current_target); using scan_state_type - = reduce_by_key::lookback_scan_state_t; + = reduce_by_key::lookback_scan_state_t; const unsigned int block_size = params.kernel_config.block_size; const unsigned int items_per_block = block_size * params.kernel_config.items_per_thread; @@ -162,7 +161,7 @@ hipError_t reduce_by_key_impl_wrapped_config(void* temporary // The number of segment heads in previous launches. std::size_t* d_global_head_count = nullptr; // The running accumulation across the launch boundary. - accumulator_type* d_previous_accumulated = nullptr; + AccumulatorType* d_previous_accumulated = nullptr; detail::temp_storage::layout layout{}; ROCPRIM_RETURN_ON_ERROR( @@ -291,6 +290,7 @@ hipError_t reduce_by_key_impl_wrapped_config(void* temporary template; - using accumulator_type = reduce_by_key::accumulator_type_t; - using Selector = reduce_by_key_config_selector; - - return detail::reduce_by_key_impl_wrapped_config( - temporary_storage, - storage_size, - keys_input, - values_input, - size, - unique_output, - aggregates_output, - unique_count_output, - reduce_op, - key_compare_op, - stream, - debug_synchronous); + using key_type = ::rocprim::detail::value_type_t; + using Selector = reduce_by_key_config_selector; + + return detail:: + reduce_by_key_impl_wrapped_config( + temporary_storage, + storage_size, + keys_input, + values_input, + size, + unique_output, + aggregates_output, + unique_count_output, + reduce_op, + key_compare_op, + stream, + debug_synchronous); } } // namespace detail @@ -466,19 +466,22 @@ inline hipError_t reduce_by_key(void* temporary_storage, hipStream_t stream = 0, bool debug_synchronous = false) { - return detail::reduce_by_key_impl(temporary_storage, - storage_size, - keys_input, - values_input, - size, - unique_output, - aggregates_output, - unique_count_output, - reduce_op, - key_compare_op, - stream, - debug_synchronous); + return detail::reduce_by_key_impl< + detail::lookback_scan_determinism::default_determinism, + Config, + detail::reduce_by_key::accumulator_type_t>( + temporary_storage, + storage_size, + keys_input, + values_input, + size, + unique_output, + aggregates_output, + unique_count_output, + reduce_op, + key_compare_op, + stream, + debug_synchronous); } /// \brief Bitwise-reproducible parallel reduce-by-key primitive for device level. @@ -512,7 +515,10 @@ inline hipError_t deterministic_reduce_by_key(void* temporar hipStream_t stream = 0, bool debug_synchronous = false) { - return detail::reduce_by_key_impl( + return detail::reduce_by_key_impl< + detail::lookback_scan_determinism::deterministic, + Config, + detail::reduce_by_key::accumulator_type_t>( temporary_storage, storage_size, keys_input, diff --git a/projects/hipccl/hipccl3/rocprim/rocprim/include/rocprim/device/device_run_length_encode.hpp b/projects/hipccl/hipccl3/rocprim/rocprim/include/rocprim/device/device_run_length_encode.hpp index 0cd5c5326fb1..7eb9545795dd 100644 --- a/projects/hipccl/hipccl3/rocprim/rocprim/include/rocprim/device/device_run_length_encode.hpp +++ b/projects/hipccl/hipccl3/rocprim/rocprim/include/rocprim/device/device_run_length_encode.hpp @@ -83,18 +83,19 @@ hipError_t run_length_encode_impl(void* temporary_storage, return detail::reduce_by_key_impl_wrapped_config< detail::lookback_scan_determinism::nondeterministic, typename select_reduce_by_key_config::type, - Selector>(temporary_storage, - storage_size, - keys_input, - values_input, - size, - unique_output, - aggregates_output, - unique_count_output, - reduce_op, - key_compare_op, - stream, - debug_synchronous); + Selector, + accumulator_type>(temporary_storage, + storage_size, + keys_input, + values_input, + size, + unique_output, + aggregates_output, + unique_count_output, + reduce_op, + key_compare_op, + stream, + debug_synchronous); } template #include diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/adjacent_difference/custom.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/adjacent_difference/custom.cu index 50793dde5c36..fdbe30a6ec3b 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/adjacent_difference/custom.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/adjacent_difference/custom.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 #include #include diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/adjacent_difference/in_place.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/adjacent_difference/in_place.cu index 5d0d79db1c39..7797861818bc 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/adjacent_difference/in_place.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/adjacent_difference/in_place.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 #include #include diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/copy/basic.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/copy/basic.cu index 4deefb2f01fa..b9cbc10674ca 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/copy/basic.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/copy/basic.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 // Benchmark utils #include "bench_utils.hpp" diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/copy/if.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/copy/if.cu index b8e394a4f60d..a877bc327d25 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/copy/if.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/copy/if.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 #include #include diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/fill/basic.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/fill/basic.cu index 779d7a4d9acb..ff0da0b81b4c 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/fill/basic.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/fill/basic.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 // Benchmark utils #include "bench_utils.hpp" diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/for_each/basic.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/for_each/basic.cu index 184d4a9470c4..01782ce27ea5 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/for_each/basic.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/for_each/basic.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 #include #include diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/generate/basic.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/generate/basic.cu new file mode 100644 index 000000000000..0f999693ef50 --- /dev/null +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/generate/basic.cu @@ -0,0 +1,82 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// Benchmark utils +#include "bench_utils.hpp" + +// rocThrust +#include +#include +#include + +template +struct generator +{ + THRUST_DEVICE auto operator()() const -> T + { + return T{42}; + } +}; + +template +struct generate_benchmark : public primbench::benchmark_interface +{ + generate_benchmark(size_t items) + : m_items(items) + {} + + primbench::json meta() const override + { + return primbench::json{} + .add("algo", "generate") + .add("subalgo", "basic") + .add("input_type", primbench::name()) + .add("elements", m_items); + } + + void run(primbench::state& state) override + { + bench_utils::caching_allocator_t alloc{}; + thrust::detail::device_t policy{}; + + thrust::device_vector output(m_items); + + state.set_items(m_items); + state.add_writes(m_items); + + state.run([&] { + thrust::generate(policy(alloc).on(state.stream), output.begin(), output.end(), generator{}); + }); + } + +private: + size_t m_items; +}; + +#define QUEUE(T) \ + for (size_t size : bench_utils::sizes(sizeof(T))) \ + executor.queue>(size); + +int main(int argc, char* argv[]) +{ + primbench::settings settings; + settings.size = 1; // bench_utils::sizes() calculates it later. + settings.min_gpu_ms_per_batch = 100; + settings.batch_window_size = 3; + primbench::executor executor(argc, argv, settings, primbench::flags::sync); + + QUEUE(int8_t) + QUEUE(int16_t) + QUEUE(int32_t) + QUEUE(int64_t) + +#ifndef _MSC_VER + QUEUE(int128_t) +#endif + + QUEUE(float) + QUEUE(double) + + executor.run(); +} diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/inner_product/basic.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/inner_product/basic.cu index 06268ab1b3b0..a8a04bf179a6 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/inner_product/basic.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/inner_product/basic.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 // Benchmark utils #include "bench_utils.hpp" diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/merge/basic.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/merge/basic.cu index 90599ea6fbbd..ec1626b61715 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/merge/basic.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/merge/basic.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 #include #include diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/partition/basic.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/partition/basic.cu index 3226acb93c4c..c133efd713f3 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/partition/basic.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/partition/basic.cu @@ -1,35 +1,13 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 #include #include #include +#include _THRUST_STD_INCLUDE(iterator) + #include #include "bench_utils.hpp" @@ -74,7 +52,7 @@ struct partition_benchmark : public primbench::benchmark_interface in.cbegin(), in.cend(), out.begin(), - thrust::make_reverse_iterator(out.begin() + m_items), + _THRUST_STD::make_reverse_iterator(out.begin() + m_items), select_op); }); } diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/reduce/basic.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/reduce/basic.cu index c8cf0f7fefd1..a37680a861de 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/reduce/basic.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/reduce/basic.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 // Benchmark utils #include "bench_utils.hpp" diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/reduce/by_key.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/reduce/by_key.cu index 5e44e40dd6d5..b641202e4c61 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/reduce/by_key.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/reduce/by_key.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 // Benchmark utils #include "bench_utils.hpp" diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/exclusive/by_key.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/exclusive/by_key.cu index e7d57061fd67..db3d31d76a41 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/exclusive/by_key.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/exclusive/by_key.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 // Benchmark utils #include "bench_utils.hpp" diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/exclusive/max.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/exclusive/max.cu index e98dba6c439c..40328cd5134c 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/exclusive/max.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/exclusive/max.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 // Benchmark utils #include "bench_utils.hpp" diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/exclusive/sum.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/exclusive/sum.cu index b21bfda681bf..a9794992aba1 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/exclusive/sum.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/exclusive/sum.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 // Benchmark utils #include "bench_utils.hpp" diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/inclusive/by_key.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/inclusive/by_key.cu index 739fbd334c75..5266392199fc 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/inclusive/by_key.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/inclusive/by_key.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 // Benchmark utils #include "bench_utils.hpp" diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/inclusive/max.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/inclusive/max.cu index fbeb0b3b9a4a..4dbbab91c9f2 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/inclusive/max.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/inclusive/max.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 // Benchmark utils #include "bench_utils.hpp" diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/inclusive/sum.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/inclusive/sum.cu index 16e7f35b3862..e6177e5e1b14 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/inclusive/sum.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/scan/inclusive/sum.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 // Benchmark utils #include "bench_utils.hpp" diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/set_operations/base.hpp b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/set_operations/base.hpp index 8c8214e0c3a6..8b81ad9f706e 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/set_operations/base.hpp +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/set_operations/base.hpp @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 #pragma once diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/set_operations/by_key.hpp b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/set_operations/by_key.hpp index 8b8e5f062c77..21a026a55e4a 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/set_operations/by_key.hpp +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/set_operations/by_key.hpp @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 #pragma once diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/shuffle/basic.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/shuffle/basic.cu index 0634e2142955..bf371fd2baf4 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/shuffle/basic.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/shuffle/basic.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 // Benchmark utils #include "bench_utils.hpp" diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/sort/keys.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/sort/keys.cu index c694ad1e3f60..5f8bcf0e78c3 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/sort/keys.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/sort/keys.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 #include #include diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/sort/keys_custom.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/sort/keys_custom.cu index f51235a30901..e989329244b9 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/sort/keys_custom.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/sort/keys_custom.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 #include #include diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/sort/pairs.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/sort/pairs.cu index 8ab57197e4ec..50f25c693407 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/sort/pairs.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/sort/pairs.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 #include #include diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/sort/pairs_custom.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/sort/pairs_custom.cu index c3265801ef44..90847adb6dec 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/sort/pairs_custom.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/sort/pairs_custom.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 #include #include diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/tabulate/basic.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/tabulate/basic.cu index 4f0d762e40e0..3b59b07ac2b0 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/tabulate/basic.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/tabulate/basic.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 // Benchmark utils #include "bench_utils.hpp" @@ -32,10 +8,47 @@ // rocThrust #include #include +#include #include #include "thrust/detail/raw_pointer_cast.h" +template +struct sequence_benchmark : public primbench::benchmark_interface +{ + sequence_benchmark(size_t items) + : m_items(items) + {} + + primbench::json meta() const override + { + return primbench::json{} + .add("algo", "tabulate") + .add("subalgo", "sequence") + .add("input_type", primbench::name()) + .add("elements", m_items); + } + + void run(primbench::state& state) override + { + bench_utils::caching_allocator_t alloc{}; + thrust::detail::device_t policy{}; + + thrust::device_vector out(m_items, thrust::no_init); + + state.set_items(m_items); + state.add_writes(m_items); + + state.run([&] { + // sequence is implemented via thrust::tabulate + thrust::sequence(policy(alloc).on(state.stream), out.begin(), out.end()); + }); + } + +private: + size_t m_items; +}; + template struct seg_size_t { @@ -44,7 +57,7 @@ struct seg_size_t template __device__ T operator()(OffsetT i) { - return d_offsets[i + 1] - d_offsets[i]; + return static_cast(d_offsets[i + 1] - d_offsets[i]); } }; @@ -70,7 +83,7 @@ struct tabulate_benchmark : public primbench::benchmark_interface thrust::detail::device_t policy{}; thrust::device_vector in(m_items + 1); - thrust::device_vector out(m_items); + thrust::device_vector out(m_items, thrust::no_init); seg_size_t op{thrust::raw_pointer_cast(in.data())}; @@ -89,7 +102,10 @@ private: #define QUEUE(T) \ for (size_t size : bench_utils::sizes(2 * sizeof(T))) \ - executor.queue>(size); + { \ + executor.queue>(size); \ + executor.queue>(size); \ + } int main(int argc, char* argv[]) { @@ -99,8 +115,10 @@ int main(int argc, char* argv[]) settings.batch_window_size = 3; primbench::executor executor(argc, argv, settings, primbench::flags::sync); - QUEUE(uint32_t) - QUEUE(uint64_t) + QUEUE(int8_t) + QUEUE(int16_t) + QUEUE(int32_t) + QUEUE(int64_t) executor.run(); } diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/transform/basic.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/transform/basic.cu index e96824daf69d..1324aeb5aaa5 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/transform/basic.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/transform/basic.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 // Benchmark utils #include "bench_utils.hpp" diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/transform/if.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/transform/if.cu new file mode 100644 index 000000000000..6881be1afb18 --- /dev/null +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/transform/if.cu @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include +#include + +#include + +#include "bench_utils.hpp" + +template +struct negate_if_benchmark : public primbench::benchmark_interface +{ + negate_if_benchmark(size_t items, size_t entropy) + : m_items(items) + , entropy(entropy) + {} + + primbench::json meta() const override + { + return primbench::json{} + .add("algo", "transform") + .add("subalgo", "if") + .add("input_type", primbench::name()) + .add("elements", m_items) + .add("entropy", std::to_string(bench_utils::get_entropy_percentage(entropy))); + } + + void run(primbench::state& state) override + { + const T val = bench_utils::lerp_min_max(bench_utils::get_entropy_percentage(entropy)); + auto transform_op = thrust::negate{}; + bench_utils::less_then_t select_op{val}; + + bench_utils::caching_allocator_t alloc{}; // transform_if shouldn't allocate, but let's be consistent + thrust::detail::device_t policy{}; + + thrust::device_vector in = bench_utils::generate(m_items, state.seed); + thrust::device_vector out(m_items); + + const auto selected_elements = thrust::count_if(in.cbegin(), in.cend(), select_op); + + state.set_items(m_items); + state.add_reads(m_items); + state.add_writes(selected_elements); + + state.run([&] { + thrust::transform_if(policy(alloc).on(state.stream), in.begin(), in.end(), out.begin(), transform_op, select_op); + }); + } + +private: + size_t m_items; + size_t entropy; +}; + +#define QUEUE(T) \ + for (size_t size : bench_utils::sizes(2 * sizeof(T))) \ + { \ + executor.queue>(size, 0); \ + executor.queue>(size, 2); \ + executor.queue>(size, 4200); \ + } + +int main(int argc, char* argv[]) +{ + primbench::settings settings; + settings.size = 1; // bench_utils::sizes() calculates it later. + settings.min_gpu_ms_per_batch = 300; + settings.batch_window_size = 3; + primbench::executor executor(argc, argv, settings, primbench::flags::sync); + + QUEUE(int8_t) + QUEUE(int16_t) + QUEUE(int32_t) + QUEUE(int64_t) + + executor.run(); +} diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/transform/if_stencil.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/transform/if_stencil.cu new file mode 100644 index 000000000000..4e542a222810 --- /dev/null +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/transform/if_stencil.cu @@ -0,0 +1,86 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include +#include + +#include + +#include "bench_utils.hpp" + +template +struct negate_if_stencil_benchmark : public primbench::benchmark_interface +{ + negate_if_stencil_benchmark(size_t items, size_t entropy) + : m_items(items) + , entropy(entropy) + {} + + primbench::json meta() const override + { + return primbench::json{} + .add("algo", "transform") + .add("subalgo", "if_stencil") + .add("input_type", primbench::name()) + .add("elements", m_items) + .add("entropy", std::to_string(bench_utils::get_entropy_percentage(entropy))); + } + + void run(primbench::state& state) override + { + const T val = bench_utils::lerp_min_max(bench_utils::get_entropy_percentage(entropy)); + auto transform_op = thrust::negate{}; + bench_utils::less_then_t select_op{val}; + + bench_utils::caching_allocator_t alloc{}; // transform_if shouldn't allocate, but let's be consistent + thrust::detail::device_t policy{}; + + thrust::device_vector in = bench_utils::generate(m_items, state.seed); + thrust::device_vector stencil = bench_utils::generate(m_items, state.seed + 1); + thrust::device_vector out(m_items); + + const auto selected_elements = thrust::count_if(in.cbegin(), in.cend(), select_op); + + state.set_items(m_items); + state.add_reads(m_items + selected_elements); + state.add_writes(selected_elements); + + state.run([&] { + thrust::transform_if( + policy(alloc).on(state.stream), in.begin(), in.end(), stencil.begin(), out.begin(), transform_op, select_op); + }); + } + +private: + size_t m_items; + size_t entropy; +}; + +#define QUEUE(T) \ + for (size_t size : bench_utils::sizes(3 * sizeof(T))) \ + { \ + executor.queue>(size, 0); \ + executor.queue>(size, 2); \ + executor.queue>(size, 4200); \ + } + +int main(int argc, char* argv[]) +{ + primbench::settings settings; + settings.size = 1; // bench_utils::sizes() calculates it later. + settings.min_gpu_ms_per_batch = 300; + settings.batch_window_size = 3; + primbench::executor executor(argc, argv, settings, primbench::flags::sync); + + QUEUE(int8_t) + QUEUE(int16_t) + QUEUE(int32_t) + QUEUE(int64_t) + + executor.run(); +} diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/transform/zip_transform.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/transform/zip_transform.cu new file mode 100644 index 000000000000..98fb688a5c2a --- /dev/null +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/transform/zip_transform.cu @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// Benchmark utils +#include "bench_utils.hpp" + +// rocThrust +#include +#include +#include + +#include _THRUST_LIBCXX_INCLUDE(__functional/address_stability.h) +#include _THRUST_LIBCXX_INCLUDE(iterator) + +#include + +// This benchmark is intended to be compared to the babelstream.nstream benchmark from basic.cu, so we can: +// * detect regressions in the unpacking of a zip_transform_iterator + +// same variables as in basic.cu so we can compare results +constexpr auto startA = 1; // BabelStream: 0.1 +constexpr auto startB = 2; // BabelStream: 0.2 +constexpr auto startC = 3; // BabelStream: 0.1 +constexpr auto startScalar = 4; // BabelStream: 0.4 + +template +inline bool is_valid(const size_t size) +{ + try + { + thrust::device_vector a(size); + thrust::device_vector b(size); + thrust::device_vector c(size); + return true; + } + catch (const ::thrust::system::detail::bad_alloc& e) + { + return false; + } +} + +template +struct nstream_zip_transform_benchmark : public primbench::benchmark_interface +{ + nstream_zip_transform_benchmark(size_t items) + : m_items(items) + {} + + primbench::json meta() const override + { + return primbench::json{} + .add("algo", "transform") + .add("subalgo", "zip_transform.nstream") + .add("input_type", primbench::name()) + .add("elements", m_items); + } + + void run(primbench::state& state) override + { + bench_utils::caching_allocator_t alloc{}; // transform shouldn't allocate, but let's be consistent + thrust::detail::device_t policy{}; + + thrust::device_vector a(m_items, T{startA}); + thrust::device_vector b(m_items, T{startB}); + thrust::device_vector c(m_items, T{startC}); + + state.set_items(m_items); + state.add_reads(3 * m_items); + state.add_writes(m_items); + + const T scalar = startScalar; + auto lambda = _THRUST_LIBCXX::proclaim_copyable_arguments( + [=] THRUST_DEVICE(const T& ai, const T& bi, const T& ci) -> T { + return ai + bi + scalar * ci; + }); + _THRUST_LIBCXX::zip_transform_iterator begin{lambda, a.begin(), b.begin(), c.begin()}; + _THRUST_LIBCXX::zip_transform_iterator end{lambda, a.end(), b.end(), c.end()}; + + state.run([&] { + thrust::copy(policy(alloc).on(state.stream), begin, end, a.begin()); + }); + } + +private: + size_t m_items; +}; + +#define QUEUE_SIZE(T, S) \ + { \ + if (is_valid(S)) \ + executor.queue>(S); \ + } + +#define QUEUE(T) \ + QUEUE_SIZE(T, 1u << 25) \ + QUEUE_SIZE(T, 1u << 31) + +int main(int argc, char* argv[]) +{ + primbench::settings settings; + settings.size = 1; // sizes are set explicitly by QUEUE. + settings.min_gpu_ms_per_batch = 300; + settings.batch_window_size = 5; + primbench::executor executor(argc, argv, settings, primbench::flags::sync); + + QUEUE(int8_t) + QUEUE(int16_t) + QUEUE(float) + QUEUE(double) +#ifndef _MSC_VER + QUEUE(int128_t) +#endif + + executor.run(); +} diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/transform_reduce/sum.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/transform_reduce/sum.cu index 3197072164b9..8c235e8256e4 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/transform_reduce/sum.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/transform_reduce/sum.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 // Benchmark utils #include "bench_utils.hpp" diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/uninitialized_copy/basic.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/uninitialized_copy/basic.cu new file mode 100644 index 000000000000..504b1c8dbd1a --- /dev/null +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/uninitialized_copy/basic.cu @@ -0,0 +1,108 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3-Clause + +// Benchmark utils +#include "bench_utils.hpp" + +// rocThrust +#include +#include +#include +#include + +#include +#include + +// Not allowed to be copied using std::memcpy or hipMemcpy. +struct no_copy +{ + uint32_t a; + + no_copy() = default; + + THRUST_HOST_DEVICE no_copy(uint32_t i) + : a(i) + {} + + // the user-defined copy constructor prevents the type from being trivially copyable + THRUST_HOST_DEVICE no_copy(const no_copy& nt) + : a(nt.a) + {} +}; + +static_assert(std::is_trivially_default_constructible_v); +static_assert(!std::is_trivially_copyable_v); // as required by the C++ standard +static_assert(!thrust::is_trivially_relocatable_v); // thrust uses this check internally + +// Requires use of placement new +struct no_construct +{ + uint32_t a = 1337; +}; + +static_assert(!std::is_trivially_default_constructible_v); +static_assert(std::is_trivially_copyable_v); // as required by the C++ standard +static_assert(thrust::is_trivially_relocatable_v); // thrust uses this check internally + +PRIMBENCH_REGISTER_TYPE(no_copy, "no_copy") +PRIMBENCH_REGISTER_TYPE(no_construct, "no_construct") + +template +struct uninitialized_copy_benchmark : public primbench::benchmark_interface +{ + uninitialized_copy_benchmark(size_t items) + : m_items(items) + {} + + primbench::json meta() const override + { + return primbench::json{} + .add("algo", "uninitialized_copy") + .add("subalgo", "basic") + .add("input_type", primbench::name()) + .add("elements", m_items); + } + + void run(primbench::state& state) override + { + bench_utils::caching_allocator_t alloc{}; + thrust::detail::device_t policy{}; + + thrust::device_vector input(m_items, T{0xAA}); + thrust::device_vector output(m_items, thrust::default_init); + + state.set_items(m_items); + state.add_reads(m_items); + state.add_writes(m_items); + + state.run([&] { + thrust::uninitialized_copy(policy(alloc).on(state.stream), input.cbegin(), input.cend(), output.begin()); + }); + } + +private: + size_t m_items; +}; + +#define QUEUE(T) \ + for (size_t size : bench_utils::sizes(2 * sizeof(T))) \ + executor.queue>(size); + +int main(int argc, char* argv[]) +{ + primbench::settings settings; + settings.size = 1; // bench_utils::sizes() calculates it later. + settings.min_gpu_ms_per_batch = 100; + settings.batch_window_size = 3; + primbench::executor executor(argc, argv, settings, primbench::flags::sync); + + QUEUE(uint8_t) + QUEUE(uint16_t) + QUEUE(uint32_t) + QUEUE(uint64_t) + QUEUE(no_copy) + QUEUE(no_construct) + + executor.run(); +} diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/unique/basic.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/unique/basic.cu index f8d3b119c6c3..e886801cf865 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/unique/basic.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/unique/basic.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 // Benchmark utils #include "bench_utils.hpp" diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/unique/by_key.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/unique/by_key.cu index bd96a5b390fe..efe210d1577d 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/unique/by_key.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/unique/by_key.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 // rocThrust #include diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/vectorized_search/basic.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/vectorized_search/basic.cu index 1eb3eeb36cd2..afedf3157b80 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/vectorized_search/basic.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/vectorized_search/basic.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 // Benchmark utils #include "bench_utils.hpp" diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/vectorized_search/lower_bound.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/vectorized_search/lower_bound.cu index 463a6f8aa8bf..678a96d148f8 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/vectorized_search/lower_bound.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/vectorized_search/lower_bound.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 // Benchmark utils #include "bench_utils.hpp" diff --git a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/vectorized_search/upper_bound.cu b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/vectorized_search/upper_bound.cu index 676fda74e035..3d95ee60bede 100644 --- a/projects/hipccl/hipccl3/rocthrust/benchmark/bench/vectorized_search/upper_bound.cu +++ b/projects/hipccl/hipccl3/rocthrust/benchmark/bench/vectorized_search/upper_bound.cu @@ -1,30 +1,6 @@ -/****************************************************************************** - * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ +// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: BSD-3 // Benchmark utils #include "bench_utils.hpp" diff --git a/projects/hipccl/hipccl3/rocthrust/examples/CMakeLists.txt b/projects/hipccl/hipccl3/rocthrust/examples/CMakeLists.txt index 8f30561f1a3a..24c9b91b83c9 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/CMakeLists.txt +++ b/projects/hipccl/hipccl3/rocthrust/examples/CMakeLists.txt @@ -100,6 +100,7 @@ add_thrust_example("sorting_aos_vs_soa") add_thrust_example("sparse_vector") add_thrust_example("stream_compaction") add_thrust_example("strided_range") +add_thrust_example("sum_columns") add_thrust_example("sum_rows") add_thrust_example("sum") add_thrust_example("summary_statistics") diff --git a/projects/hipccl/hipccl3/rocthrust/examples/arbitrary_transformation.cu b/projects/hipccl/hipccl3/rocthrust/examples/arbitrary_transformation.cu index e27c08fba6d1..1edfc75b1e22 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/arbitrary_transformation.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/arbitrary_transformation.cu @@ -82,21 +82,12 @@ struct arbitrary_functor2 int main() { - // allocate storage - thrust::device_vector A(5); - thrust::device_vector B(5); - thrust::device_vector C(5); + // allocate and initialize + thrust::device_vector A{3, 4, 0, 8, 2}; + thrust::device_vector B{6, 7, 2, 1, 8}; + thrust::device_vector C{2, 5, 7, 4, 3}; thrust::device_vector D1(5); - // clang-format off - // initialize input vectors - A[0] = 3; B[0] = 6; C[0] = 2; - A[1] = 4; B[1] = 7; C[1] = 5; - A[2] = 0; B[2] = 2; C[2] = 7; - A[3] = 8; B[3] = 1; C[3] = 4; - A[4] = 2; B[4] = 8; C[4] = 3; - // clang-format on - // apply the transformation thrust::for_each(thrust::make_zip_iterator(thrust::make_tuple(A.begin(), B.begin(), C.begin(), D1.begin())), thrust::make_zip_iterator(thrust::make_tuple(A.end(), B.end(), C.end(), D1.end())), @@ -104,7 +95,7 @@ int main() // print the output std::cout << "Tuple functor" << std::endl; - for (int i = 0; i < 5; i++) + for (size_t i = 0; i < A.size(); i++) { std::cout << A[i] << " + " << B[i] << " * " << C[i] << " = " << D1[i] << std::endl; } @@ -117,7 +108,7 @@ int main() // print the output std::cout << "N-ary functor" << std::endl; - for (int i = 0; i < 5; i++) + for (size_t i = 0; i < A.size(); i++) { std::cout << A[i] << " + " << B[i] << " * " << C[i] << " = " << D2[i] << std::endl; } diff --git a/projects/hipccl/hipccl3/rocthrust/examples/basic_vector.cu b/projects/hipccl/hipccl3/rocthrust/examples/basic_vector.cu index 51cb4b205450..53f2bb76fbec 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/basic_vector.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/basic_vector.cu @@ -25,14 +25,8 @@ int main(void) { - // H has storage for 4 integers - thrust::host_vector H(4); - - // initialize individual elements - H[0] = 14; - H[1] = 20; - H[2] = 38; - H[3] = 46; + // H holds 4 integers + thrust::host_vector H{14, 20, 38, 46}; // H.size() returns the size of vector H std::cout << "H has size " << H.size() << std::endl; diff --git a/projects/hipccl/hipccl3/rocthrust/examples/bounding_box.cu b/projects/hipccl/hipccl3/rocthrust/examples/bounding_box.cu index a489cbfcd9e2..ad74dc43e060 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/bounding_box.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/bounding_box.cu @@ -20,10 +20,11 @@ #include #include -#include #include #include +#include _THRUST_STD_INCLUDE(utility) + #include "include/host_device.h" // This example shows how to compute a bounding box @@ -74,7 +75,7 @@ struct bbox }; // reduce a pair of bounding boxes (a,b) to a bounding box containing a and b -struct bbox_reduction +struct bbox_union { __host__ __device__ bbox operator()(bbox a, bbox b) { @@ -91,13 +92,13 @@ struct bbox_reduction int main(void) { const size_t N = 40; - thrust::default_random_engine rng; - thrust::uniform_real_distribution u01(0.0f, 1.0f); // allocate storage for points thrust::device_vector points(N); // generate some random points in the unit square + thrust::default_random_engine rng; + thrust::uniform_real_distribution u01(0.0f, 1.0f); for (size_t i = 0; i < N; i++) { float x = u01(rng); @@ -106,13 +107,10 @@ int main(void) } // initial bounding box contains first point - bbox init = bbox(points[0], points[0]); - - // binary reduction operation - bbox_reduction binary_op; + bbox init(points[0], points[0]); // compute the bounding box for the point set - bbox result = thrust::reduce(points.begin(), points.end(), init, binary_op); + bbox result = thrust::reduce(points.begin(), points.end(), init, bbox_union{}); // print output std::cout << "bounding box " << std::fixed; diff --git a/projects/hipccl/hipccl3/rocthrust/examples/constant_iterator.cu b/projects/hipccl/hipccl3/rocthrust/examples/constant_iterator.cu index 5e90f358eb26..33fc4f8ba576 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/constant_iterator.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/constant_iterator.cu @@ -29,11 +29,7 @@ int main(void) { - thrust::device_vector data(4); - data[0] = 3; - data[1] = 7; - data[2] = 2; - data[3] = 5; + thrust::device_vector data{3, 7, 2, 5}; // add 10 to all values in data thrust::transform( diff --git a/projects/hipccl/hipccl3/rocthrust/examples/counting_iterator.cu b/projects/hipccl/hipccl3/rocthrust/examples/counting_iterator.cu index 83d8c7007ecb..a62802e4d1d2 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/counting_iterator.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/counting_iterator.cu @@ -31,15 +31,7 @@ int main() // this example computes indices for all the nonzero values in a sequence // sequence of zero and nonzero values - thrust::device_vector stencil(8); - stencil[0] = 0; - stencil[1] = 1; - stencil[2] = 1; - stencil[3] = 0; - stencil[4] = 0; - stencil[5] = 1; - stencil[6] = 0; - stencil[7] = 1; + thrust::device_vector stencil{0, 1, 1, 0, 0, 1, 0, 1}; // storage for the nonzero indices thrust::device_vector indices(8); @@ -55,7 +47,7 @@ int main() // indices now contains [1,2,5,7] // print result - std::cout << "found " << (indices_end - indices.begin()) << " nonzero values at indices:\n"; + std::cout << "found " << _THRUST_STD::distance(indices.begin(), indices_end) << " nonzero values at indices:\n"; thrust::copy(indices.begin(), indices_end, std::ostream_iterator(std::cout, "\n")); return 0; diff --git a/projects/hipccl/hipccl3/rocthrust/examples/cuda/range_view.cu b/projects/hipccl/hipccl3/rocthrust/examples/cuda/range_view.cu index ce747e095869..dd5563785380 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/cuda/range_view.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/cuda/range_view.cu @@ -27,8 +27,6 @@ #include -#include "../include/host_device.h" - // This example demonstrates the use of a view: a non-owning wrapper for an // iterator range which presents a container-like interface to the user. // @@ -89,21 +87,21 @@ public: return last; } - __host__ __device__ thrust::reverse_iterator rbegin() + __host__ __device__ cuda::std::reverse_iterator rbegin() { - return thrust::reverse_iterator(end()); + return cuda::std::reverse_iterator(end()); } - __host__ __device__ const thrust::reverse_iterator crbegin() const + __host__ __device__ const cuda::std::reverse_iterator crbegin() const { - return thrust::reverse_iterator(cend()); + return cuda::std::reverse_iterator(cend()); } - __host__ __device__ thrust::reverse_iterator rend() + __host__ __device__ cuda::std::reverse_iterator rend() { - return thrust::reverse_iterator(begin()); + return cuda::std::reverse_iterator(begin()); } - __host__ __device__ const thrust::reverse_iterator crend() const + __host__ __device__ const cuda::std::reverse_iterator crend() const { - return thrust::reverse_iterator(cbegin()); + return cuda::std::reverse_iterator(cbegin()); } __host__ __device__ reference front() { diff --git a/projects/hipccl/hipccl3/rocthrust/examples/device_ptr.cu b/projects/hipccl/hipccl3/rocthrust/examples/device_ptr.cu index 75c658adf653..4b4d926799c8 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/device_ptr.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/device_ptr.cu @@ -35,7 +35,7 @@ int main() // device_ptr supports pointer arithmetic thrust::device_ptr first = d_ptr; thrust::device_ptr last = d_ptr + 10; - std::cout << "device array contains " << (last - first) << " values\n"; + std::cout << "device array contains " << _THRUST_STD::distance(first, last) << " values\n"; // algorithms work as expected thrust::sequence(first, last); diff --git a/projects/hipccl/hipccl3/rocthrust/examples/discrete_voronoi.cu b/projects/hipccl/hipccl3/rocthrust/examples/discrete_voronoi.cu index b7f313f8ed39..248472005eea 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/discrete_voronoi.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/discrete_voronoi.cu @@ -46,11 +46,11 @@ // Tuple = -struct minFunctor +struct voronoi_site_selector { int m, n, k; - __host__ __device__ minFunctor(int m, int n, int k) + __host__ __device__ voronoi_site_selector(int m, int n, int k) : m(m) , n(n) , k(k) @@ -219,7 +219,7 @@ void jfa(thrust::device_vector& in, thrust::device_vector& out, unsign thrust::counting_iterator(0))) + n * m, out.begin(), - minFunctor(m, n, k)); + voronoi_site_selector(m, n, k)); } /********************************************/ diff --git a/projects/hipccl/hipccl3/rocthrust/examples/dot_products_with_zip.cu b/projects/hipccl/hipccl3/rocthrust/examples/dot_products_with_zip.cu index 37634b23571f..c88f28f5c218 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/dot_products_with_zip.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/dot_products_with_zip.cu @@ -53,7 +53,7 @@ thrust::host_vector random_vector(const size_t N, unsigned int seed = thr thrust::default_random_engine rng(seed); thrust::uniform_real_distribution u01(0.0f, 1.0f); thrust::host_vector temp(N); - for (size_t i = 0; i < N; i++) + for (size_t i = 0; i < N; ++i) { temp[i] = u01(rng); } diff --git a/projects/hipccl/hipccl3/rocthrust/examples/expand.cu b/projects/hipccl/hipccl3/rocthrust/examples/expand.cu index 33d86e93e29b..5ea518ef48bc 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/expand.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/expand.cu @@ -62,7 +62,7 @@ OutputIterator expand(InputIterator1 first1, InputIterator1 last1, InputIterator // compute max-scan over the output indices, filling in the holes thrust::inclusive_scan( - output_indices.begin(), output_indices.end(), output_indices.begin(), _THRUST_LIBCXX::maximum()); + output_indices.begin(), output_indices.end(), output_indices.begin(), _THRUST_LIBCXX::maximum{}); // gather input values according to index array (output = first2[output_indices]) thrust::gather(output_indices.begin(), output_indices.end(), first2, output); @@ -84,15 +84,10 @@ void print(const std::string& s, const Vector& v) int main() { - int counts[] = {3, 5, 2, 0, 1, 3, 4, 2, 4}; - int values[] = {1, 2, 3, 4, 5, 6, 7, 8, 9}; + thrust::device_vector d_counts = {3, 5, 2, 0, 1, 3, 4, 2, 4}; + thrust::device_vector d_values = {1, 2, 3, 4, 5, 6, 7, 8, 9}; - size_t input_size = sizeof(counts) / sizeof(int); - size_t output_size = thrust::reduce(counts, counts + input_size); - - // copy inputs to device - thrust::device_vector d_counts(counts, counts + input_size); - thrust::device_vector d_values(values, values + input_size); + const size_t output_size = thrust::reduce(d_counts.begin(), d_counts.end()); thrust::device_vector d_output(output_size); // expand values according to counts diff --git a/projects/hipccl/hipccl3/rocthrust/examples/lambda.cu b/projects/hipccl/hipccl3/rocthrust/examples/lambda.cu index 370640f4e932..65f9b657d070 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/lambda.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/lambda.cu @@ -61,14 +61,14 @@ struct saxpy_functor int main(void) { // input data - float a = 2.0f; - float x[4] = {1, 2, 3, 4}; - float y[4] = {1, 1, 1, 1}; + float a = 2.0f; + thrust::device_vector x_data = {1, 2, 3, 4}; + thrust::device_vector y_data = {1, 1, 1, 1}; // SAXPY implemented with a functor (function object) { - thrust::device_vector X(x, x + 4); - thrust::device_vector Y(y, y + 4); + thrust::device_vector X = x_data; + thrust::device_vector Y = y_data; thrust::transform( X.begin(), @@ -78,16 +78,16 @@ int main(void) saxpy_functor(a)); // functor std::cout << "SAXPY (functor method)" << std::endl; - for (size_t i = 0; i < 4; i++) + for (size_t i = 0; i < Y.size(); i++) { - std::cout << a << " * " << x[i] << " + " << y[i] << " = " << Y[i] << std::endl; + std::cout << a << " * " << x_data[i] << " + " << y_data[i] << " = " << Y[i] << std::endl; } } // SAXPY implemented with a placeholders { - thrust::device_vector X(x, x + 4); - thrust::device_vector Y(y, y + 4); + thrust::device_vector X = x_data; + thrust::device_vector Y = y_data; thrust::transform( X.begin(), @@ -97,9 +97,9 @@ int main(void) a * _1 + _2); // placeholder expression std::cout << "SAXPY (placeholder method)" << std::endl; - for (size_t i = 0; i < 4; i++) + for (size_t i = 0; i < Y.size(); i++) { - std::cout << a << " * " << x[i] << " + " << y[i] << " = " << Y[i] << std::endl; + std::cout << a << " * " << x_data[i] << " + " << y_data[i] << " = " << Y[i] << std::endl; } } diff --git a/projects/hipccl/hipccl3/rocthrust/examples/lexicographical_sort.cu b/projects/hipccl/hipccl3/rocthrust/examples/lexicographical_sort.cu index 372e1cef9ca1..a28f6320e501 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/lexicographical_sort.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/lexicographical_sort.cu @@ -79,7 +79,7 @@ int main() thrust::device_vector lower = random_vector(N); std::cout << "Unsorted Keys" << std::endl; - for (size_t i = 0; i < N; i++) + for (size_t i = 0; i < upper.size(); i++) { std::cout << "(" << upper[i] << "," << middle[i] << "," << lower[i] << ")" << std::endl; } @@ -102,7 +102,7 @@ int main() apply_permutation(upper, permutation); std::cout << "Sorted Keys" << std::endl; - for (size_t i = 0; i < N; i++) + for (size_t i = 0; i < upper.size(); i++) { std::cout << "(" << upper[i] << "," << middle[i] << "," << lower[i] << ")" << std::endl; } diff --git a/projects/hipccl/hipccl3/rocthrust/examples/max_abs_diff.cu b/projects/hipccl/hipccl3/rocthrust/examples/max_abs_diff.cu index 441ce7f10991..c481672e8e53 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/max_abs_diff.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/max_abs_diff.cu @@ -22,6 +22,8 @@ #include #include +#include _THRUST_LIBCXX_INCLUDE(functional) + #include #include @@ -41,21 +43,14 @@ struct abs_diff int main(void) { - thrust::device_vector d_a(4); - thrust::device_vector d_b(4); - - // clang-format off - d_a[0] = 1.0; d_b[0] = 2.0; - d_a[1] = 2.0; d_b[1] = 4.0; - d_a[2] = 3.0; d_b[2] = 3.0; - d_a[3] = 4.0; d_b[3] = 0.0; - // clang-format on + thrust::device_vector d_a = {1.0, 2.0, 3.0, 4.0}; + thrust::device_vector d_b = {2.0, 4.0, 3.0, 0.0}; // initial value of the reduction float init = 0; // binary operations - _THRUST_LIBCXX::maximum binary_op1; + _THRUST_LIBCXX::maximum binary_op1{}; abs_diff binary_op2; float max_abs_diff = thrust::inner_product(d_a.begin(), d_a.end(), d_b.begin(), init, binary_op1, binary_op2); diff --git a/projects/hipccl/hipccl3/rocthrust/examples/minimal_custom_backend.cu b/projects/hipccl/hipccl3/rocthrust/examples/minimal_custom_backend.cu index 2c294d13ebdf..4e119aceae66 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/minimal_custom_backend.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/minimal_custom_backend.cu @@ -25,9 +25,6 @@ #include #include -#if !_THRUST_HAS_DEVICE_SYSTEM_STD -# include -#endif // This example demonstrates how to build a minimal custom // Thrust backend by intercepting for_each's dispatch. diff --git a/projects/hipccl/hipccl3/rocthrust/examples/minmax.cu b/projects/hipccl/hipccl3/rocthrust/examples/minmax.cu index a023f6f39e6b..acb007df8901 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/minmax.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/minmax.cu @@ -79,11 +79,12 @@ int main(void) thrust::uniform_int_distribution dist(10, 99); // initialize data on host - thrust::device_vector data(N); - for (size_t i = 0; i < data.size(); i++) + thrust::host_vector host_data(N); + for (auto& e : host_data) { - data[i] = dist(rng); + e = dist(rng); } + thrust::device_vector data = host_data; // setup arguments minmax_unary_op unary_op; @@ -97,9 +98,9 @@ int main(void) // print results std::cout << "[ "; - for (size_t i = 0; i < N; i++) + for (auto& e : host_data) { - std::cout << data[i] << " "; + std::cout << e << " "; } std::cout << "]" << std::endl; diff --git a/projects/hipccl/hipccl3/rocthrust/examples/mode.cu b/projects/hipccl/hipccl3/rocthrust/examples/mode.cu index f8a1234a6652..5130f699bdf4 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/mode.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/mode.cu @@ -46,9 +46,9 @@ int main() // generate random data on the host thrust::host_vector h_data(N); - for (size_t i = 0; i < N; i++) + for (auto& e : h_data) { - h_data[i] = dist(rng); + e = dist(rng); } // transfer data to device @@ -90,7 +90,7 @@ int main() thrust::device_vector::iterator mode_iter; mode_iter = thrust::max_element(d_output_counts.begin(), d_output_counts.end()); - int mode = d_output_keys[mode_iter - d_output_counts.begin()]; + int mode = d_output_keys[_THRUST_STD::distance(d_output_counts.begin(), mode_iter)]; int occurrences = *mode_iter; std::cout << "Modal value " << mode << " occurs " << occurrences << " times " << std::endl; diff --git a/projects/hipccl/hipccl3/rocthrust/examples/norm.cu b/projects/hipccl/hipccl3/rocthrust/examples/norm.cu index 548a21f1960d..c4bfc2fcaf44 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/norm.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/norm.cu @@ -51,11 +51,8 @@ struct square int main() { - // initialize host array - float x[4] = {1.0, 2.0, 3.0, 4.0}; - - // transfer to device - thrust::device_vector d_x(x, x + 4); + // initialize device vector directly + thrust::device_vector d_x = {1.0, 2.0, 3.0, 4.0}; // setup arguments square unary_op; diff --git a/projects/hipccl/hipccl3/rocthrust/examples/permutation_iterator.cu b/projects/hipccl/hipccl3/rocthrust/examples/permutation_iterator.cu index d9838bde6d42..2a35e105b88a 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/permutation_iterator.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/permutation_iterator.cu @@ -30,20 +30,10 @@ int main(void) { // gather locations - thrust::device_vector map(4); - map[0] = 3; - map[1] = 1; - map[2] = 0; - map[3] = 5; + thrust::device_vector map = {3, 1, 0, 5}; // array to gather from - thrust::device_vector source(6); - source[0] = 10; - source[1] = 20; - source[2] = 30; - source[3] = 40; - source[4] = 50; - source[5] = 60; + thrust::device_vector source = {10, 20, 30, 40, 50, 60}; // fuse gather with reduction: // sum = source[map[0]] + source[map[1]] + ... diff --git a/projects/hipccl/hipccl3/rocthrust/examples/remove_points2d.cu b/projects/hipccl/hipccl3/rocthrust/examples/remove_points2d.cu index 264897bc7c6b..5d0c83c6f410 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/remove_points2d.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/remove_points2d.cu @@ -70,7 +70,7 @@ int main(void) // print the initial points std::cout << std::fixed; std::cout << "Generated " << N << " points" << std::endl; - for (size_t i = 0; i < N; i++) + for (size_t i = 0; i < x.size(); i++) { std::cout << "(" << x[i] << "," << y[i] << ")" << std::endl; } @@ -89,7 +89,7 @@ int main(void) // print the filtered points std::cout << "After stream compaction, " << new_size << " points remain" << std::endl; - for (size_t i = 0; i < new_size; i++) + for (size_t i = 0; i < x.size(); i++) { std::cout << "(" << x[i] << "," << y[i] << ")" << std::endl; } diff --git a/projects/hipccl/hipccl3/rocthrust/examples/repeated_range.cu b/projects/hipccl/hipccl3/rocthrust/examples/repeated_range.cu index ea6afb7aa84d..7586fb4f4acb 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/repeated_range.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/repeated_range.cu @@ -30,10 +30,6 @@ #include "include/host_device.h" -#if !_THRUST_HAS_DEVICE_SYSTEM_STD -# include -#endif - // this example illustrates how to make repeated access to a range of values // examples: // repeated_range([0, 1, 2, 3], 1) -> [0, 1, 2, 3] @@ -95,11 +91,7 @@ protected: int main() { - thrust::device_vector data(4); - data[0] = 10; - data[1] = 20; - data[2] = 30; - data[3] = 40; + thrust::device_vector data{10, 20, 30, 40}; // print the initial data std::cout << "range "; diff --git a/projects/hipccl/hipccl3/rocthrust/examples/saxpy.cu b/projects/hipccl/hipccl3/rocthrust/examples/saxpy.cu index 1ab0f34905be..862c622aa192 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/saxpy.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/saxpy.cu @@ -75,13 +75,13 @@ void saxpy_slow(float A, thrust::device_vector& X, thrust::device_vector< int main(void) { // initialize host arrays - float x[4] = {1.0, 1.0, 1.0, 1.0}; - float y[4] = {1.0, 2.0, 3.0, 4.0}; + thrust::host_vector x{1.0, 1.0, 1.0, 1.0}; + thrust::host_vector y{1.0, 2.0, 3.0, 4.0}; { // transfer to device - thrust::device_vector X(x, x + 4); - thrust::device_vector Y(y, y + 4); + thrust::device_vector X(x); + thrust::device_vector Y(y); // slow method saxpy_slow(2.0, X, Y); @@ -89,8 +89,8 @@ int main(void) { // transfer to device - thrust::device_vector X(x, x + 4); - thrust::device_vector Y(y, y + 4); + thrust::device_vector X(x); + thrust::device_vector Y(y); // fast method saxpy_fast(2.0, X, Y); diff --git a/projects/hipccl/hipccl3/rocthrust/examples/scan_by_key.cu b/projects/hipccl/hipccl3/rocthrust/examples/scan_by_key.cu index 4aa70be3d2f6..3736debd485d 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/scan_by_key.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/scan_by_key.cu @@ -40,11 +40,11 @@ struct head_flag_predicate template void print(const Vector& v) { - for (size_t i = 0; i < v.size(); i++) + for (const auto& e : v) { - std::cout << v[i] << " "; + std::cout << e << " "; } - std::cout << "\n"; + std::cout << std::endl; } int main(void) diff --git a/projects/hipccl/hipccl3/rocthrust/examples/set_operations.cu b/projects/hipccl/hipccl3/rocthrust/examples/set_operations.cu index d384c9b435ff..0b4569f17e47 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/set_operations.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/set_operations.cu @@ -57,11 +57,11 @@ template void print(const String& s, const Vector& v) { std::cout << s << " ["; - for (size_t i = 0; i < v.size(); i++) + for (const auto& e : v) { - std::cout << " " << v[i]; + std::cout << " " << e; } - std::cout << " ]\n"; + std::cout << " ]" << std::endl; } template diff --git a/projects/hipccl/hipccl3/rocthrust/examples/simple_moving_average.cu b/projects/hipccl/hipccl3/rocthrust/examples/simple_moving_average.cu index 153b9781a62c..da2c076ac7c5 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/simple_moving_average.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/simple_moving_average.cu @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -88,13 +89,14 @@ int main(void) size_t w = 4; // generate random data series - thrust::device_vector data(n); + thrust::host_vector host_data(n); thrust::default_random_engine rng; thrust::uniform_int_distribution dist(0, 10); - for (size_t i = 0; i < n; i++) + for (auto& e : host_data) { - data[i] = static_cast(dist(rng)); + e = static_cast(dist(rng)); } + thrust::device_vector data = host_data; // allocate storage for averages thrust::device_vector averages(data.size() - (w - 1)); @@ -104,9 +106,9 @@ int main(void) // print data series std::cout << "data series: [ "; - for (size_t i = 0; i < data.size(); i++) + for (const auto& value : data) { - std::cout << data[i] << " "; + std::cout << value << " "; } std::cout << "]" << std::endl; diff --git a/projects/hipccl/hipccl3/rocthrust/examples/sort.cu b/projects/hipccl/hipccl3/rocthrust/examples/sort.cu index 6c52bdcf14c0..c109dd6a3784 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/sort.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/sort.cu @@ -19,11 +19,16 @@ // SOFTWARE. #include +#include #include +#include #include +#include _THRUST_STD_INCLUDE(utility) + #include #include +#include #include "include/host_device.h" @@ -33,69 +38,77 @@ void initialize(thrust::device_vector& v) { thrust::default_random_engine rng(123456); thrust::uniform_int_distribution dist(10, 99); - for (size_t i = 0; i < v.size(); i++) + thrust::host_vector host_data(v.size()); + for (auto& e : host_data) { - v[i] = dist(rng); + e = dist(rng); } + v = host_data; } void initialize(thrust::device_vector& v) { thrust::default_random_engine rng(123456); thrust::uniform_int_distribution dist(2, 19); - for (size_t i = 0; i < v.size(); i++) + thrust::host_vector host_data(v.size()); + for (auto& e : host_data) { - v[i] = dist(rng) / 2.0f; + e = dist(rng) / 2.0f; } + v = host_data; } -void initialize(thrust::device_vector>& v) +void initialize(thrust::device_vector<_THRUST_STD::pair>& v) { thrust::default_random_engine rng(123456); thrust::uniform_int_distribution dist(0, 9); - for (size_t i = 0; i < v.size(); i++) + thrust::host_vector<_THRUST_STD::pair> host_data(v.size()); + for (auto& e : host_data) { int a = dist(rng); int b = dist(rng); - v[i] = thrust::make_pair(a, b); + e = _THRUST_STD::make_pair(a, b); } + v = host_data; } void initialize(thrust::device_vector& v1, thrust::device_vector& v2) { thrust::default_random_engine rng(123456); thrust::uniform_int_distribution dist(10, 99); - for (size_t i = 0; i < v1.size(); i++) + thrust::host_vector host_data(v1.size()); + for (auto& e : host_data) { - v1[i] = dist(rng); - v2[i] = static_cast(i); + e = dist(rng); } + v1 = host_data; + thrust::sequence(v2.begin(), v2.end(), 0); } void print(const thrust::device_vector& v) { - for (size_t i = 0; i < v.size(); i++) + for (const auto& value : v) { - std::cout << " " << v[i]; + std::cout << " " << value; } std::cout << "\n"; } void print(const thrust::device_vector& v) { - for (size_t i = 0; i < v.size(); i++) + for (const auto& value : v) { - std::cout << " " << std::fixed << std::setprecision(1) << v[i]; + std::cout << " " << std::fixed << std::setprecision(1) << value; } std::cout << "\n"; } -void print(const thrust::device_vector>& v) +void print(const thrust::device_vector<_THRUST_STD::pair>& v) { - for (size_t i = 0; i < v.size(); i++) + for (const auto& p : v) { - thrust::pair p = v[i]; - std::cout << " (" << p.first << "," << p.second << ")"; + _THRUST_STD::pair local_p = p; + std::cout << " (" << local_p.first << "," << local_p.second << ")"; } std::cout << "\n"; } @@ -172,7 +185,7 @@ int main(void) std::cout << "\nsorting pairs\n"; { - thrust::device_vector> keys(N); + thrust::device_vector<_THRUST_STD::pair> keys(N); initialize(keys); print(keys); thrust::sort(keys.begin(), keys.end()); diff --git a/projects/hipccl/hipccl3/rocthrust/examples/sorting_aos_vs_soa.cu b/projects/hipccl/hipccl3/rocthrust/examples/sorting_aos_vs_soa.cu index 5d058c89e26c..d9888d9c840f 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/sorting_aos_vs_soa.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/sorting_aos_vs_soa.cu @@ -54,9 +54,9 @@ void initialize_keys(thrust::device_vector& keys) thrust::host_vector h_keys(keys.size()); - for (size_t i = 0; i < h_keys.size(); i++) + for (auto& e : h_keys) { - h_keys[i] = dist(rng); + e = dist(rng); } keys = h_keys; diff --git a/projects/hipccl/hipccl3/rocthrust/examples/strided_range.cu b/projects/hipccl/hipccl3/rocthrust/examples/strided_range.cu index 484b0575d0d0..5c52a74fc20c 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/strided_range.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/strided_range.cu @@ -30,10 +30,6 @@ #include "include/host_device.h" -#if !_THRUST_HAS_DEVICE_SYSTEM_STD -# include -#endif - // this example illustrates how to make strided access to a range of values // examples: // strided_range([0, 1, 2, 3, 4, 5, 6], 1) -> [0, 1, 2, 3, 4, 5, 6] diff --git a/projects/hipccl/hipccl3/rocthrust/examples/sum_columns.cu b/projects/hipccl/hipccl3/rocthrust/examples/sum_columns.cu new file mode 100644 index 000000000000..991be54c4d3a --- /dev/null +++ b/projects/hipccl/hipccl3/rocthrust/examples/sum_columns.cu @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-FileCopyrightText: Modifications Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include _THRUST_LIBCXX_INCLUDE(iterator) +#include _THRUST_STD_INCLUDE(mdspan) + +#include +#include + +#include "include/host_device.h" + +int main() +{ + const int rows = 32; + const int cols = 16; + + // Create a 2D multidimensional array of ints. + thrust::universal_vector data(rows * cols, 42); + _THRUST_STD::mdspan M(thrust::raw_pointer_cast(data.data()), rows, cols); + + // Create an iterator to the flat linear index space for the multidimensional array. + auto flat_idx = _THRUST_LIBCXX::counting_iterator(0); + + // Fill the array with pseudorandom inputs in parallel on the device. + thrust::tabulate(thrust::device, M.data_handle(), M.data_handle() + M.size(), [] __host__ __device__(int flat) { + thrust::default_random_engine rng; + thrust::uniform_int_distribution dist(0, 3); + rng.discard(flat); // Advance to the current element's position. + return dist(rng); + }); + + // Create a range to the column index of each element. + auto col_idx_begin = thrust::make_transform_iterator(flat_idx, [=] __host__ __device__(int flat) { + return flat / rows; + }); + auto col_idx_end = col_idx_begin + M.size(); + + // Create a transposed view of the multidimensional array. + auto M_transposed = thrust::make_permutation_iterator( + M.data_handle(), + thrust::make_transform_iterator(_THRUST_LIBCXX::counting_iterator(0), [=] __host__ __device__(int flat) { + int i = flat / cols; + int j = flat % cols; + return i + j * rows; + })); + + // Sum each column, storing the result in a new vector. + thrust::universal_vector sums(cols); + thrust::reduce_by_key( + thrust::device, col_idx_begin, col_idx_end, M_transposed, thrust::make_discard_iterator(), sums.begin()); + + // Output the result. + thrust::for_each_n(thrust::seq, flat_idx, rows, [&](int i) { + std::cout << "[ "; + thrust::for_each_n(thrust::seq, flat_idx, cols, [&](int j) { + std::cout << std::setw(2) << M(i, j) << " "; + }); + std::cout << "]\n"; + }); + + std::cout << " "; + thrust::for_each_n(thrust::seq, flat_idx, cols, [&](int) { + std::cout << " = "; + }); + std::cout << "\n"; + + std::cout << " "; + thrust::for_each_n(thrust::seq, flat_idx, cols, [&](int j) { + std::cout << std::setw(2) << sums[j] << " "; + }); + std::cout << "\n"; +} diff --git a/projects/hipccl/hipccl3/rocthrust/examples/sum_rows.cu b/projects/hipccl/hipccl3/rocthrust/examples/sum_rows.cu index 3db98a232a01..d5901d7d3452 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/sum_rows.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/sum_rows.cu @@ -18,71 +18,61 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include +#include +#include +#include _THRUST_LIBCXX_INCLUDE(iterator) +#include _THRUST_STD_INCLUDE(mdspan) + +#include #include #include "include/host_device.h" -// convert a linear index to a row index -template -struct linear_index_to_row_index +int main() { - T C; // number of columns - - __host__ __device__ linear_index_to_row_index(T C) - : C(C) - {} + const int rows = 32; + const int cols = 16; - __host__ __device__ T operator()(T i) - { - return i / C; - } -}; + // Create a 2D multidimensional array of ints. + thrust::universal_vector data(rows * cols, 42); + _THRUST_STD::mdspan M(thrust::raw_pointer_cast(data.data()), rows, cols); -int main(void) -{ - int R = 5; // number of rows - int C = 8; // number of columns - thrust::default_random_engine rng; - thrust::uniform_int_distribution dist(10, 99); + // Create an iterator to the flat linear index space for the multidimensional array. + auto flat_idx = _THRUST_LIBCXX::counting_iterator(0); - // initialize data - thrust::device_vector array(R * C); - for (size_t i = 0; i < array.size(); i++) - { - array[i] = dist(rng); - } + // Fill the array with pseudorandom inputs in parallel on the device. + thrust::tabulate(thrust::device, M.data_handle(), M.data_handle() + M.size(), [] __host__ __device__(int flat) { + thrust::default_random_engine rng; + thrust::uniform_int_distribution dist(0, 3); + rng.discard(flat); // Advance to the current element's position. + return dist(rng); + }); - // allocate storage for row sums and indices - thrust::device_vector row_sums(R); - thrust::device_vector row_indices(R); + // Create a range to the row index of each element. + auto row_idx_begin = thrust::make_transform_iterator(flat_idx, [=] __host__ __device__(int flat) { + return flat / cols; + }); + auto row_idx_end = row_idx_begin + M.size(); - // compute row sums by summing values with equal row indices + // Sum each row, storing the result in a new vector. + thrust::universal_vector sums(rows); thrust::reduce_by_key( - thrust::make_transform_iterator(thrust::counting_iterator(0), linear_index_to_row_index(C)), - thrust::make_transform_iterator(thrust::counting_iterator(0), linear_index_to_row_index(C)) + (R * C), - array.begin(), - row_indices.begin(), - row_sums.begin(), - _THRUST_STD::equal_to(), - _THRUST_STD::plus()); + thrust::device, row_idx_begin, row_idx_end, M.data_handle(), thrust::make_discard_iterator(), sums.begin()); - // print data - for (int i = 0; i < R; i++) - { + // Output the result. + thrust::for_each_n(thrust::seq, flat_idx, rows, [&](int i) { std::cout << "[ "; - for (int j = 0; j < C; j++) - { - std::cout << array[i * C + j] << " "; - } - std::cout << "] = " << row_sums[i] << "\n"; - } - - return 0; + thrust::for_each_n(thrust::seq, flat_idx, cols, [&](int j) { + std::cout << std::setw(2) << M(i, j) << " "; + }); + std::cout << "] = " << sums[i] << "\n"; + }); } diff --git a/projects/hipccl/hipccl3/rocthrust/examples/summary_statistics.cu b/projects/hipccl/hipccl3/rocthrust/examples/summary_statistics.cu index ef32e1cbd2a1..54cf22787884 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/summary_statistics.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/summary_statistics.cu @@ -155,10 +155,10 @@ int main() using T = float; // initialize host array - T h_x[] = {4, 7, 13, 16}; + thrust::host_vector h_x{4, 7, 13, 16}; // transfer to device - thrust::device_vector d_x(h_x, h_x + sizeof(h_x) / sizeof(T)); + thrust::device_vector d_x(h_x); // setup arguments summary_stats_unary_op unary_op; diff --git a/projects/hipccl/hipccl3/rocthrust/examples/tiled_range.cu b/projects/hipccl/hipccl3/rocthrust/examples/tiled_range.cu index e658a7b10ae9..ba5b707c7e30 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/tiled_range.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/tiled_range.cu @@ -30,10 +30,6 @@ #include "include/host_device.h" -#if !_THRUST_HAS_DEVICE_SYSTEM_STD -# include -#endif - // this example illustrates how to tile a range multiple times // examples: // tiled_range([0, 1, 2, 3], 1) -> [0, 1, 2, 3] @@ -95,11 +91,7 @@ protected: int main() { - thrust::device_vector data(4); - data[0] = 10; - data[1] = 20; - data[2] = 30; - data[3] = 40; + thrust::device_vector data{10, 20, 30, 40}; // print the initial data std::cout << "range "; diff --git a/projects/hipccl/hipccl3/rocthrust/examples/transform_input_output_iterator.cu b/projects/hipccl/hipccl3/rocthrust/examples/transform_input_output_iterator.cu index f1cca6ceb594..15a30a4e9aa6 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/transform_input_output_iterator.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/transform_input_output_iterator.cu @@ -129,9 +129,9 @@ int main(void) std::cout << "] \n"; std::cout << "Result [ "; - for (size_t i = 0; i < size; i++) + for (const auto& value : C_h) { - std::cout << C_h[i] << " "; + std::cout << value << " "; } std::cout << "] \n"; diff --git a/projects/hipccl/hipccl3/rocthrust/examples/transform_output_iterator.cu b/projects/hipccl/hipccl3/rocthrust/examples/transform_output_iterator.cu index 5967ba328c22..e9f718c24bb2 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/transform_output_iterator.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/transform_output_iterator.cu @@ -21,11 +21,13 @@ #include #include #include +#include #include #include #include "include/host_device.h" + struct Functor { template @@ -39,15 +41,15 @@ struct Functor int main(void) { - float u[4] = {4, 3, 2, 1}; - float v[4] = {-1, 1, 1, -1}; - int idx[3] = {3, 0, 1}; - float w[3] = {0, 0, 0}; + thrust::host_vector u{4, 3, 2, 1}; + thrust::host_vector v{-1, 1, 1, -1}; + thrust::host_vector idx{3, 0, 1}; + thrust::host_vector w{0, 0, 0}; - thrust::device_vector U(u, u + 4); - thrust::device_vector V(v, v + 4); - thrust::device_vector IDX(idx, idx + 3); - thrust::device_vector W(w, w + 3); + thrust::device_vector U(u); + thrust::device_vector V(v); + thrust::device_vector IDX(idx); + thrust::device_vector W(w); // gather multiple elements and apply a function before writing result in memory thrust::gather(IDX.begin(), @@ -56,9 +58,9 @@ int main(void) thrust::make_transform_output_iterator(W.begin(), Functor())); std::cout << "result= [ "; - for (size_t i = 0; i < 3; i++) + for (const auto& value : W) { - std::cout << W[i] << " "; + std::cout << value << " "; } std::cout << "] \n"; diff --git a/projects/hipccl/hipccl3/rocthrust/examples/weld_vertices.cu b/projects/hipccl/hipccl3/rocthrust/examples/weld_vertices.cu index 403540bf228f..112ae3cf29ae 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/weld_vertices.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/weld_vertices.cu @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -61,15 +62,18 @@ int main(void) // allocate memory for input mesh representation thrust::device_vector input(9); - input[0] = vec2(0, 0); // First Triangle - input[1] = vec2(1, 0); - input[2] = vec2(0, 1); - input[3] = vec2(1, 0); // Second Triangle - input[4] = vec2(1, 1); - input[5] = vec2(0, 1); - input[6] = vec2(1, 0); // Third Triangle - input[7] = vec2(2, 0); - input[8] = vec2(1, 1); + thrust::host_vector h_input{ + vec2(0, 0), + vec2(1, 0), + vec2(0, 1), // First Triangle + vec2(1, 0), + vec2(1, 1), + vec2(0, 1), // Second Triangle + vec2(1, 0), + vec2(2, 0), + vec2(1, 1) // Third Triangle + }; + input = h_input; // allocate space for output mesh representation thrust::device_vector vertices = input; diff --git a/projects/hipccl/hipccl3/rocthrust/examples/word_count.cu b/projects/hipccl/hipccl3/rocthrust/examples/word_count.cu index f74c27d14e56..341c93677f04 100644 --- a/projects/hipccl/hipccl3/rocthrust/examples/word_count.cu +++ b/projects/hipccl/hipccl3/rocthrust/examples/word_count.cu @@ -20,6 +20,7 @@ #include #include +#include #include #include diff --git a/projects/hipccl/hipccl3/rocthrust/test/CMakeLists.txt b/projects/hipccl/hipccl3/rocthrust/test/CMakeLists.txt index 8f3d72d1a091..b1a862f4b6cc 100644 --- a/projects/hipccl/hipccl3/rocthrust/test/CMakeLists.txt +++ b/projects/hipccl/hipccl3/rocthrust/test/CMakeLists.txt @@ -229,7 +229,6 @@ if(BUILD_TEST) add_rocthrust_test("merge") add_rocthrust_test("merge_by_key") add_rocthrust_test("merge_key_value") - add_rocthrust_test("metaprogramming") add_rocthrust_test("min_and_max") add_rocthrust_test("min_element") add_rocthrust_test("minimum_system") @@ -334,6 +333,7 @@ if(BUILD_TEST) add_rocthrust_test("vector_allocators") add_rocthrust_test("vector_insert") add_rocthrust_test("vector_manipulation") + add_rocthrust_test("xor_combine_engine_max") add_rocthrust_test("zip_function") add_rocthrust_test("zip_iterator") add_rocthrust_test("zip_iterator_reduce") diff --git a/projects/hipccl/hipccl3/rocthrust/test/test_allocator_aware_policies.cpp b/projects/hipccl/hipccl3/rocthrust/test/test_allocator_aware_policies.cpp index a272fce9852e..b0448c9a8ff4 100644 --- a/projects/hipccl/hipccl3/rocthrust/test/test_allocator_aware_policies.cpp +++ b/projects/hipccl/hipccl3/rocthrust/test/test_allocator_aware_policies.cpp @@ -17,10 +17,10 @@ #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include "test_param_fixtures.hpp" #include "test_utils.hpp" @@ -117,9 +117,9 @@ struct TestAllocatorAttachment }; using sequential_info = policy_info; -using cpp_par_info = policy_info; -using omp_par_info = policy_info; -using tbb_par_info = policy_info; +using cpp_par_info = policy_info; +using omp_par_info = policy_info; +using tbb_par_info = policy_info; using hip_par_info = policy_info; diff --git a/projects/hipccl/hipccl3/rocthrust/test/test_complex.cpp b/projects/hipccl/hipccl3/rocthrust/test/test_complex.cpp index 8bfa087b22b3..923436f1df90 100644 --- a/projects/hipccl/hipccl3/rocthrust/test/test_complex.cpp +++ b/projects/hipccl/hipccl3/rocthrust/test/test_complex.cpp @@ -971,11 +971,11 @@ TYPED_TEST(ComplexPairsTests, TestCompoundPlusOperator) run_compound_tests( [=](std::complex& lhs, const std::complex& rhs) { - using type = ::internal::common_type_t; + using type = _THRUST_STD::common_type_t; lhs = std::complex(lhs.real() + rhs.real(), lhs.imag() + rhs.imag()); }, [=](std::complex& lhs, const U& rhs) { - using type = ::internal::common_type_t; + using type = _THRUST_STD::common_type_t; lhs = std::complex(lhs.real() + rhs, lhs.imag()); }, [=](thrust::complex& lhs, const thrust::complex& rhs) { @@ -995,11 +995,11 @@ TYPED_TEST(ComplexPairsTests, TestCompoundMinusOperator) run_compound_tests( [=](std::complex& lhs, const std::complex& rhs) { - using type = ::internal::common_type_t; + using type = _THRUST_STD::common_type_t; lhs = std::complex(lhs.real() - rhs.real(), lhs.imag() - rhs.imag()); }, [=](std::complex& lhs, const U& rhs) { - using type = ::internal::common_type_t; + using type = _THRUST_STD::common_type_t; lhs = std::complex(lhs.real() - rhs, lhs.imag()); }, [=](thrust::complex& lhs, const thrust::complex& rhs) { @@ -1020,7 +1020,7 @@ TYPED_TEST(ComplexPairsTests, TestCompoundMultiplyOperator) run_compound_tests( [=](std::complex& lhs, const std::complex& rhs) { // (a + bi)(c + di) = ac + i(ad + bc) - bd - using type = ::internal::common_type_t; + using type = _THRUST_STD::common_type_t; type a = static_cast(lhs.real()); type b = static_cast(lhs.imag()); @@ -1054,7 +1054,7 @@ TYPED_TEST(ComplexPairsTests, TestCompoundDivisionOperator) run_compound_tests( [=](std::complex& lhs, const std::complex& rhs) { - using type = ::internal::common_type_t; + using type = _THRUST_STD::common_type_t; // (a + bi) / (c + di) = ((ac + bd) + (bc -ad)i) / (c^2 + d^2) type a = static_cast(lhs.real()); diff --git a/projects/hipccl/hipccl3/rocthrust/test/test_copy.cpp b/projects/hipccl/hipccl3/rocthrust/test/test_copy.cpp index b2c7e18f0fb2..8680aee857c8 100644 --- a/projects/hipccl/hipccl3/rocthrust/test/test_copy.cpp +++ b/projects/hipccl/hipccl3/rocthrust/test/test_copy.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -1130,3 +1131,37 @@ TEST(CopyTests, TestCopyIfDevice) } } } + +// trivial_copy_from_device / trivial_copy_to_device use hipMemcpyDefault, so the runtime determines the copy +// direction. With explicit directions the copy can fail when a device-accessible pointer is actually host memory. +TEST(CopyTests, TestTrivialCopyFromDeviceHostSource) +{ + SCOPED_TRACE(testing::Message() << "with device_id= " << test::set_device_from_ctest()); + + int src[] = {0, 10, 20, 30, 40}; + int dst[5] = {}; + + hipError_t status = thrust::hip_rocprim::trivial_copy_from_device(dst, src, 5, hipStreamDefault); + + ASSERT_EQ(status, hipSuccess); + for (int i = 0; i < 5; i++) + { + ASSERT_EQ(dst[i], i * 10); + } +} + +TEST(CopyTests, TestTrivialCopyToDeviceHostDest) +{ + SCOPED_TRACE(testing::Message() << "with device_id= " << test::set_device_from_ctest()); + + int src[] = {0, 100, 200, 300, 400}; + int dst[5] = {}; + + hipError_t status = thrust::hip_rocprim::trivial_copy_to_device(dst, src, 5, hipStreamDefault); + + ASSERT_EQ(status, hipSuccess); + for (int i = 0; i < 5; i++) + { + ASSERT_EQ(dst[i], i * 100); + } +} diff --git a/projects/hipccl/hipccl3/rocthrust/test/test_device_delete.cpp b/projects/hipccl/hipccl3/rocthrust/test/test_device_delete.cpp index 4d4c6173b0fc..247eae995827 100644 --- a/projects/hipccl/hipccl3/rocthrust/test/test_device_delete.cpp +++ b/projects/hipccl/hipccl3/rocthrust/test/test_device_delete.cpp @@ -25,38 +25,85 @@ struct Foo { - THRUST_HOST_DEVICE Foo() - : set_me_upon_destruction{nullptr} + THRUST_DEVICE Foo() + : destroyed{nullptr} {} - THRUST_HOST_DEVICE ~Foo() + Foo(const Foo&) = delete; + Foo& operator=(const Foo&) = delete; + + THRUST_DEVICE ~Foo() { - _THRUST_IF_TARGET(_THRUST_IS_DEVICE, - (if (set_me_upon_destruction != nullptr) { *set_me_upon_destruction = true; })); + if (destroyed != nullptr) + { + *destroyed = true; + } } - bool* set_me_upon_destruction; + bool* destroyed = nullptr; }; -#if !defined(__QNX__) TEST(DeviceDelete, TestDeviceDeleteDestructorInvocation) { - // SCOPED_TRACE(testing::Message() << "with device_id= " << test::set_device_from_ctest()); - // - // KNOWN_FAILURE; - // - // thrust::device_vector destructor_flag(1, false); - // - // thrust::device_ptr foo_ptr = thrust::device_new(); - // - // Foo exemplar; - // exemplar.set_me_upon_destruction = thrust::raw_pointer_cast(&destructor_flag[0]); - // *foo_ptr = exemplar; - // - // ASSERT_EQ(false, destructor_flag[0]); - // - // thrust::device_delete(foo_ptr); - // - // ASSERT_EQ(true, destructor_flag[0]); + SCOPED_TRACE(testing::Message() << "with device_id= " << test::set_device_from_ctest()); + + thrust::device_ptr foo_ptr = thrust::device_new(); + + thrust::device_vector destructor_flag(1, false); + *thrust::device_ptr(&foo_ptr.get()->destroyed) = destructor_flag.data().get(); + + ASSERT_EQ(false, destructor_flag[0]); + thrust::device_delete(foo_ptr); + ASSERT_EQ(true, destructor_flag[0]); +} + +// based on: https://github.com/NVIDIA/cccl/issues/6132 +struct base +{ + THRUST_HOST_DEVICE virtual ~base() + { + if (base_destroyed != nullptr) + { + *base_destroyed = true; + } + } + + // Thrust does not support abstract class types inside vectors, so f() has a body instead of being abstract + THRUST_HOST_DEVICE virtual void f() {} + + bool* base_destroyed = nullptr; +}; + +struct derived : base +{ + THRUST_HOST_DEVICE ~derived() override + { + if (derived_destroyed != nullptr) + { + *derived_destroyed = true; + } + } + + THRUST_HOST_DEVICE void f() override {} + + bool* derived_destroyed = nullptr; +}; + +TEST(DeviceDelete, TestDeviceDeleteVirtualDestructorInvocation) +{ + SCOPED_TRACE(testing::Message() << "with device_id= " << test::set_device_from_ctest()); + + thrust::device_ptr ptr = thrust::device_new(); + + thrust::device_vector destructor_flags(2, false); + *thrust::device_ptr(&ptr.get()->base_destroyed) = destructor_flags.data().get() + 0; + *thrust::device_ptr(&ptr.get()->derived_destroyed) = destructor_flags.data().get() + 1; + + thrust::device_ptr base_ptr = ptr; + + ASSERT_EQ(false, destructor_flags[0]); + ASSERT_EQ(false, destructor_flags[1]); + thrust::device_delete(base_ptr); // delete through the base pointer + ASSERT_EQ(true, destructor_flags[0]); + ASSERT_EQ(true, destructor_flags[1]); } -#endif diff --git a/projects/hipccl/hipccl3/rocthrust/test/test_find.cpp b/projects/hipccl/hipccl3/rocthrust/test/test_find.cpp index 32bc47fa5722..311a9668060f 100644 --- a/projects/hipccl/hipccl3/rocthrust/test/test_find.cpp +++ b/projects/hipccl/hipccl3/rocthrust/test/test_find.cpp @@ -28,50 +28,7 @@ TESTS_DEFINE(FindTestsVector, FullTestsParams); TESTS_DEFINE(FindTests, NumericalTestsParams); -template -struct equal_to_value_pred -{ - T value; - - equal_to_value_pred(T value) - : value(value) - {} - - THRUST_HOST_DEVICE bool operator()(T v) const - { - return v == value; - } -}; - -template -struct not_equal_to_value_pred -{ - T value; - - not_equal_to_value_pred(T value) - : value(value) - {} - - THRUST_HOST_DEVICE bool operator()(T v) const - { - return v != value; - } -}; - -template -struct less_than_value_pred -{ - T value; - - less_than_value_pred(T value) - : value(value) - {} - - THRUST_HOST_DEVICE bool operator()(T v) const - { - return v < value; - } -}; +using thrust::placeholders::_1; TYPED_TEST(FindTestsVector, TestFindSimple) { @@ -135,12 +92,12 @@ TYPED_TEST(FindTestsVector, TestFindIfSimple) Vector vec{1, 2, 3, 3, 5}; - ASSERT_EQ(thrust::find_if(vec.begin(), vec.end(), equal_to_value_pred(0)) - vec.begin(), 5); - ASSERT_EQ(thrust::find_if(vec.begin(), vec.end(), equal_to_value_pred(1)) - vec.begin(), 0); - ASSERT_EQ(thrust::find_if(vec.begin(), vec.end(), equal_to_value_pred(2)) - vec.begin(), 1); - ASSERT_EQ(thrust::find_if(vec.begin(), vec.end(), equal_to_value_pred(3)) - vec.begin(), 2); - ASSERT_EQ(thrust::find_if(vec.begin(), vec.end(), equal_to_value_pred(4)) - vec.begin(), 5); - ASSERT_EQ(thrust::find_if(vec.begin(), vec.end(), equal_to_value_pred(5)) - vec.begin(), 4); + ASSERT_EQ(thrust::find_if(vec.begin(), vec.end(), _1 == T(0)) - vec.begin(), 5); + ASSERT_EQ(thrust::find_if(vec.begin(), vec.end(), _1 == T(1)) - vec.begin(), 0); + ASSERT_EQ(thrust::find_if(vec.begin(), vec.end(), _1 == T(2)) - vec.begin(), 1); + ASSERT_EQ(thrust::find_if(vec.begin(), vec.end(), _1 == T(3)) - vec.begin(), 2); + ASSERT_EQ(thrust::find_if(vec.begin(), vec.end(), _1 == T(4)) - vec.begin(), 5); + ASSERT_EQ(thrust::find_if(vec.begin(), vec.end(), _1 == T(5)) - vec.begin(), 4); } template @@ -189,12 +146,12 @@ TYPED_TEST(FindTestsVector, TestFindIfNotSimple) Vector vec{0, 1, 2, 3, 4}; - ASSERT_EQ(0, thrust::find_if_not(vec.begin(), vec.end(), less_than_value_pred(0)) - vec.begin()); - ASSERT_EQ(1, thrust::find_if_not(vec.begin(), vec.end(), less_than_value_pred(1)) - vec.begin()); - ASSERT_EQ(2, thrust::find_if_not(vec.begin(), vec.end(), less_than_value_pred(2)) - vec.begin()); - ASSERT_EQ(3, thrust::find_if_not(vec.begin(), vec.end(), less_than_value_pred(3)) - vec.begin()); - ASSERT_EQ(4, thrust::find_if_not(vec.begin(), vec.end(), less_than_value_pred(4)) - vec.begin()); - ASSERT_EQ(5, thrust::find_if_not(vec.begin(), vec.end(), less_than_value_pred(5)) - vec.begin()); + ASSERT_EQ(0, thrust::find_if_not(vec.begin(), vec.end(), _1 < T(0)) - vec.begin()); + ASSERT_EQ(1, thrust::find_if_not(vec.begin(), vec.end(), _1 < T(1)) - vec.begin()); + ASSERT_EQ(2, thrust::find_if_not(vec.begin(), vec.end(), _1 < T(2)) - vec.begin()); + ASSERT_EQ(3, thrust::find_if_not(vec.begin(), vec.end(), _1 < T(3)) - vec.begin()); + ASSERT_EQ(4, thrust::find_if_not(vec.begin(), vec.end(), _1 < T(4)) - vec.begin()); + ASSERT_EQ(5, thrust::find_if_not(vec.begin(), vec.end(), _1 < T(5)) - vec.begin()); } template @@ -291,15 +248,15 @@ TYPED_TEST(FindTests, TestFindIf) typename thrust::host_vector::iterator h_iter; typename thrust::device_vector::iterator d_iter; - h_iter = thrust::find_if(h_data.begin(), h_data.end(), equal_to_value_pred(0)); - d_iter = thrust::find_if(d_data.begin(), d_data.end(), equal_to_value_pred(0)); + h_iter = thrust::find_if(h_data.begin(), h_data.end(), _1 == T(0)); + d_iter = thrust::find_if(d_data.begin(), d_data.end(), _1 == T(0)); ASSERT_EQ(h_iter - h_data.begin(), d_iter - d_data.begin()); for (size_t i = 1; i < size; i *= 2) { T sample = h_data[i]; - h_iter = thrust::find_if(h_data.begin(), h_data.end(), equal_to_value_pred(sample)); - d_iter = thrust::find_if(d_data.begin(), d_data.end(), equal_to_value_pred(sample)); + h_iter = thrust::find_if(h_data.begin(), h_data.end(), _1 == T(sample)); + d_iter = thrust::find_if(d_data.begin(), d_data.end(), _1 == T(sample)); ASSERT_EQ(h_iter - h_data.begin(), d_iter - d_data.begin()); } } @@ -327,15 +284,15 @@ TYPED_TEST(FindTests, TestFindIfNot) typename thrust::host_vector::iterator h_iter; typename thrust::device_vector::iterator d_iter; - h_iter = thrust::find_if_not(h_data.begin(), h_data.end(), not_equal_to_value_pred(0)); - d_iter = thrust::find_if_not(d_data.begin(), d_data.end(), not_equal_to_value_pred(0)); + h_iter = thrust::find_if_not(h_data.begin(), h_data.end(), _1 != T(0)); + d_iter = thrust::find_if_not(d_data.begin(), d_data.end(), _1 != T(0)); ASSERT_EQ(h_iter - h_data.begin(), d_iter - d_data.begin()); for (size_t i = 1; i < size; i *= 2) { T sample = h_data[i]; - h_iter = thrust::find_if_not(h_data.begin(), h_data.end(), not_equal_to_value_pred(sample)); - d_iter = thrust::find_if_not(d_data.begin(), d_data.end(), not_equal_to_value_pred(sample)); + h_iter = thrust::find_if_not(h_data.begin(), h_data.end(), _1 != T(sample)); + d_iter = thrust::find_if_not(d_data.begin(), d_data.end(), _1 != T(sample)); ASSERT_EQ(h_iter - h_data.begin(), d_iter - d_data.begin()); } } diff --git a/projects/hipccl/hipccl3/rocthrust/test/test_gather.cpp b/projects/hipccl/hipccl3/rocthrust/test/test_gather.cpp index 718fb170a316..89b9b99a0c6f 100644 --- a/projects/hipccl/hipccl3/rocthrust/test/test_gather.cpp +++ b/projects/hipccl/hipccl3/rocthrust/test/test_gather.cpp @@ -28,6 +28,14 @@ #include "test_real_assertions.hpp" #include "test_utils.hpp" +// There is an unfortunate miscompilation of the gcc-11 vectorizer leading to OOB writes +// Adding this attribute suffices that this miscompilation does not appear anymore +#if THRUST_COMPILER(GCC, >=, 11) +# define THRUST_DISABLE_BROKEN_GCC_VECTORIZER __attribute__((optimize("no-tree-vectorize"))) +#else +# define THRUST_DISABLE_BROKEN_GCC_VECTORIZER +#endif + TESTS_DEFINE(GatherTests, FullTestsParams); TESTS_DEFINE(PrimitiveGatherTests, NumericalTestsParams); @@ -418,7 +426,7 @@ TYPED_TEST(PrimitiveGatherTests, TestGatherIfToDiscardIterator) } } -TYPED_TEST(GatherTests, TestGatherCountingIterator) +TYPED_TEST(GatherTests, TestGatherCountingIterator) THRUST_DISABLE_BROKEN_GCC_VECTORIZER { using Vector = typename TestFixture::input_type; diff --git a/projects/hipccl/hipccl3/rocthrust/test/test_libcxx_iterators.cpp b/projects/hipccl/hipccl3/rocthrust/test/test_libcxx_iterators.cpp index 3a1afd8407d3..54a1d4bef9c6 100644 --- a/projects/hipccl/hipccl3/rocthrust/test/test_libcxx_iterators.cpp +++ b/projects/hipccl/hipccl3/rocthrust/test/test_libcxx_iterators.cpp @@ -46,35 +46,47 @@ TEST(LibcxxIteratorTests, TestLibcxxConstantIterator) { { // device system thrust::device_vector vec{1, 2, 3, 4}; + thrust::device_vector expected{42, 42, 42, 42}; thrust::copy(_THRUST_LIBCXX::constant_iterator{42, 0}, _THRUST_LIBCXX::constant_iterator{42, 4}, vec.begin()); + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); } { // host system thrust::host_vector vec{1, 2, 3, 4}; + thrust::host_vector expected{42, 42, 42, 42}; thrust::copy(_THRUST_LIBCXX::constant_iterator{42, 0}, _THRUST_LIBCXX::constant_iterator{42, 4}, vec.begin()); + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); } { // plain std::vector std::vector vec{1, 2, 3, 4}; + std::vector expected{42, 42, 42, 42}; thrust::copy(_THRUST_LIBCXX::constant_iterator{42, 0}, _THRUST_LIBCXX::constant_iterator{42, 4}, vec.begin()); + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); } } TEST(LibcxxIteratorTests, TestLibcxxCountingIterator) { { // device system - thrust::device_vector vec{1, 2, 3, 4}; + thrust::device_vector vec{-1, -1, -1, -1}; + thrust::device_vector expected{0, 1, 2, 3}; thrust::copy(_THRUST_LIBCXX::counting_iterator{0}, _THRUST_LIBCXX::counting_iterator{4}, vec.begin()); + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); } { // host system - thrust::host_vector vec{1, 2, 3, 4}; + thrust::host_vector vec{-1, -1, -1, -1}; + thrust::host_vector expected{0, 1, 2, 3}; thrust::copy(_THRUST_LIBCXX::counting_iterator{0}, _THRUST_LIBCXX::counting_iterator{4}, vec.begin()); + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); } { // plain std::vector - std::vector vec{1, 2, 3, 4}; + std::vector vec{-1, -1, -1, -1}; + std::vector expected{0, 1, 2, 3}; thrust::copy(_THRUST_LIBCXX::counting_iterator{0}, _THRUST_LIBCXX::counting_iterator{4}, vec.begin()); + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); } } @@ -111,25 +123,82 @@ TEST(LibcxxIteratorTests, TestLibcxxPermutationIterator) } } +struct fake_bijection +{ + using index_type = _THRUST_STD::uint32_t; + + constexpr fake_bijection() = default; + + [[nodiscard]] THRUST_HOST_DEVICE constexpr index_type size() const noexcept + { + return 5; + } + + [[nodiscard]] THRUST_HOST_DEVICE constexpr index_type operator()(index_type n) const noexcept + { + return __random_indices[n]; + } + + _THRUST_STD::uint32_t __random_indices[5] = {4, 1, 2, 0, 3}; +}; + +TEST(LibcxxIteratorTests, TestLibcxxShuffleIterator) +{ + { // device system + thrust::device_vector vec{1, 2, 3, 4}; + thrust::device_vector expected{4, 1, 2, 0}; + thrust::copy(_THRUST_LIBCXX::shuffle_iterator{fake_bijection{}, 0}, + _THRUST_LIBCXX::shuffle_iterator{fake_bijection{}, 4}, + vec.begin()); + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); + } + + { // host system + thrust::host_vector vec{1, 2, 3, 4}; + thrust::host_vector expected{4, 1, 2, 0}; + thrust::copy(_THRUST_LIBCXX::shuffle_iterator{fake_bijection{}, 0}, + _THRUST_LIBCXX::shuffle_iterator{fake_bijection{}, 4}, + vec.begin()); + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); + } + + { // plain std::vector + std::vector vec{1, 2, 3, 4}; + std::vector expected{4, 1, 2, 0}; + thrust::copy(_THRUST_LIBCXX::shuffle_iterator{fake_bijection{}, 0}, + _THRUST_LIBCXX::shuffle_iterator{fake_bijection{}, 4}, + vec.begin()); + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); + } +} + TEST(LibcxxIteratorTests, TestLibcxxStridedIterator) { - auto discard = _THRUST_LIBCXX::discard_iterator{}; { // device system - thrust::device_vector vec{1, 2, 3, 4, 5, 6}; - thrust::copy( - _THRUST_LIBCXX::strided_iterator{vec.begin(), 2}, _THRUST_LIBCXX::strided_iterator{vec.end(), 2}, discard); + thrust::device_vector vec{-1, -1, -1, -1}; + thrust::device_vector expected{0, 2, 4, 6}; + thrust::copy(_THRUST_LIBCXX::strided_iterator{_THRUST_LIBCXX::counting_iterator{0}, 2}, + _THRUST_LIBCXX::strided_iterator{_THRUST_LIBCXX::counting_iterator{8}, 2}, + vec.begin()); + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); } { // host system - thrust::host_vector vec{1, 2, 3, 4, 5, 6}; - thrust::copy( - _THRUST_LIBCXX::strided_iterator{vec.begin(), 2}, _THRUST_LIBCXX::strided_iterator{vec.end(), 2}, discard); + thrust::host_vector vec{-1, -1, -1, -1}; + thrust::host_vector expected{0, 2, 4, 6}; + thrust::copy(_THRUST_LIBCXX::strided_iterator{_THRUST_LIBCXX::counting_iterator{0}, 2}, + _THRUST_LIBCXX::strided_iterator{_THRUST_LIBCXX::counting_iterator{8}, 2}, + vec.begin()); + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); } { // plain std::vector - std::vector vec{1, 2, 3, 4, 5, 6}; - thrust::copy( - _THRUST_LIBCXX::strided_iterator{vec.begin(), 2}, _THRUST_LIBCXX::strided_iterator{vec.end(), 2}, discard); + std::vector vec{-1, -1, -1, -1}; + std::vector expected{0, 2, 4, 6}; + thrust::copy(_THRUST_LIBCXX::strided_iterator{_THRUST_LIBCXX::counting_iterator{0}, 2}, + _THRUST_LIBCXX::strided_iterator{_THRUST_LIBCXX::counting_iterator{8}, 2}, + vec.begin()); + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); } } @@ -168,58 +237,184 @@ struct plus_one } }; +TEST(LibcxxIteratorTests, TestLibcxxTransformInputOutputIterator) +{ + { // device system + thrust::device_vector vec{-1, -1, -1, -1, -1}; + auto iter = _THRUST_LIBCXX::transform_input_output_iterator(vec.begin(), plus_one{}, plus_one{}); + thrust::copy(_THRUST_LIBCXX::counting_iterator{3}, _THRUST_LIBCXX::counting_iterator{8}, iter); + + // Ensure we did write the right output, sequence starts at 3 + 1 == 4 + thrust::device_vector expected{4, 5, 6, 7, 8}; + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); + + // Ensure we did read the right input, output starts 4 + 1 == 5 + thrust::copy(iter, iter + 5, _THRUST_LIBCXX::make_tabulate_output_iterator(is_equal_index{}, 5)); + } + + { // host system + thrust::host_vector vec{-1, -1, -1, -1, -1}; + auto iter = _THRUST_LIBCXX::transform_input_output_iterator(vec.begin(), plus_one{}, plus_one{}); + thrust::copy(_THRUST_LIBCXX::counting_iterator{3}, _THRUST_LIBCXX::counting_iterator{8}, iter); + + // Ensure we did write the right output, sequence starts at 3 + 1 == 4 + thrust::host_vector expected{4, 5, 6, 7, 8}; + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); + + // Ensure we did read the right input, output starts 4 + 1 == 5 + thrust::copy(iter, iter + 5, _THRUST_LIBCXX::make_tabulate_output_iterator(is_equal_index{}, 5)); + } + + { // plain std::vector + std::vector vec{-1, -1, -1, -1, -1}; + auto iter = _THRUST_LIBCXX::transform_input_output_iterator(vec.begin(), plus_one{}, plus_one{}); + thrust::copy(_THRUST_LIBCXX::counting_iterator{3}, _THRUST_LIBCXX::counting_iterator{8}, iter); + + // Ensure we did write the right output, sequence starts at 3 + 1 == 4 + std::vector expected{4, 5, 6, 7, 8}; + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); + + // Ensure we did read the right input, output starts 4 + 1 == 5 + thrust::copy(iter, iter + 5, _THRUST_LIBCXX::make_tabulate_output_iterator(is_equal_index{}, 5)); + } +} + TEST(LibcxxIteratorTests, TestLibcxxTransformOutputIterator) { { // device system thrust::device_vector vec{-1, -1, -1, -1, -1}; + thrust::device_vector expected{1, 2, 3, 4, 5}; thrust::copy(_THRUST_LIBCXX::counting_iterator{0}, _THRUST_LIBCXX::counting_iterator{5}, _THRUST_LIBCXX::make_transform_output_iterator(vec.begin(), plus_one{})); - thrust::device_vector expected{1, 2, 3, 4, 5}; ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); } { // host system thrust::host_vector vec{-1, -1, -1, -1, -1}; + thrust::host_vector expected{1, 2, 3, 4, 5}; thrust::copy(_THRUST_LIBCXX::counting_iterator{0}, _THRUST_LIBCXX::counting_iterator{5}, _THRUST_LIBCXX::make_transform_output_iterator(vec.begin(), plus_one{})); - thrust::host_vector expected{1, 2, 3, 4, 5}; ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); } { // plain std::vector std::vector vec{-1, -1, -1, -1, -1}; + std::vector expected{1, 2, 3, 4, 5}; thrust::copy(_THRUST_LIBCXX::counting_iterator{0}, _THRUST_LIBCXX::counting_iterator{5}, _THRUST_LIBCXX::make_transform_output_iterator(vec.begin(), plus_one{})); - std::vector expected{1, 2, 3, 4, 5}; ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); } } TEST(LibcxxIteratorTests, TestLibcxxTransformIterator) { - auto discard = _THRUST_LIBCXX::discard_iterator{}; { // device system - thrust::device_vector vec{1, 2, 3, 4}; - thrust::copy(_THRUST_LIBCXX::make_transform_iterator(vec.begin(), plus_one{}), - _THRUST_LIBCXX::make_transform_iterator(vec.end(), plus_one{}), - discard); + thrust::device_vector vec{-1, -1, -1, -1}; + thrust::device_vector expected{1, 2, 3, 4}; + thrust::copy(_THRUST_LIBCXX::transform_iterator(_THRUST_LIBCXX::counting_iterator{0}, plus_one{}), + _THRUST_LIBCXX::transform_iterator(_THRUST_LIBCXX::counting_iterator{4}, plus_one{}), + vec.begin()); + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); } { // host system - thrust::host_vector vec{1, 2, 3, 4}; - thrust::copy(_THRUST_LIBCXX::make_transform_iterator(vec.begin(), plus_one{}), - _THRUST_LIBCXX::make_transform_iterator(vec.end(), plus_one{}), - discard); + thrust::host_vector vec{-1, -1, -1, -1}; + thrust::host_vector expected{1, 2, 3, 4}; + thrust::copy(_THRUST_LIBCXX::transform_iterator(_THRUST_LIBCXX::counting_iterator{0}, plus_one{}), + _THRUST_LIBCXX::transform_iterator(_THRUST_LIBCXX::counting_iterator{4}, plus_one{}), + vec.begin()); + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); } { // plain std::vector - std::vector vec{1, 2, 3, 4}; - thrust::copy(_THRUST_LIBCXX::make_transform_iterator(vec.begin(), plus_one{}), - _THRUST_LIBCXX::make_transform_iterator(vec.end(), plus_one{}), - discard); + std::vector vec{-1, -1, -1, -1}; + std::vector expected{1, 2, 3, 4}; + thrust::copy(_THRUST_LIBCXX::transform_iterator(_THRUST_LIBCXX::counting_iterator{0}, plus_one{}), + _THRUST_LIBCXX::transform_iterator(_THRUST_LIBCXX::counting_iterator{4}, plus_one{}), + vec.begin()); + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); + } +} + +TEST(LibcxxIteratorTests, TestLibcxxZipIterator) +{ + _THRUST_LIBCXX::zip_function<_THRUST_STD::plus> fun{}; + { // device system + thrust::device_vector vec{-1, -1, -1, -1}; + auto iter = _THRUST_LIBCXX::make_transform_iterator(_THRUST_LIBCXX::make_zip_iterator(vec.begin(), _THRUST_LIBCXX::counting_iterator{4}), fun); + thrust::copy(iter, iter + 4, vec.begin()); + thrust::device_vector expected{3, 4, 5, 6}; + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); + } + + { // host system + thrust::host_vector vec{-1, -1, -1, -1}; + auto iter = _THRUST_LIBCXX::make_transform_iterator(_THRUST_LIBCXX::make_zip_iterator(vec.begin(), _THRUST_LIBCXX::counting_iterator{4}), fun); + thrust::copy(iter, iter + 4, vec.begin()); + thrust::host_vector expected{3, 4, 5, 6}; + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); + } + + { // plain std::vector + std::vector vec{-1, -1, -1, -1}; + auto iter = _THRUST_LIBCXX::make_transform_iterator(_THRUST_LIBCXX::make_zip_iterator(vec.begin(), _THRUST_LIBCXX::counting_iterator{4}), fun); + thrust::copy(iter, iter + 4, vec.begin()); + std::vector expected{3, 4, 5, 6}; + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); + } +} + +TEST(LibcxxIteratorTests, TestLibcxxZipTransformIterator) +{ + { // device system + thrust::device_vector vec{-1, -1, -1, -1}; + auto iter = _THRUST_LIBCXX::make_zip_transform_iterator(_THRUST_STD::plus{}, vec.begin(), _THRUST_LIBCXX::counting_iterator{4}); + thrust::copy(iter, iter + 4, vec.begin()); + thrust::device_vector expected{3, 4, 5, 6}; + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); + } + + { // host system + thrust::host_vector vec{-1, -1, -1, -1}; + auto iter = _THRUST_LIBCXX::make_zip_transform_iterator(_THRUST_STD::plus{}, vec.begin(), _THRUST_LIBCXX::counting_iterator{4}); + thrust::copy(iter, iter + 4, vec.begin()); + thrust::host_vector expected{3, 4, 5, 6}; + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); + } + + { // plain std::vector + std::vector vec{-1, -1, -1, -1}; + auto iter = _THRUST_LIBCXX::make_zip_transform_iterator(_THRUST_STD::plus{}, vec.begin(), _THRUST_LIBCXX::counting_iterator{4}); + thrust::copy(iter, iter + 4, vec.begin()); + std::vector expected{3, 4, 5, 6}; + ASSERT_TRUE(thrust::equal(vec.begin(), vec.end(), expected.begin())); + } +} + +TEST(LibcxxIteratorTests, TestLibcxxReverseIterator) +{ + { // device system + thrust::device_vector vec{1, 2, 3, 4, 5, 6, 7, 8, 9}; + thrust::device_vector expected{9, 8, 7, 6, 5, 4, 3, 2, 1}; + ASSERT_TRUE(thrust::equal( + _THRUST_STD::reverse_iterator{vec.end()}, _THRUST_STD::reverse_iterator{vec.begin()}, expected.begin())); + } + + { // host system + thrust::host_vector vec{1, 2, 3, 4, 5, 6, 7, 8, 9}; + thrust::host_vector expected{9, 8, 7, 6, 5, 4, 3, 2, 1}; + ASSERT_TRUE(thrust::equal( + _THRUST_STD::reverse_iterator{vec.end()}, _THRUST_STD::reverse_iterator{vec.begin()}, expected.begin())); + } + + { // plain std::vector + std::vector vec{1, 2, 3, 4, 5, 6, 7, 8, 9}; + std::vector expected{9, 8, 7, 6, 5, 4, 3, 2, 1}; + ASSERT_TRUE(thrust::equal( + _THRUST_STD::reverse_iterator{vec.end()}, _THRUST_STD::reverse_iterator{vec.begin()}, expected.begin())); } } diff --git a/projects/hipccl/hipccl3/rocthrust/test/test_metaprogramming.cpp b/projects/hipccl/hipccl3/rocthrust/test/test_metaprogramming.cpp deleted file mode 100644 index 818d60d3f2de..000000000000 --- a/projects/hipccl/hipccl3/rocthrust/test/test_metaprogramming.cpp +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2008-2013 NVIDIA Corporation - * Modifications Copyright© 2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include "test_param_fixtures.hpp" -#include "test_utils.hpp" - -TESTS_DEFINE(MetaprogrammingTests, FullTestsParams); - -TEST(MetaprogrammingTests, TestLog2) -{ - SCOPED_TRACE(testing::Message() << "with device_id= " << test::set_device_from_ctest()); - - unsigned int result; - - result = thrust::detail::mpl::math::log2<1>::value; - ASSERT_EQ(result, 0lu); - result = thrust::detail::mpl::math::log2<2>::value; - ASSERT_EQ(result, 1lu); - result = thrust::detail::mpl::math::log2<3>::value; - ASSERT_EQ(result, 1lu); - result = thrust::detail::mpl::math::log2<4>::value; - ASSERT_EQ(result, 2lu); - result = thrust::detail::mpl::math::log2<5>::value; - ASSERT_EQ(result, 2lu); - result = thrust::detail::mpl::math::log2<6>::value; - ASSERT_EQ(result, 2lu); - result = thrust::detail::mpl::math::log2<7>::value; - ASSERT_EQ(result, 2lu); - result = thrust::detail::mpl::math::log2<8>::value; - ASSERT_EQ(result, 3lu); - result = thrust::detail::mpl::math::log2<9>::value; - ASSERT_EQ(result, 3lu); - result = thrust::detail::mpl::math::log2<15>::value; - ASSERT_EQ(result, 3lu); - result = thrust::detail::mpl::math::log2<16>::value; - ASSERT_EQ(result, 4lu); - result = thrust::detail::mpl::math::log2<17>::value; - ASSERT_EQ(result, 4lu); - result = thrust::detail::mpl::math::log2<127>::value; - ASSERT_EQ(result, 6lu); - result = thrust::detail::mpl::math::log2<128>::value; - ASSERT_EQ(result, 7lu); - result = thrust::detail::mpl::math::log2<129>::value; - ASSERT_EQ(result, 7lu); - result = thrust::detail::mpl::math::log2<256>::value; - ASSERT_EQ(result, 8lu); - result = thrust::detail::mpl::math::log2<511>::value; - ASSERT_EQ(result, 8lu); - result = thrust::detail::mpl::math::log2<512>::value; - ASSERT_EQ(result, 9lu); -} diff --git a/projects/hipccl/hipccl3/rocthrust/test/test_reduce_by_key.cpp b/projects/hipccl/hipccl3/rocthrust/test/test_reduce_by_key.cpp index 52f383a50c36..0b1a54f46a6b 100644 --- a/projects/hipccl/hipccl3/rocthrust/test/test_reduce_by_key.cpp +++ b/projects/hipccl/hipccl3/rocthrust/test/test_reduce_by_key.cpp @@ -15,11 +15,20 @@ * limitations under the License. */ +#include +#include +#include +#include #include #include #include #include +#include +#include +#include +#include + #include "test_param_fixtures.hpp" #include "test_utils.hpp" @@ -449,3 +458,133 @@ TEST(ReduceByKeyTests, TestReduceByKeyWithBigIndexes) } } } + +// Checks whether the equality operator is ever invoked on out-of-bounds items +struct check_valid_item_op +{ + std::uint32_t* error_counter{}; + int expected_upper_bound{}; + + __device__ bool operator()(const int lhs, const int rhs) const + { + if (lhs > expected_upper_bound || rhs > expected_upper_bound) + { + if (error_counter) + { + atomicAdd(error_counter, 1); + } + return false; + } + return lhs == rhs; + } +}; + +template +struct check_accumulator_t_op +{ + std::uint32_t* error_counter{}; + + template && !std::is_same_v>> + __device__ ReturnedT operator()(const T lhs, const U& rhs) const + { + return static_cast(lhs + rhs); + } + + template + __device__ ReturnedT operator()(const ReturnedT lhs, const T& rhs) const + { + atomicAdd(error_counter, 1); + return lhs + static_cast(rhs); + } +}; + +TEST(ReduceByKeyTests, TestReduceByKeyWithCustomEqualityOp) +{ + SCOPED_TRACE(testing::Message() << "with device_id= " << test::set_device_from_ctest()); + + using key_vector_t = thrust::device_vector; + using val_vector_t = thrust::device_vector; + using key_t = key_vector_t::value_type; + using val_t = val_vector_t::value_type; + + constexpr int num_items = 1000; + auto keys = thrust::make_counting_iterator(key_t{0}); + auto values = thrust::make_counting_iterator(val_t{42}); + + thrust::device_vector error_counter(1, 0); + auto const error_counter_ptr = thrust::raw_pointer_cast(error_counter.data()); + + key_vector_t unique_out(num_items); + val_vector_t aggregates_out(num_items); + auto [unique_out_end, aggregates_out_end] = thrust::reduce_by_key( + keys, + keys + num_items, + values, + unique_out.begin(), + aggregates_out.begin(), + check_valid_item_op{error_counter_ptr, num_items - 1}); + + // Verify that the number of unique keys is correct + const auto num_unique_out = std::distance(unique_out.begin(), unique_out_end); + const auto num_aggregates_out = std::distance(aggregates_out.begin(), aggregates_out_end); + ASSERT_EQ(num_unique_out, num_items); + ASSERT_EQ(num_aggregates_out, num_items); + + // Verify that the equality operator was never invoked on out-of-bounds items + ASSERT_EQ(error_counter[0], std::uint32_t{0}); + + // Verify that unique keys are correct + ASSERT_TRUE(thrust::equal(unique_out.cbegin(), unique_out.cend(), keys)); + + // Verify that the aggregates are correct + ASSERT_TRUE(thrust::equal(aggregates_out.cbegin(), aggregates_out.cend(), values)); +} + +TEST(ReduceByKeyTests, TestReduceByKeyWithDifferentAccumulatorT) +{ + SCOPED_TRACE(testing::Message() << "with device_id= " << test::set_device_from_ctest()); + + using key_t = std::uint32_t; + using val_t = std::uint8_t; + using reduction_op_t = check_accumulator_t_op; + + constexpr int num_items = 20000; + constexpr int expected_num_uniques = 1; + constexpr auto unique_key = key_t{42U}; + auto keys = thrust::make_constant_iterator(unique_key); + auto values = thrust::make_counting_iterator(val_t{0}); + + thrust::device_vector error_counter(1, 0); + auto const error_counter_ptr = thrust::raw_pointer_cast(error_counter.data()); + + thrust::device_vector unique_out(expected_num_uniques); + thrust::device_vector aggregates_out(expected_num_uniques); + auto [unique_out_end, aggregates_out_end] = thrust::reduce_by_key( + keys, + keys + num_items, + values, + unique_out.begin(), + aggregates_out.begin(), + thrust::equal_to<>{}, + reduction_op_t{error_counter_ptr}); + + // Verify that the number of unique keys is correct + auto num_unique_out = std::distance(unique_out.begin(), unique_out_end); + auto num_aggregates_out = std::distance(aggregates_out.begin(), aggregates_out_end); + ASSERT_EQ(num_unique_out, expected_num_uniques); + ASSERT_EQ(num_aggregates_out, expected_num_uniques); + + // Verify that the reduction operator was never invoked with the accumulator type + ASSERT_EQ(error_counter[0], std::uint32_t{0}); + + // Verify that the unique key is correct + ASSERT_EQ(unique_out[0], unique_key); + + // Verify that the aggregate is correct + constexpr auto mod_val = 0x01 << std::numeric_limits::digits; + constexpr auto sum = ((num_items * (num_items - 1)) / 2); + constexpr auto expected_aggregate = static_cast(sum % mod_val); + ASSERT_EQ(aggregates_out[0], expected_aggregate); +} diff --git a/projects/hipccl/hipccl3/rocthrust/test/test_scan.cpp b/projects/hipccl/hipccl3/rocthrust/test/test_scan.cpp index 424981d55d4a..3913fd222071 100644 --- a/projects/hipccl/hipccl3/rocthrust/test/test_scan.cpp +++ b/projects/hipccl/hipccl3/rocthrust/test/test_scan.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include "test_param_fixtures.hpp" #include "test_real_assertions.hpp" @@ -39,6 +40,8 @@ # include #endif +#include + TESTS_DEFINE(ScanTests, FullTestsParams); TESTS_DEFINE(ScanVariablesTests, NumericalTestsParams); @@ -748,15 +751,23 @@ struct only_set_when_expected_it { return *this; } + + THRUST_HOST_DEVICE only_set_when_expected_it operator++(int) const + { + return *this; + } + THRUST_HOST_DEVICE only_set_when_expected_it operator*() const { return *this; } + template THRUST_HOST_DEVICE only_set_when_expected_it operator+(Difference) const { return *this; } + template THRUST_HOST_DEVICE only_set_when_expected_it operator[](Index) const { @@ -872,24 +883,39 @@ struct Int THRUST_HOST_DEVICE explicit Int(int num) : i(num) {} - THRUST_HOST_DEVICE Int() - : i{} - {} - THRUST_HOST_DEVICE Int operator+(Int const& o) const + + Int() = default; + + THRUST_HOST_DEVICE friend Int operator+(Int const& a, Int const& b) + { + return Int{a.i + b.i}; + } + + THRUST_HOST_DEVICE friend bool operator==(Int const& a, Int const& b) { - return Int{this->i + o.i}; + return a.i == b.i; + } + + friend std::ostream& operator<<(std::ostream& os, Int const& a) + { + return os << "Int{" << a.i << "}"; } }; -TEST(ScanTests, TestInclusiveScanWithUserDefinedType) +TEST(ScanTests, TestScanWithUserDefinedTypeAndInit) { SCOPED_TRACE(testing::Message() << "with device_id= " << test::set_device_from_ctest()); - thrust::device_vector vec(5, Int{1}); - - thrust::inclusive_scan(thrust::device, vec.cbegin(), vec.cend(), vec.begin()); - - ASSERT_EQ(static_cast(vec.back()).i, 5); + { + thrust::device_vector vec(5, Int{1}); + thrust::exclusive_scan(thrust::device, vec.cbegin(), vec.cend(), vec.begin(), Int{100}, _THRUST_STD::plus()); + ASSERT_EQ(vec, (thrust::device_vector{Int{100}, Int{101}, Int{102}, Int{103}, Int{104}})); + } + { + thrust::device_vector vec(5, Int{1}); + thrust::inclusive_scan(thrust::device, vec.cbegin(), vec.cend(), vec.begin(), Int{100}, _THRUST_STD::plus()); + ASSERT_EQ(vec, (thrust::device_vector{Int{101}, Int{102}, Int{103}, Int{104}, Int{105}})); + } } // Represents a permutation as a tuple of integers, see also: https://en.wikipedia.org/wiki/Permutation @@ -998,3 +1024,87 @@ TEST(ScanTests, TestInclusiveScanWithNonCommutativeOp) {0, 1, 4, 2, 3}, {4, 0, 2, 1, 3}})); } + +// Adapted from issue: https://github.com/NVIDIA/cccl/issues/6317 +TEST(ScanTests, TestScanBug6317) +{ + SCOPED_TRACE(testing::Message() << "with device_id= " << test::set_device_from_ctest()); + + using T = unsigned int; + const int n = 1729; + for (auto seed : get_seeds()) + { + SCOPED_TRACE(testing::Message() << "with seed= " << seed); + { + thrust::host_vector h_s = get_random_data(n, get_default_limits::min(), get_default_limits::max(), seed); + thrust::device_vector s = h_s; + thrust::device_vector d(n); + const auto r = thrust::inclusive_scan(s.cbegin(), s.cend(), d.begin(), _THRUST_STD::multiplies<>{}); + ASSERT_EQ((d.end() == r), true); + std::partial_sum(h_s.cbegin(), h_s.cend(), h_s.begin(), std::multiplies<>{}); + ASSERT_EQ(h_s, d); + } + { + thrust::host_vector h_s = get_random_data(n, get_default_limits::min(), get_default_limits::max(), seed); + thrust::device_vector s = h_s; + thrust::device_vector d(n); + const auto r = thrust::exclusive_scan(s.cbegin(), s.cend(), d.begin(), 42, _THRUST_STD::multiplies<>{}); + ASSERT_EQ((d.end() == r), true); + thrust::exclusive_scan(h_s.cbegin(), h_s.cend(), h_s.begin(), 42, _THRUST_STD::multiplies<>{}); + ASSERT_EQ(h_s, d); + } + } +} + +// Test edge cases for parallel scan with non-additive operations +TEST(ScanTests, TestScanEdgeCases) +{ + SCOPED_TRACE(testing::Message() << "with device_id= " << test::set_device_from_ctest()); + + using thrust::placeholders::_1; + + // Large array, at, and below the OMP parallel_scan_threshold (1024) + auto check = [](int n, auto gen, int init, bool inclusive) { + thrust::device_vector d_input(n); + thrust::tabulate(d_input.begin(), d_input.end(), gen); + + thrust::device_vector d_output(n); + auto r = inclusive + ? thrust::inclusive_scan(d_input.begin(), d_input.end(), d_output.begin(), init, _THRUST_STD::multiplies<>{}) + : thrust::exclusive_scan(d_input.begin(), d_input.end(), d_output.begin(), init, _THRUST_STD::multiplies<>{}); + ASSERT_EQ((d_output.end() == r), true); + + thrust::host_vector h_input = d_input; + thrust::host_vector h_output(n); + if (inclusive) + { + thrust::inclusive_scan(h_input.begin(), h_input.end(), h_output.begin(), init, _THRUST_STD::multiplies<>{}); + } + else + { + thrust::exclusive_scan(h_input.begin(), h_input.end(), h_output.begin(), init, _THRUST_STD::multiplies<>{}); + } + ASSERT_EQ(d_output, h_output); + }; + check(10000, (_1 % 5) + 1, 2, true); + check(1024, _1 + 1, 10, true); + check(1023, (_1 % 3) + 1, 5, true); + check(10000, (_1 % 3) + 1, 5, false); + check(1024, (_1 % 5) + 1, 3, false); + + // Very small arrays (edge case for block distribution) + { + thrust::device_vector d_input = {3, 7}; + thrust::device_vector d_output(2); + auto r = thrust::inclusive_scan(d_input.begin(), d_input.end(), d_output.begin(), 2, _THRUST_STD::multiplies<>{}); + ASSERT_EQ((d_output.end() == r), true); + ASSERT_EQ(d_output, (thrust::device_vector{6, 42})); + } + { + thrust::device_vector d_input = {2, 4}; + thrust::device_vector d_output(2); + auto r = thrust::exclusive_scan(d_input.begin(), d_input.end(), d_output.begin(), 3, _THRUST_STD::multiplies<>{}); + ASSERT_EQ((d_output.end() == r), true); + ASSERT_EQ(d_output, (thrust::device_vector{3, 6})); + } +} diff --git a/projects/hipccl/hipccl3/rocthrust/test/test_shuffle.cpp b/projects/hipccl/hipccl3/rocthrust/test/test_shuffle.cpp index 0b957f63a391..0968980fc7ff 100644 --- a/projects/hipccl/hipccl3/rocthrust/test/test_shuffle.cpp +++ b/projects/hipccl/hipccl3/rocthrust/test/test_shuffle.cpp @@ -26,6 +26,7 @@ #include #include +#include #include #include @@ -39,382 +40,6 @@ TESTS_DEFINE(ShuffleVariablesTests, NumericalTestsParams); TESTS_DEFINE(ShuffleVectorTests, VectorSignedIntegerTestsParams); -// Functions for performing statistical tests of randomness -// From NIST-Statistical-Test-Suite -// Licence: -// "This software was developed at the National Institute of Standards and -// Technology by employees of the Federal Government in the course of their -// official duties. Pursuant to title 17 Section 105 of the United States Code -// this software is not subject to copyright protection and is in the public -// domain. The NIST Statistical Test Suite is an experimental system. NIST -// assumes no responsibility whatsoever for its use by other parties, and makes -// no guarantees, expressed or implied, about its quality, reliability, or any -// other characteristic. We would appreciate acknowledgment if the software is -// used." -class CephesFunctions -{ -public: - static double cephes_igamc(double a, double x) - { - double ans, ax, c, yc, r, t, y, z; - double pk, pkm1, pkm2, qk, qkm1, qkm2; - - if ((x <= 0) || (a <= 0)) - { - return (1.0); - } - - if ((x < 1.0) || (x < a)) - { - return (1.e0 - cephes_igam(a, x)); - } - - ax = a * log(x) - x - cephes_lgam(a); - - if (ax < -MAXLOG) - { - printf("igamc: UNDERFLOW\n"); - return 0.0; - } - ax = exp(ax); - - /* continued fraction */ - y = 1.0 - a; - z = x + y + 1.0; - c = 0.0; - pkm2 = 1.0; - qkm2 = x; - pkm1 = x + 1.0; - qkm1 = z * x; - ans = pkm1 / qkm1; - - do - { - c += 1.0; - y += 1.0; - z += 2.0; - yc = y * c; - pk = pkm1 * z - pkm2 * yc; - qk = qkm1 * z - qkm2 * yc; - if (qk != 0) - { - r = pk / qk; - t = fabs((ans - r) / r); - ans = r; - } - else - { - t = 1.0; - } - pkm2 = pkm1; - pkm1 = pk; - qkm2 = qkm1; - qkm1 = qk; - if (fabs(pk) > big) - { - pkm2 *= biginv; - pkm1 *= biginv; - qkm2 *= biginv; - qkm1 *= biginv; - } - } while (t > MACHEP); - - return ans * ax; - } - -private: - static constexpr double rel_error = 1E-12; - - static constexpr double MACHEP = 1.11022302462515654042E-16; // 2**-53 - static constexpr double MAXLOG = 7.09782712893383996732224E2; // log(MAXNUM) - static constexpr double MAXNUM = 1.7976931348623158E308; // 2**1024*(1-MACHEP) - static constexpr double PI = 3.14159265358979323846; - - static constexpr double big = 4.503599627370496e15; - static constexpr double biginv = 2.22044604925031308085e-16; - - static int sgngam; - - static double cephes_igam(double a, double x) - { - double ans, ax, c, r; - - if ((x <= 0) || (a <= 0)) - { - return 0.0; - } - - if ((x > 1.0) && (x > a)) - { - return 1.e0 - cephes_igamc(a, x); - } - - /* Compute x**a * exp(-x) / gamma(a) */ - ax = a * log(x) - x - cephes_lgam(a); - if (ax < -MAXLOG) - { - printf("igam: UNDERFLOW\n"); - return 0.0; - } - ax = exp(ax); - - /* power series */ - r = a; - c = 1.0; - ans = 1.0; - - do - { - r += 1.0; - c *= x / r; - ans += c; - } while (c / ans > MACHEP); - - return ans * ax / a; - } - - /* A[]: Stirling's formula expansion of log gamma - * B[], C[]: log gamma function between 2 and 3 - */ - static constexpr double A[] = { - 0.000811614167470508488140545910738410384510643780, - -0.000595061904284301438315674115386855191900394857, - 0.000793650340457716942620114419781884862459264696, - -0.002777777777300996942672073330982129846233874559, - 0.083333333333333189929525985917280195280909538269}; - static constexpr double B[] = { - -1378.251525691208598800585605204105377197265625, - -38801.631513463784358464181423187255859375, - -331612.9927388711948879063129425048828125, - -1162370.97492762305773794651031494140625, - -1721737.00820839661173522472381591796875, - -853555.66424576542340219020843505859375}; - static constexpr double C[] = { - -351.8157014365234545039129443466663360595703125, - -17064.21066518811494461260735988616943359375, - -220528.59055385444662533700466156005859375, - -1139334.44367982516996562480926513671875, - -2532523.07177582941949367523193359375, - -2018891.4143353276886045932769775390625}; - - static constexpr double MAXLGM = 2.556348e305; - - /* Logarithm of gamma function */ - static double cephes_lgam(double x) - { - double p, q, u, w, z; - int i; - - sgngam = 1; - - if (x < -34.0) - { - q = -x; - w = cephes_lgam(q); /* note this modifies sgngam! */ - p = floor(q); - if (p == q) - { - lgsing: - goto loverf; - } - i = (int) p; - if ((i & 1) == 0) - { - sgngam = -1; - } - else - { - sgngam = 1; - } - z = q - p; - if (z > 0.5) - { - p += 1.0; - z = p - q; - } - z = q * sin(PI * z); - if (z == 0.0) - { - goto lgsing; - } - /* z = log(PI) - log( z ) - w;*/ - z = log(PI) - log(z) - w; - return z; - } - - if (x < 13.0) - { - z = 1.0; - p = 0.0; - u = x; - while (u >= 3.0) - { - p -= 1.0; - u = x + p; - z *= u; - } - while (u < 2.0) - { - if (u == 0.0) - { - goto lgsing; - } - z /= u; - p += 1.0; - u = x + p; - } - if (z < 0.0) - { - sgngam = -1; - z = -z; - } - else - { - sgngam = 1; - } - if (u == 2.0) - { - return (log(z)); - } - p -= 2.0; - x = x + p; - p = x * cephes_polevl(x, B, 5) / cephes_p1evl(x, C, 6); - - return log(z) + p; - } - - if (x > MAXLGM) - { - loverf: - printf("lgam: OVERFLOW\n"); - - return sgngam * MAXNUM; - } - - q = (x - 0.5) * log(x) - x + log(sqrt(2 * PI)); - if (x > 1.0e8) - { - return q; - } - - p = 1.0 / (x * x); - if (x >= 1000.0) - { - q += ((7.9365079365079365079365e-4 * p - 2.7777777777777777777778e-3) * p + 0.0833333333333333333333) / x; - } - else - { - q += cephes_polevl(p, A, 4) / x; - } - - return q; - } - - static double cephes_polevl(double x, const double* coef, int N) - { - const double* p = coef; - double ans = *p++; - int i = N; - do - { - ans = ans * x + *p++; - } while (--i); - - return ans; - } - - static double cephes_p1evl(double x, const double* coef, int N) - { - const double* p = coef; - double ans = x + *p++; - int i = N - 1; - - do - { - ans = ans * x + *p++; - } while (--i); - - return ans; - } - - static double cephes_erf(double x) - { - static const double two_sqrtpi = 1.128379167095512574; - double sum = x, term = x, xsqr = x * x; - int j = 1; - - if (fabs(x) > 2.2) - { - return 1.0 - cephes_erfc(x); - } - - do - { - term *= xsqr / j; - sum -= term / (2 * j + 1); - j++; - term *= xsqr / j; - sum += term / (2 * j + 1); - j++; - } while (fabs(term) / sum > rel_error); - - return two_sqrtpi * sum; - } - - static double cephes_erfc(double x) - { - static const double one_sqrtpi = 0.564189583547756287; - double a = 1, b = x, c = x, d = x * x + 0.5; - double q1, q2 = b / d, n = 1.0, t; - - if (fabs(x) < 2.2) - { - return 1.0 - cephes_erf(x); - } - if (x < 0) - { - return 2.0 - cephes_erfc(-x); - } - - do - { - t = a * n + b * x; - a = b; - b = t; - t = c * n + d * x; - c = d; - d = t; - n += 0.5; - q1 = q2; - q2 = b / d; - } while (fabs(q1 - q2) / q2 > rel_error); - - return one_sqrtpi * exp(-x * x) * q2; - } - - static double cephes_normal(double x) - { - double arg, result, sqrt2 = 1.414213562373095048801688724209698078569672; - - if (x > 0) - { - arg = x / sqrt2; - result = 0.5 * (1 + erf(arg)); - } - else - { - arg = -x / sqrt2; - result = 0.5 * (1 - erf(arg)); - } - - return (result); - } -}; -int CephesFunctions::sgngam = 0; -constexpr double CephesFunctions::A[]; -constexpr double CephesFunctions::B[]; -constexpr double CephesFunctions::C[]; - struct iterator_shuffle_copy { template @@ -563,7 +188,7 @@ void TestFunctionIsBijectionBase() { return; } - ASSERT_LE(total_length, (std::max)(size * 2, size_t(16))); // Check the rounded up size is at most double the input + ASSERT_LE(total_length, (std::max)(size * 2, size_t(256))); // Check the rounded up size is at most double the input auto device_result_it = thrust::make_transform_iterator(thrust::make_counting_iterator(T(0)), device_f); @@ -601,17 +226,17 @@ TYPED_TEST(ShuffleVariablesTests, TestFeistelBijectionLength) thrust::default_random_engine g(0xD5); - uint64_t m = 31; + uint64_t m = 345; thrust::detail::feistel_bijection f(m, g); - ASSERT_EQ(f.nearest_power_of_two(), uint64_t(32)); + ASSERT_EQ(f.size(), uint64_t(512)); - m = 32; + m = 256; f = thrust::detail::feistel_bijection(m, g); - ASSERT_EQ(f.nearest_power_of_two(), uint64_t(32)); + ASSERT_EQ(f.size(), uint64_t(256)); m = 1; f = thrust::detail::feistel_bijection(m, g); - ASSERT_EQ(f.nearest_power_of_two(), uint64_t(16)); + ASSERT_EQ(f.size(), uint64_t(256)); } TYPED_TEST(ShuffleVariablesTests, TestShuffleIteratorConstructibleFromBijection) { @@ -623,13 +248,12 @@ TYPED_TEST(ShuffleVariablesTests, TestShuffleIteratorConstructibleFromBijection) thrust::shuffle_iterator it(f); g.seed(0xD5); - thrust::detail::random_bijection f2(32, g); + thrust::detail::random_bijection f2(f.size(), g); thrust::shuffle_iterator it2(f2); g.seed(0xD5); - thrust::shuffle_iterator it3(32, g); + thrust::shuffle_iterator it3(f.size(), g); - ASSERT_EQ(f.size(), f2.size()); ASSERT_EQ(true, thrust::equal(thrust::device, it, it + f.size(), it2)); ASSERT_EQ(true, thrust::equal(thrust::device, it, it + f.size(), it3)); } @@ -663,42 +287,56 @@ TYPED_TEST(ShuffleVariablesTests, TestShuffleIteratorStateless) ASSERT_EQ(*(it - 1), *(it - 1)); } +double inverse_erf(double x) +{ + double tt1, tt2, lnx, sgn; + sgn = (x < 0) ? -1.0 : 1.0; + + x = (1 - x) * (1 + x); + lnx = std::log(x); + + tt1 = 2 / (3.14159265358979323846 * 0.147) + 0.5f * lnx; + tt2 = 1 / (0.147) * lnx; + + return (sgn * std::sqrt(-tt1 + std::sqrt(tt1 * tt1 - tt2))); +} + // Individual input keys should be permuted to output locations with uniform -// probability. Perform chi-squared test with confidence 99.9%. +// probability. Perform chi-squared test with confidence 95%. template void TestShuffleKeyPositionBase() { - using T = typename Vector::value_type; - size_t m = 20; - size_t num_samples = 100; - thrust::host_vector index_sum(m, 0); - thrust::host_vector sequence(m); - thrust::sequence(sequence.begin(), sequence.end(), T(0)); - + using T = typename Vector::value_type; + const int num_samples = 1000; + const int n = 20; thrust::default_random_engine g(0xD5); - for (size_t i = 0; i < num_samples; i++) + thrust::host_vector expected_value(n); + thrust::host_vector sequence(n); + thrust::sequence(sequence.begin(), sequence.end(), T(0)); + for (size_t i = 0; i < num_samples; ++i) { Vector shuffled(sequence.begin(), sequence.end()); ShuffleFunc{}(shuffled.begin(), shuffled.end(), g); thrust::host_vector tmp(shuffled.begin(), shuffled.end()); - - for (auto j = 0ull; j < m; j++) + for (size_t j = 0; j < n; ++j) { - index_sum[tmp[j]] += j; + expected_value[j] += static_cast(tmp[j]); } } - double expected_average_position = static_cast(m - 1) / 2; - double chi_squared = 0.0; - for (auto j = 0ull; j < m; j++) + double mu = (n - 1) / 2.0; + double sigma = std::sqrt((n * n - 1) / 12.0); + double zmax = 0.0; + for (size_t i = 0; i < n; ++i) { - double average_position = static_cast(index_sum[j]) / num_samples; - chi_squared += std::pow(expected_average_position - average_position, 2) / expected_average_position; + double mean = expected_value[i] / double(num_samples); + double z = std::abs(mean - mu) / (sigma / std::sqrt(double(num_samples))); + zmax = std::max(zmax, std::abs(z)); } - // Tabulated chi-squared critical value for m-1=19 degrees of freedom - // and 99.9% confidence - double confidence_threshold = 43.82; - ASSERT_LT(chi_squared, confidence_threshold); + + double alpha = 0.05; + double zcrit = inverse_erf(1.0 - alpha / (2.0 * n)) * std::sqrt(2.0); + ASSERT_LT(zmax, zcrit); } TYPED_TEST(ShuffleVectorTests, TestShuffleKeyPosition) @@ -765,8 +403,9 @@ void TestShuffleUniformPermutationBase() { chi_squared += std::pow(expected_count - kv.second, 2) / expected_count; } - double p_score = CephesFunctions::cephes_igamc((double) (total_permutations - 1) / 2.0, chi_squared / 2.0); - ASSERT_GT(p_score, 0.01); + // 119 degrees of freedom, 95% confidence + const double critical_value = 145.461; + ASSERT_LT(chi_squared, critical_value); } TYPED_TEST(ShuffleVectorTests, TestShuffleUniformPermutation) @@ -782,132 +421,3 @@ TYPED_TEST(ShuffleVectorTests, TestShuffleUniformPermutationIterator) using Vector = typename TestFixture::input_type; TestShuffleUniformPermutationBase(); } -template -void TestShuffleEvenSpacingBetweenOccurancesBase() -{ - using T = typename Vector::value_type; - const uint64_t shuffle_size = 10; - const uint64_t num_samples = 1000; - - thrust::host_vector h_results; - Vector sequence(shuffle_size); - thrust::sequence(sequence.begin(), sequence.end(), 0); - thrust::default_random_engine g(0xD6); - for (auto i = 0ull; i < num_samples; i++) - { - ShuffleFunc{}(sequence.begin(), sequence.end(), g); - thrust::host_vector tmp(sequence.begin(), sequence.end()); - h_results.insert(h_results.end(), sequence.begin(), sequence.end()); - } - - std::vector>> distance_between( - num_samples, std::vector>(num_samples, std::vector(shuffle_size, 0))); - - for (uint64_t sample = 0; sample < num_samples; sample++) - { - for (uint64_t i = 0; i < shuffle_size - 1; i++) - { - for (uint64_t j = 1; j < shuffle_size - i; j++) - { - T val_1 = h_results[sample * shuffle_size + i]; - T val_2 = h_results[sample * shuffle_size + i + j]; - distance_between[val_1][val_2][j]++; - distance_between[val_2][val_1][shuffle_size - j]++; - } - } - } - - const double expected_occurances = (double) num_samples / (shuffle_size - 1); - for (uint64_t val_1 = 0; val_1 < shuffle_size; val_1++) - { - for (uint64_t val_2 = val_1 + 1; val_2 < shuffle_size; val_2++) - { - double chi_squared = 0.0; - auto& distances = distance_between[val_1][val_2]; - for (uint64_t i = 1; i < shuffle_size; i++) - { - chi_squared += std::pow((double) distances[i] - expected_occurances, 2) / expected_occurances; - } - - double p_score = CephesFunctions::cephes_igamc((double) (shuffle_size - 2) / 2.0, chi_squared / 2.0); - ASSERT_GT(p_score, 0.01); - } - } -} - -TYPED_TEST(ShuffleVectorTests, TestShuffleEvenSpacingBetweenOccurances) -{ - SCOPED_TRACE(testing::Message() << "with device_id= " << test::set_device_from_ctest()); - using Vector = typename TestFixture::input_type; - TestShuffleEvenSpacingBetweenOccurancesBase(); -} - -TYPED_TEST(ShuffleVectorTests, TestShuffleEvenSpacingBetweenOccurancesIterator) -{ - SCOPED_TRACE(testing::Message() << "with device_id= " << test::set_device_from_ctest()); - using Vector = typename TestFixture::input_type; - TestShuffleEvenSpacingBetweenOccurancesBase(); -} - -template -void TestShuffleEvenDistributionBase() -{ - using T = typename Vector::value_type; - const uint64_t shuffle_sizes[] = {10, 100, 500}; - thrust::default_random_engine g(0xD5); - for (auto shuffle_size : shuffle_sizes) - { - if (shuffle_size > (uint64_t) std::numeric_limits::max()) - { - continue; - } - const uint64_t num_samples = shuffle_size == 500 ? 1000 : 200; - - std::vector counts(shuffle_size * shuffle_size, 0); - Vector sequence(shuffle_size); - for (auto i = 0ull; i < num_samples; i++) - { - thrust::sequence(sequence.begin(), sequence.end(), 0); - ShuffleFunc{}(sequence.begin(), sequence.end(), g); - thrust::host_vector tmp(sequence.begin(), sequence.end()); - for (uint64_t j = 0; j < shuffle_size; j++) - { - assert(j < tmp.size()); - counts.at(j * shuffle_size + tmp[j])++; - } - } - - const double expected_occurances = (double) num_samples / shuffle_size; - for (uint64_t i = 0; i < shuffle_size; i++) - { - double chi_squared_pos = 0.0; - double chi_squared_num = 0.0; - for (uint64_t j = 0; j < shuffle_size; j++) - { - auto count_pos = counts.at(i * shuffle_size + j); - auto count_num = counts.at(j * shuffle_size + i); - chi_squared_pos += std::pow((double) count_pos - expected_occurances, 2) / expected_occurances; - chi_squared_num += std::pow((double) count_num - expected_occurances, 2) / expected_occurances; - } - - double p_score_pos = CephesFunctions::cephes_igamc((double) (shuffle_size - 1) / 2.0, chi_squared_pos / 2.0); - ASSERT_GT(p_score_pos, 0.001 / (double) shuffle_size); - - double p_score_num = CephesFunctions::cephes_igamc((double) (shuffle_size - 1) / 2.0, chi_squared_num / 2.0); - ASSERT_GT(p_score_num, 0.001 / (double) shuffle_size); - } - } -} - -TYPED_TEST(ShuffleVectorTests, TestShuffleEvenDistribution) -{ - SCOPED_TRACE(testing::Message() << "with device_id= " << test::set_device_from_ctest()); - using Vector = typename TestFixture::input_type; - TestShuffleEvenDistributionBase(); -} -TYPED_TEST(ShuffleVectorTests, TestShuffleEvenDistributionIterator) -{ - SCOPED_TRACE(testing::Message() << "with device_id= " << test::set_device_from_ctest()); - using Vector = typename TestFixture::input_type; - TestShuffleEvenDistributionBase(); -} diff --git a/projects/hipccl/hipccl3/rocthrust/test/test_tabulate_output_iterator.cpp b/projects/hipccl/hipccl3/rocthrust/test/test_tabulate_output_iterator.cpp index 0ba3ce624a11..cb7648c77770 100644 --- a/projects/hipccl/hipccl3/rocthrust/test/test_tabulate_output_iterator.cpp +++ b/projects/hipccl/hipccl3/rocthrust/test/test_tabulate_output_iterator.cpp @@ -15,6 +15,8 @@ * limitations under the License. */ +#define CCCL_IGNORE_DEPRECATED_API + #include #include #include diff --git a/projects/hipccl/hipccl3/rocthrust/test/test_transform_input_output_iterator.cpp b/projects/hipccl/hipccl3/rocthrust/test/test_transform_input_output_iterator.cpp index de201b58949c..0ffd2fee506d 100644 --- a/projects/hipccl/hipccl3/rocthrust/test/test_transform_input_output_iterator.cpp +++ b/projects/hipccl/hipccl3/rocthrust/test/test_transform_input_output_iterator.cpp @@ -32,11 +32,11 @@ # include #endif -// There is an unfortunate miscompilation of the gcc-13 vectorizer leading to OOB writes +// There is an unfortunate miscompilation of the gcc-12/gcc-13 vectorizer leading to OOB writes // Adding this attribute suffices that this miscompilation does not appear anymore -#if THRUST_COMPILER(GCC, >=, 13) +#if THRUST_COMPILER(GCC, >=, 12) # define THRUST_DISABLE_BROKEN_GCC_VECTORIZER __attribute__((optimize("no-tree-vectorize"))) -#else // THRUST_COMPILER(GCC, <, 13) +#else // THRUST_COMPILER(GCC, <, 12) # define THRUST_DISABLE_BROKEN_GCC_VECTORIZER #endif diff --git a/projects/hipccl/hipccl3/rocthrust/test/test_transform_scan.cpp b/projects/hipccl/hipccl3/rocthrust/test/test_transform_scan.cpp index eec9f1bb951d..4df7d69ef3ed 100644 --- a/projects/hipccl/hipccl3/rocthrust/test/test_transform_scan.cpp +++ b/projects/hipccl/hipccl3/rocthrust/test/test_transform_scan.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "test_param_fixtures.hpp" @@ -464,3 +465,129 @@ TYPED_TEST(TransformScanVariablesTests, TestValueCategoryDeduction) ASSERT_EQ((thrust::device_vector{0, 5, 5, 5, 8, 8, 8, 8, 8, 8}), vec); } + +// User-defined Int type with poison default constructor +struct TransformInt +{ + int value; + + THRUST_HOST_DEVICE TransformInt() + : value(999) + {} + THRUST_HOST_DEVICE TransformInt(int v) + : value(v) + {} + + THRUST_HOST_DEVICE friend TransformInt operator+(TransformInt a, TransformInt b) + { + return TransformInt(a.value + b.value); + } + THRUST_HOST_DEVICE friend TransformInt operator*(TransformInt a, TransformInt b) + { + return TransformInt(a.value * b.value); + } + THRUST_HOST_DEVICE friend bool operator==(TransformInt a, TransformInt b) + { + return a.value == b.value; + } + friend std::ostream& operator<<(std::ostream& os, TransformInt i) + { + return os << i.value; + } +}; + +// Test edge cases for parallel transform_scan with non-additive operations +TEST(TransformScanTests, TestTransformScanEdgeCases) +{ + SCOPED_TRACE(testing::Message() << "with device_id= " << test::set_device_from_ctest()); + + using thrust::placeholders::_1; + + // Large array, transform_inclusive_scan, multiplies, and init + { + const int n = 10000; + thrust::device_vector d_input(n); + thrust::tabulate(d_input.begin(), d_input.end(), (_1 % 5) + 1); + + thrust::device_vector d_output(n); + auto r = thrust::transform_inclusive_scan( + d_input.begin(), d_input.end(), d_output.begin(), _THRUST_STD::negate(), 2, _THRUST_STD::multiplies<>{}); + ASSERT_EQ((d_output.end() == r), true); + + thrust::host_vector h_input = d_input; + thrust::host_vector h_output(n); + thrust::transform_inclusive_scan( + h_input.begin(), h_input.end(), h_output.begin(), _THRUST_STD::negate(), 2, _THRUST_STD::multiplies<>{}); + ASSERT_EQ(d_output, h_output); + } + + // transform_exclusive_scan with multiplies on large array + { + const int n = 10000; + thrust::device_vector d_input(n); + thrust::tabulate(d_input.begin(), d_input.end(), (_1 % 3) + 1); + + thrust::device_vector d_output(n); + auto r = thrust::transform_exclusive_scan( + d_input.begin(), d_input.end(), d_output.begin(), _THRUST_STD::negate(), 5, _THRUST_STD::multiplies<>{}); + ASSERT_EQ((d_output.end() == r), true); + + thrust::host_vector h_input = d_input; + thrust::host_vector h_output(n); + thrust::transform_exclusive_scan( + h_input.begin(), h_input.end(), h_output.begin(), _THRUST_STD::negate(), 5, _THRUST_STD::multiplies<>{}); + ASSERT_EQ(d_output, h_output); + } + + // User-defined type with poison default constructor + { + thrust::device_vector vec(5, TransformInt{1}); + thrust::transform_exclusive_scan( + vec.begin(), + vec.end(), + vec.begin(), + _THRUST_STD::identity{}, + TransformInt{100}, + _THRUST_STD::plus()); + ASSERT_EQ(vec, + (thrust::device_vector{ + TransformInt{100}, TransformInt{101}, TransformInt{102}, TransformInt{103}, TransformInt{104}})); + } + + // In-place transform_exclusive_scan with multiplies + { + thrust::device_vector vec = {2, 3, 4, 5}; + auto r = thrust::transform_exclusive_scan( + vec.begin(), vec.end(), vec.begin(), _THRUST_STD::negate(), 10, _THRUST_STD::multiplies<>{}); + ASSERT_EQ((vec.end() == r), true); + ASSERT_EQ(vec, (thrust::device_vector{10, -20, 60, -240})); + } + + // Boundary case at OMP parallel_scan_threshold (1024 elements) + { + const int n = 1024; + thrust::device_vector d_input(n); + thrust::tabulate(d_input.begin(), d_input.end(), _1 + 1); + + thrust::device_vector d_output(n); + auto r = thrust::transform_inclusive_scan( + d_input.begin(), d_input.end(), d_output.begin(), thrust::square(), 1, _THRUST_STD::multiplies<>{}); + ASSERT_EQ((d_output.end() == r), true); + + thrust::host_vector h_input = d_input; + thrust::host_vector h_output(n); + thrust::transform_inclusive_scan( + h_input.begin(), h_input.end(), h_output.begin(), thrust::square(), 1, _THRUST_STD::multiplies<>{}); + ASSERT_EQ(d_output, h_output); + } + + // Very small array (2 elements) + { + thrust::device_vector d_input = {3, 7}; + thrust::device_vector d_output(2); + auto r = thrust::transform_inclusive_scan( + d_input.begin(), d_input.end(), d_output.begin(), _THRUST_STD::negate(), 2, _THRUST_STD::multiplies<>{}); + ASSERT_EQ((d_output.end() == r), true); + ASSERT_EQ(d_output, (thrust::device_vector{-6, 42})); + } +} diff --git a/projects/hipccl/hipccl3/rocthrust/test/test_type_traits.cpp b/projects/hipccl/hipccl3/rocthrust/test/test_type_traits.cpp index 1779efa6ca9e..28973201d101 100644 --- a/projects/hipccl/hipccl3/rocthrust/test/test_type_traits.cpp +++ b/projects/hipccl/hipccl3/rocthrust/test/test_type_traits.cpp @@ -18,8 +18,13 @@ #include #include #include +#include +#include +#include #include #include +#include +#include #include #include #include @@ -37,9 +42,9 @@ #include _THRUST_STD_INCLUDE(tuple) #include _THRUST_STD_INCLUDE(utility) -#if !_THRUST_HAS_DEVICE_SYSTEM_STD -# include -#endif +#include + +#include _THRUST_LIBCXX_INCLUDE(iterator) TESTS_DEFINE(TypeTraitsTests, FullTestsParams); @@ -218,12 +223,10 @@ TEST(TypeTraitsTests, TestTriviallyRelocatable) static_assert(thrust::is_trivially_relocatable<_THRUST_STD::tuple, char>>::value, ""); #endif // THRUST_COMPILER(GCC, >=, 7) -#if _THRUST_HAS_DEVICE_SYSTEM_STD static_assert(thrust::is_trivially_relocatable< _THRUST_STD::tuple>>, thrust::tuple<_THRUST_STD::pair>, int>>>::value, ""); -#endif static_assert(!thrust::is_trivially_relocatable>::value, ""); static_assert(!thrust::is_trivially_relocatable<_THRUST_STD::pair>::value, ""); @@ -232,12 +235,206 @@ TEST(TypeTraitsTests, TestTriviallyRelocatable) // test propagation of relocatability through pair and tuple static_assert(thrust::is_trivially_relocatable::value, ""); -#if _THRUST_HAS_DEVICE_SYSTEM_STD static_assert(thrust::is_trivially_relocatable>::value, ""); -#endif static_assert(thrust::is_trivially_relocatable<_THRUST_STD::pair>::value, ""); -#if _THRUST_HAS_DEVICE_SYSTEM_STD static_assert(thrust::is_trivially_relocatable>::value, ""); -#endif static_assert(thrust::is_trivially_relocatable<_THRUST_STD::tuple>::value, ""); } + +template +inline constexpr bool system_is = _THRUST_STD::is_same_v, Tag>; + +template +inline constexpr bool traversal_is = + _THRUST_STD::is_same_v, Traversal>; + +TEST(TypeTraitsTests, TestIteratorCategoryToSystem) +{ + static_assert(system_is); + static_assert(system_is); + static_assert(system_is); + static_assert(system_is); + static_assert(system_is); + + static_assert(system_is); + static_assert(system_is); + static_assert(system_is); + static_assert(system_is); + static_assert(system_is); +} + +TEST(TypeTraitsTests, TestIteratorCategoryToTraversal) +{ + static_assert(traversal_is); + static_assert(traversal_is); + static_assert(traversal_is); + static_assert(traversal_is); + static_assert(traversal_is); + + static_assert(traversal_is); + static_assert(traversal_is); + static_assert(traversal_is); + static_assert(traversal_is); + static_assert(traversal_is); +} + +struct thrust_make_counting_iterator +{ + template + auto operator()(Args&&... args) const + { + return thrust::make_counting_iterator(_THRUST_STD::forward(args)...); + } +}; + +struct thrust_make_transform_iterator +{ + template + auto operator()(Args&&... args) const + { + return thrust::make_transform_iterator(_THRUST_STD::forward(args)...); + } +}; + +struct thrust_make_zip_iterator +{ + template + auto operator()(Args&&... args) const + { + return thrust::make_zip_iterator(_THRUST_STD::forward(args)...); + } +}; + +struct libcxx_make_counting_iterator +{ + template + auto operator()(Args&&... args) const + { + return _THRUST_LIBCXX::make_counting_iterator(_THRUST_STD::forward(args)...); + } +}; + +struct libcxx_make_transform_iterator +{ + template + auto operator()(Args&&... args) const + { + return _THRUST_LIBCXX::make_transform_iterator(_THRUST_STD::forward(args)...); + } +}; + +struct libcxx_make_zip_iterator +{ + template + auto operator()(Args&&... args) const + { + return _THRUST_LIBCXX::make_zip_iterator(_THRUST_STD::forward(args)...); + } +}; + +template +inline constexpr bool has_random_access_traversal = + _THRUST_STD::is_same_v, thrust::random_access_traversal_tag>; + +auto expected_tag(thrust::device_vector) -> thrust::device_system_tag; +auto expected_tag(thrust::host_vector) -> thrust::host_system_tag; +auto expected_tag(std::vector) -> thrust::host_system_tag; + +template +void check_any_system_propagation() +{ + auto counting_it = MakeCounting{}(0); + static_assert(_THRUST_STD::is_same_v, thrust::any_system_tag>); + static_assert(has_random_access_traversal); + + auto transform_it = MakeTransform{}(counting_it, thrust::square<>{}); + static_assert(_THRUST_STD::is_same_v, thrust::any_system_tag>); + static_assert(has_random_access_traversal); + + [[maybe_unused]] auto zip_it = MakeZip{}(counting_it, transform_it); + static_assert(_THRUST_STD::is_same_v, thrust::any_system_tag>); + static_assert(has_random_access_traversal); +} + +template +void check_vector_propagation() +{ + Vector vec{}; + using tag = decltype(expected_tag(vec)); + + auto vec_it = vec.begin(); + static_assert(_THRUST_STD::is_same_v, tag>); + static_assert(has_random_access_traversal); + + using thrust::placeholders::_1; + auto transform_it = MakeTransform{}(vec_it, _1 + 1); + static_assert(_THRUST_STD::is_same_v, tag>); + static_assert(has_random_access_traversal); + + [[maybe_unused]] auto zip_it = MakeZip{}(vec_it, transform_it); + static_assert(_THRUST_STD::is_same_v, tag>); + static_assert(has_random_access_traversal); +} + +template +void check_vector_and_any_system_propagation() +{ + Vector vec{}; + using vec_tag = decltype(expected_tag(vec)); + + auto vec_it = vec.begin(); + static_assert(_THRUST_STD::is_same_v, vec_tag>); + static_assert(has_random_access_traversal); + + auto counting_it = MakeCounting{}(0); + static_assert(_THRUST_STD::is_same_v, thrust::any_system_tag>); + static_assert(has_random_access_traversal); + + [[maybe_unused]] auto zip_it = MakeZip{}(vec_it, counting_it); + static_assert(_THRUST_STD::is_same_v, vec_tag>); + static_assert(has_random_access_traversal); +} + +TEST(TypeTraitsTests, TestIteratorSystemAndTraversalPropagationAnySystem) +{ + check_any_system_propagation(); + check_any_system_propagation(); + check_any_system_propagation(); + check_any_system_propagation(); + check_any_system_propagation(); + check_any_system_propagation(); + check_any_system_propagation(); + check_any_system_propagation(); +} + +TEST(TypeTraitsTests, TestIteratorSystemAndTraversalPropagationVectors) +{ + check_vector_propagation, thrust_make_transform_iterator, thrust_make_zip_iterator>(); + check_vector_propagation, thrust_make_transform_iterator, thrust_make_zip_iterator>(); + check_vector_propagation, thrust_make_transform_iterator, thrust_make_zip_iterator>(); + check_vector_propagation, thrust_make_transform_iterator, libcxx_make_zip_iterator>(); + check_vector_propagation, libcxx_make_transform_iterator, thrust_make_zip_iterator>(); + check_vector_propagation, libcxx_make_transform_iterator, libcxx_make_zip_iterator>(); + check_vector_propagation, thrust_make_transform_iterator, libcxx_make_zip_iterator>(); + check_vector_propagation, libcxx_make_transform_iterator, thrust_make_zip_iterator>(); + check_vector_propagation, libcxx_make_transform_iterator, libcxx_make_zip_iterator>(); + check_vector_propagation, thrust_make_transform_iterator, libcxx_make_zip_iterator>(); + check_vector_propagation, libcxx_make_transform_iterator, thrust_make_zip_iterator>(); + check_vector_propagation, libcxx_make_transform_iterator, libcxx_make_zip_iterator>(); +} + +TEST(TypeTraitsTests, TestIteratorSystemAndTraversalPropagationVectorsAndAnySystem) +{ + check_vector_and_any_system_propagation, thrust_make_counting_iterator, thrust_make_zip_iterator>(); + check_vector_and_any_system_propagation, thrust_make_counting_iterator, thrust_make_zip_iterator>(); + check_vector_and_any_system_propagation, thrust_make_counting_iterator, thrust_make_zip_iterator>(); + check_vector_and_any_system_propagation, thrust_make_counting_iterator, libcxx_make_zip_iterator>(); + check_vector_and_any_system_propagation, libcxx_make_counting_iterator, thrust_make_zip_iterator>(); + check_vector_and_any_system_propagation, libcxx_make_counting_iterator, libcxx_make_zip_iterator>(); + check_vector_and_any_system_propagation, thrust_make_counting_iterator, libcxx_make_zip_iterator>(); + check_vector_and_any_system_propagation, libcxx_make_counting_iterator, thrust_make_zip_iterator>(); + check_vector_and_any_system_propagation, libcxx_make_counting_iterator, libcxx_make_zip_iterator>(); + check_vector_and_any_system_propagation, thrust_make_counting_iterator, libcxx_make_zip_iterator>(); + check_vector_and_any_system_propagation, libcxx_make_counting_iterator, thrust_make_zip_iterator>(); + check_vector_and_any_system_propagation, libcxx_make_counting_iterator, libcxx_make_zip_iterator>(); +} diff --git a/projects/hipccl/hipccl3/rocthrust/test/test_unique.cpp b/projects/hipccl/hipccl3/rocthrust/test/test_unique.cpp index c4323756022c..bb9da629e7c2 100644 --- a/projects/hipccl/hipccl3/rocthrust/test/test_unique.cpp +++ b/projects/hipccl/hipccl3/rocthrust/test/test_unique.cpp @@ -15,7 +15,9 @@ * limitations under the License. */ +#include #include +#include #include #include #include @@ -448,3 +450,70 @@ TEST(UniqueTests, TestUniqueDevice) } } } + +struct check_valid_item_op +{ + unsigned int* error_counter{}; + int expected_upper_bound{}; + + __device__ bool operator()(const int lhs, const int rhs) const + { + if (lhs > expected_upper_bound || rhs > expected_upper_bound) + { + if (error_counter) + { + atomicAdd(error_counter, 1u); + } + return false; + } + return lhs == rhs; + } +}; + +// The equality operator must not be invoked for out-of-bounds items. +TEST(UniqueTests, TestUniqueWithCustomEqualityOp) +{ + SCOPED_TRACE(testing::Message() << "with device_id= " << test::set_device_from_ctest()); + + constexpr int num_items = 1000; + auto data = thrust::make_counting_iterator(0); + + thrust::device_vector error_counter(1, 0); + auto const error_counter_ptr = thrust::raw_pointer_cast(error_counter.data()); + + thrust::device_vector unique_out(num_items); + auto unique_out_end = thrust::unique_copy( + data, data + num_items, unique_out.begin(), check_valid_item_op{error_counter_ptr, num_items - 1}); + + ASSERT_EQ(unique_out_end - unique_out.begin(), num_items); + ASSERT_EQ(error_counter[0], 0u); + ASSERT_TRUE(thrust::equal(unique_out.cbegin(), unique_out.cend(), data)); +} + +template +struct NonConstAdapter +{ + F f; + NonConstAdapter(const F& func) + : f(func) + {} + + template + __device__ auto operator()(Args&&... args) -> decltype(f(_THRUST_STD::forward(args)...)) + { + return f(_THRUST_STD::forward(args)...); + } +}; + +TEST(UniqueTests, TestUniqueWithCustomEqualityOpMutable) +{ + SCOPED_TRACE(testing::Message() << "with device_id= " << test::set_device_from_ctest()); + + thrust::device_vector in = {1, 1, 2, 3, 4, 4, 5}; + auto new_end = thrust::unique(in.begin(), in.end(), NonConstAdapter>(thrust::equal_to{})); + + thrust::device_vector expected = {1, 2, 3, 4, 5}; + ASSERT_EQ(new_end - in.begin(), 5); + in.resize(new_end - in.begin()); + ASSERT_EQ(in, expected); +} diff --git a/projects/hipccl/hipccl3/rocthrust/test/test_utils.hpp b/projects/hipccl/hipccl3/rocthrust/test/test_utils.hpp index fee7a8819b90..8d265d83c278 100644 --- a/projects/hipccl/hipccl3/rocthrust/test/test_utils.hpp +++ b/projects/hipccl/hipccl3/rocthrust/test/test_utils.hpp @@ -37,13 +37,7 @@ #include -#if _THRUST_HAS_DEVICE_SYSTEM_STD -# include _THRUST_STD_INCLUDE(__algorithm/min.h) -#else -# include - -# include -#endif +#include _THRUST_STD_INCLUDE(__algorithm/min.h) #define TEST_EVENT_WAIT(e) test_event_wait(e) diff --git a/projects/hipccl/hipccl3/rocthrust/test/test_xor_combine_engine_max.cpp b/projects/hipccl/hipccl3/rocthrust/test/test_xor_combine_engine_max.cpp new file mode 100644 index 000000000000..3a415133353f --- /dev/null +++ b/projects/hipccl/hipccl3/rocthrust/test/test_xor_combine_engine_max.cpp @@ -0,0 +1,42 @@ +/* + * Copyright 2008-2013 NVIDIA Corporation + * Modifications Copyright© 2025-2026 Advanced Micro Devices, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "test_utils.hpp" + +TEST(XorCombineEngineMaxTests, TestLog2) +{ + static_assert(thrust::random::detail::log2(1u) == 0u); + static_assert(thrust::random::detail::log2(2u) == 1u); + static_assert(thrust::random::detail::log2(3u) == 1u); + static_assert(thrust::random::detail::log2(4u) == 2u); + static_assert(thrust::random::detail::log2(5u) == 2u); + static_assert(thrust::random::detail::log2(6u) == 2u); + static_assert(thrust::random::detail::log2(7u) == 2u); + static_assert(thrust::random::detail::log2(8u) == 3u); + static_assert(thrust::random::detail::log2(9u) == 3u); + static_assert(thrust::random::detail::log2(15u) == 3u); + static_assert(thrust::random::detail::log2(16u) == 4u); + static_assert(thrust::random::detail::log2(17u) == 4u); + static_assert(thrust::random::detail::log2(127u) == 6u); + static_assert(thrust::random::detail::log2(128u) == 7u); + static_assert(thrust::random::detail::log2(129u) == 7u); + static_assert(thrust::random::detail::log2(256u) == 8u); + static_assert(thrust::random::detail::log2(511u) == 8u); + static_assert(thrust::random::detail::log2(512u) == 9u); +} diff --git a/projects/hipccl/hipccl3/rocthrust/test/test_zip_iterator_sort.cpp b/projects/hipccl/hipccl3/rocthrust/test/test_zip_iterator_sort.cpp index 61a6532d6937..89af472a1846 100644 --- a/projects/hipccl/hipccl3/rocthrust/test/test_zip_iterator_sort.cpp +++ b/projects/hipccl/hipccl3/rocthrust/test/test_zip_iterator_sort.cpp @@ -47,10 +47,10 @@ TYPED_TEST(ZipIteratorStableSortTests, TestZipIteratorStableSort) device_vector d2 = h2; // sort on host - stable_sort(make_zip_iterator(h1.begin(), h2.begin()), make_zip_iterator(h1.end(), h2.end())); + thrust::stable_sort(make_zip_iterator(h1.begin(), h2.begin()), make_zip_iterator(h1.end(), h2.end())); // sort on device - stable_sort(make_zip_iterator(d1.begin(), d2.begin()), make_zip_iterator(d1.end(), d2.end())); + thrust::stable_sort(make_zip_iterator(d1.begin(), d2.begin()), make_zip_iterator(d1.end(), d2.end())); ASSERT_EQ_QUIET(h1, d1); ASSERT_EQ_QUIET(h2, d2); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/CMakeLists.txt b/projects/hipccl/hipccl3/rocthrust/testing/CMakeLists.txt index e512d67c6fbc..a3f25817dfb5 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/CMakeLists.txt +++ b/projects/hipccl/hipccl3/rocthrust/testing/CMakeLists.txt @@ -218,7 +218,6 @@ add_thrust_test("memory") add_thrust_test("merge") add_thrust_test("merge_by_key") add_thrust_test("merge_key_value") -add_thrust_test("metaprogramming") add_thrust_test("min_and_max") add_thrust_test("min_element") add_thrust_test("minimum_system") @@ -317,6 +316,7 @@ add_thrust_test("vector") add_thrust_test("vector_allocators") add_thrust_test("vector_insert") add_thrust_test("vector_manipulation") +add_thrust_test("xor_combine_engine_max") add_thrust_test("zip_function") add_thrust_test("zip_iterator") add_thrust_test("zip_iterator_reduce") diff --git a/projects/hipccl/hipccl3/rocthrust/testing/allocator_aware_policies.cu b/projects/hipccl/hipccl3/rocthrust/testing/allocator_aware_policies.cu index 59a27bc93f2b..c5b4aa42c899 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/allocator_aware_policies.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/allocator_aware_policies.cu @@ -17,10 +17,10 @@ #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include @@ -116,9 +116,9 @@ struct TestAllocatorAttachment }; using sequential_info = policy_info; -using cpp_par_info = policy_info; -using omp_par_info = policy_info; -using tbb_par_info = policy_info; +using cpp_par_info = policy_info; +using omp_par_info = policy_info; +using tbb_par_info = policy_info; using hip_par_info = policy_info; diff --git a/projects/hipccl/hipccl3/rocthrust/testing/binary_search.cu b/projects/hipccl/hipccl3/rocthrust/testing/binary_search.cu index 9a9806bfa14e..4417786787ff 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/binary_search.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/binary_search.cu @@ -223,11 +223,11 @@ void TestScalarEqualRangeSimple() DECLARE_VECTOR_UNITTEST(TestScalarEqualRangeSimple); template -thrust::pair +_THRUST_STD::pair equal_range(my_system& system, ForwardIterator first, ForwardIterator /*last*/, const LessThanComparable& /*value*/) { system.validate_dispatch(); - return thrust::make_pair(first, first); + return _THRUST_STD::make_pair(first, first); } void TestScalarEqualRangeDispatchExplicit() @@ -242,11 +242,11 @@ void TestScalarEqualRangeDispatchExplicit() DECLARE_UNITTEST(TestScalarEqualRangeDispatchExplicit); template -thrust::pair +_THRUST_STD::pair equal_range(my_tag, ForwardIterator first, ForwardIterator /*last*/, const LessThanComparable& /*value*/) { *first = 13; - return thrust::make_pair(first, first); + return _THRUST_STD::make_pair(first, first); } void TestScalarEqualRangeDispatchImplicit() diff --git a/projects/hipccl/hipccl3/rocthrust/testing/complex.cu b/projects/hipccl/hipccl3/rocthrust/testing/complex.cu index fe003bac10f9..34fb9b8bd1a0 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/complex.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/complex.cu @@ -39,7 +39,6 @@ THRUST_DIAG_SUPPRESS_MSVC(4244) // conversion from 'const T1' to 'const T', poss namespace { - // Helper construction to create a double from float and // vice versa to test thrust::complex promoting operators. template @@ -60,7 +59,6 @@ struct other_floating_point_type template using other_floating_point_type_t = typename other_floating_point_type::type; - } // anonymous namespace template diff --git a/projects/hipccl/hipccl3/rocthrust/testing/copy.cu b/projects/hipccl/hipccl3/rocthrust/testing/copy.cu index 5272790987ee..b504905fd509 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/copy.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/copy.cu @@ -34,10 +34,6 @@ #include -#if !_THRUST_HAS_DEVICE_SYSTEM_STD -# include -#endif - #if _CCCL_COMPILER(GCC, >=, 11) # define THRUST_DISABLE_BROKEN_GCC_VECTORIZER __attribute__((optimize("no-tree-vectorize"))) #else @@ -427,7 +423,6 @@ DECLARE_INTEGRAL_VARIABLE_UNITTEST(TestCopyIfStencil); namespace { - struct object_with_non_trivial_ctor { // This struct will only properly assign if its `magic` member is @@ -471,7 +466,6 @@ struct always_true return true; } }; - } // namespace void TestCopyIfNonTrivial() @@ -757,7 +751,6 @@ struct iterator_traits }; } // namespace std -# if _THRUST_HAS_DEVICE_SYSTEM_STD _THRUST_STD_NAMESPACE_BEGIN template <> struct iterator_traits @@ -768,7 +761,6 @@ struct iterator_traits using difference_type = _THRUST_STD::ptrdiff_t; }; _THRUST_STD_NAMESPACE_END -# endif void TestCopyWithBigIndexesHelper(int magnitude) { diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cpp/adjacent_difference.cu b/projects/hipccl/hipccl3/rocthrust/testing/cpp/adjacent_difference.cu index cb1de515ef75..24dce3c3ec3f 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/cpp/adjacent_difference.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/cpp/adjacent_difference.cu @@ -24,6 +24,12 @@ struct detect_wrong_difference { + using difference_type = void; + using value_type = long long; + using pointer = void; + using reference = detect_wrong_difference; + using iterator_category = _THRUST_STD::output_iterator_tag; + bool* flag; THRUST_HOST_DEVICE detect_wrong_difference operator++() const diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cuda/binary_search.cu b/projects/hipccl/hipccl3/rocthrust/testing/cuda/binary_search.cu index 4868811ab786..7277a693a890 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/cuda/binary_search.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/cuda/binary_search.cu @@ -18,16 +18,17 @@ #include #include #include -#include #include +#include _THRUST_STD_INCLUDE(utility) + #include void TestEqualRangeOnStream() { // Regression test for GH issue #921 (nvbug 2173437) using vector_t = typename thrust::device_vector; using iterator_t = typename vector_t::iterator; - using result_t = thrust::pair; + using result_t = cuda::std::pair; vector_t input(10); thrust::sequence(thrust::device, input.begin(), input.end(), 0); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cuda/cudart.cu b/projects/hipccl/hipccl3/rocthrust/testing/cuda/cudart.cu index 03592f5f2496..ba3dce85cd1b 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/cuda/cudart.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/cuda/cudart.cu @@ -15,9 +15,9 @@ * limitations under the License. */ -#include +#include -#include +#include #include @@ -28,6 +28,6 @@ void TestCudaMallocResultAligned(const std::size_t n) cudaMalloc(&ptr, n * sizeof(T)); cudaFree(ptr); - ASSERT_EQUAL(true, thrust::detail::util::is_aligned(ptr)); + ASSERT_EQUAL(true, ::cuda::std::is_sufficiently_aligned(ptr)); } DECLARE_VARIABLE_UNITTEST(TestCudaMallocResultAligned); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cuda/find.cu b/projects/hipccl/hipccl3/rocthrust/testing/cuda/find.cu index 2668aa15a2fc..295a2f2bf8c2 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/cuda/find.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/cuda/find.cu @@ -17,54 +17,10 @@ #include #include +#include #include -template -struct equal_to_value_pred -{ - T value; - - equal_to_value_pred(T value) - : value(value) - {} - - _CCCL_HOST_DEVICE bool operator()(T v) const - { - return v == value; - } -}; - -template -struct not_equal_to_value_pred -{ - T value; - - not_equal_to_value_pred(T value) - : value(value) - {} - - _CCCL_HOST_DEVICE bool operator()(T v) const - { - return v != value; - } -}; - -template -struct less_than_value_pred -{ - T value; - - less_than_value_pred(T value) - : value(value) - {} - - _CCCL_HOST_DEVICE bool operator()(T v) const - { - return v < value; - } -}; - #ifdef THRUST_TEST_DEVICE_SIDE template __global__ void find_kernel(ExecutionPolicy exec, Iterator first, Iterator last, T value, Iterator2 result) @@ -142,9 +98,10 @@ void TestFindIfDevice(ExecutionPolicy exec) using iter_type = typename thrust::device_vector::iterator; thrust::device_vector d_result(1); - h_iter = thrust::find_if(h_data.begin(), h_data.end(), equal_to_value_pred(0)); + using thrust::placeholders::_1; + h_iter = thrust::find_if(h_data.begin(), h_data.end(), _1 == 0); - find_if_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), equal_to_value_pred(0), d_result.begin()); + find_if_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), _1 == 0, d_result.begin()); { cudaError_t const err = cudaDeviceSynchronize(); ASSERT_EQUAL(cudaSuccess, err); @@ -156,9 +113,9 @@ void TestFindIfDevice(ExecutionPolicy exec) { int sample = h_data[i]; - h_iter = thrust::find_if(h_data.begin(), h_data.end(), equal_to_value_pred(sample)); + h_iter = thrust::find_if(h_data.begin(), h_data.end(), _1 == sample); - find_if_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), equal_to_value_pred(sample), d_result.begin()); + find_if_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), _1 == sample, d_result.begin()); { cudaError_t const err = cudaDeviceSynchronize(); ASSERT_EQUAL(cudaSuccess, err); @@ -198,9 +155,10 @@ void TestFindIfNotDevice(ExecutionPolicy exec) using iter_type = typename thrust::device_vector::iterator; thrust::device_vector d_result(1); - h_iter = thrust::find_if_not(h_data.begin(), h_data.end(), not_equal_to_value_pred(0)); + using thrust::placeholders::_1; + h_iter = thrust::find_if_not(h_data.begin(), h_data.end(), _1 != 0); - find_if_not_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), not_equal_to_value_pred(0), d_result.begin()); + find_if_not_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), _1 != 0, d_result.begin()); { cudaError_t const err = cudaDeviceSynchronize(); ASSERT_EQUAL(cudaSuccess, err); @@ -212,10 +170,9 @@ void TestFindIfNotDevice(ExecutionPolicy exec) { int sample = h_data[i]; - h_iter = thrust::find_if_not(h_data.begin(), h_data.end(), not_equal_to_value_pred(sample)); + h_iter = thrust::find_if_not(h_data.begin(), h_data.end(), _1 != sample); - find_if_not_kernel<<<1, 1>>>( - exec, d_data.begin(), d_data.end(), not_equal_to_value_pred(sample), d_result.begin()); + find_if_not_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), _1 != sample, d_result.begin()); { cudaError_t const err = cudaDeviceSynchronize(); ASSERT_EQUAL(cudaSuccess, err); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cuda/memory.cu b/projects/hipccl/hipccl3/rocthrust/testing/cuda/memory.cu index b6d1cb4a0e51..7c5b37f4f690 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/cuda/memory.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/cuda/memory.cu @@ -67,7 +67,7 @@ void TestGetTemporaryBufferDeviceSeq() const std::ptrdiff_t n = 9001; using pointer = thrust::pointer; - using ptr_and_sz_type = thrust::pair; + using ptr_and_sz_type = cuda::std::pair; thrust::device_vector d_result(1); get_temporary_buffer_kernel<<<1, 1>>>(n, d_result.begin()); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cuda/merge_by_key.cu b/projects/hipccl/hipccl3/rocthrust/testing/cuda/merge_by_key.cu index b477cd3dd092..d52ebaf7cf40 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/cuda/merge_by_key.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/cuda/merge_by_key.cu @@ -72,7 +72,7 @@ void TestMergeByKeyDevice(ExecutionPolicy exec) using Iterator = typename thrust::device_vector::iterator; - thrust::device_vector> result_ends(1); + thrust::device_vector> result_ends(1); merge_by_key_kernel<<<1, 1>>>( exec, @@ -88,7 +88,7 @@ void TestMergeByKeyDevice(ExecutionPolicy exec) cudaError_t const err = cudaDeviceSynchronize(); ASSERT_EQUAL(cudaSuccess, err); - thrust::pair ends = result_ends[0]; + cuda::std::pair ends = result_ends[0]; ASSERT_EQUAL_QUIET(result_key.end(), ends.first); ASSERT_EQUAL_QUIET(result_val.end(), ends.second); @@ -137,7 +137,7 @@ void TestMergeByKeyCudaStreams() cudaStream_t s; cudaStreamCreate(&s); - thrust::pair ends = thrust::merge_by_key( + cuda::std::pair ends = thrust::merge_by_key( thrust::cuda::par.on(s), a_key.begin(), a_key.end(), diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cuda/minmax_element.cu b/projects/hipccl/hipccl3/rocthrust/testing/cuda/minmax_element.cu index bc878dbd9a82..d0a8ea35c996 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/cuda/minmax_element.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/cuda/minmax_element.cu @@ -47,7 +47,7 @@ void TestMinMaxElementDevice(ExecutionPolicy exec) typename thrust::device_vector::iterator d_max; using pair_type = - thrust::pair::iterator, typename thrust::device_vector::iterator>; + cuda::std::pair::iterator, typename thrust::device_vector::iterator>; thrust::device_vector d_result(1); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cuda/mismatch.cu b/projects/hipccl/hipccl3/rocthrust/testing/cuda/mismatch.cu index 46b12a6aaffd..8ff97fda4b1b 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/cuda/mismatch.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/cuda/mismatch.cu @@ -37,7 +37,7 @@ void TestMismatchDevice(ExecutionPolicy exec) thrust::device_vector b = {1, 2, 4, 3}; using pair_type = - thrust::pair::iterator, typename thrust::device_vector::iterator>; + cuda::std::pair::iterator, typename thrust::device_vector::iterator>; thrust::device_vector d_result(1); @@ -123,7 +123,7 @@ public: ++my_count; } - _CCCL_HOST_DEVICE bool operator==(const Wrapper& input) const + _CCCL_HOST_DEVICE bool operator==(const Wrapper&) const { return true; } diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cuda/offset_iterator.cu b/projects/hipccl/hipccl3/rocthrust/testing/cuda/offset_iterator.cu index 1a154b4b93a1..7704ce97be27 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/cuda/offset_iterator.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/cuda/offset_iterator.cu @@ -73,7 +73,12 @@ struct device_only_iterator return *this; } - _CCCL_DEVICE friend bool operator-(const device_only_iterator& a, const device_only_iterator& b) + _CCCL_DEVICE friend device_only_iterator operator+(device_only_iterator it, difference_type d) + { + return it += d; + } + + _CCCL_DEVICE friend difference_type operator-(const device_only_iterator& a, const device_only_iterator& b) { return a.m_ptr - b.m_ptr; } diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cuda/pair_sort.cu b/projects/hipccl/hipccl3/rocthrust/testing/cuda/pair_sort.cu index 0d9cc2fb7c8d..5e6c6b88c964 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/cuda/pair_sort.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/cuda/pair_sort.cu @@ -16,9 +16,10 @@ */ #include -#include #include +#include _THRUST_STD_INCLUDE(utility) + #include #ifdef THRUST_TEST_DEVICE_SIDE @@ -31,9 +32,9 @@ __global__ void stable_sort_kernel(ExecutionPolicy exec, Iterator first, Iterato struct make_pair_functor { template - THRUST_HOST_DEVICE thrust::pair operator()(const T1& x, const T2& y) + _CCCL_HOST_DEVICE cuda::std::pair operator()(const T1& x, const T2& y) { - return thrust::make_pair(x, y); + return cuda::std::make_pair(x, y); } // end operator()() }; // end make_pair_functor @@ -41,7 +42,7 @@ template void TestPairStableSortDevice(ExecutionPolicy exec) { size_t n = 10000; - using P = thrust::pair; + using P = cuda::std::pair; thrust::host_vector h_p1 = unittest::random_integers(n); thrust::host_vector h_p2 = unittest::random_integers(n); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cuda/pair_sort_by_key.cu b/projects/hipccl/hipccl3/rocthrust/testing/cuda/pair_sort_by_key.cu index 493ac533ce9d..4768e0c11410 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/cuda/pair_sort_by_key.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/cuda/pair_sort_by_key.cu @@ -16,11 +16,12 @@ */ #include -#include #include #include #include +#include _THRUST_STD_INCLUDE(utility) + #include #ifdef THRUST_TEST_DEVICE_SIDE @@ -34,9 +35,9 @@ stable_sort_by_key_kernel(ExecutionPolicy exec, Iterator1 keys_first, Iterator1 struct make_pair_functor { template - THRUST_HOST_DEVICE thrust::pair operator()(const T1& x, const T2& y) + _CCCL_HOST_DEVICE cuda::std::pair operator()(const T1& x, const T2& y) { - return thrust::make_pair(x, y); + return cuda::std::make_pair(x, y); } // end operator()() }; // end make_pair_functor @@ -44,7 +45,7 @@ template void TestPairStableSortByKeyDevice(ExecutionPolicy exec) { size_t n = 10000; - using P = thrust::pair; + using P = cuda::std::pair; // host arrays thrust::host_vector h_p1 = unittest::random_integers(n); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cuda/partition.cu b/projects/hipccl/hipccl3/rocthrust/testing/cuda/partition.cu index f9b5238e98e2..302b58b9b89d 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/cuda/partition.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/cuda/partition.cu @@ -204,7 +204,7 @@ void TestPartitionCopyDevice(ExecutionPolicy exec) thrust::device_vector true_results(2); thrust::device_vector false_results(3); - using pair_type = thrust::pair; + using pair_type = cuda::std::pair; thrust::device_vector iterators(1); partition_copy_kernel<<<1, 1>>>( @@ -290,7 +290,7 @@ void TestPartitionCopyStencilDevice(ExecutionPolicy exec) thrust::device_vector false_results(3); using iterator = typename thrust::device_vector::iterator; - using pair_type = thrust::pair; + using pair_type = cuda::std::pair; thrust::device_vector iterators(1); partition_copy_kernel<<<1, 1>>>( @@ -491,7 +491,7 @@ void TestStablePartitionCopyDevice(ExecutionPolicy exec) thrust::device_vector true_results(2); thrust::device_vector false_results(3); - using pair_type = thrust::pair; + using pair_type = cuda::std::pair; thrust::device_vector iterators(1); stable_partition_copy_kernel<<<1, 1>>>( @@ -577,7 +577,7 @@ void TestStablePartitionCopyStencilDevice(ExecutionPolicy exec) thrust::device_vector false_results(3); using iterator = typename thrust::device_vector::iterator; - using pair_type = thrust::pair; + using pair_type = cuda::std::pair; thrust::device_vector iterators(1); stable_partition_copy_kernel<<<1, 1>>>( diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cuda/reduce_by_key.cu b/projects/hipccl/hipccl3/rocthrust/testing/cuda/reduce_by_key.cu index 8a3946ae4557..f6074bff046b 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/cuda/reduce_by_key.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/cuda/reduce_by_key.cu @@ -15,12 +15,16 @@ * limitations under the License. */ +#include #include #include #include #include #include +#include _THRUST_LIBCXX_INCLUDE(iterator) +#include _THRUST_STD_INCLUDE(functional) + #include #include @@ -127,6 +131,47 @@ void initialize_values(Vector& values) values[8] = 8; } +// Checks whether the equality operator is ever invoked on out-of-bounds items +struct check_valid_item_op +{ + cuda::std::uint32_t* error_counter{}; + int expected_upper_bound{}; + + __device__ bool operator()(const int lhs, const int rhs) const + { + if (lhs > expected_upper_bound || rhs > expected_upper_bound) + { + if (error_counter) + { + atomicAdd(error_counter, 1); + } + return false; + } + return lhs == rhs; + } +}; + +template +struct check_accumulator_t_op +{ + cuda::std::uint32_t* error_counter{}; + + template && !cuda::std::is_same_v>> + _CCCL_DEVICE ReturnedT operator()(const T lhs, const U& rhs) const + { + return static_cast(lhs + rhs); + } + + template + _CCCL_DEVICE ReturnedT operator()(const ReturnedT lhs, const T& rhs) const + { + atomicAdd(error_counter, 1); + return lhs + static_cast(rhs); + } +}; + #ifdef THRUST_TEST_DEVICE_SIDE template void TestReduceByKeyDevice(ExecutionPolicy exec) @@ -137,7 +182,7 @@ void TestReduceByKeyDevice(ExecutionPolicy exec) thrust::device_vector values; using iterator_pair = - typename thrust::pair::iterator, typename thrust::device_vector::iterator>; + typename cuda::std::pair::iterator, typename thrust::device_vector::iterator>; thrust::device_vector new_last_vec(1); iterator_pair new_last; @@ -266,7 +311,7 @@ void TestReduceByKeyCudaStreams(ExecutionPolicy policy) Vector keys; Vector values; - thrust::pair new_last; + cuda::std::pair new_last; // basic test initialize_keys(keys); @@ -457,3 +502,94 @@ void TestReduceByKeyWithBigIndexes() #endif } DECLARE_UNITTEST(TestReduceByKeyWithBigIndexes); + +void TestReduceByKeyWithCustomEqualityOp() +{ + using key_vector_t = thrust::device_vector; + using val_vector_t = thrust::device_vector; + using key_t = key_vector_t::value_type; + using val_t = val_vector_t::value_type; + + auto constexpr num_items = 1000; + auto keys = cuda::make_counting_iterator(key_t{0}); + auto values = cuda::make_counting_iterator(val_t{42}); + + thrust::device_vector error_counter(1, 0); + auto const error_counter_ptr = thrust::raw_pointer_cast(error_counter.data()); + + key_vector_t unique_out(num_items); + val_vector_t aggregates_out(num_items); + auto [unique_out_end, aggregates_out_end] = thrust::reduce_by_key( + keys, + keys + num_items, + values, + unique_out.begin(), + aggregates_out.begin(), + check_valid_item_op{error_counter_ptr, num_items - 1}); + + // Verify that the number of unique keys is correct + const auto num_unique_out = cuda::std::distance(unique_out.begin(), unique_out_end); + const auto num_aggregates_out = cuda::std::distance(aggregates_out.begin(), aggregates_out_end); + ASSERT_EQUAL(num_unique_out, num_items); + ASSERT_EQUAL(num_aggregates_out, num_items); + + // Verify that the equality operator was never invoked on out-of-bounds items + ASSERT_EQUAL(error_counter[0], cuda::std::uint32_t{0}); + + // Verify that unique keys are correct + bool all_keys_correct = thrust::equal(unique_out.cbegin(), unique_out.cend(), keys); + ASSERT_EQUAL(all_keys_correct, true); + + // Verify that the aggregates are correct + bool all_values_correct = thrust::equal(aggregates_out.cbegin(), aggregates_out.cend(), values); + ASSERT_EQUAL(all_values_correct, true); +} + +DECLARE_UNITTEST(TestReduceByKeyWithCustomEqualityOp); + +void TestReduceByKeyWithDifferentAccumulatorT() +{ + using key_t = cuda::std::uint32_t; + using val_t = cuda::std::uint8_t; + using reduction_op_t = check_accumulator_t_op; + + auto constexpr num_items = 20000; + auto constexpr expected_num_uniques = 1; + constexpr auto unique_key = key_t{42U}; + auto keys = cuda::make_constant_iterator(unique_key); + auto values = cuda::make_counting_iterator(val_t{0}); + + thrust::device_vector error_counter(1, 0); + auto const error_counter_ptr = thrust::raw_pointer_cast(error_counter.data()); + + thrust::device_vector unique_out(expected_num_uniques); + thrust::device_vector aggregates_out(expected_num_uniques); + auto [unique_out_end, aggregates_out_end] = thrust::reduce_by_key( + keys, + keys + num_items, + values, + unique_out.begin(), + aggregates_out.begin(), + cuda::std::equal_to<>{}, + reduction_op_t{error_counter_ptr}); + + // Verify that the number of unique keys is correct + auto num_unique_out = cuda::std::distance(unique_out.begin(), unique_out_end); + auto num_aggregates_out = cuda::std::distance(aggregates_out.begin(), aggregates_out_end); + ASSERT_EQUAL(num_unique_out, expected_num_uniques); + ASSERT_EQUAL(num_aggregates_out, expected_num_uniques); + + // Verify that the equality operator was never invoked on out-of-bounds items + ASSERT_EQUAL(error_counter[0], cuda::std::uint32_t{0}); + + // Verify that the unique key is correct + ASSERT_EQUAL(unique_out[0], unique_key); + + // // Verify that the aggregate is correct + constexpr auto mod_val = 0x01 << cuda::std::numeric_limits::digits; + constexpr auto sum = ((num_items * (num_items - 1)) / 2); + constexpr auto expected_aggregate = static_cast(sum % mod_val); + ASSERT_EQUAL(aggregates_out[0], expected_aggregate); +} + +DECLARE_UNITTEST(TestReduceByKeyWithDifferentAccumulatorT); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cuda/set_difference_by_key.cu b/projects/hipccl/hipccl3/rocthrust/testing/cuda/set_difference_by_key.cu index 16c507e83176..6448c670d208 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/cuda/set_difference_by_key.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/cuda/set_difference_by_key.cu @@ -58,7 +58,7 @@ void TestSetDifferenceByKeyDevice(ExecutionPolicy exec) Vector ref_key{2, 5}, ref_val{0, 0}; Vector result_key(2), result_val(2); - using iter_pair = thrust::pair; + using iter_pair = cuda::std::pair; thrust::device_vector end_vec(1); @@ -111,7 +111,7 @@ void TestSetDifferenceByKeyCudaStreams() cudaStream_t s; cudaStreamCreate(&s); - thrust::pair end = thrust::set_difference_by_key( + cuda::std::pair end = thrust::set_difference_by_key( thrust::cuda::par.on(s), a_key.begin(), a_key.end(), diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cuda/set_intersection_by_key.cu b/projects/hipccl/hipccl3/rocthrust/testing/cuda/set_intersection_by_key.cu index e7a80514856d..32ca385627e5 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/cuda/set_intersection_by_key.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/cuda/set_intersection_by_key.cu @@ -55,7 +55,7 @@ void TestSetIntersectionByKeyDevice(ExecutionPolicy exec) Vector ref_key{0, 4}, ref_val{0, 0}; Vector result_key(2), result_val(2); - using iter_pair = thrust::pair; + using iter_pair = cuda::std::pair; thrust::device_vector end_vec(1); set_intersection_by_key_kernel<<<1, 1>>>( @@ -71,7 +71,7 @@ void TestSetIntersectionByKeyDevice(ExecutionPolicy exec) cudaError_t const err = cudaDeviceSynchronize(); ASSERT_EQUAL(cudaSuccess, err); - thrust::pair end = end_vec.front(); + cuda::std::pair end = end_vec.front(); ASSERT_EQUAL_QUIET(result_key.end(), end.first); ASSERT_EQUAL_QUIET(result_val.end(), end.second); @@ -115,7 +115,7 @@ void TestSetIntersectionByKeyCudaStreams(ExecutionPolicy policy) auto streampolicy = policy.on(s); - thrust::pair end = thrust::set_intersection_by_key( + cuda::std::pair end = thrust::set_intersection_by_key( streampolicy, a_key.begin(), a_key.end(), diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cuda/set_symmetric_difference_by_key.cu b/projects/hipccl/hipccl3/rocthrust/testing/cuda/set_symmetric_difference_by_key.cu index 040371b94fe9..61ff13ab68ef 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/cuda/set_symmetric_difference_by_key.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/cuda/set_symmetric_difference_by_key.cu @@ -57,7 +57,7 @@ void TestSetSymmetricDifferenceByKeyDevice(ExecutionPolicy exec) Vector ref_key{2, 3, 3, 6, 7}, ref_val{0, 1, 1, 0, 1}; Vector result_key(5), result_val(5); - using iter_pair = thrust::pair; + using iter_pair = cuda::std::pair; thrust::device_vector end_vec(1); set_symmetric_difference_by_key_kernel<<<1, 1>>>( @@ -109,7 +109,7 @@ void TestSetSymmetricDifferenceByKeyCudaStreams() cudaStream_t s; cudaStreamCreate(&s); - thrust::pair end = thrust::set_symmetric_difference_by_key( + cuda::std::pair end = thrust::set_symmetric_difference_by_key( thrust::cuda::par.on(s), a_key.begin(), a_key.end(), diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cuda/set_union_by_key.cu b/projects/hipccl/hipccl3/rocthrust/testing/cuda/set_union_by_key.cu index 9ddc547da4c9..374ea87f1e7c 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/cuda/set_union_by_key.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/cuda/set_union_by_key.cu @@ -57,7 +57,7 @@ void TestSetUnionByKeyDevice(ExecutionPolicy exec) Vector ref_key{0, 2, 3, 3, 4}, ref_val{0, 0, 1, 1, 0}; Vector result_key(5), result_val(5); - thrust::device_vector> end_vec(1); + thrust::device_vector> end_vec(1); set_union_by_key_kernel<<<1, 1>>>( exec, @@ -73,7 +73,7 @@ void TestSetUnionByKeyDevice(ExecutionPolicy exec) cudaError_t const err = cudaDeviceSynchronize(); ASSERT_EQUAL(cudaSuccess, err); - thrust::pair end = end_vec[0]; + cuda::std::pair end = end_vec[0]; ASSERT_EQUAL_QUIET(result_key.end(), end.first); ASSERT_EQUAL_QUIET(result_val.end(), end.second); @@ -108,7 +108,7 @@ void TestSetUnionByKeyCudaStreams() cudaStream_t s; cudaStreamCreate(&s); - thrust::pair end = thrust::set_union_by_key( + cuda::std::pair end = thrust::set_union_by_key( thrust::cuda::par.on(s), a_key.begin(), a_key.end(), diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cuda/sort.cu b/projects/hipccl/hipccl3/rocthrust/testing/cuda/sort.cu index 2e67de7bc174..651a8e2f9067 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/cuda/sort.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/cuda/sort.cu @@ -20,10 +20,10 @@ #include #include #include -#include #include #include +#include _THRUST_STD_INCLUDE(iterator) #include #include @@ -254,7 +254,7 @@ void TestSortWithMagnitude(int magnitude) thrust::device_vector vec(num_items); auto counting_it = thrust::make_counting_iterator(std::size_t{0}); auto key_value_it = thrust::make_transform_iterator(counting_it, index_to_key_value_op{}); - auto rev_sorted_it = thrust::make_reverse_iterator(key_value_it + num_items); + auto rev_sorted_it = _THRUST_STD::make_reverse_iterator(key_value_it + num_items); thrust::copy(rev_sorted_it, rev_sorted_it + num_items, vec.begin()); thrust::sort(vec.begin(), vec.end()); auto expected_result_it = thrust::make_transform_iterator( diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cuda/stream_wrapper.cu b/projects/hipccl/hipccl3/rocthrust/testing/cuda/stream_wrapper.cu index 536169fbb426..1b155bd0bae7 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/cuda/stream_wrapper.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/cuda/stream_wrapper.cu @@ -19,7 +19,7 @@ #include #include -#include +#include #include diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cuda/transform.cu b/projects/hipccl/hipccl3/rocthrust/testing/cuda/transform.cu index f530a976dc6f..675d3d2f65c5 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/cuda/transform.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/cuda/transform.cu @@ -18,6 +18,8 @@ #include #include +#include _THRUST_LIBCXX_INCLUDE(iterator) + #include #ifdef THRUST_TEST_DEVICE_SIDE @@ -396,11 +398,10 @@ public: } template - _CCCL_HOST_DEVICE auto operator()(Tuple&& t) const - -> decltype(detail::zip_detail::apply(std::declval(), THRUST_FWD(t))) + _CCCL_HOST_DEVICE double operator()(Tuple&& t) const { // not calling func, so we would get a wrong result if we were called - return {}; + return 0; } private: @@ -409,7 +410,7 @@ private: THRUST_NAMESPACE_END // test that the cuda_cub backend of Thrust unwraps zip_iterators/zip_functions into their input streams -void TestTransformZipIteratorUnwrapping() +void TestTransformThrustZipIteratorUnwrapping() { constexpr int num_items = 100; thrust::device_vector a(num_items, 1); @@ -442,4 +443,73 @@ void TestTransformZipIteratorUnwrapping() ASSERT_EQUAL(reference, result); } } -DECLARE_UNITTEST(TestTransformZipIteratorUnwrapping); +DECLARE_UNITTEST(TestTransformThrustZipIteratorUnwrapping); + +// we specialize zip_function for sum_five, but do nothing in the call operator so the test below would fail if the +// zip_function is actually called (and not unwrapped) +_CCCL_BEGIN_NAMESPACE_CUDA +template <> +class zip_function +{ +private: + sum_five __fun_; + +public: + zip_function() = default; + + _CCCL_HOST_DEVICE zip_function(sum_five&& func) noexcept + : __fun_(_THRUST_STD::move(func)) + {} + + template + _CCCL_HOST_DEVICE decltype(auto) operator()(_Tuple&& __tuple) const noexcept + { + // not calling func, just return a default ctored element, so we would get a wrong result if we were called + return decltype(_THRUST_STD::apply(__fun_, _THRUST_STD::forward<_Tuple>(__tuple))){}; + } + + _CCCL_HOST_DEVICE sum_five& __fun() noexcept + { + return __fun_; + } + + _CCCL_HOST_DEVICE const sum_five& __fun() const noexcept + { + return __fun_; + } +}; +_CCCL_END_NAMESPACE_CUDA + +// test that the cuda_cub backend of Thrust unwraps zip_iterators/zip_functions into their input streams +void TestTransformCudaZipIteratorUnwrapping() +{ + constexpr int num_items = 100; + thrust::device_vector a(num_items, 1); + thrust::device_vector b(num_items, 2); + thrust::device_vector c(num_items, 3); + thrust::device_vector d(num_items, 4); + thrust::device_vector e(num_items, 5); + + thrust::device_vector result(num_items); + // SECTION("once") // TODO(bgruber): enable sections when we migrate to Catch2 + { + const auto z = cuda::make_zip_iterator(a.begin(), b.begin(), c.begin(), d.begin(), e.begin()); + thrust::transform(z, z + num_items, result.begin(), cuda::zip_function(sum_five{})); + + // compute reference and verify + thrust::device_vector reference(num_items, 1 + 2 + 3 + 4 + 5); + ASSERT_EQUAL(reference, result); + } + // SECTION("trice") + { + const auto z = cuda::make_zip_iterator( + cuda::make_zip_iterator(cuda::make_zip_iterator(a.begin(), b.begin(), c.begin(), d.begin(), e.begin()))); + thrust::transform( + z, z + num_items, result.begin(), cuda::zip_function>>{}); + + // compute reference and verify + thrust::device_vector reference(num_items, 1 + 2 + 3 + 4 + 5); + ASSERT_EQUAL(reference, result); + } +} +DECLARE_UNITTEST(TestTransformCudaZipIteratorUnwrapping); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cuda/trivial_copy_memcpy_default.cu b/projects/hipccl/hipccl3/rocthrust/testing/cuda/trivial_copy_memcpy_default.cu new file mode 100644 index 000000000000..bb600d8d8d62 --- /dev/null +++ b/projects/hipccl/hipccl3/rocthrust/testing/cuda/trivial_copy_memcpy_default.cu @@ -0,0 +1,43 @@ +#include + +#include + +// These tests verify that trivial_copy_from_device and trivial_copy_to_device +// use cudaMemcpyDefault instead of cudaMemcpyDeviceToHost/cudaMemcpyHostToDevice. +// +// With explicit copy directions, cudaMemcpyAsync fails when the actual memory +// location doesn't match the expected direction (e.g., cudaMemcpyDeviceToHost +// with a host source pointer returns cudaErrorInvalidValue). +// +// With cudaMemcpyDefault, CUDA determines the correct direction at runtime, +// which is necessary when device-accessible pointers may point to host memory. + +void TestTrivialCopyFromDevice_HostSource() +{ + int src[] = {0, 10, 20, 30, 40}; + int dst[5] = {}; + + cudaError_t status = thrust::cuda_cub::trivial_copy_from_device(dst, src, 5, cudaStreamDefault); + + ASSERT_EQUAL(status, cudaSuccess); + for (int i = 0; i < 5; i++) + { + ASSERT_EQUAL(dst[i], i * 10); + } +} +DECLARE_UNITTEST(TestTrivialCopyFromDevice_HostSource); + +void TestTrivialCopyToDevice_HostDest() +{ + int src[] = {0, 100, 200, 300, 400}; + int dst[5] = {}; + + cudaError_t status = thrust::cuda_cub::trivial_copy_to_device(dst, src, 5, cudaStreamDefault); + + ASSERT_EQUAL(status, cudaSuccess); + for (int i = 0; i < 5; i++) + { + ASSERT_EQUAL(dst[i], i * 100); + } +} +DECLARE_UNITTEST(TestTrivialCopyToDevice_HostDest); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cuda/unique.cu b/projects/hipccl/hipccl3/rocthrust/testing/cuda/unique.cu index 8c6c1aa1f5b3..04675a91a097 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/cuda/unique.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/cuda/unique.cu @@ -414,12 +414,18 @@ void TestUniqueWithMagnitude(int magnitude) } void TestUniqueWithLargeNumberOfItems() +try { for (int mag : {30, 31, 32, 33}) { TestUniqueWithMagnitude(mag); } } +catch (std::bad_alloc&) +{ + // if we run out of memory, just skip the test + return; +} DECLARE_UNITTEST(TestUniqueWithLargeNumberOfItems); void TestUniqueWithCustomEqualityOp() diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cuda/unique_by_key.cu b/projects/hipccl/hipccl3/rocthrust/testing/cuda/unique_by_key.cu index da9ceceef9ac..25b8a11c6105 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/cuda/unique_by_key.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/cuda/unique_by_key.cu @@ -73,7 +73,7 @@ void TestUniqueByKeyDevice(ExecutionPolicy exec) Vector keys; Vector values; - using iter_pair = thrust::pair; + using iter_pair = cuda::std::pair; thrust::device_vector new_last_vec(1); iter_pair new_last; @@ -151,7 +151,7 @@ void TestUniqueByKeyCudaStreams(ExecutionPolicy policy) Vector keys; Vector values; - using iter_pair = thrust::pair; + using iter_pair = cuda::std::pair; iter_pair new_last; // basic test @@ -255,7 +255,7 @@ void TestUniqueCopyByKeyDevice(ExecutionPolicy exec) Vector keys; Vector values; - using iter_pair = thrust::pair; + using iter_pair = cuda::std::pair; thrust::device_vector new_last_vec(1); iter_pair new_last; @@ -345,7 +345,7 @@ void TestUniqueCopyByKeyCudaStreams(ExecutionPolicy policy) Vector keys; Vector values; - using iter_pair = thrust::pair; + using iter_pair = cuda::std::pair; iter_pair new_last; // basic test diff --git a/projects/hipccl/hipccl3/rocthrust/testing/cuda_iterators.cu b/projects/hipccl/hipccl3/rocthrust/testing/cuda_iterators.cu index 9927335fccc5..27e35e3922a2 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/cuda_iterators.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/cuda_iterators.cu @@ -48,17 +48,23 @@ void TestCUDAConstantIterator() { { // device system thrust::device_vector vec{1, 2, 3, 4}; + thrust::device_vector expected{42, 42, 42, 42}; thrust::copy(cuda::constant_iterator{42, 0}, cuda::constant_iterator{42, 4}, vec.begin()); + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); } { // host system thrust::host_vector vec{1, 2, 3, 4}; + thrust::host_vector expected{42, 42, 42, 42}; thrust::copy(cuda::constant_iterator{42, 0}, cuda::constant_iterator{42, 4}, vec.begin()); + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); } { // plain std::vector std::vector vec{1, 2, 3, 4}; + std::vector expected{42, 42, 42, 42}; thrust::copy(cuda::constant_iterator{42, 0}, cuda::constant_iterator{42, 4}, vec.begin()); + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); } } DECLARE_UNITTEST(TestCUDAConstantIterator); @@ -66,18 +72,24 @@ DECLARE_UNITTEST(TestCUDAConstantIterator); void TestCUDACountingIterator() { { // device system - thrust::device_vector vec{1, 2, 3, 4}; + thrust::device_vector vec{-1, -1, -1, -1}; + thrust::device_vector expected{0, 1, 2, 3}; thrust::copy(cuda::counting_iterator{0}, cuda::counting_iterator{4}, vec.begin()); + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); } { // host system - thrust::host_vector vec{1, 2, 3, 4}; + thrust::host_vector vec{-1, -1, -1, -1}; + thrust::host_vector expected{0, 1, 2, 3}; thrust::copy(cuda::counting_iterator{0}, cuda::counting_iterator{4}, vec.begin()); + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); } { // plain std::vector - std::vector vec{1, 2, 3, 4}; + std::vector vec{-1, -1, -1, -1}; + std::vector expected{0, 1, 2, 3}; thrust::copy(cuda::counting_iterator{0}, cuda::counting_iterator{4}, vec.begin()); + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); } } DECLARE_UNITTEST(TestCUDACountingIterator); @@ -115,22 +127,77 @@ void TestCUDAPermutationIterator() } DECLARE_UNITTEST(TestCUDAPermutationIterator); +struct fake_bijection +{ + using index_type = ::cuda::std::uint32_t; + + constexpr fake_bijection() = default; + + [[nodiscard]] _CCCL_HOST_DEVICE constexpr index_type size() const noexcept + { + return 5; + } + + [[nodiscard]] _CCCL_HOST_DEVICE constexpr index_type operator()(index_type n) const noexcept + { + return __random_indices[n]; + } + + ::cuda::std::uint32_t __random_indices[5] = {4, 1, 2, 0, 3}; +}; + +void TestCUDAShuffleIterator() +{ + { // device system + thrust::device_vector vec{1, 2, 3, 4}; + thrust::device_vector expected{4, 1, 2, 0}; + thrust::copy(cuda::shuffle_iterator{fake_bijection{}, 0}, cuda::shuffle_iterator{fake_bijection{}, 4}, vec.begin()); + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); + } + + { // host system + thrust::host_vector vec{1, 2, 3, 4}; + thrust::host_vector expected{4, 1, 2, 0}; + thrust::copy(cuda::shuffle_iterator{fake_bijection{}, 0}, cuda::shuffle_iterator{fake_bijection{}, 4}, vec.begin()); + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); + } + + { // plain std::vector + std::vector vec{1, 2, 3, 4}; + std::vector expected{4, 1, 2, 0}; + thrust::copy(cuda::shuffle_iterator{fake_bijection{}, 0}, cuda::shuffle_iterator{fake_bijection{}, 4}, vec.begin()); + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); + } +} +DECLARE_UNITTEST(TestCUDAShuffleIterator); + void TestCUDAStridedIterator() { - auto discard = cuda::discard_iterator{}; { // device system - thrust::device_vector vec{1, 2, 3, 4, 5, 6}; - thrust::copy(cuda::strided_iterator{vec.begin(), 2}, cuda::strided_iterator{vec.end(), 2}, discard); + thrust::device_vector vec{-1, -1, -1, -1}; + thrust::device_vector expected{0, 2, 4, 6}; + thrust::copy(cuda::strided_iterator{cuda::counting_iterator{0}, 2}, + cuda::strided_iterator{cuda::counting_iterator{8}, 2}, + vec.begin()); + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); } { // host system - thrust::host_vector vec{1, 2, 3, 4, 5, 6}; - thrust::copy(cuda::strided_iterator{vec.begin(), 2}, cuda::strided_iterator{vec.end(), 2}, discard); + thrust::host_vector vec{-1, -1, -1, -1}; + thrust::host_vector expected{0, 2, 4, 6}; + thrust::copy(cuda::strided_iterator{cuda::counting_iterator{0}, 2}, + cuda::strided_iterator{cuda::counting_iterator{8}, 2}, + vec.begin()); + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); } { // plain std::vector - std::vector vec{1, 2, 3, 4, 5, 6}; - thrust::copy(cuda::strided_iterator{vec.begin(), 2}, cuda::strided_iterator{vec.end(), 2}, discard); + std::vector vec{-1, -1, -1, -1}; + std::vector expected{0, 2, 4, 6}; + thrust::copy(cuda::strided_iterator{cuda::counting_iterator{0}, 2}, + cuda::strided_iterator{cuda::counting_iterator{8}, 2}, + vec.begin()); + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); } } DECLARE_UNITTEST(TestCUDAStridedIterator); @@ -171,32 +238,75 @@ struct plus_one } }; +void TestCUDATransformInputOutputIterator() +{ + { // device system + thrust::device_vector vec{-1, -1, -1, -1, -1}; + auto iter = cuda::transform_input_output_iterator(vec.begin(), plus_one{}, plus_one{}); + thrust::copy(cuda::counting_iterator{3}, cuda::counting_iterator{8}, iter); + + // Ensure we did write the right output, sequence starts at 3 + 1 == 4 + thrust::device_vector expected{4, 5, 6, 7, 8}; + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); + + // Ensure we did read the right input, output starts 4 + 1 == 5 + thrust::copy(iter, iter + 5, cuda::make_tabulate_output_iterator(is_equal_index{}, 5)); + } + + { // host system + thrust::host_vector vec{-1, -1, -1, -1, -1}; + auto iter = cuda::transform_input_output_iterator(vec.begin(), plus_one{}, plus_one{}); + thrust::copy(cuda::counting_iterator{3}, cuda::counting_iterator{8}, iter); + + // Ensure we did write the right output, sequence starts at 3 + 1 == 4 + thrust::host_vector expected{4, 5, 6, 7, 8}; + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); + + // Ensure we did read the right input, output starts 4 + 1 == 5 + thrust::copy(iter, iter + 5, cuda::make_tabulate_output_iterator(is_equal_index{}, 5)); + } + + { // plain std::vector + std::vector vec{-1, -1, -1, -1, -1}; + auto iter = cuda::transform_input_output_iterator(vec.begin(), plus_one{}, plus_one{}); + thrust::copy(cuda::counting_iterator{3}, cuda::counting_iterator{8}, iter); + + // Ensure we did write the right output, sequence starts at 3 + 1 == 4 + std::vector expected{4, 5, 6, 7, 8}; + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); + + // Ensure we did read the right input, output starts 4 + 1 == 5 + thrust::copy(iter, iter + 5, cuda::make_tabulate_output_iterator(is_equal_index{}, 5)); + } +} +DECLARE_UNITTEST(TestCUDATransformInputOutputIterator); + void TestCUDATransformOutputIterator() { { // device system thrust::device_vector vec{-1, -1, -1, -1, -1}; + thrust::device_vector expected{1, 2, 3, 4, 5}; thrust::copy(cuda::counting_iterator{0}, cuda::counting_iterator{5}, cuda::make_transform_output_iterator(vec.begin(), plus_one{})); - thrust::device_vector expected{1, 2, 3, 4, 5}; ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); } { // host system thrust::host_vector vec{-1, -1, -1, -1, -1}; + thrust::host_vector expected{1, 2, 3, 4, 5}; thrust::copy(cuda::counting_iterator{0}, cuda::counting_iterator{5}, cuda::make_transform_output_iterator(vec.begin(), plus_one{})); - thrust::host_vector expected{1, 2, 3, 4, 5}; ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); } { // plain std::vector std::vector vec{-1, -1, -1, -1, -1}; + std::vector expected{1, 2, 3, 4, 5}; thrust::copy(cuda::counting_iterator{0}, cuda::counting_iterator{5}, cuda::make_transform_output_iterator(vec.begin(), plus_one{})); - std::vector expected{1, 2, 3, 4, 5}; ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); } } @@ -204,26 +314,113 @@ DECLARE_UNITTEST(TestCUDATransformOutputIterator); void TestCUDATransformIterator() { - auto discard = cuda::discard_iterator{}; { // device system - thrust::device_vector vec{1, 2, 3, 4}; - thrust::copy(cuda::make_transform_iterator(vec.begin(), plus_one{}), - cuda::make_transform_iterator(vec.end(), plus_one{}), - discard); + thrust::device_vector vec{-1, -1, -1, -1}; + thrust::device_vector expected{1, 2, 3, 4}; + thrust::copy(cuda::transform_iterator(cuda::counting_iterator{0}, plus_one{}), + cuda::transform_iterator(cuda::counting_iterator{4}, plus_one{}), + vec.begin()); + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); } { // host system - thrust::host_vector vec{1, 2, 3, 4}; - thrust::copy(cuda::make_transform_iterator(vec.begin(), plus_one{}), - cuda::make_transform_iterator(vec.end(), plus_one{}), - discard); + thrust::host_vector vec{-1, -1, -1, -1}; + thrust::host_vector expected{1, 2, 3, 4}; + thrust::copy(cuda::transform_iterator(cuda::counting_iterator{0}, plus_one{}), + cuda::transform_iterator(cuda::counting_iterator{4}, plus_one{}), + vec.begin()); + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); } { // plain std::vector - std::vector vec{1, 2, 3, 4}; - thrust::copy(cuda::make_transform_iterator(vec.begin(), plus_one{}), - cuda::make_transform_iterator(vec.end(), plus_one{}), - discard); + std::vector vec{-1, -1, -1, -1}; + std::vector expected{1, 2, 3, 4}; + thrust::copy(cuda::transform_iterator(cuda::counting_iterator{0}, plus_one{}), + cuda::transform_iterator(cuda::counting_iterator{4}, plus_one{}), + vec.begin()); + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); } } DECLARE_UNITTEST(TestCUDATransformIterator); + +void TestCUDAZipIterator() +{ + cuda::zip_function> fun{}; + { // device system + thrust::device_vector vec{-1, -1, -1, -1}; + auto iter = cuda::make_transform_iterator(cuda::make_zip_iterator(vec.begin(), cuda::counting_iterator{4}), fun); + thrust::copy(iter, iter + 4, vec.begin()); + thrust::device_vector expected{3, 4, 5, 6}; + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); + } + + { // host system + thrust::host_vector vec{-1, -1, -1, -1}; + auto iter = cuda::make_transform_iterator(cuda::make_zip_iterator(vec.begin(), cuda::counting_iterator{4}), fun); + thrust::copy(iter, iter + 4, vec.begin()); + thrust::host_vector expected{3, 4, 5, 6}; + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); + } + + { // plain std::vector + std::vector vec{-1, -1, -1, -1}; + auto iter = cuda::make_transform_iterator(cuda::make_zip_iterator(vec.begin(), cuda::counting_iterator{4}), fun); + thrust::copy(iter, iter + 4, vec.begin()); + std::vector expected{3, 4, 5, 6}; + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); + } +} +DECLARE_UNITTEST(TestCUDAZipIterator); + +void TestCUDAZipTransformIterator() +{ + { // device system + thrust::device_vector vec{-1, -1, -1, -1}; + auto iter = cuda::make_zip_transform_iterator(cuda::std::plus{}, vec.begin(), cuda::counting_iterator{4}); + thrust::copy(iter, iter + 4, vec.begin()); + thrust::device_vector expected{3, 4, 5, 6}; + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); + } + + { // host system + thrust::host_vector vec{-1, -1, -1, -1}; + auto iter = cuda::make_zip_transform_iterator(cuda::std::plus{}, vec.begin(), cuda::counting_iterator{4}); + thrust::copy(iter, iter + 4, vec.begin()); + thrust::host_vector expected{3, 4, 5, 6}; + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); + } + + { // plain std::vector + std::vector vec{-1, -1, -1, -1}; + auto iter = cuda::make_zip_transform_iterator(cuda::std::plus{}, vec.begin(), cuda::counting_iterator{4}); + thrust::copy(iter, iter + 4, vec.begin()); + std::vector expected{3, 4, 5, 6}; + ASSERT_EQUAL(true, thrust::equal(vec.begin(), vec.end(), expected.begin())); + } +} +DECLARE_UNITTEST(TestCUDAZipTransformIterator); + +void TestCUDAReverseIterator() +{ + { // device system + thrust::device_vector vec{1, 2, 3, 4, 5, 6, 7, 8, 9}; + thrust::device_vector expected{9, 8, 7, 6, 5, 4, 3, 2, 1}; + ASSERT_EQUAL(true, thrust::equal( + cuda::std::reverse_iterator{vec.end()}, cuda::std::reverse_iterator{vec.begin()}, expected.begin())); + } + + { // host system + thrust::host_vector vec{1, 2, 3, 4, 5, 6, 7, 8, 9}; + thrust::host_vector expected{9, 8, 7, 6, 5, 4, 3, 2, 1}; + ASSERT_EQUAL(true, thrust::equal( + cuda::std::reverse_iterator{vec.end()}, cuda::std::reverse_iterator{vec.begin()}, expected.begin())); + } + + { // plain std::vector + std::vector vec{1, 2, 3, 4, 5, 6, 7, 8, 9}; + std::vector expected{9, 8, 7, 6, 5, 4, 3, 2, 1}; + ASSERT_EQUAL(true, thrust::equal( + cuda::std::reverse_iterator{vec.end()}, cuda::std::reverse_iterator{vec.begin()}, expected.begin())); + } +} +DECLARE_UNITTEST(TestCUDAReverseIterator); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/device_delete.cu b/projects/hipccl/hipccl3/rocthrust/testing/device_delete.cu index f04b00981273..c986525f846b 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/device_delete.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/device_delete.cu @@ -24,37 +24,85 @@ struct Foo { - THRUST_HOST_DEVICE Foo() - : set_me_upon_destruction{nullptr} + THRUST_DEVICE Foo() + : destroyed{nullptr} {} - THRUST_HOST_DEVICE ~Foo() + Foo(const Foo&) = delete; + Foo& operator=(const Foo&) = delete; + + THRUST_DEVICE ~Foo() { - _THRUST_IF_TARGET(_THRUST_IS_DEVICE, - (if (set_me_upon_destruction != nullptr) { *set_me_upon_destruction = true; })); + if (destroyed != nullptr) + { + *destroyed = true; + } } - bool* set_me_upon_destruction; + bool* destroyed = nullptr; }; -#if !defined(__QNX__) void TestDeviceDeleteDestructorInvocation() { - KNOWN_FAILURE; - // - // thrust::device_vector destructor_flag(1, false); - // - // thrust::device_ptr foo_ptr = thrust::device_new(); - // - // Foo exemplar; - // exemplar.set_me_upon_destruction = thrust::raw_pointer_cast(&destructor_flag[0]); - // *foo_ptr = exemplar; - // - // ASSERT_EQUAL(false, destructor_flag[0]); - // - // thrust::device_delete(foo_ptr); - // - // ASSERT_EQUAL(true, destructor_flag[0]); + thrust::device_ptr foo_ptr = thrust::device_new(); + + thrust::device_vector destructor_flag(1, false); + *thrust::device_ptr(&foo_ptr.get()->destroyed) = destructor_flag.data().get(); + + ASSERT_EQUAL(false, destructor_flag[0]); + thrust::device_delete(foo_ptr); + ASSERT_EQUAL(true, destructor_flag[0]); } DECLARE_UNITTEST(TestDeviceDeleteDestructorInvocation); -#endif + +// based on: https://github.com/NVIDIA/cccl/issues/6132 +struct base +{ + THRUST_HOST_DEVICE virtual ~base() + { + if (base_destroyed != nullptr) + { + *base_destroyed = true; + } + } + + // Thrust does not support abstract class types inside vectors, so f() has a body instead of being abstract + THRUST_HOST_DEVICE virtual void f() {} + + bool* base_destroyed = nullptr; +}; + +struct derived : base +{ + THRUST_HOST_DEVICE ~derived() override + { + if (derived_destroyed != nullptr) + { + *derived_destroyed = true; + } + } + + THRUST_HOST_DEVICE void f() override {} + + bool* derived_destroyed = nullptr; +}; + +void TestDeviceDeleteVirtualDestructorInvocation() +{ + { + thrust::device_ptr ptr = thrust::device_new(); + + thrust::device_vector destructor_flags(2, false); + *thrust::device_ptr(&ptr.get()->base_destroyed) = destructor_flags.data().get() + 0; + *thrust::device_ptr(&ptr.get()->derived_destroyed) = destructor_flags.data().get() + 1; + + thrust::device_ptr base_ptr = ptr; + + ASSERT_EQUAL(false, destructor_flags[0]); + ASSERT_EQUAL(false, destructor_flags[1]); + thrust::device_delete(base_ptr); // delete through the base pointer + ASSERT_EQUAL(true, destructor_flags[0]); + ASSERT_EQUAL(true, destructor_flags[1]); + } +} +DECLARE_UNITTEST(TestDeviceDeleteVirtualDestructorInvocation); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/docs/doxybook_test.h b/projects/hipccl/hipccl3/rocthrust/testing/docs/doxybook_test.h index cec55dc0ac07..e260c2dfd1cc 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/docs/doxybook_test.h +++ b/projects/hipccl/hipccl3/rocthrust/testing/docs/doxybook_test.h @@ -22,7 +22,6 @@ namespace thrust { - /*! \addtogroup test Test * \{ */ @@ -187,7 +186,6 @@ using test_alias = test_class; */ namespace test_namespace { - inline constexpr int test_constant = 12; /*! \brief \c nested_function is a function intended to exercise and test @@ -212,7 +210,6 @@ struct test_struct */ bool operator<(test_struct const& t); }; - } // namespace test_namespace /*! \brief \c THRUST_TEST_MACRO is a macro intended to exercise and test @@ -222,5 +219,4 @@ struct test_struct /*! \} // test */ - } // namespace thrust diff --git a/projects/hipccl/hipccl3/rocthrust/testing/find.cu b/projects/hipccl/hipccl3/rocthrust/testing/find.cu index 28aa3d239583..b24591034808 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/find.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/find.cu @@ -24,51 +24,6 @@ #include _THRUST_STD_INCLUDE(cstdint) -template -struct equal_to_value_pred -{ - T value; - - equal_to_value_pred(T value) - : value(value) - {} - - THRUST_HOST_DEVICE bool operator()(T v) const - { - return v == value; - } -}; - -template -struct not_equal_to_value_pred -{ - T value; - - not_equal_to_value_pred(T value) - : value(value) - {} - - THRUST_HOST_DEVICE bool operator()(T v) const - { - return v != value; - } -}; - -template -struct less_than_value_pred -{ - T value; - - less_than_value_pred(T value) - : value(value) - {} - - THRUST_HOST_DEVICE bool operator()(T v) const - { - return v < value; - } -}; - template void TestFindSimple() { @@ -125,12 +80,13 @@ void TestFindIfSimple() Vector vec{1, 2, 3, 3, 5}; - ASSERT_EQUAL(thrust::find_if(vec.begin(), vec.end(), equal_to_value_pred(0)) - vec.begin(), 5); - ASSERT_EQUAL(thrust::find_if(vec.begin(), vec.end(), equal_to_value_pred(1)) - vec.begin(), 0); - ASSERT_EQUAL(thrust::find_if(vec.begin(), vec.end(), equal_to_value_pred(2)) - vec.begin(), 1); - ASSERT_EQUAL(thrust::find_if(vec.begin(), vec.end(), equal_to_value_pred(3)) - vec.begin(), 2); - ASSERT_EQUAL(thrust::find_if(vec.begin(), vec.end(), equal_to_value_pred(4)) - vec.begin(), 5); - ASSERT_EQUAL(thrust::find_if(vec.begin(), vec.end(), equal_to_value_pred(5)) - vec.begin(), 4); + using thrust::placeholders::_1; + ASSERT_EQUAL(thrust::find_if(vec.begin(), vec.end(), _1 == T{0}) - vec.begin(), 5); + ASSERT_EQUAL(thrust::find_if(vec.begin(), vec.end(), _1 == T{1}) - vec.begin(), 0); + ASSERT_EQUAL(thrust::find_if(vec.begin(), vec.end(), _1 == T{2}) - vec.begin(), 1); + ASSERT_EQUAL(thrust::find_if(vec.begin(), vec.end(), _1 == T{3}) - vec.begin(), 2); + ASSERT_EQUAL(thrust::find_if(vec.begin(), vec.end(), _1 == T{4}) - vec.begin(), 5); + ASSERT_EQUAL(thrust::find_if(vec.begin(), vec.end(), _1 == T{5}) - vec.begin(), 4); } DECLARE_VECTOR_UNITTEST(TestFindIfSimple); @@ -176,12 +132,13 @@ void TestFindIfNotSimple() Vector vec{0, 1, 2, 3, 4}; - ASSERT_EQUAL(0, thrust::find_if_not(vec.begin(), vec.end(), less_than_value_pred(0)) - vec.begin()); - ASSERT_EQUAL(1, thrust::find_if_not(vec.begin(), vec.end(), less_than_value_pred(1)) - vec.begin()); - ASSERT_EQUAL(2, thrust::find_if_not(vec.begin(), vec.end(), less_than_value_pred(2)) - vec.begin()); - ASSERT_EQUAL(3, thrust::find_if_not(vec.begin(), vec.end(), less_than_value_pred(3)) - vec.begin()); - ASSERT_EQUAL(4, thrust::find_if_not(vec.begin(), vec.end(), less_than_value_pred(4)) - vec.begin()); - ASSERT_EQUAL(5, thrust::find_if_not(vec.begin(), vec.end(), less_than_value_pred(5)) - vec.begin()); + using thrust::placeholders::_1; + ASSERT_EQUAL(0, thrust::find_if_not(vec.begin(), vec.end(), _1 < T{0}) - vec.begin()); + ASSERT_EQUAL(1, thrust::find_if_not(vec.begin(), vec.end(), _1 < T{1}) - vec.begin()); + ASSERT_EQUAL(2, thrust::find_if_not(vec.begin(), vec.end(), _1 < T{2}) - vec.begin()); + ASSERT_EQUAL(3, thrust::find_if_not(vec.begin(), vec.end(), _1 < T{3}) - vec.begin()); + ASSERT_EQUAL(4, thrust::find_if_not(vec.begin(), vec.end(), _1 < T{4}) - vec.begin()); + ASSERT_EQUAL(5, thrust::find_if_not(vec.begin(), vec.end(), _1 < T{5}) - vec.begin()); } DECLARE_VECTOR_UNITTEST(TestFindIfNotSimple); @@ -257,15 +214,16 @@ struct TestFindIf typename thrust::host_vector::iterator h_iter; typename thrust::device_vector::iterator d_iter; - h_iter = thrust::find_if(h_data.begin(), h_data.end(), equal_to_value_pred(0)); - d_iter = thrust::find_if(d_data.begin(), d_data.end(), equal_to_value_pred(0)); + using thrust::placeholders::_1; + h_iter = thrust::find_if(h_data.begin(), h_data.end(), _1 == T{0}); + d_iter = thrust::find_if(d_data.begin(), d_data.end(), _1 == T{0}); ASSERT_EQUAL(h_iter - h_data.begin(), d_iter - d_data.begin()); for (size_t i = 1; i < n; i *= 2) { T sample = h_data[i]; - h_iter = thrust::find_if(h_data.begin(), h_data.end(), equal_to_value_pred(sample)); - d_iter = thrust::find_if(d_data.begin(), d_data.end(), equal_to_value_pred(sample)); + h_iter = thrust::find_if(h_data.begin(), h_data.end(), _1 == T{sample}); + d_iter = thrust::find_if(d_data.begin(), d_data.end(), _1 == T{sample}); ASSERT_EQUAL(h_iter - h_data.begin(), d_iter - d_data.begin()); } } @@ -283,15 +241,16 @@ struct TestFindIfNot typename thrust::host_vector::iterator h_iter; typename thrust::device_vector::iterator d_iter; - h_iter = thrust::find_if_not(h_data.begin(), h_data.end(), not_equal_to_value_pred(0)); - d_iter = thrust::find_if_not(d_data.begin(), d_data.end(), not_equal_to_value_pred(0)); + using thrust::placeholders::_1; + h_iter = thrust::find_if_not(h_data.begin(), h_data.end(), _1 != T{0}); + d_iter = thrust::find_if_not(d_data.begin(), d_data.end(), _1 != T{0}); ASSERT_EQUAL(h_iter - h_data.begin(), d_iter - d_data.begin()); for (size_t i = 1; i < n; i *= 2) { T sample = h_data[i]; - h_iter = thrust::find_if_not(h_data.begin(), h_data.end(), not_equal_to_value_pred(sample)); - d_iter = thrust::find_if_not(d_data.begin(), d_data.end(), not_equal_to_value_pred(sample)); + h_iter = thrust::find_if_not(h_data.begin(), h_data.end(), _1 != T{sample}); + d_iter = thrust::find_if_not(d_data.begin(), d_data.end(), _1 != T{sample}); ASSERT_EQUAL(h_iter - h_data.begin(), d_iter - d_data.begin()); } } @@ -324,7 +283,6 @@ DECLARE_UNITTEST(TestFindWithBigIndexes); namespace { - class Weird { int value; @@ -339,7 +297,6 @@ public: return x == y.value; } }; - } // namespace void TestFindAsymmetricEquality() diff --git a/projects/hipccl/hipccl3/rocthrust/testing/gather.cu b/projects/hipccl/hipccl3/rocthrust/testing/gather.cu index 238d6a525732..3ef807f50d71 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/gather.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/gather.cu @@ -26,6 +26,14 @@ #include +// There is an unfortunate miscompilation of the gcc-11 vectorizer leading to OOB writes +// Adding this attribute suffices that this miscompilation does not appear anymore +#if THRUST_COMPILER(GCC, >=, 11) +# define THRUST_DISABLE_BROKEN_GCC_VECTORIZER __attribute__((optimize("no-tree-vectorize"))) +#else +# define THRUST_DISABLE_BROKEN_GCC_VECTORIZER +#endif + THRUST_DIAG_PUSH THRUST_DIAG_SUPPRESS_MSVC(4244 4267) // possible loss of data @@ -324,7 +332,7 @@ void TestGatherIfToDiscardIterator(const size_t n) DECLARE_VARIABLE_UNITTEST(TestGatherIfToDiscardIterator); template -void TestGatherCountingIterator() +THRUST_DISABLE_BROKEN_GCC_VECTORIZER void TestGatherCountingIterator() { Vector source(10); thrust::sequence(source.begin(), source.end(), 0); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/memory.cu b/projects/hipccl/hipccl3/rocthrust/testing/memory.cu index 395337d8222b..8012c8af04a9 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/memory.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/memory.cu @@ -18,11 +18,12 @@ #include #include #include -#include #include #include #include +#include _THRUST_STD_INCLUDE(utility) + #include #include @@ -76,17 +77,16 @@ private: namespace my_old_namespace { - struct my_old_temporary_allocation_system : public thrust::device_execution_policy {}; template -thrust::pair, std::ptrdiff_t> +_THRUST_STD::pair, std::ptrdiff_t> get_temporary_buffer(my_old_temporary_allocation_system, std::ptrdiff_t) { thrust::pointer const result(reinterpret_cast(4217)); - return thrust::make_pair(result, 314); + return _THRUST_STD::make_pair(result, 314); } template @@ -95,22 +95,20 @@ void return_temporary_buffer(my_old_temporary_allocation_system, Pointer p) using RP = typename thrust::detail::pointer_traits::raw_pointer; ASSERT_EQUAL(p.get(), reinterpret_cast(4217)); } - } // namespace my_old_namespace namespace my_new_namespace { - struct my_new_temporary_allocation_system : public thrust::device_execution_policy {}; template -thrust::pair, std::ptrdiff_t> +_THRUST_STD::pair, std::ptrdiff_t> get_temporary_buffer(my_new_temporary_allocation_system, std::ptrdiff_t) { thrust::pointer const result(reinterpret_cast(1742)); - return thrust::make_pair(result, 413); + return _THRUST_STD::make_pair(result, 413); } template @@ -128,7 +126,6 @@ void return_temporary_buffer(my_new_temporary_allocation_system, Pointer p, std: ASSERT_EQUAL(p.get(), reinterpret_cast(1742)); ASSERT_EQUAL(n, 413); } - } // namespace my_new_namespace template @@ -188,7 +185,7 @@ void TestGetTemporaryBuffer() thrust::device_system_tag dev_tag; using pointer = thrust::pointer; - thrust::pair ptr_and_sz = thrust::get_temporary_buffer(dev_tag, n); + _THRUST_STD::pair ptr_and_sz = thrust::get_temporary_buffer(dev_tag, n); ASSERT_EQUAL(ptr_and_sz.second, n); @@ -258,15 +255,15 @@ void TestFreeDispatchExplicit() DECLARE_UNITTEST(TestFreeDispatchExplicit); template -thrust::pair, std::ptrdiff_t> +_THRUST_STD::pair, std::ptrdiff_t> get_temporary_buffer(my_memory_system& system, std::ptrdiff_t n) { system.validate_dispatch(); thrust::device_system_tag device_sys; - thrust::pair, std::ptrdiff_t> result = + _THRUST_STD::pair, std::ptrdiff_t> result = thrust::get_temporary_buffer(device_sys, n); - return thrust::make_pair(thrust::pointer(result.first.get()), result.second); + return _THRUST_STD::make_pair(thrust::pointer(result.first.get()), result.second); } // [NON-CCCL PARITY BEGIN] @@ -288,7 +285,7 @@ void TestGetTemporaryBufferDispatchExplicit() my_memory_system sys(0); using pointer = thrust::pointer; - thrust::pair ptr_and_sz = thrust::get_temporary_buffer(sys, n); + _THRUST_STD::pair ptr_and_sz = thrust::get_temporary_buffer(sys, n); ASSERT_EQUAL(ptr_and_sz.second, n); ASSERT_EQUAL(true, sys.is_valid()); @@ -337,7 +334,7 @@ void TestTemporaryBufferOldCustomization() { using system = my_old_namespace::my_old_temporary_allocation_system; using pointer = thrust::pointer; - using pointer_and_size = thrust::pair; + using pointer_and_size = _THRUST_STD::pair; system sys; @@ -357,7 +354,7 @@ void TestTemporaryBufferNewCustomization() { using system = my_new_namespace::my_new_temporary_allocation_system; using pointer = thrust::pointer; - using pointer_and_size = thrust::pair; + using pointer_and_size = _THRUST_STD::pair; system sys; diff --git a/projects/hipccl/hipccl3/rocthrust/testing/merge_by_key.cu b/projects/hipccl/hipccl3/rocthrust/testing/merge_by_key.cu index 2526fe27694e..42683990fd49 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/merge_by_key.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/merge_by_key.cu @@ -61,7 +61,7 @@ template -thrust::pair merge_by_key( +_THRUST_STD::pair merge_by_key( my_system& system, InputIterator1, InputIterator1, @@ -73,7 +73,7 @@ thrust::pair merge_by_key( OutputIterator2 values_result) { system.validate_dispatch(); - return thrust::make_pair(keys_result, values_result); + return _THRUST_STD::make_pair(keys_result, values_result); } void TestMergeByKeyDispatchExplicit() @@ -94,7 +94,7 @@ template -thrust::pair merge_by_key( +_THRUST_STD::pair merge_by_key( my_tag, InputIterator1, InputIterator1, @@ -106,7 +106,7 @@ thrust::pair merge_by_key( OutputIterator2 values_result) { *keys_result = 13; - return thrust::make_pair(keys_result, values_result); + return _THRUST_STD::make_pair(keys_result, values_result); } void TestMergeByKeyDispatchImplicit() @@ -239,7 +239,7 @@ void TestMergeByKeyToDiscardIterator(size_t n) const thrust::device_vector d_a_vals = h_a_vals; const thrust::device_vector d_b_vals = h_b_vals; - using discard_pair = thrust::pair, thrust::discard_iterator<>>; + using discard_pair = _THRUST_STD::pair, thrust::discard_iterator<>>; const discard_pair h_result = thrust::merge_by_key( h_a_keys.begin(), diff --git a/projects/hipccl/hipccl3/rocthrust/testing/metaprogramming.cu b/projects/hipccl/hipccl3/rocthrust/testing/metaprogramming.cu deleted file mode 100644 index 6f05dcdc1c11..000000000000 --- a/projects/hipccl/hipccl3/rocthrust/testing/metaprogramming.cu +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2008-2013 NVIDIA Corporation - * Modifications Copyright© 2019-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include - -void TestLog2() -{ - unsigned int result; - - result = thrust::detail::mpl::math::log2<1>::value; - ASSERT_EQUAL(result, 0lu); - result = thrust::detail::mpl::math::log2<2>::value; - ASSERT_EQUAL(result, 1lu); - result = thrust::detail::mpl::math::log2<3>::value; - ASSERT_EQUAL(result, 1lu); - result = thrust::detail::mpl::math::log2<4>::value; - ASSERT_EQUAL(result, 2lu); - result = thrust::detail::mpl::math::log2<5>::value; - ASSERT_EQUAL(result, 2lu); - result = thrust::detail::mpl::math::log2<6>::value; - ASSERT_EQUAL(result, 2lu); - result = thrust::detail::mpl::math::log2<7>::value; - ASSERT_EQUAL(result, 2lu); - result = thrust::detail::mpl::math::log2<8>::value; - ASSERT_EQUAL(result, 3lu); - result = thrust::detail::mpl::math::log2<9>::value; - ASSERT_EQUAL(result, 3lu); - result = thrust::detail::mpl::math::log2<15>::value; - ASSERT_EQUAL(result, 3lu); - result = thrust::detail::mpl::math::log2<16>::value; - ASSERT_EQUAL(result, 4lu); - result = thrust::detail::mpl::math::log2<17>::value; - ASSERT_EQUAL(result, 4lu); - result = thrust::detail::mpl::math::log2<127>::value; - ASSERT_EQUAL(result, 6lu); - result = thrust::detail::mpl::math::log2<128>::value; - ASSERT_EQUAL(result, 7lu); - result = thrust::detail::mpl::math::log2<129>::value; - ASSERT_EQUAL(result, 7lu); - result = thrust::detail::mpl::math::log2<256>::value; - ASSERT_EQUAL(result, 8lu); - result = thrust::detail::mpl::math::log2<511>::value; - ASSERT_EQUAL(result, 8lu); - result = thrust::detail::mpl::math::log2<512>::value; - ASSERT_EQUAL(result, 9lu); -} -DECLARE_UNITTEST(TestLog2); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/minmax_element.cu b/projects/hipccl/hipccl3/rocthrust/testing/minmax_element.cu index 1a260fce2538..f7b260513507 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/minmax_element.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/minmax_element.cu @@ -80,10 +80,11 @@ void TestMinMaxElement(const size_t n) DECLARE_VARIABLE_UNITTEST(TestMinMaxElement); template -thrust::pair minmax_element(my_system& system, ForwardIterator first, ForwardIterator) +_THRUST_STD::pair +minmax_element(my_system& system, ForwardIterator first, ForwardIterator) { system.validate_dispatch(); - return thrust::make_pair(first, first); + return _THRUST_STD::make_pair(first, first); } void TestMinMaxElementDispatchExplicit() @@ -98,10 +99,10 @@ void TestMinMaxElementDispatchExplicit() DECLARE_UNITTEST(TestMinMaxElementDispatchExplicit); template -thrust::pair minmax_element(my_tag, ForwardIterator first, ForwardIterator) +_THRUST_STD::pair minmax_element(my_tag, ForwardIterator first, ForwardIterator) { *first = 13; - return thrust::make_pair(first, first); + return _THRUST_STD::make_pair(first, first); } void TestMinMaxElementDispatchImplicit() @@ -121,7 +122,7 @@ void TestMinMaxElementWithBigIndexesHelper(int magnitude) Iter end = begin + (1ll << magnitude); ASSERT_EQUAL(_THRUST_STD::distance(begin, end), 1ll << magnitude); - thrust::pair result = thrust::minmax_element(thrust::device, begin, end); + _THRUST_STD::pair result = thrust::minmax_element(thrust::device, begin, end); ASSERT_EQUAL(*result.first, 1); ASSERT_EQUAL(*result.second, (1ll << magnitude)); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/mismatch.cu b/projects/hipccl/hipccl3/rocthrust/testing/mismatch.cu index 60bc69eab07a..567af422d6f3 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/mismatch.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/mismatch.cu @@ -41,11 +41,11 @@ void TestMismatchSimple() DECLARE_VECTOR_UNITTEST(TestMismatchSimple); template -thrust::pair +_THRUST_STD::pair mismatch(my_system& system, InputIterator1 first, InputIterator1, InputIterator2) { system.validate_dispatch(); - return thrust::make_pair(first, first); + return _THRUST_STD::make_pair(first, first); } void TestMismatchDispatchExplicit() @@ -60,10 +60,10 @@ void TestMismatchDispatchExplicit() DECLARE_UNITTEST(TestMismatchDispatchExplicit); template -thrust::pair mismatch(my_tag, InputIterator1 first, InputIterator1, InputIterator2) +_THRUST_STD::pair mismatch(my_tag, InputIterator1 first, InputIterator1, InputIterator2) { *first = 13; - return thrust::make_pair(first, first); + return _THRUST_STD::make_pair(first, first); } void TestMismatchDispatchImplicit() diff --git a/projects/hipccl/hipccl3/rocthrust/testing/mr_disjoint_pool.cu b/projects/hipccl/hipccl3/rocthrust/testing/mr_disjoint_pool.cu index 46e5877c582c..aadba800980f 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/mr_disjoint_pool.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/mr_disjoint_pool.cu @@ -64,7 +64,6 @@ struct pointer_traits return reinterpret_cast(id.alignment); } }; - } // end namespace detail THRUST_NAMESPACE_END diff --git a/projects/hipccl/hipccl3/rocthrust/testing/pair_reduce.cu b/projects/hipccl/hipccl3/rocthrust/testing/pair_reduce.cu index 5dd6e6e2ce64..a5ae4e1a0c2e 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/pair_reduce.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/pair_reduce.cu @@ -15,18 +15,19 @@ * limitations under the License. */ -#include #include #include +#include _THRUST_STD_INCLUDE(utility) + #include struct make_pair_functor { template - THRUST_HOST_DEVICE thrust::pair operator()(const T1& x, const T2& y) + THRUST_HOST_DEVICE _THRUST_STD::pair operator()(const T1& x, const T2& y) { - return thrust::make_pair(x, y); + return _THRUST_STD::make_pair(x, y); } // end operator()() }; // end make_pair_functor @@ -38,7 +39,7 @@ struct add_pairs // Need cast to undo integer promotion, decltype(char{} + char{}) == int using P1T1 = typename Pair1::first_type; using P1T2 = typename Pair1::second_type; - return thrust::make_pair(static_cast(x.first + y.first), static_cast(x.second + y.second)); + return _THRUST_STD::make_pair(static_cast(x.first + y.first), static_cast(x.second + y.second)); } // end operator() }; // end add_pairs @@ -47,7 +48,7 @@ struct TestPairReduce { void operator()(const size_t n) { - using P = thrust::pair; + using P = _THRUST_STD::pair; thrust::host_vector h_p1 = unittest::random_integers(n); thrust::host_vector h_p2 = unittest::random_integers(n); @@ -60,7 +61,7 @@ struct TestPairReduce thrust::device_vector d_p2 = h_p2; thrust::device_vector

d_pairs = h_pairs; - P init = thrust::make_pair(T{13}, T{13}); + P init = _THRUST_STD::make_pair(T{13}, T{13}); // reduce on the host P h_result = thrust::reduce(h_pairs.begin(), h_pairs.end(), init, add_pairs()); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/pair_scan.cu b/projects/hipccl/hipccl3/rocthrust/testing/pair_scan.cu index 662345b03608..863b7d583d96 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/pair_scan.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/pair_scan.cu @@ -15,19 +15,20 @@ * limitations under the License. */ -#include #include #include +#include _THRUST_STD_INCLUDE(utility) + #include #include struct make_pair_functor { template - THRUST_HOST_DEVICE thrust::pair operator()(const T1& x, const T2& y) + THRUST_HOST_DEVICE _THRUST_STD::pair operator()(const T1& x, const T2& y) { - return thrust::make_pair(x, y); + return _THRUST_STD::make_pair(x, y); } // end operator()() }; // end make_pair_functor @@ -36,7 +37,7 @@ struct add_pairs template THRUST_HOST_DEVICE Pair1 operator()(const Pair1& x, const Pair2& y) { - return thrust::make_pair(x.first + y.first, x.second + y.second); + return _THRUST_STD::make_pair(x.first + y.first, x.second + y.second); } // end operator() }; // end add_pairs @@ -45,7 +46,7 @@ struct TestPairScan { void operator()(const size_t n) { - using P = thrust::pair; + using P = _THRUST_STD::pair; thrust::host_vector h_p1 = unittest::random_integers(n); thrust::host_vector h_p2 = unittest::random_integers(n); @@ -60,7 +61,7 @@ struct TestPairScan thrust::device_vector

d_pairs = h_pairs; thrust::device_vector

d_output(n); - P init = thrust::make_pair(13, 13); + P init = _THRUST_STD::make_pair(13, 13); // scan with plus thrust::inclusive_scan(h_pairs.begin(), h_pairs.end(), h_output.begin(), add_pairs()); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/pair_scan_by_key.cu b/projects/hipccl/hipccl3/rocthrust/testing/pair_scan_by_key.cu index ff4ed73f6d23..25d39cf7d2c6 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/pair_scan_by_key.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/pair_scan_by_key.cu @@ -15,18 +15,19 @@ * limitations under the License. */ -#include #include #include +#include _THRUST_STD_INCLUDE(utility) + #include struct make_pair_functor { template - THRUST_HOST_DEVICE thrust::pair operator()(const T1& x, const T2& y) + THRUST_HOST_DEVICE _THRUST_STD::pair operator()(const T1& x, const T2& y) { - return thrust::make_pair(x, y); + return _THRUST_STD::make_pair(x, y); } // end operator()() }; // end make_pair_functor @@ -38,7 +39,7 @@ struct add_pairs // Need cast to undo integer promotion, decltype(char{} + char{}) == int using P1T1 = typename Pair1::first_type; using P1T2 = typename Pair1::second_type; - return thrust::make_pair(static_cast(x.first + y.first), static_cast(x.second + y.second)); + return _THRUST_STD::make_pair(static_cast(x.first + y.first), static_cast(x.second + y.second)); } // end operator() }; // end add_pairs @@ -47,7 +48,7 @@ struct TestPairScanByKey { void operator()(const size_t n) { - using P = thrust::pair; + using P = _THRUST_STD::pair; thrust::host_vector h_p1 = unittest::random_integers(n); thrust::host_vector h_p2 = unittest::random_integers(n); @@ -63,7 +64,7 @@ struct TestPairScanByKey thrust::host_vector h_keys = unittest::random_integers(n); thrust::device_vector d_keys = h_keys; - P init = thrust::make_pair(T{13}, T{13}); + P init = _THRUST_STD::make_pair(T{13}, T{13}); // scan on the host thrust::exclusive_scan_by_key( diff --git a/projects/hipccl/hipccl3/rocthrust/testing/pair_sort.cu b/projects/hipccl/hipccl3/rocthrust/testing/pair_sort.cu index 68277db6e760..da52e1f881d8 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/pair_sort.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/pair_sort.cu @@ -15,18 +15,19 @@ * limitations under the License. */ -#include #include #include +#include _THRUST_STD_INCLUDE(utility) + #include struct make_pair_functor { template - THRUST_HOST_DEVICE thrust::pair operator()(const T1& x, const T2& y) + THRUST_HOST_DEVICE _THRUST_STD::pair operator()(const T1& x, const T2& y) { - return thrust::make_pair(x, y); + return _THRUST_STD::make_pair(x, y); } // end operator()() }; // end make_pair_functor @@ -35,7 +36,7 @@ struct TestPairStableSortByKey { void operator()(const size_t n) { - using P = thrust::pair; + using P = _THRUST_STD::pair; // host arrays thrust::host_vector h_p1 = unittest::random_integers(n); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/pair_sort_by_key.cu b/projects/hipccl/hipccl3/rocthrust/testing/pair_sort_by_key.cu index b9385b95c0cf..b686a9e964cd 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/pair_sort_by_key.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/pair_sort_by_key.cu @@ -15,17 +15,18 @@ * limitations under the License. */ -#include #include +#include _THRUST_STD_INCLUDE(utility) + #include struct make_pair_functor { template - THRUST_HOST_DEVICE thrust::pair operator()(const T1& x, const T2& y) + THRUST_HOST_DEVICE _THRUST_STD::pair operator()(const T1& x, const T2& y) { - return thrust::make_pair(x, y); + return _THRUST_STD::make_pair(x, y); } // end operator()() }; // end make_pair_functor @@ -34,7 +35,7 @@ struct TestPairStableSort { void operator()(const size_t n) { - using P = thrust::pair; + using P = _THRUST_STD::pair; thrust::host_vector h_p1 = unittest::random_integers(n); thrust::host_vector h_p2 = unittest::random_integers(n); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/pair_transform.cu b/projects/hipccl/hipccl3/rocthrust/testing/pair_transform.cu index b486daac5fde..4e2f18d8f32b 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/pair_transform.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/pair_transform.cu @@ -16,10 +16,11 @@ */ #include -#include #include #include +#include _THRUST_STD_INCLUDE(utility) + #include #if !_THRUST_HAS_DEVICE_SYSTEM_STD @@ -29,9 +30,9 @@ struct make_pair_functor { template - THRUST_HOST_DEVICE thrust::pair operator()(const T1& x, const T2& y) + THRUST_HOST_DEVICE _THRUST_STD::pair operator()(const T1& x, const T2& y) { - return thrust::make_pair(x, y); + return _THRUST_STD::make_pair(x, y); } // end operator()() }; // end make_pair_functor @@ -43,7 +44,7 @@ struct add_pairs using T1 = typename _THRUST_STD::common_type::type; using T2 = typename _THRUST_STD::common_type::type; - return thrust::make_pair(static_cast(x.first + y.first), static_cast(x.second + y.second)); + return _THRUST_STD::make_pair(static_cast(x.first + y.first), static_cast(x.second + y.second)); } // end operator() }; // end add_pairs @@ -52,7 +53,7 @@ struct TestPairTransform { void operator()(const size_t n) { - using P = thrust::pair; + using P = _THRUST_STD::pair; thrust::host_vector h_p1 = unittest::random_integers(n); thrust::host_vector h_p2 = unittest::random_integers(n); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/partition.cu b/projects/hipccl/hipccl3/rocthrust/testing/partition.cu index f8a2098736bc..86b614ed09a5 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/partition.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/partition.cu @@ -96,7 +96,7 @@ void TestPartitionCopySimple() Vector true_results(2); Vector false_results(3); - thrust::pair ends = + _THRUST_STD::pair ends = thrust::partition_copy(data.begin(), data.end(), true_results.begin(), false_results.begin(), is_even()); Vector true_ref(2, 2); @@ -122,7 +122,7 @@ void TestPartitionCopyStencilSimple() Vector true_results(2); Vector false_results(3); - thrust::pair ends = thrust::partition_copy( + _THRUST_STD::pair ends = thrust::partition_copy( data.begin(), data.end(), stencil.begin(), true_results.begin(), false_results.begin(), is_even()); Vector true_ref(2, 1); @@ -181,7 +181,7 @@ void TestStablePartitionCopySimple() Vector true_results(2); Vector false_results(3); - thrust::pair ends = + _THRUST_STD::pair ends = thrust::stable_partition_copy(data.begin(), data.end(), true_results.begin(), false_results.begin(), is_even()); Vector true_ref(2, 2); @@ -204,7 +204,7 @@ void TestStablePartitionCopyStencilSimple() Vector true_results(2); Vector false_results(3); - thrust::pair ends = thrust::stable_partition_copy( + _THRUST_STD::pair ends = thrust::stable_partition_copy( data.begin(), data.end(), stencil.begin(), true_results.begin(), false_results.begin(), _THRUST_STD::identity{}); Vector true_ref(2, 2); @@ -297,11 +297,11 @@ struct TestPartitionCopy thrust::device_vector d_true_results(n_true, 0); thrust::device_vector d_false_results(n_false, 0); - thrust::pair::iterator, typename thrust::host_vector::iterator> h_ends = + _THRUST_STD::pair::iterator, typename thrust::host_vector::iterator> h_ends = thrust::partition_copy( h_data.begin(), h_data.end(), h_true_results.begin(), h_false_results.begin(), is_even()); - thrust::pair::iterator, typename thrust::device_vector::iterator> d_ends = + _THRUST_STD::pair::iterator, typename thrust::device_vector::iterator> d_ends = thrust::partition_copy( d_data.begin(), d_data.end(), d_true_results.begin(), d_false_results.begin(), is_even()); @@ -342,11 +342,11 @@ struct TestPartitionCopyStencil thrust::device_vector d_true_results(n_true, 0); thrust::device_vector d_false_results(n_false, 0); - thrust::pair::iterator, typename thrust::host_vector::iterator> h_ends = + _THRUST_STD::pair::iterator, typename thrust::host_vector::iterator> h_ends = thrust::partition_copy( h_data.begin(), h_data.end(), h_stencil.begin(), h_true_results.begin(), h_false_results.begin(), is_even()); - thrust::pair::iterator, typename thrust::device_vector::iterator> d_ends = + _THRUST_STD::pair::iterator, typename thrust::device_vector::iterator> d_ends = thrust::partition_copy( d_data.begin(), d_data.end(), d_stencil.begin(), d_true_results.begin(), d_false_results.begin(), is_even()); @@ -387,11 +387,11 @@ struct TestStablePartitionCopyStencil thrust::device_vector d_true_results(n_true, 0); thrust::device_vector d_false_results(n_false, 0); - thrust::pair::iterator, typename thrust::host_vector::iterator> h_ends = + _THRUST_STD::pair::iterator, typename thrust::host_vector::iterator> h_ends = thrust::stable_partition_copy( h_data.begin(), h_data.end(), h_stencil.begin(), h_true_results.begin(), h_false_results.begin(), is_even()); - thrust::pair::iterator, typename thrust::device_vector::iterator> d_ends = + _THRUST_STD::pair::iterator, typename thrust::device_vector::iterator> d_ends = thrust::stable_partition_copy( d_data.begin(), d_data.end(), d_stencil.begin(), d_true_results.begin(), d_false_results.begin(), is_even()); @@ -425,14 +425,14 @@ struct TestPartitionCopyToDiscardIterator std::ptrdiff_t n_false = n - n_true; // mask both ranges - thrust::pair, thrust::discard_iterator<>> h_result1 = thrust::partition_copy( + _THRUST_STD::pair, thrust::discard_iterator<>> h_result1 = thrust::partition_copy( h_data.begin(), h_data.end(), thrust::make_discard_iterator(), thrust::make_discard_iterator(), is_even()); - thrust::pair, thrust::discard_iterator<>> d_result1 = thrust::partition_copy( + _THRUST_STD::pair, thrust::discard_iterator<>> d_result1 = thrust::partition_copy( d_data.begin(), d_data.end(), thrust::make_discard_iterator(), thrust::make_discard_iterator(), is_even()); - thrust::pair, thrust::discard_iterator<>> reference1 = - thrust::make_pair(thrust::make_discard_iterator(n_true), thrust::make_discard_iterator(n_false)); + _THRUST_STD::pair, thrust::discard_iterator<>> reference1 = + _THRUST_STD::make_pair(thrust::make_discard_iterator(n_true), thrust::make_discard_iterator(n_false)); ASSERT_EQUAL_QUIET(reference1, h_result1); ASSERT_EQUAL_QUIET(reference1, d_result1); @@ -441,19 +441,19 @@ struct TestPartitionCopyToDiscardIterator thrust::host_vector h_trues(n_true); thrust::device_vector d_trues(n_true); - thrust::pair::iterator, thrust::discard_iterator<>> h_result2 = + _THRUST_STD::pair::iterator, thrust::discard_iterator<>> h_result2 = thrust::partition_copy( h_data.begin(), h_data.end(), h_trues.begin(), thrust::make_discard_iterator(), is_even()); - thrust::pair::iterator, thrust::discard_iterator<>> d_result2 = + _THRUST_STD::pair::iterator, thrust::discard_iterator<>> d_result2 = thrust::partition_copy( d_data.begin(), d_data.end(), d_trues.begin(), thrust::make_discard_iterator(), is_even()); - thrust::pair::iterator, thrust::discard_iterator<>> h_reference2 = - thrust::make_pair(h_trues.begin() + n_true, thrust::make_discard_iterator(n_false)); + _THRUST_STD::pair::iterator, thrust::discard_iterator<>> h_reference2 = + _THRUST_STD::make_pair(h_trues.begin() + n_true, thrust::make_discard_iterator(n_false)); - thrust::pair::iterator, thrust::discard_iterator<>> d_reference2 = - thrust::make_pair(d_trues.begin() + n_true, thrust::make_discard_iterator(n_false)); + _THRUST_STD::pair::iterator, thrust::discard_iterator<>> d_reference2 = + _THRUST_STD::make_pair(d_trues.begin() + n_true, thrust::make_discard_iterator(n_false)); ASSERT_EQUAL(h_trues, d_trues); ASSERT_EQUAL_QUIET(h_reference2, h_result2); @@ -463,19 +463,19 @@ struct TestPartitionCopyToDiscardIterator thrust::host_vector h_falses(n_false); thrust::device_vector d_falses(n_false); - thrust::pair, typename thrust::host_vector::iterator> h_result3 = + _THRUST_STD::pair, typename thrust::host_vector::iterator> h_result3 = thrust::partition_copy( h_data.begin(), h_data.end(), thrust::make_discard_iterator(), h_falses.begin(), is_even()); - thrust::pair, typename thrust::device_vector::iterator> d_result3 = + _THRUST_STD::pair, typename thrust::device_vector::iterator> d_result3 = thrust::partition_copy( d_data.begin(), d_data.end(), thrust::make_discard_iterator(), d_falses.begin(), is_even()); - thrust::pair, typename thrust::host_vector::iterator> h_reference3 = - thrust::make_pair(thrust::make_discard_iterator(n_true), h_falses.begin() + n_false); + _THRUST_STD::pair, typename thrust::host_vector::iterator> h_reference3 = + _THRUST_STD::make_pair(thrust::make_discard_iterator(n_true), h_falses.begin() + n_false); - thrust::pair, typename thrust::device_vector::iterator> d_reference3 = - thrust::make_pair(thrust::make_discard_iterator(n_true), d_falses.begin() + n_false); + _THRUST_STD::pair, typename thrust::device_vector::iterator> d_reference3 = + _THRUST_STD::make_pair(thrust::make_discard_iterator(n_true), d_falses.begin() + n_false); ASSERT_EQUAL(h_falses, d_falses); ASSERT_EQUAL_QUIET(h_reference3, h_result3); @@ -499,7 +499,7 @@ struct TestPartitionCopyStencilToDiscardIterator std::ptrdiff_t n_false = n - n_true; // mask both ranges - thrust::pair, thrust::discard_iterator<>> h_result1 = thrust::partition_copy( + _THRUST_STD::pair, thrust::discard_iterator<>> h_result1 = thrust::partition_copy( h_data.begin(), h_data.end(), h_stencil.begin(), @@ -507,7 +507,7 @@ struct TestPartitionCopyStencilToDiscardIterator thrust::make_discard_iterator(), is_even()); - thrust::pair, thrust::discard_iterator<>> d_result1 = thrust::partition_copy( + _THRUST_STD::pair, thrust::discard_iterator<>> d_result1 = thrust::partition_copy( d_data.begin(), d_data.end(), d_stencil.begin(), @@ -515,8 +515,8 @@ struct TestPartitionCopyStencilToDiscardIterator thrust::make_discard_iterator(), is_even()); - thrust::pair, thrust::discard_iterator<>> reference1 = - thrust::make_pair(thrust::make_discard_iterator(n_true), thrust::make_discard_iterator(n_false)); + _THRUST_STD::pair, thrust::discard_iterator<>> reference1 = + _THRUST_STD::make_pair(thrust::make_discard_iterator(n_true), thrust::make_discard_iterator(n_false)); ASSERT_EQUAL_QUIET(reference1, h_result1); ASSERT_EQUAL_QUIET(reference1, d_result1); @@ -525,19 +525,19 @@ struct TestPartitionCopyStencilToDiscardIterator thrust::host_vector h_trues(n_true); thrust::device_vector d_trues(n_true); - thrust::pair::iterator, thrust::discard_iterator<>> h_result2 = + _THRUST_STD::pair::iterator, thrust::discard_iterator<>> h_result2 = thrust::partition_copy( h_data.begin(), h_data.end(), h_stencil.begin(), h_trues.begin(), thrust::make_discard_iterator(), is_even()); - thrust::pair::iterator, thrust::discard_iterator<>> d_result2 = + _THRUST_STD::pair::iterator, thrust::discard_iterator<>> d_result2 = thrust::partition_copy( d_data.begin(), d_data.end(), d_stencil.begin(), d_trues.begin(), thrust::make_discard_iterator(), is_even()); - thrust::pair::iterator, thrust::discard_iterator<>> h_reference2 = - thrust::make_pair(h_trues.begin() + n_true, thrust::make_discard_iterator(n_false)); + _THRUST_STD::pair::iterator, thrust::discard_iterator<>> h_reference2 = + _THRUST_STD::make_pair(h_trues.begin() + n_true, thrust::make_discard_iterator(n_false)); - thrust::pair::iterator, thrust::discard_iterator<>> d_reference2 = - thrust::make_pair(d_trues.begin() + n_true, thrust::make_discard_iterator(n_false)); + _THRUST_STD::pair::iterator, thrust::discard_iterator<>> d_reference2 = + _THRUST_STD::make_pair(d_trues.begin() + n_true, thrust::make_discard_iterator(n_false)); ASSERT_EQUAL(h_trues, d_trues); ASSERT_EQUAL_QUIET(h_reference2, h_result2); @@ -547,7 +547,7 @@ struct TestPartitionCopyStencilToDiscardIterator thrust::host_vector h_falses(n_false); thrust::device_vector d_falses(n_false); - thrust::pair, typename thrust::host_vector::iterator> h_result3 = + _THRUST_STD::pair, typename thrust::host_vector::iterator> h_result3 = thrust::partition_copy( h_data.begin(), h_data.end(), @@ -556,7 +556,7 @@ struct TestPartitionCopyStencilToDiscardIterator h_falses.begin(), is_even()); - thrust::pair, typename thrust::device_vector::iterator> d_result3 = + _THRUST_STD::pair, typename thrust::device_vector::iterator> d_result3 = thrust::partition_copy( d_data.begin(), d_data.end(), @@ -565,11 +565,11 @@ struct TestPartitionCopyStencilToDiscardIterator d_falses.begin(), is_even()); - thrust::pair, typename thrust::host_vector::iterator> h_reference3 = - thrust::make_pair(thrust::make_discard_iterator(n_true), h_falses.begin() + n_false); + _THRUST_STD::pair, typename thrust::host_vector::iterator> h_reference3 = + _THRUST_STD::make_pair(thrust::make_discard_iterator(n_true), h_falses.begin() + n_false); - thrust::pair, typename thrust::device_vector::iterator> d_reference3 = - thrust::make_pair(thrust::make_discard_iterator(n_true), d_falses.begin() + n_false); + _THRUST_STD::pair, typename thrust::device_vector::iterator> d_reference3 = + _THRUST_STD::make_pair(thrust::make_discard_iterator(n_true), d_falses.begin() + n_false); ASSERT_EQUAL(h_falses, d_falses); ASSERT_EQUAL_QUIET(h_reference3, h_result3); @@ -649,11 +649,11 @@ struct TestStablePartitionCopy thrust::device_vector d_true_results(n_true, 0); thrust::device_vector d_false_results(n_false, 0); - thrust::pair::iterator, typename thrust::host_vector::iterator> h_ends = + _THRUST_STD::pair::iterator, typename thrust::host_vector::iterator> h_ends = thrust::stable_partition_copy( h_data.begin(), h_data.end(), h_true_results.begin(), h_false_results.begin(), is_even()); - thrust::pair::iterator, typename thrust::device_vector::iterator> d_ends = + _THRUST_STD::pair::iterator, typename thrust::device_vector::iterator> d_ends = thrust::stable_partition_copy( d_data.begin(), d_data.end(), d_true_results.begin(), d_false_results.begin(), is_even()); @@ -683,14 +683,14 @@ struct TestStablePartitionCopyToDiscardIterator std::ptrdiff_t n_false = n - n_true; // mask both ranges - thrust::pair, thrust::discard_iterator<>> h_result1 = thrust::stable_partition_copy( + _THRUST_STD::pair, thrust::discard_iterator<>> h_result1 = thrust::stable_partition_copy( h_data.begin(), h_data.end(), thrust::make_discard_iterator(), thrust::make_discard_iterator(), is_even()); - thrust::pair, thrust::discard_iterator<>> d_result1 = thrust::stable_partition_copy( + _THRUST_STD::pair, thrust::discard_iterator<>> d_result1 = thrust::stable_partition_copy( d_data.begin(), d_data.end(), thrust::make_discard_iterator(), thrust::make_discard_iterator(), is_even()); - thrust::pair, thrust::discard_iterator<>> reference1 = - thrust::make_pair(thrust::make_discard_iterator(n_true), thrust::make_discard_iterator(n_false)); + _THRUST_STD::pair, thrust::discard_iterator<>> reference1 = + _THRUST_STD::make_pair(thrust::make_discard_iterator(n_true), thrust::make_discard_iterator(n_false)); ASSERT_EQUAL_QUIET(reference1, h_result1); ASSERT_EQUAL_QUIET(reference1, d_result1); @@ -699,19 +699,19 @@ struct TestStablePartitionCopyToDiscardIterator thrust::host_vector h_trues(n_true); thrust::device_vector d_trues(n_true); - thrust::pair::iterator, thrust::discard_iterator<>> h_result2 = + _THRUST_STD::pair::iterator, thrust::discard_iterator<>> h_result2 = thrust::stable_partition_copy( h_data.begin(), h_data.end(), h_trues.begin(), thrust::make_discard_iterator(), is_even()); - thrust::pair::iterator, thrust::discard_iterator<>> d_result2 = + _THRUST_STD::pair::iterator, thrust::discard_iterator<>> d_result2 = thrust::stable_partition_copy( d_data.begin(), d_data.end(), d_trues.begin(), thrust::make_discard_iterator(), is_even()); - thrust::pair::iterator, thrust::discard_iterator<>> h_reference2 = - thrust::make_pair(h_trues.begin() + n_true, thrust::make_discard_iterator(n_false)); + _THRUST_STD::pair::iterator, thrust::discard_iterator<>> h_reference2 = + _THRUST_STD::make_pair(h_trues.begin() + n_true, thrust::make_discard_iterator(n_false)); - thrust::pair::iterator, thrust::discard_iterator<>> d_reference2 = - thrust::make_pair(d_trues.begin() + n_true, thrust::make_discard_iterator(n_false)); + _THRUST_STD::pair::iterator, thrust::discard_iterator<>> d_reference2 = + _THRUST_STD::make_pair(d_trues.begin() + n_true, thrust::make_discard_iterator(n_false)); ASSERT_EQUAL(h_trues, d_trues); ASSERT_EQUAL_QUIET(h_reference2, h_result2); @@ -721,19 +721,19 @@ struct TestStablePartitionCopyToDiscardIterator thrust::host_vector h_falses(n_false); thrust::device_vector d_falses(n_false); - thrust::pair, typename thrust::host_vector::iterator> h_result3 = + _THRUST_STD::pair, typename thrust::host_vector::iterator> h_result3 = thrust::stable_partition_copy( h_data.begin(), h_data.end(), thrust::make_discard_iterator(), h_falses.begin(), is_even()); - thrust::pair, typename thrust::device_vector::iterator> d_result3 = + _THRUST_STD::pair, typename thrust::device_vector::iterator> d_result3 = thrust::stable_partition_copy( d_data.begin(), d_data.end(), thrust::make_discard_iterator(), d_falses.begin(), is_even()); - thrust::pair, typename thrust::host_vector::iterator> h_reference3 = - thrust::make_pair(thrust::make_discard_iterator(n_true), h_falses.begin() + n_false); + _THRUST_STD::pair, typename thrust::host_vector::iterator> h_reference3 = + _THRUST_STD::make_pair(thrust::make_discard_iterator(n_true), h_falses.begin() + n_false); - thrust::pair, typename thrust::device_vector::iterator> d_reference3 = - thrust::make_pair(thrust::make_discard_iterator(n_true), d_falses.begin() + n_false); + _THRUST_STD::pair, typename thrust::device_vector::iterator> d_reference3 = + _THRUST_STD::make_pair(thrust::make_discard_iterator(n_true), d_falses.begin() + n_false); ASSERT_EQUAL(h_falses, d_falses); ASSERT_EQUAL_QUIET(h_reference3, h_result3); @@ -758,7 +758,7 @@ struct TestStablePartitionCopyStencilToDiscardIterator std::ptrdiff_t n_false = n - n_true; // mask both ranges - thrust::pair, thrust::discard_iterator<>> h_result1 = thrust::stable_partition_copy( + _THRUST_STD::pair, thrust::discard_iterator<>> h_result1 = thrust::stable_partition_copy( h_data.begin(), h_data.end(), h_stencil.begin(), @@ -766,7 +766,7 @@ struct TestStablePartitionCopyStencilToDiscardIterator thrust::make_discard_iterator(), is_even()); - thrust::pair, thrust::discard_iterator<>> d_result1 = thrust::stable_partition_copy( + _THRUST_STD::pair, thrust::discard_iterator<>> d_result1 = thrust::stable_partition_copy( d_data.begin(), d_data.end(), d_stencil.begin(), @@ -774,8 +774,8 @@ struct TestStablePartitionCopyStencilToDiscardIterator thrust::make_discard_iterator(), is_even()); - thrust::pair, thrust::discard_iterator<>> reference1 = - thrust::make_pair(thrust::make_discard_iterator(n_true), thrust::make_discard_iterator(n_false)); + _THRUST_STD::pair, thrust::discard_iterator<>> reference1 = + _THRUST_STD::make_pair(thrust::make_discard_iterator(n_true), thrust::make_discard_iterator(n_false)); ASSERT_EQUAL_QUIET(reference1, h_result1); ASSERT_EQUAL_QUIET(reference1, d_result1); @@ -784,19 +784,19 @@ struct TestStablePartitionCopyStencilToDiscardIterator thrust::host_vector h_trues(n_true); thrust::device_vector d_trues(n_true); - thrust::pair::iterator, thrust::discard_iterator<>> h_result2 = + _THRUST_STD::pair::iterator, thrust::discard_iterator<>> h_result2 = thrust::stable_partition_copy( h_data.begin(), h_data.end(), h_stencil.begin(), h_trues.begin(), thrust::make_discard_iterator(), is_even()); - thrust::pair::iterator, thrust::discard_iterator<>> d_result2 = + _THRUST_STD::pair::iterator, thrust::discard_iterator<>> d_result2 = thrust::stable_partition_copy( d_data.begin(), d_data.end(), d_stencil.begin(), d_trues.begin(), thrust::make_discard_iterator(), is_even()); - thrust::pair::iterator, thrust::discard_iterator<>> h_reference2 = - thrust::make_pair(h_trues.begin() + n_true, thrust::make_discard_iterator(n_false)); + _THRUST_STD::pair::iterator, thrust::discard_iterator<>> h_reference2 = + _THRUST_STD::make_pair(h_trues.begin() + n_true, thrust::make_discard_iterator(n_false)); - thrust::pair::iterator, thrust::discard_iterator<>> d_reference2 = - thrust::make_pair(d_trues.begin() + n_true, thrust::make_discard_iterator(n_false)); + _THRUST_STD::pair::iterator, thrust::discard_iterator<>> d_reference2 = + _THRUST_STD::make_pair(d_trues.begin() + n_true, thrust::make_discard_iterator(n_false)); ASSERT_EQUAL(h_trues, d_trues); ASSERT_EQUAL_QUIET(h_reference2, h_result2); @@ -806,7 +806,7 @@ struct TestStablePartitionCopyStencilToDiscardIterator thrust::host_vector h_falses(n_false); thrust::device_vector d_falses(n_false); - thrust::pair, typename thrust::host_vector::iterator> h_result3 = + _THRUST_STD::pair, typename thrust::host_vector::iterator> h_result3 = thrust::stable_partition_copy( h_data.begin(), h_data.end(), @@ -815,7 +815,7 @@ struct TestStablePartitionCopyStencilToDiscardIterator h_falses.begin(), is_even()); - thrust::pair, typename thrust::device_vector::iterator> d_result3 = + _THRUST_STD::pair, typename thrust::device_vector::iterator> d_result3 = thrust::stable_partition_copy( d_data.begin(), d_data.end(), @@ -824,11 +824,11 @@ struct TestStablePartitionCopyStencilToDiscardIterator d_falses.begin(), is_even()); - thrust::pair, typename thrust::host_vector::iterator> h_reference3 = - thrust::make_pair(thrust::make_discard_iterator(n_true), h_falses.begin() + n_false); + _THRUST_STD::pair, typename thrust::host_vector::iterator> h_reference3 = + _THRUST_STD::make_pair(thrust::make_discard_iterator(n_true), h_falses.begin() + n_false); - thrust::pair, typename thrust::device_vector::iterator> d_reference3 = - thrust::make_pair(thrust::make_discard_iterator(n_true), d_falses.begin() + n_false); + _THRUST_STD::pair, typename thrust::device_vector::iterator> d_reference3 = + _THRUST_STD::make_pair(thrust::make_discard_iterator(n_true), d_falses.begin() + n_false); ASSERT_EQUAL(h_falses, d_falses); ASSERT_EQUAL_QUIET(h_reference3, h_result3); @@ -1013,11 +1013,11 @@ void TestPartitionStencilDispatchImplicit() DECLARE_UNITTEST(TestPartitionStencilDispatchImplicit); template -thrust::pair partition_copy( +_THRUST_STD::pair partition_copy( my_system& system, InputIterator, InputIterator, OutputIterator1 out_true, OutputIterator2 out_false, Predicate) { system.validate_dispatch(); - return thrust::make_pair(out_true, out_false); + return _THRUST_STD::make_pair(out_true, out_false); } void TestPartitionCopyDispatchExplicit() @@ -1036,7 +1036,7 @@ template -thrust::pair partition_copy( +_THRUST_STD::pair partition_copy( my_system& system, InputIterator1, InputIterator1, @@ -1046,7 +1046,7 @@ thrust::pair partition_copy( Predicate) { system.validate_dispatch(); - return thrust::make_pair(out_true, out_false); + return _THRUST_STD::make_pair(out_true, out_false); } void TestPartitionCopyStencilDispatchExplicit() @@ -1061,11 +1061,11 @@ void TestPartitionCopyStencilDispatchExplicit() DECLARE_UNITTEST(TestPartitionCopyStencilDispatchExplicit); template -thrust::pair partition_copy( +_THRUST_STD::pair partition_copy( my_tag, InputIterator first, InputIterator, OutputIterator1 out_true, OutputIterator2 out_false, Predicate) { *first = 13; - return thrust::make_pair(out_true, out_false); + return _THRUST_STD::make_pair(out_true, out_false); } void TestPartitionCopyDispatchImplicit() @@ -1088,7 +1088,7 @@ template -thrust::pair partition_copy( +_THRUST_STD::pair partition_copy( my_tag, InputIterator1 first, InputIterator1, @@ -1098,7 +1098,7 @@ thrust::pair partition_copy( Predicate) { *first = 13; - return thrust::make_pair(out_true, out_false); + return _THRUST_STD::make_pair(out_true, out_false); } void TestPartitionCopyStencilDispatchImplicit() @@ -1189,11 +1189,11 @@ void TestStablePartitionStencilDispatchImplicit() DECLARE_UNITTEST(TestStablePartitionStencilDispatchImplicit); template -thrust::pair stable_partition_copy( +_THRUST_STD::pair stable_partition_copy( my_system& system, InputIterator, InputIterator, OutputIterator1 out_true, OutputIterator2 out_false, Predicate) { system.validate_dispatch(); - return thrust::make_pair(out_true, out_false); + return _THRUST_STD::make_pair(out_true, out_false); } void TestStablePartitionCopyDispatchExplicit() @@ -1212,7 +1212,7 @@ template -thrust::pair stable_partition_copy( +_THRUST_STD::pair stable_partition_copy( my_system& system, InputIterator1, InputIterator1, @@ -1222,7 +1222,7 @@ thrust::pair stable_partition_copy( Predicate) { system.validate_dispatch(); - return thrust::make_pair(out_true, out_false); + return _THRUST_STD::make_pair(out_true, out_false); } void TestStablePartitionCopyStencilDispatchExplicit() @@ -1237,11 +1237,11 @@ void TestStablePartitionCopyStencilDispatchExplicit() DECLARE_UNITTEST(TestStablePartitionCopyStencilDispatchExplicit); template -thrust::pair stable_partition_copy( +_THRUST_STD::pair stable_partition_copy( my_tag, InputIterator first, InputIterator, OutputIterator1 out_true, OutputIterator2 out_false, Predicate) { *first = 13; - return thrust::make_pair(out_true, out_false); + return _THRUST_STD::make_pair(out_true, out_false); } void TestStablePartitionCopyDispatchImplicit() @@ -1264,7 +1264,7 @@ template -thrust::pair stable_partition_copy( +_THRUST_STD::pair stable_partition_copy( my_tag, InputIterator1 first, InputIterator1, @@ -1274,7 +1274,7 @@ thrust::pair stable_partition_copy( Predicate) { *first = 13; - return thrust::make_pair(out_true, out_false); + return _THRUST_STD::make_pair(out_true, out_false); } void TestStablePartitionCopyStencilDispatchImplicit() diff --git a/projects/hipccl/hipccl3/rocthrust/testing/reduce_by_key.cu b/projects/hipccl/hipccl3/rocthrust/testing/reduce_by_key.cu index 29963c4993d4..73a2fc039a30 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/reduce_by_key.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/reduce_by_key.cu @@ -53,7 +53,7 @@ void TestReduceByKeySimple() Vector keys; Vector values; - typename thrust::pair new_last; + typename _THRUST_STD::pair new_last; // basic test initialize_keys(keys); @@ -144,8 +144,8 @@ struct TestReduceByKey using DeviceKeyIterator = typename thrust::device_vector::iterator; using DeviceValIterator = typename thrust::device_vector::iterator; - using HostIteratorPair = typename thrust::pair; - using DeviceIteratorPair = typename thrust::pair; + using HostIteratorPair = typename _THRUST_STD::pair; + using DeviceIteratorPair = typename _THRUST_STD::pair; HostIteratorPair h_last = thrust::reduce_by_key(h_keys.begin(), h_keys.end(), h_vals.begin(), h_keys_output.begin(), h_vals_output.begin()); @@ -212,7 +212,7 @@ struct TestReduceByKeyToDiscardIterator VariableUnitTest TestReduceByKeyToDiscardIteratorInstance; template -thrust::pair reduce_by_key( +_THRUST_STD::pair reduce_by_key( my_system& system, InputIterator1, InputIterator1, @@ -221,7 +221,7 @@ thrust::pair reduce_by_key( OutputIterator2 values_output) { system.validate_dispatch(); - return thrust::make_pair(keys_output, values_output); + return _THRUST_STD::make_pair(keys_output, values_output); } void TestReduceByKeyDispatchExplicit() @@ -236,11 +236,11 @@ void TestReduceByKeyDispatchExplicit() DECLARE_UNITTEST(TestReduceByKeyDispatchExplicit); template -thrust::pair reduce_by_key( +_THRUST_STD::pair reduce_by_key( my_tag, InputIterator1, InputIterator1, InputIterator2, OutputIterator1 keys_output, OutputIterator2 values_output) { *keys_output = 13; - return thrust::make_pair(keys_output, values_output); + return _THRUST_STD::make_pair(keys_output, values_output); } void TestReduceByKeyDispatchImplicit() diff --git a/projects/hipccl/hipccl3/rocthrust/testing/reverse_iterator.cu b/projects/hipccl/hipccl3/rocthrust/testing/reverse_iterator.cu index 9d18a93d7aa6..9b5ad3c9f4d4 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/reverse_iterator.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/reverse_iterator.cu @@ -22,12 +22,9 @@ #include +#include _THRUST_STD_INCLUDE(iterator) #include _THRUST_STD_INCLUDE(type_traits) -#if !_THRUST_HAS_DEVICE_SYSTEM_STD -# include -#endif - // ensure that we properly support thrust::reverse_iterator from _THRUST_STD void TestReverseIteratorTraits() { @@ -47,14 +44,12 @@ void TestReverseIteratorTraits() static_assert(_THRUST_STD::__has_random_access_traversal); -#if _THRUST_HAS_DEVICE_SYSTEM_STD || THRUST_STD_VER >= 2020 static_assert(_THRUST_STD::output_iterator); static_assert(_THRUST_STD::input_iterator); static_assert(_THRUST_STD::forward_iterator); static_assert(_THRUST_STD::bidirectional_iterator); static_assert(_THRUST_STD::random_access_iterator); static_assert(!_THRUST_STD::contiguous_iterator); -#endif } DECLARE_UNITTEST(TestReverseIteratorTraits); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/scan.cu b/projects/hipccl/hipccl3/rocthrust/testing/scan.cu index c4d0ea1edc53..4ae51b6fbcbd 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/scan.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/scan.cu @@ -25,17 +25,14 @@ #include #include #include +#include #include -#if _THRUST_HAS_DEVICE_SYSTEM_STD -# include _THRUST_LIBCXX_INCLUDE(functional) -#endif +#include _THRUST_LIBCXX_INCLUDE(functional) #include _THRUST_STD_INCLUDE(array) -#if !_THRUST_HAS_DEVICE_SYSTEM_STD -# include -#endif +#include template void TestScanSimple() @@ -567,15 +564,23 @@ struct only_set_when_expected_it { return *this; } + + THRUST_HOST_DEVICE only_set_when_expected_it operator++(int) const + { + return *this; + } + THRUST_HOST_DEVICE only_set_when_expected_it operator*() const { return *this; } + template THRUST_HOST_DEVICE only_set_when_expected_it operator+(Difference) const { return *this; } + template THRUST_HOST_DEVICE only_set_when_expected_it operator[](Index) const { @@ -602,7 +607,6 @@ struct iterator_traits }; } // namespace std -#if _THRUST_HAS_DEVICE_SYSTEM_STD _THRUST_STD_NAMESPACE_BEGIN template <> struct iterator_traits @@ -613,7 +617,6 @@ struct iterator_traits using difference_type = _THRUST_STD::ptrdiff_t; }; _THRUST_STD_NAMESPACE_END -#endif void TestInclusiveScanWithBigIndexesHelper(int magnitude) { @@ -687,24 +690,40 @@ struct Int THRUST_HOST_DEVICE explicit Int(int num) : i(num) {} - THRUST_HOST_DEVICE Int() - : i{} - {} - THRUST_HOST_DEVICE Int operator+(Int const& o) const + + // TODO(bgruber): I think we should not need this constructors, but CUB fails to compile without it + Int() = default; + + THRUST_HOST_DEVICE friend Int operator+(Int const& a, Int const& b) { - return Int{this->i + o.i}; + return Int{a.i + b.i}; } -}; -void TestInclusiveScanWithUserDefinedType() -{ - thrust::device_vector vec(5, Int{1}); + THRUST_HOST_DEVICE friend bool operator==(Int const& a, Int const& b) + { + return a.i == b.i; + } - thrust::inclusive_scan(thrust::device, vec.cbegin(), vec.cend(), vec.begin()); + friend std::ostream& operator<<(std::ostream& os, Int const& a) + { + return os << "Int{" << a.i << "}"; + } +}; - ASSERT_EQUAL(static_cast(vec.back()).i, 5); +void TestScanWithUserDefinedTypeAndInit() +{ + { + thrust::device_vector vec(5, Int{1}); + thrust::exclusive_scan(thrust::device, vec.cbegin(), vec.cend(), vec.begin(), Int{100}, _THRUST_STD::plus()); + ASSERT_EQUAL(vec, (thrust::device_vector{Int{100}, Int{101}, Int{102}, Int{103}, Int{104}})); + } + { + thrust::device_vector vec(5, Int{1}); + thrust::inclusive_scan(thrust::device, vec.cbegin(), vec.cend(), vec.begin(), Int{100}, _THRUST_STD::plus()); + ASSERT_EQUAL(vec, (thrust::device_vector{Int{101}, Int{102}, Int{103}, Int{104}, Int{105}})); + } } -DECLARE_UNITTEST(TestInclusiveScanWithUserDefinedType); +DECLARE_UNITTEST(TestScanWithUserDefinedTypeAndInit); // Represents a permutation as a tuple of integers, see also: https://en.wikipedia.org/wiki/Permutation // We need a distinct type (instead of an alias) for operator<< to be found via ADL @@ -797,3 +816,148 @@ void TestInclusiveScanWithNonCommutativeOp() {4, 0, 2, 1, 3}})); } DECLARE_UNITTEST(TestInclusiveScanWithNonCommutativeOp); + +// Adapted from issue: https://github.com/NVIDIA/cccl/issues/6317 +void TestScanBug6317() +{ + using T = unsigned int; + const int n = 1729; + { + thrust::device_vector s = unittest::random_integers(n); + thrust::device_vector d(n); + const auto r = thrust::inclusive_scan(s.cbegin(), s.cend(), d.begin(), _THRUST_STD::multiplies<>{}); + ASSERT_EQUAL((d.end() == r), true); + std::partial_sum(s.cbegin(), s.cend(), s.begin(), std::multiplies<>{}); + ASSERT_EQUAL(s, d); + } + { + thrust::device_vector s = unittest::random_integers(n); + thrust::device_vector d(n); + const auto r = thrust::exclusive_scan(s.cbegin(), s.cend(), d.begin(), 42, _THRUST_STD::multiplies<>{}); + ASSERT_EQUAL((d.end() == r), true); + thrust::exclusive_scan(s.cbegin(), s.cend(), s.begin(), 42, _THRUST_STD::multiplies<>{}); + ASSERT_EQUAL(s, d); + } +} +DECLARE_UNITTEST(TestScanBug6317); + +// Test edge cases for parallel scan with non-additive operations +void TestScanEdgeCases() +{ + // Test 1: Large array with inclusive_scan, multiplies, and init + // Tests Step 1 fix with many threads + { + const int n = 10000; + thrust::device_vector d_input(n); + using thrust::placeholders::_1; + thrust::tabulate(d_input.begin(), d_input.end(), (_1 % 5) + 1); + + thrust::device_vector d_output(n); + auto r = thrust::inclusive_scan(d_input.begin(), d_input.end(), d_output.begin(), 2, _THRUST_STD::multiplies<>{}); + ASSERT_EQUAL((d_output.end() == r), true); + + // Verify with host scan + thrust::host_vector h_input = d_input; + thrust::host_vector h_output(n); + thrust::inclusive_scan(h_input.begin(), h_input.end(), h_output.begin(), 2, _THRUST_STD::multiplies<>{}); + + ASSERT_EQUAL(d_output, h_output); + } + + // Test 2: Boundary exactly at parallel_scan_threshold (1024) + { + const int n = 1024; + thrust::device_vector d_input(n); + using thrust::placeholders::_1; + thrust::tabulate(d_input.begin(), d_input.end(), _1 + 1); + + thrust::device_vector d_output(n); + auto r = thrust::inclusive_scan(d_input.begin(), d_input.end(), d_output.begin(), 10, _THRUST_STD::multiplies<>{}); + ASSERT_EQUAL((d_output.end() == r), true); + + thrust::host_vector h_input = d_input; + thrust::host_vector h_output(n); + thrust::inclusive_scan(h_input.begin(), h_input.end(), h_output.begin(), 10, _THRUST_STD::multiplies<>{}); + + ASSERT_EQUAL(d_output, h_output); + } + + // Test 3: Below threshold (1023) should use serial path + { + const int n = 1023; + thrust::device_vector d_input(n); + using thrust::placeholders::_1; + thrust::tabulate(d_input.begin(), d_input.end(), (_1 % 3) + 1); + + thrust::device_vector d_output(n); + auto r = thrust::inclusive_scan(d_input.begin(), d_input.end(), d_output.begin(), 5, _THRUST_STD::multiplies<>{}); + ASSERT_EQUAL((d_output.end() == r), true); + + thrust::host_vector h_input = d_input; + thrust::host_vector h_output(n); + thrust::inclusive_scan(h_input.begin(), h_input.end(), h_output.begin(), 5, _THRUST_STD::multiplies<>{}); + + ASSERT_EQUAL(d_output, h_output); + } + + // Test 4: Very small array (edge case for block distribution) + { + thrust::device_vector d_input = {3, 7}; + thrust::device_vector d_output(2); + + auto r = thrust::inclusive_scan(d_input.begin(), d_input.end(), d_output.begin(), 2, _THRUST_STD::multiplies<>{}); + ASSERT_EQUAL((d_output.end() == r), true); + + thrust::device_vector expected = {6, 42}; + ASSERT_EQUAL(d_output, expected); + } + + // Test 5: exclusive_scan with large array and multiplies + { + const int n = 10000; + thrust::device_vector d_input(n); + using thrust::placeholders::_1; + thrust::tabulate(d_input.begin(), d_input.end(), (_1 % 3) + 1); + + thrust::device_vector d_output(n); + auto r = thrust::exclusive_scan(d_input.begin(), d_input.end(), d_output.begin(), 5, _THRUST_STD::multiplies<>{}); + ASSERT_EQUAL((d_output.end() == r), true); + + thrust::host_vector h_input = d_input; + thrust::host_vector h_output(n); + thrust::exclusive_scan(h_input.begin(), h_input.end(), h_output.begin(), 5, _THRUST_STD::multiplies<>{}); + + ASSERT_EQUAL(d_output, h_output); + } + + // Test 6: exclusive_scan at boundary (1024 elements) + { + const int n = 1024; + thrust::device_vector d_input(n); + using thrust::placeholders::_1; + thrust::tabulate(d_input.begin(), d_input.end(), (_1 % 5) + 1); + + thrust::device_vector d_output(n); + auto r = thrust::exclusive_scan(d_input.begin(), d_input.end(), d_output.begin(), 3, _THRUST_STD::multiplies<>{}); + ASSERT_EQUAL((d_output.end() == r), true); + + thrust::host_vector h_input = d_input; + thrust::host_vector h_output(n); + thrust::exclusive_scan(h_input.begin(), h_input.end(), h_output.begin(), 3, _THRUST_STD::multiplies<>{}); + + ASSERT_EQUAL(d_output, h_output); + } + + // Test 7: exclusive_scan with very small array + { + thrust::device_vector d_input = {2, 4}; + thrust::device_vector d_output(2); + + auto r = thrust::exclusive_scan(d_input.begin(), d_input.end(), d_output.begin(), 3, _THRUST_STD::multiplies<>{}); + ASSERT_EQUAL((d_output.end() == r), true); + + thrust::device_vector expected = {3, 6}; + ASSERT_EQUAL(d_output, expected); + } +} +DECLARE_UNITTEST(TestScanEdgeCases); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/set_difference_by_key.cu b/projects/hipccl/hipccl3/rocthrust/testing/set_difference_by_key.cu index 1de71b603daf..ba901fa88d8b 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/set_difference_by_key.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/set_difference_by_key.cu @@ -28,7 +28,7 @@ template -thrust::pair set_difference_by_key( +_THRUST_STD::pair set_difference_by_key( my_system& system, InputIterator1, InputIterator1, @@ -40,7 +40,7 @@ thrust::pair set_difference_by_key( OutputIterator2 values_result) { system.validate_dispatch(); - return thrust::make_pair(keys_result, values_result); + return _THRUST_STD::make_pair(keys_result, values_result); } void TestSetDifferenceByKeyDispatchExplicit() @@ -61,7 +61,7 @@ template -thrust::pair set_difference_by_key( +_THRUST_STD::pair set_difference_by_key( my_tag, InputIterator1, InputIterator1, @@ -73,7 +73,7 @@ thrust::pair set_difference_by_key( OutputIterator2 values_result) { *keys_result = 13; - return thrust::make_pair(keys_result, values_result); + return _THRUST_STD::make_pair(keys_result, values_result); } void TestSetDifferenceByKeyDispatchImplicit() @@ -110,7 +110,7 @@ void TestSetDifferenceByKeySimple() Vector result_key(2), result_val(2); - thrust::pair end = thrust::set_difference_by_key( + _THRUST_STD::pair end = thrust::set_difference_by_key( a_key.begin(), a_key.end(), b_key.begin(), @@ -161,9 +161,9 @@ void TestSetDifferenceByKey(const size_t n) thrust::device_vector d_result_keys(n); thrust::device_vector d_result_vals(n); - thrust::pair::iterator, typename thrust::host_vector::iterator> h_end; + _THRUST_STD::pair::iterator, typename thrust::host_vector::iterator> h_end; - thrust::pair::iterator, typename thrust::device_vector::iterator> d_end; + _THRUST_STD::pair::iterator, typename thrust::device_vector::iterator> d_end; h_end = thrust::set_difference_by_key( h_a_keys.begin(), @@ -216,9 +216,9 @@ void TestSetDifferenceByKeyEquivalentRanges(const size_t n) thrust::host_vector h_result_key(n), h_result_val(n); thrust::device_vector d_result_key(n), d_result_val(n); - thrust::pair::iterator, typename thrust::host_vector::iterator> h_end; + _THRUST_STD::pair::iterator, typename thrust::host_vector::iterator> h_end; - thrust::pair::iterator, typename thrust::device_vector::iterator> d_end; + _THRUST_STD::pair::iterator, typename thrust::device_vector::iterator> d_end; h_end = thrust::set_difference_by_key( h_a_key.begin(), @@ -280,9 +280,9 @@ void TestSetDifferenceByKeyMultiset(const size_t n) thrust::host_vector h_result_key(n), h_result_val(n); thrust::device_vector d_result_key(n), d_result_val(n); - thrust::pair::iterator, typename thrust::host_vector::iterator> h_end; + _THRUST_STD::pair::iterator, typename thrust::host_vector::iterator> h_end; - thrust::pair::iterator, typename thrust::device_vector::iterator> d_end; + _THRUST_STD::pair::iterator, typename thrust::device_vector::iterator> d_end; h_end = thrust::set_difference_by_key( h_a_key.begin(), diff --git a/projects/hipccl/hipccl3/rocthrust/testing/set_difference_by_key_descending.cu b/projects/hipccl/hipccl3/rocthrust/testing/set_difference_by_key_descending.cu index 2ec484463d7d..4414bdc96084 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/set_difference_by_key_descending.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/set_difference_by_key_descending.cu @@ -34,7 +34,7 @@ void TestSetDifferenceByKeyDescendingSimple() Vector result_key(2), result_val(2); - thrust::pair end = thrust::set_difference_by_key( + _THRUST_STD::pair end = thrust::set_difference_by_key( a_key.begin(), a_key.end(), b_key.begin(), @@ -74,9 +74,9 @@ void TestSetDifferenceByKeyDescending(const size_t n) thrust::host_vector h_result_key(n), h_result_val(n); thrust::device_vector d_result_key(n), d_result_val(n); - thrust::pair::iterator, typename thrust::host_vector::iterator> h_end; + _THRUST_STD::pair::iterator, typename thrust::host_vector::iterator> h_end; - thrust::pair::iterator, typename thrust::device_vector::iterator> d_end; + _THRUST_STD::pair::iterator, typename thrust::device_vector::iterator> d_end; h_end = thrust::set_difference_by_key( h_a_key.begin(), diff --git a/projects/hipccl/hipccl3/rocthrust/testing/set_intersection_by_key.cu b/projects/hipccl/hipccl3/rocthrust/testing/set_intersection_by_key.cu index 3fcac17fb6cf..3f333a1bb2f9 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/set_intersection_by_key.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/set_intersection_by_key.cu @@ -27,7 +27,7 @@ template -thrust::pair set_intersection_by_key( +_THRUST_STD::pair set_intersection_by_key( my_system& system, InputIterator1, InputIterator1, @@ -38,7 +38,7 @@ thrust::pair set_intersection_by_key( OutputIterator2 values_result) { system.validate_dispatch(); - return thrust::make_pair(keys_result, values_result); + return _THRUST_STD::make_pair(keys_result, values_result); } void TestSetIntersectionByKeyDispatchExplicit() @@ -58,7 +58,7 @@ template -thrust::pair set_intersection_by_key( +_THRUST_STD::pair set_intersection_by_key( my_tag, InputIterator1, InputIterator1, @@ -69,7 +69,7 @@ thrust::pair set_intersection_by_key( OutputIterator2 values_result) { *keys_result = 13; - return thrust::make_pair(keys_result, values_result); + return _THRUST_STD::make_pair(keys_result, values_result); } void TestSetIntersectionByKeyDispatchImplicit() @@ -100,7 +100,7 @@ void TestSetIntersectionByKeySimple() Vector ref_key{0, 4}, ref_val{0, 0}; Vector result_key(2), result_val(2); - thrust::pair end = thrust::set_intersection_by_key( + _THRUST_STD::pair end = thrust::set_intersection_by_key( a_key.begin(), a_key.end(), b_key.begin(), b_key.end(), a_val.begin(), result_key.begin(), result_val.begin()); ASSERT_EQUAL_QUIET(result_key.end(), end.first); @@ -142,9 +142,9 @@ void TestSetIntersectionByKey(const size_t n) thrust::device_vector d_result_keys(n); thrust::device_vector d_result_vals(n); - thrust::pair::iterator, typename thrust::host_vector::iterator> h_end; + _THRUST_STD::pair::iterator, typename thrust::host_vector::iterator> h_end; - thrust::pair::iterator, typename thrust::device_vector::iterator> d_end; + _THRUST_STD::pair::iterator, typename thrust::device_vector::iterator> d_end; h_end = thrust::set_intersection_by_key( h_a_keys.begin(), @@ -193,9 +193,9 @@ void TestSetIntersectionByKeyEquivalentRanges(const size_t n) thrust::host_vector h_result_key(n), h_result_val(n); thrust::device_vector d_result_key(n), d_result_val(n); - thrust::pair::iterator, typename thrust::host_vector::iterator> h_end; + _THRUST_STD::pair::iterator, typename thrust::host_vector::iterator> h_end; - thrust::pair::iterator, typename thrust::device_vector::iterator> d_end; + _THRUST_STD::pair::iterator, typename thrust::device_vector::iterator> d_end; h_end = thrust::set_intersection_by_key( h_a_key.begin(), @@ -253,9 +253,9 @@ void TestSetIntersectionByKeyMultiset(const size_t n) thrust::host_vector h_result_key(n), h_result_val(n); thrust::device_vector d_result_key(n), d_result_val(n); - thrust::pair::iterator, typename thrust::host_vector::iterator> h_end; + _THRUST_STD::pair::iterator, typename thrust::host_vector::iterator> h_end; - thrust::pair::iterator, typename thrust::device_vector::iterator> d_end; + _THRUST_STD::pair::iterator, typename thrust::device_vector::iterator> d_end; h_end = thrust::set_intersection_by_key( h_a_key.begin(), diff --git a/projects/hipccl/hipccl3/rocthrust/testing/set_intersection_by_key_descending.cu b/projects/hipccl/hipccl3/rocthrust/testing/set_intersection_by_key_descending.cu index a8c61ed7abdd..128c7ca109f9 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/set_intersection_by_key_descending.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/set_intersection_by_key_descending.cu @@ -33,7 +33,7 @@ void TestSetIntersectionByKeyDescendingSimple() Vector ref_key{4, 0}, ref_val{0, 0}; Vector result_key(2), result_val(2); - thrust::pair end = thrust::set_intersection_by_key( + _THRUST_STD::pair end = thrust::set_intersection_by_key( a_key.begin(), a_key.end(), b_key.begin(), @@ -70,9 +70,9 @@ void TestSetIntersectionByKeyDescending(const size_t n) thrust::host_vector h_result_key(n), h_result_val(n); thrust::device_vector d_result_key(n), d_result_val(n); - thrust::pair::iterator, typename thrust::host_vector::iterator> h_end; + _THRUST_STD::pair::iterator, typename thrust::host_vector::iterator> h_end; - thrust::pair::iterator, typename thrust::device_vector::iterator> d_end; + _THRUST_STD::pair::iterator, typename thrust::device_vector::iterator> d_end; h_end = thrust::set_intersection_by_key( h_a_key.begin(), diff --git a/projects/hipccl/hipccl3/rocthrust/testing/set_symmetric_difference_by_key.cu b/projects/hipccl/hipccl3/rocthrust/testing/set_symmetric_difference_by_key.cu index 4471ba6e382a..c9109f7fb668 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/set_symmetric_difference_by_key.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/set_symmetric_difference_by_key.cu @@ -28,7 +28,7 @@ template -thrust::pair set_symmetric_difference_by_key( +_THRUST_STD::pair set_symmetric_difference_by_key( my_system& system, InputIterator1, InputIterator1, @@ -40,7 +40,7 @@ thrust::pair set_symmetric_difference_by_key( OutputIterator2 values_result) { system.validate_dispatch(); - return thrust::make_pair(keys_result, values_result); + return _THRUST_STD::make_pair(keys_result, values_result); } void TestSetSymmetricDifferenceByKeyDispatchExplicit() @@ -61,7 +61,7 @@ template -thrust::pair set_symmetric_difference_by_key( +_THRUST_STD::pair set_symmetric_difference_by_key( my_tag, InputIterator1, InputIterator1, @@ -73,7 +73,7 @@ thrust::pair set_symmetric_difference_by_key( OutputIterator2 values_result) { *keys_result = 13; - return thrust::make_pair(keys_result, values_result); + return _THRUST_STD::make_pair(keys_result, values_result); } void TestSetSymmetricDifferenceByKeyDispatchImplicit() @@ -105,7 +105,7 @@ void TestSetSymmetricDifferenceByKeySimple() Vector ref_key{2, 3, 3, 6, 7}, ref_val{0, 1, 1, 0, 1}; Vector result_key(5), result_val(5); - thrust::pair end = thrust::set_symmetric_difference_by_key( + _THRUST_STD::pair end = thrust::set_symmetric_difference_by_key( a_key.begin(), a_key.end(), b_key.begin(), @@ -158,9 +158,9 @@ void TestSetSymmetricDifferenceByKey(const size_t n) thrust::device_vector d_result_keys(max_size); thrust::device_vector d_result_vals(max_size); - thrust::pair::iterator, typename thrust::host_vector::iterator> h_end; + _THRUST_STD::pair::iterator, typename thrust::host_vector::iterator> h_end; - thrust::pair::iterator, typename thrust::device_vector::iterator> d_end; + _THRUST_STD::pair::iterator, typename thrust::device_vector::iterator> d_end; h_end = thrust::set_symmetric_difference_by_key( h_a_keys.begin(), @@ -215,9 +215,9 @@ void TestSetSymmetricDifferenceByKeyEquivalentRanges(const size_t n) thrust::host_vector h_result_key(max_size), h_result_val(max_size); thrust::device_vector d_result_key(max_size), d_result_val(max_size); - thrust::pair::iterator, typename thrust::host_vector::iterator> h_end; + _THRUST_STD::pair::iterator, typename thrust::host_vector::iterator> h_end; - thrust::pair::iterator, typename thrust::device_vector::iterator> d_end; + _THRUST_STD::pair::iterator, typename thrust::device_vector::iterator> d_end; h_end = thrust::set_symmetric_difference_by_key( h_a_key.begin(), @@ -280,9 +280,9 @@ void TestSetSymmetricDifferenceByKeyMultiset(const size_t n) thrust::host_vector h_result_key(max_size), h_result_val(max_size); thrust::device_vector d_result_key(max_size), d_result_val(max_size); - thrust::pair::iterator, typename thrust::host_vector::iterator> h_end; + _THRUST_STD::pair::iterator, typename thrust::host_vector::iterator> h_end; - thrust::pair::iterator, typename thrust::device_vector::iterator> d_end; + _THRUST_STD::pair::iterator, typename thrust::device_vector::iterator> d_end; h_end = thrust::set_symmetric_difference_by_key( h_a_key.begin(), diff --git a/projects/hipccl/hipccl3/rocthrust/testing/set_symmetric_difference_by_key_descending.cu b/projects/hipccl/hipccl3/rocthrust/testing/set_symmetric_difference_by_key_descending.cu index eee7ebc99861..8f259c4076cb 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/set_symmetric_difference_by_key_descending.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/set_symmetric_difference_by_key_descending.cu @@ -33,7 +33,7 @@ void TestSetSymmetricDifferenceByKeyDescendingSimple() Vector ref_key{7, 6, 3, 3, 2}, ref_val{1, 0, 1, 1, 0}; Vector result_key(5), result_val(5); - thrust::pair end = thrust::set_symmetric_difference_by_key( + _THRUST_STD::pair end = thrust::set_symmetric_difference_by_key( a_key.begin(), a_key.end(), b_key.begin(), @@ -74,9 +74,9 @@ void TestSetSymmetricDifferenceByKeyDescending(const size_t n) thrust::host_vector h_result_key(max_size), h_result_val(max_size); thrust::device_vector d_result_key(max_size), d_result_val(max_size); - thrust::pair::iterator, typename thrust::host_vector::iterator> h_end; + _THRUST_STD::pair::iterator, typename thrust::host_vector::iterator> h_end; - thrust::pair::iterator, typename thrust::device_vector::iterator> d_end; + _THRUST_STD::pair::iterator, typename thrust::device_vector::iterator> d_end; h_end = thrust::set_symmetric_difference_by_key( h_a_key.begin(), diff --git a/projects/hipccl/hipccl3/rocthrust/testing/set_union_by_key.cu b/projects/hipccl/hipccl3/rocthrust/testing/set_union_by_key.cu index e1586782ef64..0a12738c6448 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/set_union_by_key.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/set_union_by_key.cu @@ -28,7 +28,7 @@ template -thrust::pair set_union_by_key( +_THRUST_STD::pair set_union_by_key( my_system& system, InputIterator1, InputIterator1, @@ -40,7 +40,7 @@ thrust::pair set_union_by_key( OutputIterator2 values_result) { system.validate_dispatch(); - return thrust::make_pair(keys_result, values_result); + return _THRUST_STD::make_pair(keys_result, values_result); } void TestSetUnionByKeyDispatchExplicit() @@ -61,7 +61,7 @@ template -thrust::pair set_union_by_key( +_THRUST_STD::pair set_union_by_key( my_tag, InputIterator1, InputIterator1, @@ -73,7 +73,7 @@ thrust::pair set_union_by_key( OutputIterator2 values_result) { *keys_result = 13; - return thrust::make_pair(keys_result, values_result); + return _THRUST_STD::make_pair(keys_result, values_result); } void TestSetUnionByKeyDispatchImplicit() @@ -105,7 +105,7 @@ void TestSetUnionByKeySimple() Vector ref_key{0, 2, 3, 3, 4}, ref_val{0, 0, 1, 1, 0}; Vector result_key(5), result_val(5); - thrust::pair end = thrust::set_union_by_key( + _THRUST_STD::pair end = thrust::set_union_by_key( a_key.begin(), a_key.end(), b_key.begin(), @@ -158,9 +158,9 @@ void TestSetUnionByKey(const size_t n) thrust::device_vector d_result_keys(max_size); thrust::device_vector d_result_vals(max_size); - thrust::pair::iterator, typename thrust::host_vector::iterator> h_end; + _THRUST_STD::pair::iterator, typename thrust::host_vector::iterator> h_end; - thrust::pair::iterator, typename thrust::device_vector::iterator> d_end; + _THRUST_STD::pair::iterator, typename thrust::device_vector::iterator> d_end; h_end = thrust::set_union_by_key( h_a_keys.begin(), @@ -215,9 +215,9 @@ void TestSetUnionByKeyEquivalentRanges(const size_t n) thrust::host_vector h_result_key(max_size), h_result_val(max_size); thrust::device_vector d_result_key(max_size), d_result_val(max_size); - thrust::pair::iterator, typename thrust::host_vector::iterator> h_end; + _THRUST_STD::pair::iterator, typename thrust::host_vector::iterator> h_end; - thrust::pair::iterator, typename thrust::device_vector::iterator> d_end; + _THRUST_STD::pair::iterator, typename thrust::device_vector::iterator> d_end; h_end = thrust::set_union_by_key( h_a_key.begin(), @@ -280,9 +280,9 @@ void TestSetUnionByKeyMultiset(const size_t n) thrust::host_vector h_result_key(max_size), h_result_val(max_size); thrust::device_vector d_result_key(max_size), d_result_val(max_size); - thrust::pair::iterator, typename thrust::host_vector::iterator> h_end; + _THRUST_STD::pair::iterator, typename thrust::host_vector::iterator> h_end; - thrust::pair::iterator, typename thrust::device_vector::iterator> d_end; + _THRUST_STD::pair::iterator, typename thrust::device_vector::iterator> d_end; h_end = thrust::set_union_by_key( h_a_key.begin(), diff --git a/projects/hipccl/hipccl3/rocthrust/testing/set_union_by_key_descending.cu b/projects/hipccl/hipccl3/rocthrust/testing/set_union_by_key_descending.cu index 90156aa73adf..7ebeb17e0b9c 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/set_union_by_key_descending.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/set_union_by_key_descending.cu @@ -33,7 +33,7 @@ void TestSetUnionByKeyDescendingSimple() Vector ref_key{4, 3, 3, 2, 0}, ref_val{0, 1, 1, 0, 0}; Vector result_key(5), result_val(5); - thrust::pair end = thrust::set_union_by_key( + _THRUST_STD::pair end = thrust::set_union_by_key( a_key.begin(), a_key.end(), b_key.begin(), @@ -74,9 +74,9 @@ void TestSetUnionByKeyDescending(const size_t n) thrust::host_vector h_result_key(max_size), h_result_val(max_size); thrust::device_vector d_result_key(max_size), d_result_val(max_size); - thrust::pair::iterator, typename thrust::host_vector::iterator> h_end; + _THRUST_STD::pair::iterator, typename thrust::host_vector::iterator> h_end; - thrust::pair::iterator, typename thrust::device_vector::iterator> d_end; + _THRUST_STD::pair::iterator, typename thrust::device_vector::iterator> d_end; h_end = thrust::set_union_by_key( h_a_key.begin(), diff --git a/projects/hipccl/hipccl3/rocthrust/testing/shuffle.cu b/projects/hipccl/hipccl3/rocthrust/testing/shuffle.cu index 624a206e7a10..50e2dc6af38d 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/shuffle.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/shuffle.cu @@ -25,388 +25,15 @@ #include #include +#include _THRUST_STD_INCLUDE(algorithm) +#include _THRUST_STD_INCLUDE(cmath) + #include #include #include #include -// Functions for performing statistical tests of randomness -// From NIST-Statistical-Test-Suite -// Licence: -// "This software was developed at the National Institute of Standards and -// Technology by employees of the Federal Government in the course of their -// official duties. Pursuant to title 17 Section 105 of the United States Code -// this software is not subject to copyright protection and is in the public -// domain. The NIST Statistical Test Suite is an experimental system. NIST -// assumes no responsibility whatsoever for its use by other parties, and makes -// no guarantees, expressed or implied, about its quality, reliability, or any -// other characteristic. We would appreciate acknowledgment if the software is -// used." -class CephesFunctions -{ -public: - static double cephes_igamc(double a, double x) - { - double ans, ax, c, yc, r, t, y, z; - double pk, pkm1, pkm2, qk, qkm1, qkm2; - - if ((x <= 0) || (a <= 0)) - { - return (1.0); - } - - if ((x < 1.0) || (x < a)) - { - return (1.e0 - cephes_igam(a, x)); - } - - ax = a * log(x) - x - cephes_lgam(a); - - if (ax < -MAXLOG) - { - printf("igamc: UNDERFLOW\n"); - return 0.0; - } - ax = exp(ax); - - /* continued fraction */ - y = 1.0 - a; - z = x + y + 1.0; - c = 0.0; - pkm2 = 1.0; - qkm2 = x; - pkm1 = x + 1.0; - qkm1 = z * x; - ans = pkm1 / qkm1; - - do - { - c += 1.0; - y += 1.0; - z += 2.0; - yc = y * c; - pk = pkm1 * z - pkm2 * yc; - qk = qkm1 * z - qkm2 * yc; - if (qk != 0) - { - r = pk / qk; - t = fabs((ans - r) / r); - ans = r; - } - else - { - t = 1.0; - } - pkm2 = pkm1; - pkm1 = pk; - qkm2 = qkm1; - qkm1 = qk; - if (fabs(pk) > big) - { - pkm2 *= biginv; - pkm1 *= biginv; - qkm2 *= biginv; - qkm1 *= biginv; - } - } while (t > MACHEP); - - return ans * ax; - } - -private: - static constexpr double rel_error = 1E-12; - - static constexpr double MACHEP = 1.11022302462515654042E-16; // 2**-53 - static constexpr double MAXLOG = 7.09782712893383996732224E2; // log(MAXNUM) - static constexpr double MAXNUM = 1.7976931348623158E308; // 2**1024*(1-MACHEP) - static constexpr double PI = 3.14159265358979323846; - - static constexpr double big = 4.503599627370496e15; - static constexpr double biginv = 2.22044604925031308085e-16; - - static int sgngam; - - static double cephes_igam(double a, double x) - { - double ans, ax, c, r; - - if ((x <= 0) || (a <= 0)) - { - return 0.0; - } - - if ((x > 1.0) && (x > a)) - { - return 1.e0 - cephes_igamc(a, x); - } - - /* Compute x**a * exp(-x) / gamma(a) */ - ax = a * log(x) - x - cephes_lgam(a); - if (ax < -MAXLOG) - { - printf("igam: UNDERFLOW\n"); - return 0.0; - } - ax = exp(ax); - - /* power series */ - r = a; - c = 1.0; - ans = 1.0; - - do - { - r += 1.0; - c *= x / r; - ans += c; - } while (c / ans > MACHEP); - - return ans * ax / a; - } - - /* A[]: Stirling's formula expansion of log gamma - * B[], C[]: log gamma function between 2 and 3 - */ - static constexpr double A[] = { - 0.000811614167470508488140545910738410384510643780, - -0.000595061904284301438315674115386855191900394857, - 0.000793650340457716942620114419781884862459264696, - -0.002777777777300996942672073330982129846233874559, - 0.083333333333333189929525985917280195280909538269}; - static constexpr double B[] = { - -1378.251525691208598800585605204105377197265625, - -38801.631513463784358464181423187255859375, - -331612.9927388711948879063129425048828125, - -1162370.97492762305773794651031494140625, - -1721737.00820839661173522472381591796875, - -853555.66424576542340219020843505859375}; - static constexpr double C[] = { - -351.8157014365234545039129443466663360595703125, - -17064.21066518811494461260735988616943359375, - -220528.59055385444662533700466156005859375, - -1139334.44367982516996562480926513671875, - -2532523.07177582941949367523193359375, - -2018891.4143353276886045932769775390625}; - - static constexpr double MAXLGM = 2.556348e305; - - /* Logarithm of gamma function */ - static double cephes_lgam(double x) - { - double p, q, u, w, z; - int i; - - sgngam = 1; - - if (x < -34.0) - { - q = -x; - w = cephes_lgam(q); /* note this modifies sgngam! */ - p = floor(q); - if (p == q) - { - lgsing: - goto loverf; - } - i = (int) p; - if ((i & 1) == 0) - { - sgngam = -1; - } - else - { - sgngam = 1; - } - z = q - p; - if (z > 0.5) - { - p += 1.0; - z = p - q; - } - z = q * sin(PI * z); - if (z == 0.0) - { - goto lgsing; - } - /* z = log(PI) - log( z ) - w;*/ - z = log(PI) - log(z) - w; - return z; - } - - if (x < 13.0) - { - z = 1.0; - p = 0.0; - u = x; - while (u >= 3.0) - { - p -= 1.0; - u = x + p; - z *= u; - } - while (u < 2.0) - { - if (u == 0.0) - { - goto lgsing; - } - z /= u; - p += 1.0; - u = x + p; - } - if (z < 0.0) - { - sgngam = -1; - z = -z; - } - else - { - sgngam = 1; - } - if (u == 2.0) - { - return (log(z)); - } - p -= 2.0; - x = x + p; - p = x * cephes_polevl(x, B, 5) / cephes_p1evl(x, C, 6); - - return log(z) + p; - } - - if (x > MAXLGM) - { - loverf: - printf("lgam: OVERFLOW\n"); - - return sgngam * MAXNUM; - } - - q = (x - 0.5) * log(x) - x + log(sqrt(2 * PI)); - if (x > 1.0e8) - { - return q; - } - - p = 1.0 / (x * x); - if (x >= 1000.0) - { - q += ((7.9365079365079365079365e-4 * p - 2.7777777777777777777778e-3) * p + 0.0833333333333333333333) / x; - } - else - { - q += cephes_polevl(p, A, 4) / x; - } - - return q; - } - - static double cephes_polevl(double x, const double* coef, int N) - { - const double* p = coef; - double ans = *p++; - int i = N; - do - { - ans = ans * x + *p++; - } while (--i); - - return ans; - } - - static double cephes_p1evl(double x, const double* coef, int N) - { - const double* p = coef; - double ans = x + *p++; - int i = N - 1; - - do - { - ans = ans * x + *p++; - } while (--i); - - return ans; - } - - static double cephes_erf(double x) - { - static const double two_sqrtpi = 1.128379167095512574; - double sum = x, term = x, xsqr = x * x; - int j = 1; - - if (fabs(x) > 2.2) - { - return 1.0 - cephes_erfc(x); - } - - do - { - term *= xsqr / j; - sum -= term / (2 * j + 1); - j++; - term *= xsqr / j; - sum += term / (2 * j + 1); - j++; - } while (fabs(term) / sum > rel_error); - - return two_sqrtpi * sum; - } - - static double cephes_erfc(double x) - { - static const double one_sqrtpi = 0.564189583547756287; - double a = 1, b = x, c = x, d = x * x + 0.5; - double q1, q2 = b / d, n = 1.0, t; - - if (fabs(x) < 2.2) - { - return 1.0 - cephes_erf(x); - } - if (x < 0) - { - return 2.0 - cephes_erfc(-x); - } - - do - { - t = a * n + b * x; - a = b; - b = t; - t = c * n + d * x; - c = d; - d = t; - n += 0.5; - q1 = q2; - q2 = b / d; - } while (fabs(q1 - q2) / q2 > rel_error); - - return one_sqrtpi * exp(-x * x) * q2; - } - - static double cephes_normal(double x) - { - double arg, result, sqrt2 = 1.414213562373095048801688724209698078569672; - - if (x > 0) - { - arg = x / sqrt2; - result = 0.5 * (1 + erf(arg)); - } - else - { - arg = -x / sqrt2; - result = 0.5 * (1 - erf(arg)); - } - - return (result); - } -}; -int CephesFunctions::sgngam = 0; -constexpr double CephesFunctions::A[]; -constexpr double CephesFunctions::B[]; -constexpr double CephesFunctions::C[]; - struct iterator_shuffle_copy { template @@ -539,7 +166,7 @@ void TestFunctionIsBijectionBase(size_t m) { return; } - ASSERT_LEQUAL(total_length, (std::max)(m * 2, size_t(16))); // Check the rounded up size is at most double the input + ASSERT_LEQUAL(total_length, (std::max) (m * 2, size_t(256))); // Check the rounded up size is at most double the input auto device_result_it = thrust::make_transform_iterator(thrust::make_counting_iterator(T(0)), device_f); @@ -571,17 +198,17 @@ void TestFeistelBijectionLength() { thrust::default_random_engine g(0xD5); - uint64_t m = 31; + uint64_t m = 345; thrust::detail::feistel_bijection f(m, g); - ASSERT_EQUAL(f.nearest_power_of_two(), uint64_t(32)); + ASSERT_EQUAL(f.size(), uint64_t(512)); - m = 32; + m = 256; f = thrust::detail::feistel_bijection(m, g); - ASSERT_EQUAL(f.nearest_power_of_two(), uint64_t(32)); + ASSERT_EQUAL(f.size(), uint64_t(256)); m = 1; f = thrust::detail::feistel_bijection(m, g); - ASSERT_EQUAL(f.nearest_power_of_two(), uint64_t(16)); + ASSERT_EQUAL(f.size(), uint64_t(256)); } DECLARE_UNITTEST(TestFeistelBijectionLength); @@ -593,13 +220,12 @@ void TestShuffleIteratorConstructibleFromBijection() thrust::shuffle_iterator it(f); g.seed(0xD5); - thrust::detail::random_bijection f2(32, g); + thrust::detail::random_bijection f2(f.size(), g); thrust::shuffle_iterator it2(f2); g.seed(0xD5); - thrust::shuffle_iterator it3(32, g); + thrust::shuffle_iterator it3(f.size(), g); - ASSERT_EQUAL(f.size(), f2.size()); ASSERT_EQUAL(true, thrust::equal(thrust::device, it, it + f.size(), it2)); ASSERT_EQUAL(true, thrust::equal(thrust::device, it, it + f.size(), it3)); } @@ -636,42 +262,56 @@ void TestShuffleIteratorStateless() } DECLARE_UNITTEST(TestShuffleIteratorStateless); +double inverse_erf(double x) +{ + double tt1, tt2, lnx, sgn; + sgn = (x < 0) ? -1.0 : 1.0; + + x = (1 - x) * (1 + x); + lnx = _THRUST_STD::log(x); + + tt1 = 2 / (3.14159265358979323846 * 0.147) + 0.5f * lnx; + tt2 = 1 / (0.147) * lnx; + + return (sgn * _THRUST_STD::sqrt(-tt1 + _THRUST_STD::sqrt(tt1 * tt1 - tt2))); +} + // Individual input keys should be permuted to output locations with uniform -// probability. Perform chi-squared test with confidence 99.9%. +// probability. Perform chi-squared test with confidence 95%. template void TestShuffleKeyPositionBase() { - using T = typename Vector::value_type; - size_t m = 20; - size_t num_samples = 100; - thrust::host_vector index_sum(m, 0); - thrust::host_vector sequence(m); - thrust::sequence(sequence.begin(), sequence.end(), T(0)); - + using T = typename Vector::value_type; + const int num_samples = 1000; + const int n = 20; thrust::default_random_engine g(0xD5); - for (size_t i = 0; i < num_samples; i++) + thrust::host_vector expected_value(n); + thrust::host_vector sequence(n); + thrust::sequence(sequence.begin(), sequence.end(), T(0)); + for (size_t i = 0; i < num_samples; ++i) { Vector shuffled(sequence.begin(), sequence.end()); ShuffleFunc{}(shuffled.begin(), shuffled.end(), g); thrust::host_vector tmp(shuffled.begin(), shuffled.end()); - - for (auto j = 0ull; j < m; j++) + for (size_t j = 0; j < n; ++j) { - index_sum[tmp[j]] += j; + expected_value[j] += static_cast(tmp[j]); } } - double expected_average_position = static_cast(m - 1) / 2; - double chi_squared = 0.0; - for (auto j = 0ull; j < m; j++) + double mu = (n - 1) / 2.0; + double sigma = _THRUST_STD::sqrt((n * n - 1) / 12.0); + double zmax = 0.0; + for (size_t i = 0; i < n; ++i) { - double average_position = static_cast(index_sum[j]) / num_samples; - chi_squared += std::pow(expected_average_position - average_position, 2) / expected_average_position; + double mean = expected_value[i] / double(num_samples); + double z = _THRUST_STD::abs(mean - mu) / (sigma / _THRUST_STD::sqrt(double(num_samples))); + zmax = _THRUST_STD::max(zmax, _THRUST_STD::abs(z)); } - // Tabulated chi-squared critical value for m-1=19 degrees of freedom - // and 99.9% confidence - double confidence_threshold = 43.82; - ASSERT_LESS(chi_squared, confidence_threshold); + + double alpha = 0.05; + double zcrit = inverse_erf(1.0 - alpha / (2.0 * n)) * _THRUST_STD::sqrt(2.0); + ASSERT_LESS(zmax, zcrit); } template void TestShuffleKeyPosition() @@ -736,8 +376,9 @@ void TestShuffleUniformPermutationBase() { chi_squared += std::pow(expected_count - kv.second, 2) / expected_count; } - double p_score = CephesFunctions::cephes_igamc((double) (total_permutations - 1) / 2.0, chi_squared / 2.0); - ASSERT_GREATER(p_score, 0.01); + // 119 degrees of freedom, 95% confidence + const double critical_value = 145.461; + ASSERT_LESS(chi_squared, critical_value); } template void TestShuffleUniformPermutation() @@ -751,131 +392,3 @@ void TestShuffleUniformPermutationIterator() } DECLARE_VECTOR_UNITTEST(TestShuffleUniformPermutation); DECLARE_VECTOR_UNITTEST(TestShuffleUniformPermutationIterator); - -template -void TestShuffleEvenSpacingBetweenOccurancesBase() -{ - using T = typename Vector::value_type; - const uint64_t shuffle_size = 10; - const uint64_t num_samples = 1000; - - thrust::host_vector h_results; - Vector sequence(shuffle_size); - thrust::sequence(sequence.begin(), sequence.end(), 0); - thrust::default_random_engine g(0xD6); - for (auto i = 0ull; i < num_samples; i++) - { - ShuffleFunc{}(sequence.begin(), sequence.end(), g); - thrust::host_vector tmp(sequence.begin(), sequence.end()); - h_results.insert(h_results.end(), sequence.begin(), sequence.end()); - } - - std::vector>> distance_between( - num_samples, std::vector>(num_samples, std::vector(shuffle_size, 0))); - - for (uint64_t sample = 0; sample < num_samples; sample++) - { - for (uint64_t i = 0; i < shuffle_size - 1; i++) - { - for (uint64_t j = 1; j < shuffle_size - i; j++) - { - T val_1 = h_results[sample * shuffle_size + i]; - T val_2 = h_results[sample * shuffle_size + i + j]; - distance_between[val_1][val_2][j]++; - distance_between[val_2][val_1][shuffle_size - j]++; - } - } - } - - const double expected_occurances = (double) num_samples / (shuffle_size - 1); - for (uint64_t val_1 = 0; val_1 < shuffle_size; val_1++) - { - for (uint64_t val_2 = val_1 + 1; val_2 < shuffle_size; val_2++) - { - double chi_squared = 0.0; - auto& distances = distance_between[val_1][val_2]; - for (uint64_t i = 1; i < shuffle_size; i++) - { - chi_squared += std::pow((double) distances[i] - expected_occurances, 2) / expected_occurances; - } - - double p_score = CephesFunctions::cephes_igamc((double) (shuffle_size - 2) / 2.0, chi_squared / 2.0); - ASSERT_GREATER(p_score, 0.01); - } - } -} -template -void TestShuffleEvenSpacingBetweenOccurances() -{ - TestShuffleEvenSpacingBetweenOccurancesBase(); -} -template -void TestShuffleEvenSpacingBetweenOccurancesIterator() -{ - TestShuffleEvenSpacingBetweenOccurancesBase(); -} -DECLARE_INTEGRAL_VECTOR_UNITTEST(TestShuffleEvenSpacingBetweenOccurances); -DECLARE_INTEGRAL_VECTOR_UNITTEST(TestShuffleEvenSpacingBetweenOccurancesIterator); - -template -void TestShuffleEvenDistributionBase() -{ - using T = typename Vector::value_type; - const uint64_t shuffle_sizes[] = {10, 100, 500}; - thrust::default_random_engine g(0xD5); - for (auto shuffle_size : shuffle_sizes) - { - if (shuffle_size > (uint64_t) std::numeric_limits::max()) - { - continue; - } - const uint64_t num_samples = shuffle_size == 500 ? 1000 : 200; - - std::vector counts(shuffle_size * shuffle_size, 0); - Vector sequence(shuffle_size); - for (auto i = 0ull; i < num_samples; i++) - { - thrust::sequence(sequence.begin(), sequence.end(), 0); - ShuffleFunc{}(sequence.begin(), sequence.end(), g); - thrust::host_vector tmp(sequence.begin(), sequence.end()); - for (uint64_t j = 0; j < shuffle_size; j++) - { - assert(j < tmp.size()); - counts.at(j * shuffle_size + tmp[j])++; - } - } - - const double expected_occurances = (double) num_samples / shuffle_size; - for (uint64_t i = 0; i < shuffle_size; i++) - { - double chi_squared_pos = 0.0; - double chi_squared_num = 0.0; - for (uint64_t j = 0; j < shuffle_size; j++) - { - auto count_pos = counts.at(i * shuffle_size + j); - auto count_num = counts.at(j * shuffle_size + i); - chi_squared_pos += std::pow((double) count_pos - expected_occurances, 2) / expected_occurances; - chi_squared_num += std::pow((double) count_num - expected_occurances, 2) / expected_occurances; - } - - double p_score_pos = CephesFunctions::cephes_igamc((double) (shuffle_size - 1) / 2.0, chi_squared_pos / 2.0); - ASSERT_GREATER(p_score_pos, 0.001 / (double) shuffle_size); - - double p_score_num = CephesFunctions::cephes_igamc((double) (shuffle_size - 1) / 2.0, chi_squared_num / 2.0); - ASSERT_GREATER(p_score_num, 0.001 / (double) shuffle_size); - } - } -} -template -void TestShuffleEvenDistribution() -{ - TestShuffleEvenDistributionBase(); -} -template -void TestShuffleEvenDistributionIterator() -{ - TestShuffleEvenDistributionBase(); -} - -DECLARE_INTEGRAL_VECTOR_UNITTEST(TestShuffleEvenDistribution); -DECLARE_INTEGRAL_VECTOR_UNITTEST(TestShuffleEvenDistributionIterator); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/tabulate_output_iterator.cu b/projects/hipccl/hipccl3/rocthrust/testing/tabulate_output_iterator.cu index f98e03aa19d9..e7ca2f6c8104 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/tabulate_output_iterator.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/tabulate_output_iterator.cu @@ -15,6 +15,8 @@ * limitations under the License. */ +#define CCCL_IGNORE_DEPRECATED_API + #include #include #include @@ -32,10 +34,6 @@ #include _THRUST_STD_INCLUDE(type_traits) -#if !_THRUST_HAS_DEVICE_SYSTEM_STD -# include -#endif - template struct host_write_op { @@ -98,7 +96,7 @@ struct index_to_gather_index_op } }; -// ensure that we properly support thrust::reverse_iterator from _THRUST_STD +// ensure that we properly support thrust::tabulate_output_iterator from _THRUST_STD void TestTabulateOutputIteratorTraits() { using base_it = thrust::host_vector::iterator; @@ -122,14 +120,12 @@ void TestTabulateOutputIteratorTraits() static_assert(_THRUST_STD::__has_random_access_traversal); // FIXME(bgruber): all up to and including random access should be true -#if _THRUST_HAS_DEVICE_SYSTEM_STD || THRUST_STD_VER >= 2020 static_assert(!_THRUST_STD::output_iterator); static_assert(!_THRUST_STD::input_iterator); static_assert(!_THRUST_STD::forward_iterator); static_assert(!_THRUST_STD::bidirectional_iterator); static_assert(!_THRUST_STD::random_access_iterator); static_assert(!_THRUST_STD::contiguous_iterator); -#endif } DECLARE_UNITTEST(TestTabulateOutputIteratorTraits); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/transform.cu b/projects/hipccl/hipccl3/rocthrust/testing/transform.cu index 34484688c373..899b799d2520 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/transform.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/transform.cu @@ -19,10 +19,11 @@ #include #include #include -#include #include #include +#include _THRUST_STD_INCLUDE(utility) + #include // There is a unfortunate miscompilation of the gcc-11 vectorizer leading to OOB writes @@ -410,9 +411,9 @@ DECLARE_VARIABLE_UNITTEST(TestTransformUnaryToDiscardIterator); struct repeat2 { template - THRUST_HOST_DEVICE thrust::pair operator()(T x) + THRUST_HOST_DEVICE _THRUST_STD::pair operator()(T x) { - return thrust::make_pair(x, x); + return _THRUST_STD::make_pair(x, x); } }; diff --git a/projects/hipccl/hipccl3/rocthrust/testing/transform_input_output_iterator.cu b/projects/hipccl/hipccl3/rocthrust/testing/transform_input_output_iterator.cu index ac464f9e1523..fac96ee32d97 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/transform_input_output_iterator.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/transform_input_output_iterator.cu @@ -25,20 +25,15 @@ #include -#if !_THRUST_HAS_DEVICE_SYSTEM_STD -# include -# include -#endif - -// There is an unfortunate miscompilation of the gcc-13 vectorizer leading to OOB writes +// There is an unfortunate miscompilation of the gcc-12/gcc-13 vectorizer leading to OOB writes // Adding this attribute suffices that this miscompilation does not appear anymore -#if THRUST_COMPILER(GCC, >=, 13) +#if THRUST_COMPILER(GCC, >=, 12) # define THRUST_DISABLE_BROKEN_GCC_VECTORIZER __attribute__((optimize("no-tree-vectorize"))) -#else // THRUST_COMPILER(GCC, <, 13) +#else // THRUST_COMPILER(GCC, <, 12) # define THRUST_DISABLE_BROKEN_GCC_VECTORIZER #endif -// ensure that we properly support thrust::reverse_iterator from _THRUST_STD +// ensure that we properly support thrust::transform_input_output_iterator from _THRUST_STD void TestTransformInputOutputIteratorTraits() { using input_func = _THRUST_STD::negate; @@ -59,14 +54,12 @@ void TestTransformInputOutputIteratorTraits() static_assert(_THRUST_STD::__has_random_access_traversal); -#if _THRUST_HAS_DEVICE_SYSTEM_STD || THRUST_STD_VER >= 2020 static_assert(!_THRUST_STD::output_iterator); static_assert(_THRUST_STD::input_iterator); static_assert(_THRUST_STD::forward_iterator); static_assert(_THRUST_STD::bidirectional_iterator); static_assert(_THRUST_STD::random_access_iterator); static_assert(!_THRUST_STD::contiguous_iterator); -#endif } DECLARE_UNITTEST(TestTransformInputOutputIteratorTraits); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/transform_iterator.cu b/projects/hipccl/hipccl3/rocthrust/testing/transform_iterator.cu index 4a07ba92d189..0e5df73211ab 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/transform_iterator.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/transform_iterator.cu @@ -29,13 +29,7 @@ #include -#if !_THRUST_HAS_DEVICE_SYSTEM_STD -# include -# include -# include -#endif - -// ensure that we properly support thrust::reverse_iterator from _THRUST_STD +// ensure that we properly support thrust::transform_iterator from _THRUST_STD void TestTransformIteratorTraits() { using func = _THRUST_STD::negate; @@ -54,14 +48,12 @@ void TestTransformIteratorTraits() static_assert(_THRUST_STD::__has_random_access_traversal); -#if _THRUST_HAS_DEVICE_SYSTEM_STD || THRUST_STD_VER >= 2020 static_assert(!_THRUST_STD::output_iterator); static_assert(_THRUST_STD::input_iterator); static_assert(_THRUST_STD::forward_iterator); static_assert(_THRUST_STD::bidirectional_iterator); static_assert(_THRUST_STD::random_access_iterator); static_assert(!_THRUST_STD::contiguous_iterator); -#endif } DECLARE_UNITTEST(TestTransformIteratorTraits); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/transform_output_iterator.cu b/projects/hipccl/hipccl3/rocthrust/testing/transform_output_iterator.cu index fa272bd2e9a3..4843c4ca26cc 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/transform_output_iterator.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/transform_output_iterator.cu @@ -27,12 +27,7 @@ #include -#if !_THRUST_HAS_DEVICE_SYSTEM_STD -# include -# include -#endif - -// ensure that we properly support thrust::reverse_iterator from _THRUST_STD +// ensure that we properly support thrust::transform_output_iterator from _THRUST_STD void TestTransformOutputIteratorTraits() { using func = _THRUST_STD::negate; @@ -52,7 +47,6 @@ void TestTransformOutputIteratorTraits() static_assert(_THRUST_STD::__has_random_access_traversal); -#if _THRUST_HAS_DEVICE_SYSTEM_STD || THRUST_STD_VER >= 2020 static_assert(!_THRUST_STD::output_iterator); // FIXME(bgruber): all up to and including random access should be true static_assert(!_THRUST_STD::input_iterator); @@ -60,7 +54,6 @@ void TestTransformOutputIteratorTraits() static_assert(!_THRUST_STD::bidirectional_iterator); static_assert(!_THRUST_STD::random_access_iterator); static_assert(!_THRUST_STD::contiguous_iterator); -#endif } DECLARE_UNITTEST(TestTransformOutputIteratorTraits); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/transform_scan.cu b/projects/hipccl/hipccl3/rocthrust/testing/transform_scan.cu index 427fa18d3065..fc3fe03f5796 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/transform_scan.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/transform_scan.cu @@ -20,8 +20,12 @@ #include #include #include +#include #include +#include +#include + #include template @@ -417,3 +421,139 @@ void TestValueCategoryDeduction() ASSERT_EQUAL((thrust::device_vector{0, 5, 5, 5, 8, 8, 8, 8, 8, 8}), vec); } DECLARE_GENERIC_UNITTEST(TestValueCategoryDeduction); + +// User-defined Int type with poison default constructor +struct TransformInt +{ + int value; + + THRUST_HOST_DEVICE TransformInt() + : value(999) + {} + THRUST_HOST_DEVICE TransformInt(int v) + : value(v) + {} + + THRUST_HOST_DEVICE friend TransformInt operator+(TransformInt a, TransformInt b) + { + return TransformInt(a.value + b.value); + } + THRUST_HOST_DEVICE friend TransformInt operator*(TransformInt a, TransformInt b) + { + return TransformInt(a.value * b.value); + } + THRUST_HOST_DEVICE friend bool operator==(TransformInt a, TransformInt b) + { + return a.value == b.value; + } + friend std::ostream& operator<<(std::ostream& os, TransformInt i) + { + return os << i.value; + } +}; + +// Test edge cases for parallel transform_scan with non-additive operations +void TestTransformScanEdgeCases() +{ + // Test 1: Large array with transform_inclusive_scan, multiplies, and init + { + const int n = 10000; + thrust::device_vector d_input(n); + using thrust::placeholders::_1; + thrust::tabulate(d_input.begin(), d_input.end(), (_1 % 5) + 1); + + thrust::device_vector d_output(n); + auto r = thrust::transform_inclusive_scan( + d_input.begin(), d_input.end(), d_output.begin(), _THRUST_STD::negate(), 2, _THRUST_STD::multiplies<>{}); + ASSERT_EQUAL((d_output.end() == r), true); + + // Verify with host scan + thrust::host_vector h_input = d_input; + thrust::host_vector h_output(n); + thrust::transform_inclusive_scan( + h_input.begin(), h_input.end(), h_output.begin(), _THRUST_STD::negate(), 2, _THRUST_STD::multiplies<>{}); + + ASSERT_EQUAL(d_output, h_output); + } + + // Test 2: transform_exclusive_scan with multiplies on large array + { + const int n = 10000; + thrust::device_vector d_input(n); + using thrust::placeholders::_1; + thrust::tabulate(d_input.begin(), d_input.end(), (_1 % 3) + 1); + + thrust::device_vector d_output(n); + auto r = thrust::transform_exclusive_scan( + d_input.begin(), d_input.end(), d_output.begin(), _THRUST_STD::negate(), 5, _THRUST_STD::multiplies<>{}); + ASSERT_EQUAL((d_output.end() == r), true); + + thrust::host_vector h_input = d_input; + thrust::host_vector h_output(n); + thrust::transform_exclusive_scan( + h_input.begin(), h_input.end(), h_output.begin(), _THRUST_STD::negate(), 5, _THRUST_STD::multiplies<>{}); + + ASSERT_EQUAL(d_output, h_output); + } + + // Test 3: User-defined type with poison default constructor + { + thrust::device_vector vec(5, TransformInt{1}); + thrust::transform_exclusive_scan( + vec.begin(), + vec.end(), + vec.begin(), + _THRUST_STD::identity{}, + TransformInt{100}, + _THRUST_STD::plus()); + + ASSERT_EQUAL(vec, + (thrust::device_vector{ + TransformInt{100}, TransformInt{101}, TransformInt{102}, TransformInt{103}, TransformInt{104}})); + } + + // Test 4: In-place transform_exclusive_scan with multiplies + { + thrust::device_vector vec = {2, 3, 4, 5}; + auto r = thrust::transform_exclusive_scan( + vec.begin(), vec.end(), vec.begin(), _THRUST_STD::negate(), 10, _THRUST_STD::multiplies<>{}); + ASSERT_EQUAL((vec.end() == r), true); + + thrust::device_vector expected = {10, -20, 60, -240}; + ASSERT_EQUAL(vec, expected); + } + + // Test 5: Boundary case at threshold (1024 elements) + { + const int n = 1024; + thrust::device_vector d_input(n); + using thrust::placeholders::_1; + thrust::tabulate(d_input.begin(), d_input.end(), _1 + 1); + + thrust::device_vector d_output(n); + auto r = thrust::transform_inclusive_scan( + d_input.begin(), d_input.end(), d_output.begin(), thrust::square(), 1, _THRUST_STD::multiplies<>{}); + ASSERT_EQUAL((d_output.end() == r), true); + + thrust::host_vector h_input = d_input; + thrust::host_vector h_output(n); + thrust::transform_inclusive_scan( + h_input.begin(), h_input.end(), h_output.begin(), thrust::square(), 1, _THRUST_STD::multiplies<>{}); + + ASSERT_EQUAL(d_output, h_output); + } + + // Test 6: Very small array (2 elements) + { + thrust::device_vector d_input = {3, 7}; + thrust::device_vector d_output(2); + + auto r = thrust::transform_inclusive_scan( + d_input.begin(), d_input.end(), d_output.begin(), _THRUST_STD::negate(), 2, _THRUST_STD::multiplies<>{}); + ASSERT_EQUAL((d_output.end() == r), true); + + thrust::device_vector expected = {-6, 42}; + ASSERT_EQUAL(d_output, expected); + } +} +DECLARE_UNITTEST(TestTransformScanEdgeCases); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/type_traits.cu b/projects/hipccl/hipccl3/rocthrust/testing/type_traits.cu index 5aa572e14c72..d8aecb7b5f02 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/type_traits.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/type_traits.cu @@ -18,12 +18,16 @@ #include #include #include +#include +#include +#include #include #include +#include +#include #include #include #include -#include #include #include @@ -36,9 +40,9 @@ #include _THRUST_STD_INCLUDE(tuple) #include _THRUST_STD_INCLUDE(utility) -#if !_THRUST_HAS_DEVICE_SYSTEM_STD -# include -#endif +#include + +#include _THRUST_LIBCXX_INCLUDE(iterator) void TestIsContiguousIterator() { @@ -205,33 +209,230 @@ void TestTriviallyRelocatable() #if THRUST_COMPILER(GCC, >=, 7) static_assert(thrust::is_trivially_relocatable>::value, ""); static_assert(thrust::is_trivially_relocatable<_THRUST_STD::complex>::value, ""); - static_assert(thrust::is_trivially_relocatable>>::value, ""); + static_assert(thrust::is_trivially_relocatable<_THRUST_STD::pair>>::value, ""); static_assert(thrust::is_trivially_relocatable<_THRUST_STD::pair>>::value, ""); static_assert(thrust::is_trivially_relocatable, char>>::value, ""); static_assert(thrust::is_trivially_relocatable<_THRUST_STD::tuple, char>>::value, ""); #endif // THRUST_COMPILER(GCC, >=, 7) -#if _THRUST_HAS_DEVICE_SYSTEM_STD static_assert(thrust::is_trivially_relocatable< - _THRUST_STD::tuple>>, + _THRUST_STD::tuple<_THRUST_STD::pair>>, thrust::tuple<_THRUST_STD::pair>, int>>>::value, ""); -#endif - static_assert(!thrust::is_trivially_relocatable>::value, ""); + static_assert(!thrust::is_trivially_relocatable<_THRUST_STD::pair>::value, ""); static_assert(!thrust::is_trivially_relocatable<_THRUST_STD::pair>::value, ""); static_assert(!thrust::is_trivially_relocatable>::value, ""); static_assert(!thrust::is_trivially_relocatable<_THRUST_STD::tuple>::value, ""); // test propagation of relocatability through pair and tuple static_assert(thrust::is_trivially_relocatable::value, ""); -#if _THRUST_HAS_DEVICE_SYSTEM_STD - static_assert(thrust::is_trivially_relocatable>::value, ""); -#endif static_assert(thrust::is_trivially_relocatable<_THRUST_STD::pair>::value, ""); -#if _THRUST_HAS_DEVICE_SYSTEM_STD + static_assert(thrust::is_trivially_relocatable<_THRUST_STD::pair>::value, ""); static_assert(thrust::is_trivially_relocatable>::value, ""); -#endif static_assert(thrust::is_trivially_relocatable<_THRUST_STD::tuple>::value, ""); }; DECLARE_UNITTEST(TestTriviallyRelocatable); + +template +inline constexpr bool system_is = _THRUST_STD::is_same_v, Tag>; + +template +inline constexpr bool traversal_is = + _THRUST_STD::is_same_v, Traversal>; + +void TestIteratorCategoryToSystem() +{ + static_assert(system_is); + static_assert(system_is); + static_assert(system_is); + static_assert(system_is); + static_assert(system_is); + + static_assert(system_is); + static_assert(system_is); + static_assert(system_is); + static_assert(system_is); + static_assert(system_is); +}; +DECLARE_UNITTEST(TestIteratorCategoryToSystem); + +void TestIteratorCategoryToTraversal() +{ + static_assert(traversal_is); + static_assert(traversal_is); + static_assert(traversal_is); + static_assert(traversal_is); + static_assert(traversal_is); + + static_assert(traversal_is); + static_assert(traversal_is); + static_assert(traversal_is); + static_assert(traversal_is); + static_assert(traversal_is); +}; +DECLARE_UNITTEST(TestIteratorCategoryToTraversal); + +struct thrust_make_counting_iterator +{ + template + auto operator()(Args&&... args) const + { + return thrust::make_counting_iterator(_THRUST_STD::forward(args)...); + } +}; + +struct thrust_make_transform_iterator +{ + template + auto operator()(Args&&... args) const + { + return thrust::make_transform_iterator(_THRUST_STD::forward(args)...); + } +}; + +struct thrust_make_zip_iterator +{ + template + auto operator()(Args&&... args) const + { + return thrust::make_zip_iterator(_THRUST_STD::forward(args)...); + } +}; + +struct libcxx_make_counting_iterator +{ + template + auto operator()(Args&&... args) const + { + return _THRUST_LIBCXX::make_counting_iterator(_THRUST_STD::forward(args)...); + } +}; + +struct libcxx_make_transform_iterator +{ + template + auto operator()(Args&&... args) const + { + return _THRUST_LIBCXX::make_transform_iterator(_THRUST_STD::forward(args)...); + } +}; + +struct libcxx_make_zip_iterator +{ + template + auto operator()(Args&&... args) const + { + return _THRUST_LIBCXX::make_zip_iterator(_THRUST_STD::forward(args)...); + } +}; + +template +inline constexpr bool has_random_access_traversal = + _THRUST_STD::is_same_v, thrust::random_access_traversal_tag>; + +auto expected_tag(thrust::device_vector) -> thrust::device_system_tag; +auto expected_tag(thrust::host_vector) -> thrust::host_system_tag; +auto expected_tag(std::vector) -> thrust::host_system_tag; + +template +void check_any_system_propagation() +{ + auto counting_it = MakeCounting{}(0); + static_assert(_THRUST_STD::is_same_v, thrust::any_system_tag>); + static_assert(has_random_access_traversal); + + auto transform_it = MakeTransform{}(counting_it, thrust::square<>{}); + static_assert(_THRUST_STD::is_same_v, thrust::any_system_tag>); + static_assert(has_random_access_traversal); + + [[maybe_unused]] auto zip_it = MakeZip{}(counting_it, transform_it); + static_assert(_THRUST_STD::is_same_v, thrust::any_system_tag>); + static_assert(has_random_access_traversal); +} + +template +void check_vector_propagation() +{ + Vector vec{}; + using tag = decltype(expected_tag(vec)); + + auto vec_it = vec.begin(); + static_assert(_THRUST_STD::is_same_v, tag>); + static_assert(has_random_access_traversal); + + using thrust::placeholders::_1; + auto transform_it = MakeTransform{}(vec_it, _1 + 1); + static_assert(_THRUST_STD::is_same_v, tag>); + static_assert(has_random_access_traversal); + + [[maybe_unused]] auto zip_it = MakeZip{}(vec_it, transform_it); + static_assert(_THRUST_STD::is_same_v, tag>); + static_assert(has_random_access_traversal); +} + +template +void check_vector_and_any_system_propagation() +{ + Vector vec{}; + using vec_tag = decltype(expected_tag(vec)); + + auto vec_it = vec.begin(); + static_assert(_THRUST_STD::is_same_v, vec_tag>); + static_assert(has_random_access_traversal); + + auto counting_it = MakeCounting{}(0); + static_assert(_THRUST_STD::is_same_v, thrust::any_system_tag>); + static_assert(has_random_access_traversal); + + [[maybe_unused]] auto zip_it = MakeZip{}(vec_it, counting_it); + static_assert(_THRUST_STD::is_same_v, vec_tag>); + static_assert(has_random_access_traversal); +} + +void TestIteratorSystemAndTraversalPropagationAnySystem() +{ + check_any_system_propagation(); + check_any_system_propagation(); + check_any_system_propagation(); + check_any_system_propagation(); + check_any_system_propagation(); + check_any_system_propagation(); + check_any_system_propagation(); + check_any_system_propagation(); +}; +DECLARE_UNITTEST(TestIteratorSystemAndTraversalPropagationAnySystem); + +void TestIteratorSystemAndTraversalPropagationVectors() +{ + check_vector_propagation, thrust_make_transform_iterator, thrust_make_zip_iterator>(); + check_vector_propagation, thrust_make_transform_iterator, thrust_make_zip_iterator>(); + check_vector_propagation, thrust_make_transform_iterator, thrust_make_zip_iterator>(); + check_vector_propagation, thrust_make_transform_iterator, libcxx_make_zip_iterator>(); + check_vector_propagation, libcxx_make_transform_iterator, thrust_make_zip_iterator>(); + check_vector_propagation, libcxx_make_transform_iterator, libcxx_make_zip_iterator>(); + check_vector_propagation, thrust_make_transform_iterator, libcxx_make_zip_iterator>(); + check_vector_propagation, libcxx_make_transform_iterator, thrust_make_zip_iterator>(); + check_vector_propagation, libcxx_make_transform_iterator, libcxx_make_zip_iterator>(); + check_vector_propagation, thrust_make_transform_iterator, libcxx_make_zip_iterator>(); + check_vector_propagation, libcxx_make_transform_iterator, thrust_make_zip_iterator>(); + check_vector_propagation, libcxx_make_transform_iterator, libcxx_make_zip_iterator>(); +}; +DECLARE_UNITTEST(TestIteratorSystemAndTraversalPropagationVectors); + +void TestIteratorSystemAndTraversalPropagationVectorsAndAnySystem() +{ + check_vector_and_any_system_propagation, thrust_make_counting_iterator, thrust_make_zip_iterator>(); + check_vector_and_any_system_propagation, thrust_make_counting_iterator, thrust_make_zip_iterator>(); + check_vector_and_any_system_propagation, thrust_make_counting_iterator, thrust_make_zip_iterator>(); + check_vector_and_any_system_propagation, thrust_make_counting_iterator, libcxx_make_zip_iterator>(); + check_vector_and_any_system_propagation, libcxx_make_counting_iterator, thrust_make_zip_iterator>(); + check_vector_and_any_system_propagation, libcxx_make_counting_iterator, libcxx_make_zip_iterator>(); + check_vector_and_any_system_propagation, thrust_make_counting_iterator, libcxx_make_zip_iterator>(); + check_vector_and_any_system_propagation, libcxx_make_counting_iterator, thrust_make_zip_iterator>(); + check_vector_and_any_system_propagation, libcxx_make_counting_iterator, libcxx_make_zip_iterator>(); + check_vector_and_any_system_propagation, thrust_make_counting_iterator, libcxx_make_zip_iterator>(); + check_vector_and_any_system_propagation, libcxx_make_counting_iterator, thrust_make_zip_iterator>(); + check_vector_and_any_system_propagation, libcxx_make_counting_iterator, libcxx_make_zip_iterator>(); +}; +DECLARE_UNITTEST(TestIteratorSystemAndTraversalPropagationVectorsAndAnySystem); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/unique_by_key.cu b/projects/hipccl/hipccl3/rocthrust/testing/unique_by_key.cu index f1cade3d18d1..a1cb86e436ad 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/unique_by_key.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/unique_by_key.cu @@ -40,11 +40,11 @@ struct index_to_value_t }; template -thrust::pair +_THRUST_STD::pair unique_by_key(my_system& system, ForwardIterator1 keys_first, ForwardIterator1, ForwardIterator2 values_first) { system.validate_dispatch(); - return thrust::make_pair(keys_first, values_first); + return _THRUST_STD::make_pair(keys_first, values_first); } void TestUniqueByKeyDispatchExplicit() @@ -59,11 +59,11 @@ void TestUniqueByKeyDispatchExplicit() DECLARE_UNITTEST(TestUniqueByKeyDispatchExplicit); template -thrust::pair +_THRUST_STD::pair unique_by_key(my_tag, ForwardIterator1 keys_first, ForwardIterator1, ForwardIterator2 values_first) { *keys_first = 13; - return thrust::make_pair(keys_first, values_first); + return _THRUST_STD::make_pair(keys_first, values_first); } void TestUniqueByKeyDispatchImplicit() @@ -78,7 +78,7 @@ void TestUniqueByKeyDispatchImplicit() DECLARE_UNITTEST(TestUniqueByKeyDispatchImplicit); template -thrust::pair unique_by_key_copy( +_THRUST_STD::pair unique_by_key_copy( my_system& system, InputIterator1, InputIterator1, @@ -87,7 +87,7 @@ thrust::pair unique_by_key_copy( OutputIterator2 values_output) { system.validate_dispatch(); - return thrust::make_pair(keys_output, values_output); + return _THRUST_STD::make_pair(keys_output, values_output); } void TestUniqueByKeyCopyDispatchExplicit() @@ -102,11 +102,11 @@ void TestUniqueByKeyCopyDispatchExplicit() DECLARE_UNITTEST(TestUniqueByKeyCopyDispatchExplicit); template -thrust::pair unique_by_key_copy( +_THRUST_STD::pair unique_by_key_copy( my_tag, InputIterator1, InputIterator1, InputIterator2, OutputIterator1 keys_output, OutputIterator2 values_output) { *keys_output = 13; - return thrust::make_pair(keys_output, values_output); + return _THRUST_STD::make_pair(keys_output, values_output); } void TestUniqueByKeyCopyDispatchImplicit() @@ -155,7 +155,7 @@ void TestUniqueByKeySimple() Vector keys; Vector values; - typename thrust::pair new_last; + typename _THRUST_STD::pair new_last; // basic test initialize_keys(keys); @@ -201,7 +201,7 @@ void TestUniqueCopyByKeySimple() Vector keys; Vector values; - typename thrust::pair new_last; + typename _THRUST_STD::pair new_last; // basic test initialize_keys(keys); @@ -262,8 +262,8 @@ struct TestUniqueByKey using DeviceKeyIterator = typename thrust::device_vector::iterator; using DeviceValIterator = typename thrust::device_vector::iterator; - using HostIteratorPair = typename thrust::pair; - using DeviceIteratorPair = typename thrust::pair; + using HostIteratorPair = typename _THRUST_STD::pair; + using DeviceIteratorPair = typename _THRUST_STD::pair; HostIteratorPair h_last = thrust::unique_by_key(h_keys.begin(), h_keys.end(), h_vals.begin()); DeviceIteratorPair d_last = thrust::unique_by_key(d_keys.begin(), d_keys.end(), d_vals.begin()); @@ -308,8 +308,8 @@ struct TestUniqueCopyByKey using DeviceKeyIterator = typename thrust::device_vector::iterator; using DeviceValIterator = typename thrust::device_vector::iterator; - using HostIteratorPair = typename thrust::pair; - using DeviceIteratorPair = typename thrust::pair; + using HostIteratorPair = typename _THRUST_STD::pair; + using DeviceIteratorPair = typename _THRUST_STD::pair; HostIteratorPair h_last = thrust::unique_by_key_copy( h_keys.begin(), h_keys.end(), h_vals.begin(), h_keys_output.begin(), h_vals_output.begin()); @@ -358,51 +358,51 @@ struct TestUniqueCopyByKeyToDiscardIterator size_t num_unique_keys = h_unique_keys.size(); // mask both outputs - thrust::pair, thrust::discard_iterator<>> h_result1 = thrust::unique_by_key_copy( + _THRUST_STD::pair, thrust::discard_iterator<>> h_result1 = thrust::unique_by_key_copy( h_keys.begin(), h_keys.end(), h_vals.begin(), thrust::make_discard_iterator(), thrust::make_discard_iterator()); - thrust::pair, thrust::discard_iterator<>> d_result1 = thrust::unique_by_key_copy( + _THRUST_STD::pair, thrust::discard_iterator<>> d_result1 = thrust::unique_by_key_copy( d_keys.begin(), d_keys.end(), d_vals.begin(), thrust::make_discard_iterator(), thrust::make_discard_iterator()); - thrust::pair, thrust::discard_iterator<>> reference1 = - thrust::make_pair(thrust::make_discard_iterator(num_unique_keys), thrust::make_discard_iterator(num_unique_keys)); + _THRUST_STD::pair, thrust::discard_iterator<>> reference1 = _THRUST_STD::make_pair( + thrust::make_discard_iterator(num_unique_keys), thrust::make_discard_iterator(num_unique_keys)); ASSERT_EQUAL_QUIET(reference1, h_result1); ASSERT_EQUAL_QUIET(reference1, d_result1); // mask values output - thrust::pair::iterator, thrust::discard_iterator<>> h_result2 = + _THRUST_STD::pair::iterator, thrust::discard_iterator<>> h_result2 = thrust::unique_by_key_copy( h_keys.begin(), h_keys.end(), h_vals.begin(), h_keys_output.begin(), thrust::make_discard_iterator()); - thrust::pair::iterator, thrust::discard_iterator<>> d_result2 = + _THRUST_STD::pair::iterator, thrust::discard_iterator<>> d_result2 = thrust::unique_by_key_copy( d_keys.begin(), d_keys.end(), d_vals.begin(), d_keys_output.begin(), thrust::make_discard_iterator()); - thrust::pair::iterator, thrust::discard_iterator<>> h_reference2 = - thrust::make_pair(h_keys_output.begin() + num_unique_keys, thrust::make_discard_iterator(num_unique_keys)); + _THRUST_STD::pair::iterator, thrust::discard_iterator<>> h_reference2 = + _THRUST_STD::make_pair(h_keys_output.begin() + num_unique_keys, thrust::make_discard_iterator(num_unique_keys)); - thrust::pair::iterator, thrust::discard_iterator<>> d_reference2 = - thrust::make_pair(d_keys_output.begin() + num_unique_keys, thrust::make_discard_iterator(num_unique_keys)); + _THRUST_STD::pair::iterator, thrust::discard_iterator<>> d_reference2 = + _THRUST_STD::make_pair(d_keys_output.begin() + num_unique_keys, thrust::make_discard_iterator(num_unique_keys)); ASSERT_EQUAL(h_keys_output, d_keys_output); ASSERT_EQUAL_QUIET(h_reference2, h_result2); ASSERT_EQUAL_QUIET(d_reference2, d_result2); // mask keys output - thrust::pair, typename thrust::host_vector::iterator> h_result3 = + _THRUST_STD::pair, typename thrust::host_vector::iterator> h_result3 = thrust::unique_by_key_copy( h_keys.begin(), h_keys.end(), h_vals.begin(), thrust::make_discard_iterator(), h_vals_output.begin()); - thrust::pair, typename thrust::device_vector::iterator> d_result3 = + _THRUST_STD::pair, typename thrust::device_vector::iterator> d_result3 = thrust::unique_by_key_copy( d_keys.begin(), d_keys.end(), d_vals.begin(), thrust::make_discard_iterator(), d_vals_output.begin()); - thrust::pair, typename thrust::host_vector::iterator> h_reference3 = - thrust::make_pair(thrust::make_discard_iterator(num_unique_keys), h_vals_output.begin() + num_unique_keys); + _THRUST_STD::pair, typename thrust::host_vector::iterator> h_reference3 = + _THRUST_STD::make_pair(thrust::make_discard_iterator(num_unique_keys), h_vals_output.begin() + num_unique_keys); - thrust::pair, typename thrust::device_vector::iterator> d_reference3 = - thrust::make_pair(thrust::make_discard_iterator(num_unique_keys), d_vals_output.begin() + num_unique_keys); + _THRUST_STD::pair, typename thrust::device_vector::iterator> d_reference3 = + _THRUST_STD::make_pair(thrust::make_discard_iterator(num_unique_keys), d_vals_output.begin() + num_unique_keys); ASSERT_EQUAL(h_vals_output, d_vals_output); ASSERT_EQUAL_QUIET(h_reference3, h_result3); @@ -504,7 +504,7 @@ struct Entry void TestKeysWithoutEqualityOperator() { - using Key = thrust::pair; + using Key = _THRUST_STD::pair; const auto k1 = Key{1, {}}; const auto k2 = Key{2, {}}; diff --git a/projects/hipccl/hipccl3/rocthrust/testing/unittest/assertions.h b/projects/hipccl/hipccl3/rocthrust/testing/unittest/assertions.h index ba871cef6737..25d0f1cc6054 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/unittest/assertions.h +++ b/projects/hipccl/hipccl3/rocthrust/testing/unittest/assertions.h @@ -119,7 +119,6 @@ namespace unittest { - size_t const MAX_OUTPUT_LINES = 10; double const DEFAULT_RELATIVE_TOL = 1e-4; @@ -334,7 +333,6 @@ bool almost_equal(double a, double b, double a_tol, double r_tol) namespace { // anonymous namespace - template struct is_complex : public THRUST_NS_QUALIFIER::false_type {}; @@ -346,7 +344,6 @@ struct is_complex> : public THRUST_NS_QUALIFIER: template struct is_complex> : public THRUST_NS_QUALIFIER::true_type {}; - } // namespace template @@ -789,5 +786,4 @@ void check_assert_throws( } } } - } // end namespace unittest diff --git a/projects/hipccl/hipccl3/rocthrust/testing/unittest/exceptions.h b/projects/hipccl/hipccl3/rocthrust/testing/unittest/exceptions.h index 08f949536821..7573a05c6e9b 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/unittest/exceptions.h +++ b/projects/hipccl/hipccl3/rocthrust/testing/unittest/exceptions.h @@ -23,7 +23,6 @@ namespace unittest { - class UnitTestException { public: @@ -75,5 +74,4 @@ class UnitTestKnownFailure : public UnitTestException : UnitTestException(msg) {} }; - } // end namespace unittest diff --git a/projects/hipccl/hipccl3/rocthrust/testing/unittest/meta.h b/projects/hipccl/hipccl3/rocthrust/testing/unittest/meta.h index ad331a0dba8d..7a6130a671ba 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/unittest/meta.h +++ b/projects/hipccl/hipccl3/rocthrust/testing/unittest/meta.h @@ -25,7 +25,6 @@ namespace unittest { - // mark the absence of a type struct null_type {}; @@ -188,5 +187,4 @@ struct concat, L, Ls...> { using type = concat, Ls...>; }; - } // namespace unittest diff --git a/projects/hipccl/hipccl3/rocthrust/testing/unittest/random.h b/projects/hipccl/hipccl3/rocthrust/testing/unittest/random.h index fcfeafb540ef..73966c5d2610 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/unittest/random.h +++ b/projects/hipccl/hipccl3/rocthrust/testing/unittest/random.h @@ -28,7 +28,6 @@ namespace unittest { - inline unsigned int hash(unsigned int a) { a = (a + 0x7ed55d16) + (a << 12); @@ -40,58 +39,35 @@ inline unsigned int hash(unsigned int a) return a; } -template -struct generate_random_integer; - -template -struct generate_random_integer< - T, - typename THRUST_NS_QUALIFIER::detail::disable_if::value>::type> -{ - T operator()(unsigned int i) const - { - THRUST_NS_QUALIFIER::default_random_engine rng(hash(i)); - - return static_cast(rng()); - } -}; - -template -struct generate_random_integer::value>> -{ - T operator()(unsigned int i) const - { - THRUST_NS_QUALIFIER::default_random_engine rng(hash(i)); - THRUST_NS_QUALIFIER::uniform_int_distribution dist; - - return static_cast(dist(rng)); - } -}; - template -struct generate_random_integer::value>> +struct generate_random_integer { T operator()(unsigned int i) const - { - T const min = std::numeric_limits::min(); - T const max = std::numeric_limits::max(); - - THRUST_NS_QUALIFIER::default_random_engine rng(hash(i)); - THRUST_NS_QUALIFIER::uniform_real_distribution dist(min, max); - - return static_cast(dist(rng)); - } -}; - -template <> -struct generate_random_integer -{ - bool operator()(unsigned int i) const { THRUST_NS_QUALIFIER::default_random_engine rng(hash(i)); - THRUST_NS_QUALIFIER::uniform_int_distribution dist(0, 1); - - return dist(rng) == 1; + if constexpr (_THRUST_STD::is_same_v) + { + THRUST_NS_QUALIFIER::uniform_int_distribution dist(0, 1); + return dist(rng) == 1; + } + else if constexpr (_THRUST_STD::is_integral_v) + { + T const min = _THRUST_STD::numeric_limits::min(); + T const max = _THRUST_STD::numeric_limits::max(); + THRUST_NS_QUALIFIER::uniform_int_distribution dist(min, max); + return static_cast(dist(rng)); + } + else if constexpr (_THRUST_STD::is_floating_point_v) + { + T const min = _THRUST_STD::numeric_limits::lowest(); + T const max = _THRUST_STD::numeric_limits::max(); + THRUST_NS_QUALIFIER::uniform_real_distribution dist(min, max); + return static_cast(dist(rng)); + } + else + { + return static_cast(rng()); + } } }; @@ -112,8 +88,8 @@ THRUST_NS_QUALIFIER::host_vector random_integers(const size_t N) { THRUST_NS_QUALIFIER::host_vector vec(N); THRUST_NS_QUALIFIER::transform( - THRUST_NS_QUALIFIER::counting_iterator(static_cast(0)), - THRUST_NS_QUALIFIER::counting_iterator(static_cast(N)), + THRUST_NS_QUALIFIER::counting_iterator{0u}, + THRUST_NS_QUALIFIER::counting_iterator{static_cast(N)}, vec.begin(), generate_random_integer()); @@ -131,12 +107,11 @@ THRUST_NS_QUALIFIER::host_vector random_samples(const size_t N) { THRUST_NS_QUALIFIER::host_vector vec(N); THRUST_NS_QUALIFIER::transform( - THRUST_NS_QUALIFIER::counting_iterator(static_cast(0)), - THRUST_NS_QUALIFIER::counting_iterator(static_cast(N)), + THRUST_NS_QUALIFIER::counting_iterator{0u}, + THRUST_NS_QUALIFIER::counting_iterator{static_cast(N)}, vec.begin(), generate_random_sample()); return vec; } - } // end namespace unittest diff --git a/projects/hipccl/hipccl3/rocthrust/testing/unittest/special_types.h b/projects/hipccl/hipccl3/rocthrust/testing/unittest/special_types.h index d4b8a2a8c50b..33ad92476e8e 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/unittest/special_types.h +++ b/projects/hipccl/hipccl3/rocthrust/testing/unittest/special_types.h @@ -42,7 +42,12 @@ struct FixedVector } } - THRUST_HOST_DEVICE FixedVector operator+(const FixedVector& bs) const + THRUST_HOST_DEVICE +#if _CCCL_COMPILER(NVHPC) + __attribute__((noinline)) +#endif + FixedVector + operator+(const FixedVector& bs) const { FixedVector output; for (unsigned int i = 0; i < N; i++) @@ -183,7 +188,6 @@ struct my_tag : THRUST_NS_QUALIFIER::device_execution_policy namespace unittest { - using std::int16_t; using std::int32_t; using std::int64_t; @@ -193,5 +197,4 @@ using std::uint16_t; using std::uint32_t; using std::uint64_t; using std::uint8_t; - } // namespace unittest diff --git a/projects/hipccl/hipccl3/rocthrust/testing/unittest/testframework.h b/projects/hipccl/hipccl3/rocthrust/testing/unittest/testframework.h index 8a2042fe37e9..97d2970d87b8 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/unittest/testframework.h +++ b/projects/hipccl/hipccl3/rocthrust/testing/unittest/testframework.h @@ -28,14 +28,12 @@ #include #include +#include #include #include #include #include #include -#if _THRUST_HAS_DEVICE_SYSTEM_STD -# include -#endif #include "meta.h" #include "util.h" @@ -226,13 +224,11 @@ struct numeric_limits : numeric_limits {}; } // namespace std -#if _THRUST_HAS_DEVICE_SYSTEM_STD _THRUST_STD_NAMESPACE_BEGIN template <> struct numeric_limits : numeric_limits {}; _THRUST_STD_NAMESPACE_END -#endif using NumericTypes = unittest::type_list< char, diff --git a/projects/hipccl/hipccl3/rocthrust/testing/unittest/util.h b/projects/hipccl/hipccl3/rocthrust/testing/unittest/util.h index 4258d3216451..5544d70e5b6e 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/unittest/util.h +++ b/projects/hipccl/hipccl3/rocthrust/testing/unittest/util.h @@ -32,7 +32,6 @@ namespace unittest { - template std::string type_name() { @@ -56,7 +55,6 @@ truncate_to_max_representable(std::size_t n) { return THRUST_NS_QUALIFIER::min(static_cast(n), _THRUST_STD::numeric_limits::max()); } - } // namespace unittest template diff --git a/projects/hipccl/hipccl3/rocthrust/testing/universal_memory.cu b/projects/hipccl/hipccl3/rocthrust/testing/universal_memory.cu index 0c6eb15647a8..e0dcf9f5e23f 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/universal_memory.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/universal_memory.cu @@ -27,7 +27,6 @@ namespace { - // The managed_memory_pointer class should be identified as a // contiguous_iterator static_assert(thrust::is_contiguous_iterator_v::pointer>); @@ -51,7 +50,6 @@ struct some_object private: T m_data; }; - } // namespace template diff --git a/projects/hipccl/hipccl3/rocthrust/testing/xor_combine_engine_max.cu b/projects/hipccl/hipccl3/rocthrust/testing/xor_combine_engine_max.cu new file mode 100644 index 000000000000..4280c9c8faa9 --- /dev/null +++ b/projects/hipccl/hipccl3/rocthrust/testing/xor_combine_engine_max.cu @@ -0,0 +1,26 @@ +#include + +#include + +void TestLog2() +{ + static_assert(thrust::random::detail::log2(1u) == 0u); + static_assert(thrust::random::detail::log2(2u) == 1u); + static_assert(thrust::random::detail::log2(3u) == 1u); + static_assert(thrust::random::detail::log2(4u) == 2u); + static_assert(thrust::random::detail::log2(5u) == 2u); + static_assert(thrust::random::detail::log2(6u) == 2u); + static_assert(thrust::random::detail::log2(7u) == 2u); + static_assert(thrust::random::detail::log2(8u) == 3u); + static_assert(thrust::random::detail::log2(9u) == 3u); + static_assert(thrust::random::detail::log2(15u) == 3u); + static_assert(thrust::random::detail::log2(16u) == 4u); + static_assert(thrust::random::detail::log2(17u) == 4u); + static_assert(thrust::random::detail::log2(127u) == 6u); + static_assert(thrust::random::detail::log2(128u) == 7u); + static_assert(thrust::random::detail::log2(129u) == 7u); + static_assert(thrust::random::detail::log2(256u) == 8u); + static_assert(thrust::random::detail::log2(511u) == 8u); + static_assert(thrust::random::detail::log2(512u) == 9u); +} +DECLARE_UNITTEST(TestLog2); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/zip_iterator.cu b/projects/hipccl/hipccl3/rocthrust/testing/zip_iterator.cu index 65cd1154b963..072e9829b369 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/zip_iterator.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/zip_iterator.cu @@ -26,13 +26,9 @@ #include _THRUST_STD_INCLUDE(type_traits) -#if !_THRUST_HAS_DEVICE_SYSTEM_STD -# include -#endif - using namespace unittest; -// ensure that we properly support thrust::reverse_iterator from _THRUST_STD +// ensure that we properly support thrust::zip_iterator from _THRUST_STD void TestZipIteratorTraits() { using base_it = thrust::host_vector::iterator; @@ -52,18 +48,12 @@ void TestZipIteratorTraits() static_assert(_THRUST_STD::__has_random_access_traversal); -#if _THRUST_HAS_DEVICE_SYSTEM_STD || THRUST_STD_VER >= 2020 static_assert(!_THRUST_STD::output_iterator); -#endif -#if _THRUST_HAS_DEVICE_SYSTEM_STD // The fallback version of zip_iterator lacks these iterator concepts. static_assert(_THRUST_STD::input_iterator); static_assert(_THRUST_STD::forward_iterator); static_assert(_THRUST_STD::bidirectional_iterator); static_assert(_THRUST_STD::random_access_iterator); -#endif -#if _THRUST_HAS_DEVICE_SYSTEM_STD || THRUST_STD_VER >= 2020 static_assert(!_THRUST_STD::contiguous_iterator); -#endif } DECLARE_UNITTEST(TestZipIteratorTraits); diff --git a/projects/hipccl/hipccl3/rocthrust/testing/zip_iterator_sort.cu b/projects/hipccl/hipccl3/rocthrust/testing/zip_iterator_sort.cu index 8359b61f7128..1c361386a550 100644 --- a/projects/hipccl/hipccl3/rocthrust/testing/zip_iterator_sort.cu +++ b/projects/hipccl/hipccl3/rocthrust/testing/zip_iterator_sort.cu @@ -34,10 +34,10 @@ struct TestZipIteratorStableSort device_vector d2 = h2; // sort on host - stable_sort(make_zip_iterator(h1.begin(), h2.begin()), make_zip_iterator(h1.end(), h2.end())); + thrust::stable_sort(make_zip_iterator(h1.begin(), h2.begin()), make_zip_iterator(h1.end(), h2.end())); // sort on device - stable_sort(make_zip_iterator(d1.begin(), d2.begin()), make_zip_iterator(d1.end(), d2.end())); + thrust::stable_sort(make_zip_iterator(d1.begin(), d2.begin()), make_zip_iterator(d1.end(), d2.end())); ASSERT_EQUAL_QUIET(h1, d1); ASSERT_EQUAL_QUIET(h2, d2); diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/addressof.h b/projects/hipccl/hipccl3/rocthrust/thrust/addressof.h index cf3e948596a4..cf5a692c9538 100644 --- a/projects/hipccl/hipccl3/rocthrust/thrust/addressof.h +++ b/projects/hipccl/hipccl3/rocthrust/thrust/addressof.h @@ -15,11 +15,7 @@ # pragma system_header #endif // no system header -#if _THRUST_HAS_DEVICE_SYSTEM_STD -# include _THRUST_STD_INCLUDE(__memory/addressof.h) -#else -# include -#endif +#include _THRUST_STD_INCLUDE(__memory/addressof.h) THRUST_NAMESPACE_BEGIN using _THRUST_STD::addressof; diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/advance.h b/projects/hipccl/hipccl3/rocthrust/thrust/advance.h index 958dd2459abf..3fac8598a643 100644 --- a/projects/hipccl/hipccl3/rocthrust/thrust/advance.h +++ b/projects/hipccl/hipccl3/rocthrust/thrust/advance.h @@ -30,7 +30,10 @@ # pragma system_header #endif // no system header -#include _THRUST_STD_INCLUDE(iterator) +#include _THRUST_STD_INCLUDE(__iterator/advance.h) +#include _THRUST_STD_INCLUDE(__iterator/iterator_traits.h) +#include _THRUST_STD_INCLUDE(__iterator/next.h) +#include _THRUST_STD_INCLUDE(__iterator/prev.h) THRUST_NAMESPACE_BEGIN diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/allocate_unique.h b/projects/hipccl/hipccl3/rocthrust/thrust/allocate_unique.h index 5448a7f51f08..4344f1cf7d66 100644 --- a/projects/hipccl/hipccl3/rocthrust/thrust/allocate_unique.h +++ b/projects/hipccl/hipccl3/rocthrust/thrust/allocate_unique.h @@ -16,12 +16,13 @@ #endif // no system header #include #include -#include #include #include -#include // for internal::remove_cvref_t -#include _THRUST_STD_INCLUDE(utility) +#include _THRUST_STD_INCLUDE(__host_stdlib/memory) +#include _THRUST_STD_INCLUDE(__type_traits/remove_cvref.h) +#include _THRUST_STD_INCLUDE(__utility/move.h) +#include _THRUST_STD_INCLUDE(__utility/swap.h) THRUST_NAMESPACE_BEGIN @@ -65,7 +66,7 @@ struct allocator_delete final void operator()(pointer p) { - using traits = detail::allocator_traits<::internal::remove_cvref_t>; + using traits = detail::allocator_traits<_THRUST_STD::remove_cvref_t>; typename traits::allocator_type alloc_T(alloc_); if (nullptr != detail::pointer_traits::get(p)) @@ -141,7 +142,7 @@ struct array_allocator_delete final void operator()(pointer p) { - using traits = detail::allocator_traits<::internal::remove_cvref_t>; + using traits = detail::allocator_traits<_THRUST_STD::remove_cvref_t>; typename traits::allocator_type alloc_T(get_allocator()); if (nullptr != detail::pointer_traits::get(p)) { @@ -200,14 +201,14 @@ THRUST_HOST_DEVICE tagged_deleter make_tagged_deleter(Lambda&& //! Creates a \p std::unique_ptr holding a new object of type \p T, constructed with \p args, using \p alloc as the //! allocator. template -THRUST_HOST std::unique_ptr< - T, - allocator_delete< - T, - typename detail::allocator_traits<::internal::remove_cvref_t>::template rebind_traits::allocator_type>> +THRUST_HOST +std::unique_ptr>::template rebind_traits::allocator_type>> allocate_unique(Allocator const& alloc, Args&&... args) { - using traits = typename detail::allocator_traits<::internal::remove_cvref_t>::template rebind_traits; + using traits = typename detail::allocator_traits<_THRUST_STD::remove_cvref_t>::template rebind_traits; typename traits::allocator_type alloc_T(alloc); @@ -229,10 +230,10 @@ THRUST_HOST std::unique_ptr< T, uninitialized_allocator_delete< T, - typename detail::allocator_traits<::internal::remove_cvref_t>::template rebind_traits::allocator_type>> + typename detail::allocator_traits<_THRUST_STD::remove_cvref_t>::template rebind_traits::allocator_type>> uninitialized_allocate_unique(Allocator const& alloc) { - using traits = typename detail::allocator_traits<::internal::remove_cvref_t>::template rebind_traits; + using traits = typename detail::allocator_traits<_THRUST_STD::remove_cvref_t>::template rebind_traits; typename traits::allocator_type alloc_T(alloc); @@ -256,7 +257,7 @@ THRUST_HOST std::unique_ptr< Allocator>::type>::type>::template rebind_traits::allocator_type>> allocate_unique_n(Allocator const& alloc, Size n, Args&&... args) { - using traits = typename detail::allocator_traits<::internal::remove_cvref_t>::template rebind_traits; + using traits = typename detail::allocator_traits<_THRUST_STD::remove_cvref_t>::template rebind_traits; typename traits::allocator_type alloc_T(alloc); @@ -278,10 +279,10 @@ THRUST_HOST std::unique_ptr< T[], uninitialized_array_allocator_delete< T, - typename detail::allocator_traits<::internal::remove_cvref_t>::template rebind_traits::allocator_type>> + typename detail::allocator_traits<_THRUST_STD::remove_cvref_t>::template rebind_traits::allocator_type>> uninitialized_allocate_unique_n(Allocator const& alloc, Size n) { - using traits = typename detail::allocator_traits<::internal::remove_cvref_t>::template rebind_traits; + using traits = typename detail::allocator_traits<_THRUST_STD::remove_cvref_t>::template rebind_traits; typename traits::allocator_type alloc_T(alloc); diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/binary_search.h b/projects/hipccl/hipccl3/rocthrust/thrust/binary_search.h index 38cd22ef6a51..1bc2a9289122 100644 --- a/projects/hipccl/hipccl3/rocthrust/thrust/binary_search.h +++ b/projects/hipccl/hipccl3/rocthrust/thrust/binary_search.h @@ -30,7 +30,8 @@ # pragma system_header #endif // no system header #include -#include + +#include _THRUST_STD_INCLUDE(__utility/pair.h) THRUST_NAMESPACE_BEGIN @@ -774,7 +775,7 @@ bool binary_search(ForwardIterator first, ForwardIterator last, const T& value, * \see \p binary_search */ template -THRUST_HOST_DEVICE thrust::pair equal_range( +THRUST_HOST_DEVICE _THRUST_STD::pair equal_range( const thrust::detail::execution_policy_base& exec, ForwardIterator first, ForwardIterator last, @@ -839,7 +840,7 @@ THRUST_HOST_DEVICE thrust::pair equal_range( * \see \p binary_search */ template -thrust::pair +_THRUST_STD::pair equal_range(ForwardIterator first, ForwardIterator last, const LessThanComparable& value); /*! \p equal_range is a version of binary search: it attempts to find @@ -917,7 +918,7 @@ equal_range(ForwardIterator first, ForwardIterator last, const LessThanComparabl * \see \p binary_search */ template -THRUST_HOST_DEVICE thrust::pair equal_range( +THRUST_HOST_DEVICE _THRUST_STD::pair equal_range( const thrust::detail::execution_policy_base& exec, ForwardIterator first, ForwardIterator last, @@ -994,7 +995,7 @@ THRUST_HOST_DEVICE thrust::pair equal_range( * \see \p binary_search */ template -thrust::pair +_THRUST_STD::pair equal_range(ForwardIterator first, ForwardIterator last, const T& value, StrictWeakOrdering comp); /*! \addtogroup vectorized_binary_search Vectorized Searches diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/complex.h b/projects/hipccl/hipccl3/rocthrust/thrust/complex.h index 50c9a8d114bb..d11882c350b3 100644 --- a/projects/hipccl/hipccl3/rocthrust/thrust/complex.h +++ b/projects/hipccl/hipccl3/rocthrust/thrust/complex.h @@ -35,16 +35,13 @@ #include #include -#include _THRUST_STD_INCLUDE(cmath) +#include _THRUST_STD_INCLUDE(__type_traits/common_type.h) +#include _THRUST_STD_INCLUDE(__type_traits/remove_reference.h) #if !THRUST_COMPILER(NVRTC) # include # include #endif // !THRUST_COMPILER(NVRTC) -#if !_THRUST_HAS_DEVICE_SYSTEM_STD -# include -#endif - #define THRUST_STD_COMPLEX_REAL(z) \ reinterpret_cast::value_type(&)[2]>(z)[0] #define THRUST_STD_COMPLEX_IMAG(z) \ @@ -61,11 +58,11 @@ THRUST_NAMESPACE_BEGIN */ /*! \addtogroup numerics - * \{ + *! \{ */ /*! \addtogroup complex_numbers Complex Numbers - * \{ + *! \{ */ /*! \p complex is the Thrust equivalent to std::complex. It is @@ -396,7 +393,7 @@ THRUST_HOST_DEVICE complex conj(const complex& z); * \param theta The phase of the returned \p complex in radians. */ template -THRUST_HOST_DEVICE complex<::internal::common_type_t> polar(const T0& m, const T1& theta = T1()); +THRUST_HOST_DEVICE complex<_THRUST_STD::common_type_t> polar(const T0& m, const T1& theta = T1()); /*! Returns the projection of a \p complex on the Riemann sphere. * For all finite \p complex it returns the argument. For \p complexs @@ -419,7 +416,7 @@ THRUST_HOST_DEVICE complex proj(const T& z); * \param y The second \p complex. */ template -THRUST_HOST_DEVICE complex<::internal::common_type_t> operator+(const complex& x, const complex& y); +THRUST_HOST_DEVICE complex<_THRUST_STD::common_type_t> operator+(const complex& x, const complex& y); /*! Adds a scalar to a \p complex number. * @@ -430,7 +427,7 @@ THRUST_HOST_DEVICE complex<::internal::common_type_t> operator+(const co * \param y The scalar. */ template -THRUST_HOST_DEVICE complex<::internal::common_type_t> operator+(const complex& x, const T1& y); +THRUST_HOST_DEVICE complex<_THRUST_STD::common_type_t> operator+(const complex& x, const T1& y); /*! Adds a \p complex number to a scalar. * @@ -441,7 +438,7 @@ THRUST_HOST_DEVICE complex<::internal::common_type_t> operator+(const co * \param y The \p complex. */ template -THRUST_HOST_DEVICE complex<::internal::common_type_t> operator+(const T0& x, const complex& y); +THRUST_HOST_DEVICE complex<_THRUST_STD::common_type_t> operator+(const T0& x, const complex& y); /*! Subtracts two \p complex numbers. * @@ -452,7 +449,7 @@ THRUST_HOST_DEVICE complex<::internal::common_type_t> operator+(const T0 * \param y The second \p complex (subtrahend). */ template -THRUST_HOST_DEVICE complex<::internal::common_type_t> operator-(const complex& x, const complex& y); +THRUST_HOST_DEVICE complex<_THRUST_STD::common_type_t> operator-(const complex& x, const complex& y); /*! Subtracts a scalar from a \p complex number. * @@ -463,7 +460,7 @@ THRUST_HOST_DEVICE complex<::internal::common_type_t> operator-(const co * \param y The scalar (subtrahend). */ template -THRUST_HOST_DEVICE complex<::internal::common_type_t> operator-(const complex& x, const T1& y); +THRUST_HOST_DEVICE complex<_THRUST_STD::common_type_t> operator-(const complex& x, const T1& y); /*! Subtracts a \p complex number from a scalar. * @@ -474,7 +471,7 @@ THRUST_HOST_DEVICE complex<::internal::common_type_t> operator-(const co * \param y The \p complex (subtrahend). */ template -THRUST_HOST_DEVICE complex<::internal::common_type_t> operator-(const T0& x, const complex& y); +THRUST_HOST_DEVICE complex<_THRUST_STD::common_type_t> operator-(const T0& x, const complex& y); /*! Multiplies two \p complex numbers. * @@ -485,7 +482,7 @@ THRUST_HOST_DEVICE complex<::internal::common_type_t> operator-(const T0 * \param y The second \p complex. */ template -THRUST_HOST_DEVICE complex<::internal::common_type_t> operator*(const complex& x, const complex& y); +THRUST_HOST_DEVICE complex<_THRUST_STD::common_type_t> operator*(const complex& x, const complex& y); /*! Multiplies a \p complex number by a scalar. * @@ -493,7 +490,7 @@ THRUST_HOST_DEVICE complex<::internal::common_type_t> operator*(const co * \param y The scalar. */ template -THRUST_HOST_DEVICE complex<::internal::common_type_t> operator*(const complex& x, const T1& y); +THRUST_HOST_DEVICE complex<_THRUST_STD::common_type_t> operator*(const complex& x, const T1& y); /*! Multiplies a scalar by a \p complex number. * @@ -504,7 +501,7 @@ THRUST_HOST_DEVICE complex<::internal::common_type_t> operator*(const co * \param y The \p complex. */ template -THRUST_HOST_DEVICE complex<::internal::common_type_t> operator*(const T0& x, const complex& y); +THRUST_HOST_DEVICE complex<_THRUST_STD::common_type_t> operator*(const T0& x, const complex& y); /*! Divides two \p complex numbers. * @@ -515,7 +512,7 @@ THRUST_HOST_DEVICE complex<::internal::common_type_t> operator*(const T0 * \param y The denomimator (divisor). */ template -THRUST_HOST_DEVICE complex<::internal::common_type_t> operator/(const complex& x, const complex& y); +THRUST_HOST_DEVICE complex<_THRUST_STD::common_type_t> operator/(const complex& x, const complex& y); /*! Divides a \p complex number by a scalar. * @@ -526,7 +523,7 @@ THRUST_HOST_DEVICE complex<::internal::common_type_t> operator/(const co * \param y The scalar denomimator (divisor). */ template -THRUST_HOST_DEVICE complex<::internal::common_type_t> operator/(const complex& x, const T1& y); +THRUST_HOST_DEVICE complex<_THRUST_STD::common_type_t> operator/(const complex& x, const T1& y); /*! Divides a scalar by a \p complex number. * @@ -537,7 +534,7 @@ THRUST_HOST_DEVICE complex<::internal::common_type_t> operator/(const co * \param y The complex denomimator (divisor). */ template -THRUST_HOST_DEVICE complex<::internal::common_type_t> operator/(const T0& x, const complex& y); +THRUST_HOST_DEVICE complex<_THRUST_STD::common_type_t> operator/(const T0& x, const complex& y); /* --- Unary Arithmetic operators --- */ @@ -590,7 +587,7 @@ THRUST_HOST_DEVICE complex log10(const complex& z); * \param y The exponent. */ template -THRUST_HOST_DEVICE complex<::internal::common_type_t> pow(const complex& x, const complex& y); +THRUST_HOST_DEVICE complex<_THRUST_STD::common_type_t> pow(const complex& x, const complex& y); /*! Returns a \p complex number raised to a scalar. * @@ -601,7 +598,7 @@ THRUST_HOST_DEVICE complex<::internal::common_type_t> pow(const complex< * \param y The exponent. */ template -THRUST_HOST_DEVICE complex<::internal::common_type_t> pow(const complex& x, const T1& y); +THRUST_HOST_DEVICE complex<_THRUST_STD::common_type_t> pow(const complex& x, const T1& y); /*! Returns a scalar raised to a \p complex number. * @@ -612,7 +609,7 @@ THRUST_HOST_DEVICE complex<::internal::common_type_t> pow(const complex< * \param y The exponent. */ template -THRUST_HOST_DEVICE complex<::internal::common_type_t> pow(const T0& x, const complex& y); +THRUST_HOST_DEVICE complex<_THRUST_STD::common_type_t> pow(const T0& x, const complex& y); /*! Returns the complex square root of a \p complex number. * @@ -848,6 +845,11 @@ THRUST_HOST_DEVICE bool operator!=(const T0& x, const complex& y); template THRUST_HOST_DEVICE bool operator!=(const complex& x, const T1& y); +/*! + * \} end group complex_numbers + * \} end group numerics + */ + THRUST_NAMESPACE_END #include @@ -855,9 +857,3 @@ THRUST_NAMESPACE_END #undef THRUST_STD_COMPLEX_REAL #undef THRUST_STD_COMPLEX_IMAG #undef THRUST_STD_COMPLEX_DEVICE - -/*! \} // complex_numbers - */ - -/*! \} // numerics - */ diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/detail/adjacent_difference.inl b/projects/hipccl/hipccl3/rocthrust/thrust/detail/adjacent_difference.inl index dd3d4d323b91..9f79f93bd9c0 100644 --- a/projects/hipccl/hipccl3/rocthrust/thrust/detail/adjacent_difference.inl +++ b/projects/hipccl/hipccl3/rocthrust/thrust/detail/adjacent_difference.inl @@ -25,10 +25,24 @@ #elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) # pragma system_header #endif // no system header -#include -#include + #include +// Include all active backend system implementations (generic, sequential, host and device) +#include +#include +#include __THRUST_HOST_SYSTEM_ALGORITH_DETAIL_HEADER_INCLUDE(adjacent_difference.h) +#include __THRUST_DEVICE_SYSTEM_ALGORITH_DETAIL_HEADER_INCLUDE(adjacent_difference.h) + +// Some build systems need a hint to know which files we could include +#if 0 +# include +# include +# include +# include +# include +#endif + THRUST_NAMESPACE_BEGIN THRUST_EXEC_CHECK_DISABLE diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/detail/algorithm_wrapper.h b/projects/hipccl/hipccl3/rocthrust/thrust/detail/algorithm_wrapper.h deleted file mode 100644 index c732e106c1df..000000000000 --- a/projects/hipccl/hipccl3/rocthrust/thrust/detail/algorithm_wrapper.h +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2020 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -// When a compiler uses Thrust as part of its implementation of Standard C++ -// algorithms, a cycle of included files may result when Thrust code tries to -// use a standard algorithm. Having a macro that is defined only when Thrust -// is including an algorithms-related header gives the compiler a chance to -// detect and break the cycle of includes. - -#define THRUST_INCLUDING_ALGORITHMS_HEADER -#include -#undef THRUST_INCLUDING_ALGORITHMS_HEADER diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/detail/alignment.h b/projects/hipccl/hipccl3/rocthrust/thrust/detail/alignment.h index ddcbd4bf0ece..bef96bd91f51 100644 --- a/projects/hipccl/hipccl3/rocthrust/thrust/detail/alignment.h +++ b/projects/hipccl/hipccl3/rocthrust/thrust/detail/alignment.h @@ -29,14 +29,9 @@ # pragma system_header #endif // no system header +#include _THRUST_LIBCXX_INCLUDE(__cmath/ceil_div.h) +#include _THRUST_STD_INCLUDE(__type_traits/alignment_of.h) #include _THRUST_STD_INCLUDE(cstddef) // For `std::size_t` and `std::max_align_t`. -#include _THRUST_STD_INCLUDE(type_traits) // For `std::alignment_of`. - -#if _THRUST_HAS_DEVICE_SYSTEM_STD -# include _THRUST_LIBCXX_INCLUDE(cmath) -#elif _THRUST_USE_ROCPRIM -# include -#endif THRUST_NAMESPACE_BEGIN namespace detail @@ -80,13 +75,7 @@ THRUST_HOST_DEVICE T aligned_reinterpret_cast(U u) THRUST_HOST_DEVICE inline _THRUST_STD::size_t aligned_storage_size(_THRUST_STD::size_t n, _THRUST_STD::size_t align) { -#if _THRUST_HAS_DEVICE_SYSTEM_STD return _THRUST_LIBCXX::ceil_div(n, align) * align; -#elif _THRUST_USE_ROCPRIM - return ::rocprim::detail::ceiling_div(n, align) * align; -#else - return (n / align + (n % align > 0 ? 1 : 0)) * align; -#endif } } // end namespace detail THRUST_NAMESPACE_END diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/allocator_traits.h b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/allocator_traits.h index 19fe81ccaeed..8ea0ab2f3534 100644 --- a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/allocator_traits.h +++ b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/allocator_traits.h @@ -30,25 +30,37 @@ # pragma system_header #endif // no system header -#include +#include +#include #include #include #include +#include #include -#include _THRUST_STD_INCLUDE(type_traits) +#include _THRUST_STD_INCLUDE(__host_stdlib/memory) +#include _THRUST_STD_INCLUDE(__type_traits/add_lvalue_reference.h) +#include _THRUST_STD_INCLUDE(__type_traits/conditional.h) +#include _THRUST_STD_INCLUDE(__type_traits/is_empty.h) +#include _THRUST_STD_INCLUDE(__type_traits/make_unsigned.h) +#include _THRUST_STD_INCLUDE(__type_traits/type_identity.h) +#include _THRUST_STD_INCLUDE(__type_traits/void_t.h) +#include _THRUST_STD_INCLUDE(limits) + +#include THRUST_NAMESPACE_BEGIN namespace detail { - // forward declaration for has_member_system template struct allocator_system; +template +struct allocator_traits; + namespace allocator_traits_detail { - __THRUST_DEFINE_HAS_NESTED_TYPE(has_value_type, value_type) __THRUST_DEFINE_HAS_NESTED_TYPE(has_pointer, pointer) __THRUST_DEFINE_HAS_NESTED_TYPE(has_const_pointer, const_pointer) @@ -65,21 +77,11 @@ __THRUST_DEFINE_HAS_NESTED_TYPE(has_system_type, system_type) __THRUST_DEFINE_HAS_NESTED_TYPE(has_is_always_equal, is_always_equal) __THRUST_DEFINE_HAS_MEMBER_FUNCTION(has_member_system_impl, system) -template -struct has_rebind -{ - using yes_type = char; - using no_type = int; - - template - static yes_type test(typename S::template rebind::other*); - template - static no_type test(...); - - static bool const value = sizeof(test(0)) == sizeof(yes_type); +template +inline constexpr bool has_rebind = false; - using type = thrust::detail::integral_constant; -}; +template +inline constexpr bool has_rebind::other)>> = true; THRUST_SUPPRESS_DEPRECATED_PUSH @@ -99,8 +101,7 @@ THRUST_SPECIALIZE_DEPRECATED(has_const_reference) #undef THRUST_SPECIALIZE_DEPRECATED template -struct has_rebind, U> : false_type -{}; +inline constexpr bool has_rebind, U, void> = false; template struct nested_pointer @@ -191,7 +192,7 @@ struct has_member_system THRUST_SUPPRESS_DEPRECATED_POP -template ::value> +template > struct rebind_alloc { using type = typename Alloc::template rebind::other; @@ -209,6 +210,159 @@ struct rebind_alloc, U, false> using type = Alloc; }; +__THRUST_DEFINE_IS_CALL_POSSIBLE(has_member_allocate_with_hint_impl, allocate) + +template +class has_member_allocate_with_hint +{ + using pointer = typename allocator_traits::pointer; + using size_type = typename allocator_traits::size_type; + using const_void_pointer = typename allocator_traits::const_void_pointer; + +public: + using type = typename has_member_allocate_with_hint_impl::type; + static const bool value = type::value; +}; + +template +THRUST_HOST_DEVICE typename allocator_traits::pointer +allocate(Alloc& a, + typename allocator_traits::size_type n, + [[maybe_unused]] typename allocator_traits::const_void_pointer hint) +{ + if constexpr (has_member_allocate_with_hint::value) + { + return a.allocate(n, hint); + } + else + { + return a.allocate(n); + } +} + +__THRUST_DEFINE_IS_CALL_POSSIBLE(has_member_construct1_impl, construct) + +template +struct has_member_construct1 : has_member_construct1_impl +{}; + +THRUST_EXEC_CHECK_DISABLE +template +THRUST_HOST_DEVICE void construct(Alloc& a, T* p) +{ + if constexpr (has_member_construct1::value) + { + a.construct(p); + } + else + { + ::new (static_cast(p)) T(); + } +} + +__THRUST_DEFINE_IS_CALL_POSSIBLE(has_member_construct2_impl, construct) + +template +struct has_member_construct2 : has_member_construct2_impl +{}; + +THRUST_EXEC_CHECK_DISABLE +template +THRUST_HOST_DEVICE void construct(Alloc& a, T* p, const Arg1& arg1) +{ + if constexpr (has_member_construct2::value) + { + a.construct(p, arg1); + } + else + { + ::new (static_cast(p)) T(arg1); + } +} + +__THRUST_DEFINE_IS_CALL_POSSIBLE(has_member_constructN_impl, construct) + +template +struct has_member_constructN : has_member_constructN_impl +{}; + +THRUST_EXEC_CHECK_DISABLE +template +inline THRUST_HOST_DEVICE void construct([[maybe_unused]] Alloc& a, T* p, Args&&... args) +{ + if constexpr (has_member_constructN::value) + { + a.construct(p, THRUST_FWD(args)...); + } + else + { + ::new (static_cast(p)) T(THRUST_FWD(args)...); + } +} + +__THRUST_DEFINE_IS_CALL_POSSIBLE(has_member_destroy_impl, destroy) + +template +struct has_member_destroy : has_member_destroy_impl +{}; + +THRUST_EXEC_CHECK_DISABLE +template +THRUST_HOST_DEVICE void destroy([[maybe_unused]] Alloc& a, T* p) +{ + if constexpr (has_member_destroy::value) + { + a.destroy(p); + } + else + { + p->~T(); + } +} + +__THRUST_DEFINE_IS_CALL_POSSIBLE(has_member_max_size_impl, max_size) + +template +class has_member_max_size +{ + using size_type = typename allocator_traits::size_type; + +public: + using type = typename has_member_max_size_impl::type; + static const bool value = type::value; +}; + +template +THRUST_HOST_DEVICE typename allocator_traits::size_type max_size([[maybe_unused]] const Alloc& a) +{ + if constexpr (has_member_max_size::value) + { + return a.max_size(); + } + else + { + using size_type = typename allocator_traits::size_type; + return _THRUST_STD::numeric_limits::max(); + } +} + +// TODO(bgruber): can be return decltype(auto) here? +template +THRUST_HOST_DEVICE _THRUST_STD:: + _If::value, typename allocator_system::type&, typename allocator_system::type> + system(Alloc& a) +{ + if constexpr (has_member_system::value) + { + // return the allocator's system + return a.system(); + } + else + { + // return a copy of a value-initialized system + return typename allocator_system::type{}; + } +} } // namespace allocator_traits_detail template @@ -252,7 +406,7 @@ struct allocator_traits using size_type = typename eval_if::value, allocator_traits_detail::nested_size_type, - ::internal::make_unsigned>::type; + _THRUST_STD::make_unsigned>::type; using propagate_on_container_copy_assignment = typename eval_if::value, @@ -293,32 +447,157 @@ struct allocator_traits using other = allocator_traits; // Deprecated std::allocator aliases that we need: - using reference = typename thrust::detail::pointer_traits::reference; - using const_reference = typename thrust::detail::pointer_traits::reference; - - inline THRUST_HOST_DEVICE static pointer allocate(allocator_type& a, size_type n); + using reference = typename pointer_traits::reference; + using const_reference = typename pointer_traits::reference; - inline THRUST_HOST_DEVICE static pointer allocate(allocator_type& a, size_type n, const_void_pointer hint); + inline THRUST_HOST_DEVICE static pointer allocate(allocator_type& a, size_type n) + { + struct workaround_warnings + { + THRUST_EXEC_CHECK_DISABLE + static THRUST_HOST_DEVICE pointer allocate(Alloc& a, size_type n) + { + return a.allocate(n); + } + }; + + return workaround_warnings::allocate(a, n); + } + + inline THRUST_HOST_DEVICE static pointer allocate(allocator_type& a, size_type n, const_void_pointer hint) + { + return allocator_traits_detail::allocate(a, n, hint); + } - inline THRUST_HOST_DEVICE static void deallocate(allocator_type& a, pointer p, size_type n) noexcept; + inline THRUST_HOST_DEVICE static void deallocate(allocator_type& a, pointer p, size_type n) noexcept + { + struct workaround_warnings + { + THRUST_EXEC_CHECK_DISABLE + static THRUST_HOST_DEVICE void deallocate(Alloc& a, pointer p, size_type n) noexcept + { + return a.deallocate(p, n); + } + }; + + return workaround_warnings::deallocate(a, p, n); + } // XXX should probably change T* to pointer below and then relax later template - inline THRUST_HOST_DEVICE static void construct(allocator_type& a, T* p); + THRUST_HOST_DEVICE static void construct(allocator_type& a, T* p) + { + return allocator_traits_detail::construct(a, p); + } template - inline THRUST_HOST_DEVICE static void construct(allocator_type& a, T* p, const Arg1& arg1); + THRUST_HOST_DEVICE static void construct(allocator_type& a, T* p, const Arg1& arg1) + { + return allocator_traits_detail::construct(a, p, arg1); + } template - inline THRUST_HOST_DEVICE static void construct(allocator_type& a, T* p, Args&&... args); + THRUST_HOST_DEVICE static void construct(allocator_type& a, T* p, Args&&... args) + { + return allocator_traits_detail::construct(a, p, THRUST_FWD(args)...); + } template - inline THRUST_HOST_DEVICE static void destroy(allocator_type& a, T* p) noexcept; + THRUST_HOST_DEVICE static void destroy(allocator_type& a, T* p) noexcept + { + return allocator_traits_detail::destroy(a, p); + } - inline THRUST_HOST_DEVICE static size_type max_size(const allocator_type& a); + THRUST_HOST_DEVICE static size_type max_size(const allocator_type& a) + { + return allocator_traits_detail::max_size(a); + } }; // end allocator_traits +// std::allocator's member functions are deprecated in C++17 and removed in +// C++20, so we can't just use the generic implementation for allocator_traits +// that calls the allocator's member functions. +// Instead, specialize allocator_traits for std::allocator and defer to +// std::allocator_traits and let the STL do whatever it needs +// to for the current c++ version. Manually forward the calls to suppress +// host/device warnings. +template +struct allocator_traits> : public std::allocator_traits> +{ +private: + using superclass = std::allocator_traits>; + +public: + using allocator_type = typename superclass::allocator_type; + using value_type = typename superclass::value_type; + using pointer = typename superclass::pointer; + using const_pointer = typename superclass::const_pointer; + using void_pointer = typename superclass::void_pointer; + using const_void_pointer = typename superclass::const_void_pointer; + using difference_type = typename superclass::difference_type; + using size_type = typename superclass::size_type; + using propagate_on_container_swap = typename superclass::propagate_on_container_swap; + using propagate_on_container_copy_assignment = typename superclass::propagate_on_container_copy_assignment; + using propagate_on_container_move_assignment = typename superclass::propagate_on_container_move_assignment; + + // std::allocator_traits added this in C++17, but thrust::allocator_traits defines + // it unconditionally. + using is_always_equal = + typename eval_if::value, + allocator_traits_detail::nested_is_always_equal, + _THRUST_STD::is_empty>::type; + + // std::allocator_traits doesn't provide these, but + // thrust::detail::allocator_traits does. These used to be part of the + // std::allocator API but were deprecated in C++17. + using reference = typename pointer_traits::reference; + using const_reference = typename pointer_traits::reference; + + template + using rebind_alloc = std::allocator; + template + using rebind_traits = allocator_traits>; + + THRUST_EXEC_CHECK_DISABLE + THRUST_HOST_DEVICE static pointer allocate(allocator_type& a, size_type n) + { + return superclass::allocate(a, n); + } + + THRUST_EXEC_CHECK_DISABLE + THRUST_HOST_DEVICE static pointer allocate(allocator_type& a, size_type n, const_void_pointer hint) + { + return superclass::allocate(a, n, hint); + } + + THRUST_EXEC_CHECK_DISABLE + THRUST_HOST_DEVICE static void deallocate(allocator_type& a, pointer p, size_type n) noexcept + { + superclass::deallocate(a, p, n); + } + + THRUST_EXEC_CHECK_DISABLE + template + THRUST_HOST_DEVICE static void construct(allocator_type& a, U* p, Args&&... args) + { + superclass::construct(a, p, THRUST_FWD(args)...); + } + + THRUST_EXEC_CHECK_DISABLE + template + THRUST_HOST_DEVICE static void destroy(allocator_type& a, U* p) noexcept + { + superclass::destroy(a, p); + } + + THRUST_EXEC_CHECK_DISABLE + THRUST_HOST_DEVICE static size_type max_size(const allocator_type& a) + { + return superclass::max_size(a); + } +}; + // we consider a type an allocator if T::value_type exists // it doesn't make much sense (containers, which are not allocators, will fulfill this requirement), // but allocator_traits is specified to work for any type with that nested alias @@ -341,10 +620,10 @@ struct allocator_system _THRUST_STD::add_lvalue_reference, _THRUST_STD::type_identity>::type; - THRUST_HOST_DEVICE inline static get_result_type get(Alloc& a); + THRUST_HOST_DEVICE inline static get_result_type get(Alloc& a) + { + return allocator_traits_detail::system(a); + } }; - } // namespace detail THRUST_NAMESPACE_END - -#include diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/allocator_traits.inl b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/allocator_traits.inl deleted file mode 100644 index 20f0c3be00ed..000000000000 --- a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/allocator_traits.inl +++ /dev/null @@ -1,374 +0,0 @@ -/* - * Copyright 2008-2018 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header -#include -#include -#include -#include - -#include _THRUST_STD_INCLUDE(limits) - -#include -#if !_THRUST_HAS_DEVICE_SYSTEM_STD -# include -#endif - -THRUST_NAMESPACE_BEGIN -namespace detail -{ - -// std::allocator's member functions are deprecated in C++17 and removed in -// C++20, so we can't just use the generic implementation for allocator_traits -// that calls the allocator's member functions. -// Instead, specialize allocator_traits for std::allocator and defer to -// std::allocator_traits and let the STL do whatever it needs -// to for the current c++ version. Manually forward the calls to suppress -// host/device warnings. -template -struct allocator_traits> : public std::allocator_traits> -{ -private: - using superclass = std::allocator_traits>; - -public: - using allocator_type = typename superclass::allocator_type; - using value_type = typename superclass::value_type; - using pointer = typename superclass::pointer; - using const_pointer = typename superclass::const_pointer; - using void_pointer = typename superclass::void_pointer; - using const_void_pointer = typename superclass::const_void_pointer; - using difference_type = typename superclass::difference_type; - using size_type = typename superclass::size_type; - using propagate_on_container_swap = typename superclass::propagate_on_container_swap; - using propagate_on_container_copy_assignment = typename superclass::propagate_on_container_copy_assignment; - using propagate_on_container_move_assignment = typename superclass::propagate_on_container_move_assignment; - - // std::allocator_traits added this in C++17, but thrust::allocator_traits defines - // it unconditionally. - using is_always_equal = - typename eval_if::value, - allocator_traits_detail::nested_is_always_equal, - _THRUST_STD::is_empty>::type; - - // std::allocator_traits doesn't provide these, but - // thrust::detail::allocator_traits does. These used to be part of the - // std::allocator API but were deprecated in C++17. - using reference = typename thrust::detail::pointer_traits::reference; - using const_reference = typename thrust::detail::pointer_traits::reference; - - template - using rebind_alloc = std::allocator; - template - using rebind_traits = allocator_traits>; - - THRUST_EXEC_CHECK_DISABLE - THRUST_HOST_DEVICE static pointer allocate(allocator_type& a, size_type n) - { - return superclass::allocate(a, n); - } - - THRUST_EXEC_CHECK_DISABLE - THRUST_HOST_DEVICE static pointer allocate(allocator_type& a, size_type n, const_void_pointer hint) - { - return superclass::allocate(a, n, hint); - } - - THRUST_EXEC_CHECK_DISABLE - THRUST_HOST_DEVICE static void deallocate(allocator_type& a, pointer p, size_type n) noexcept - { - superclass::deallocate(a, p, n); - } - - THRUST_EXEC_CHECK_DISABLE - template - THRUST_HOST_DEVICE static void construct(allocator_type& a, U* p, Args&&... args) - { - superclass::construct(a, p, THRUST_FWD(args)...); - } - - THRUST_EXEC_CHECK_DISABLE - template - THRUST_HOST_DEVICE static void destroy(allocator_type& a, U* p) noexcept - { - superclass::destroy(a, p); - } - - THRUST_EXEC_CHECK_DISABLE - THRUST_HOST_DEVICE static size_type max_size(const allocator_type& a) - { - return superclass::max_size(a); - } -}; - -namespace allocator_traits_detail -{ - -__THRUST_DEFINE_IS_CALL_POSSIBLE(has_member_allocate_with_hint_impl, allocate) - -template -class has_member_allocate_with_hint -{ - using pointer = typename allocator_traits::pointer; - using size_type = typename allocator_traits::size_type; - using const_void_pointer = typename allocator_traits::const_void_pointer; - -public: - using type = typename has_member_allocate_with_hint_impl::type; - static const bool value = type::value; -}; - -template -THRUST_HOST_DEVICE _THRUST_STD::enable_if_t::value, - typename allocator_traits::pointer> -allocate(Alloc& a, - typename allocator_traits::size_type n, - typename allocator_traits::const_void_pointer hint) -{ - return a.allocate(n, hint); -} - -template -THRUST_HOST_DEVICE _THRUST_STD::enable_if_t::value, - typename allocator_traits::pointer> -allocate(Alloc& a, typename allocator_traits::size_type n, typename allocator_traits::const_void_pointer) -{ - return a.allocate(n); -} - -__THRUST_DEFINE_IS_CALL_POSSIBLE(has_member_construct1_impl, construct) - -template -struct has_member_construct1 : has_member_construct1_impl -{}; - -THRUST_EXEC_CHECK_DISABLE -template -inline THRUST_HOST_DEVICE _THRUST_STD::enable_if_t::value> construct(Alloc& a, T* p) -{ - a.construct(p); -} - -THRUST_EXEC_CHECK_DISABLE -template -inline THRUST_HOST_DEVICE _THRUST_STD::enable_if_t::value> construct(Alloc&, T* p) -{ - ::new (static_cast(p)) T(); -} - -__THRUST_DEFINE_IS_CALL_POSSIBLE(has_member_construct2_impl, construct) - -template -struct has_member_construct2 : has_member_construct2_impl -{}; - -THRUST_EXEC_CHECK_DISABLE -template -inline THRUST_HOST_DEVICE _THRUST_STD::enable_if_t::value> -construct(Alloc& a, T* p, const Arg1& arg1) -{ - a.construct(p, arg1); -} - -THRUST_EXEC_CHECK_DISABLE -template -inline THRUST_HOST_DEVICE _THRUST_STD::enable_if_t::value> -construct(Alloc&, T* p, const Arg1& arg1) -{ - ::new (static_cast(p)) T(arg1); -} - -__THRUST_DEFINE_IS_CALL_POSSIBLE(has_member_constructN_impl, construct) - -template -struct has_member_constructN : has_member_constructN_impl -{}; - -THRUST_EXEC_CHECK_DISABLE -template -inline THRUST_HOST_DEVICE _THRUST_STD::enable_if_t::value> -construct(Alloc& a, T* p, Args&&... args) -{ - a.construct(p, THRUST_FWD(args)...); -} - -THRUST_EXEC_CHECK_DISABLE -template -inline THRUST_HOST_DEVICE _THRUST_STD::enable_if_t::value> -construct(Alloc&, T* p, Args&&... args) -{ - ::new (static_cast(p)) T(THRUST_FWD(args)...); -} - -__THRUST_DEFINE_IS_CALL_POSSIBLE(has_member_destroy_impl, destroy) - -template -struct has_member_destroy : has_member_destroy_impl -{}; - -THRUST_EXEC_CHECK_DISABLE -template -inline THRUST_HOST_DEVICE _THRUST_STD::enable_if_t::value> destroy(Alloc& a, T* p) -{ - a.destroy(p); -} - -THRUST_EXEC_CHECK_DISABLE -template -inline THRUST_HOST_DEVICE _THRUST_STD::enable_if_t::value> destroy(Alloc&, T* p) -{ - p->~T(); -} - -__THRUST_DEFINE_IS_CALL_POSSIBLE(has_member_max_size_impl, max_size) - -template -class has_member_max_size -{ - using size_type = typename allocator_traits::size_type; - -public: - using type = typename has_member_max_size_impl::type; - static const bool value = type::value; -}; - -template -THRUST_HOST_DEVICE _THRUST_STD::enable_if_t::value, typename allocator_traits::size_type> -max_size(const Alloc& a) -{ - return a.max_size(); -} - -template -THRUST_HOST_DEVICE _THRUST_STD::enable_if_t::value, - typename allocator_traits::size_type> -max_size(const Alloc&) -{ - using size_type = typename allocator_traits::size_type; - return _THRUST_STD::numeric_limits::max(); -} - -template -THRUST_HOST_DEVICE _THRUST_STD::enable_if_t::value, typename allocator_system::type&> -system(Alloc& a) -{ - // return the allocator's system - return a.system(); -} - -template -THRUST_HOST_DEVICE _THRUST_STD::enable_if_t::value, typename allocator_system::type> -system(Alloc&) -{ - // return a copy of a value-initialized system - return typename allocator_system::type(); -} - -} // namespace allocator_traits_detail - -template -THRUST_HOST_DEVICE typename allocator_traits::pointer -allocator_traits::allocate(Alloc& a, typename allocator_traits::size_type n) -{ - struct workaround_warnings - { - THRUST_EXEC_CHECK_DISABLE - static THRUST_HOST_DEVICE typename allocator_traits::pointer - allocate(Alloc& a, typename allocator_traits::size_type n) - { - return a.allocate(n); - } - }; - - return workaround_warnings::allocate(a, n); -} - -template -THRUST_HOST_DEVICE typename allocator_traits::pointer allocator_traits::allocate( - Alloc& a, typename allocator_traits::size_type n, typename allocator_traits::const_void_pointer hint) -{ - return allocator_traits_detail::allocate(a, n, hint); -} - -template -THRUST_HOST_DEVICE void allocator_traits::deallocate( - Alloc& a, typename allocator_traits::pointer p, typename allocator_traits::size_type n) noexcept -{ - struct workaround_warnings - { - THRUST_EXEC_CHECK_DISABLE - static THRUST_HOST_DEVICE void deallocate( - Alloc& a, typename allocator_traits::pointer p, typename allocator_traits::size_type n) noexcept - { - return a.deallocate(p, n); - } - }; - - return workaround_warnings::deallocate(a, p, n); -} - -template -template -THRUST_HOST_DEVICE void allocator_traits::construct(allocator_type& a, T* p) -{ - return allocator_traits_detail::construct(a, p); -} - -template -template -THRUST_HOST_DEVICE void allocator_traits::construct(allocator_type& a, T* p, const Arg1& arg1) -{ - return allocator_traits_detail::construct(a, p, arg1); -} - -template -template -THRUST_HOST_DEVICE void allocator_traits::construct(allocator_type& a, T* p, Args&&... args) -{ - return allocator_traits_detail::construct(a, p, THRUST_FWD(args)...); -} - -template -template -THRUST_HOST_DEVICE void allocator_traits::destroy(allocator_type& a, T* p) noexcept -{ - return allocator_traits_detail::destroy(a, p); -} - -template -THRUST_HOST_DEVICE typename allocator_traits::size_type allocator_traits::max_size(const allocator_type& a) -{ - return allocator_traits_detail::max_size(a); -} - -template -THRUST_HOST_DEVICE typename allocator_system::get_result_type allocator_system::get(Alloc& a) -{ - return allocator_traits_detail::system(a); -} - -} // namespace detail -THRUST_NAMESPACE_END diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/copy_construct_range.h b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/copy_construct_range.h index 630d2769cc36..015135fafbf6 100644 --- a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/copy_construct_range.h +++ b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/copy_construct_range.h @@ -25,21 +25,169 @@ #elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) # pragma system_header #endif // no system header + +#include +#include #include +#include +#include +#include +#include +#include + +#include _THRUST_STD_INCLUDE(__host_stdlib/memory) +#include _THRUST_STD_INCLUDE(__iterator/advance.h) +#include _THRUST_STD_INCLUDE(__iterator/distance.h) +#include _THRUST_STD_INCLUDE(__type_traits/is_convertible.h) +#include _THRUST_STD_INCLUDE(__type_traits/is_trivially_copy_constructible.h) THRUST_NAMESPACE_BEGIN namespace detail { +template +struct copy_construct_with_allocator +{ + Allocator& a; + + template + inline THRUST_HOST_DEVICE void operator()(Tuple t) + { + const InputType& in = thrust::get<0>(t); + OutputType& out = thrust::get<1>(t); + + allocator_traits::construct(a, &out, in); + } +}; + +// we need to use allocator_traits::construct() to +// copy construct a T if either: +// 1. Allocator has a 2-argument construct() member or +// 2. T has a non-trivial copy constructor +template +inline constexpr bool needs_copy_construct_via_allocator = + allocator_traits_detail::has_member_construct2::value + || !_THRUST_STD::is_trivially_copy_constructible_v; + +// we know that std::allocator::construct's only effect is to call T's +// copy constructor, so we needn't consider or use its construct() member for copy construction +template +inline constexpr bool needs_copy_construct_via_allocator, T> = + !_THRUST_STD::is_trivially_copy_constructible_v; + +// XXX it's regrettable that this implementation is copied almost +// exactly from system::detail::generic::uninitialized_copy +// perhaps generic::uninitialized_copy could call this routine +// with a default allocator +template +THRUST_HOST_DEVICE Pointer uninitialized_copy_with_allocator( + Allocator& a, + const thrust::execution_policy& from_system, + const thrust::execution_policy& to_system, + InputIterator first, + InputIterator last, + Pointer result) +{ + if constexpr (_THRUST_STD::is_convertible_v) + { + // zip up the iterators + using IteratorTuple = thrust::tuple; + using ZipIterator = thrust::zip_iterator; + + ZipIterator begin = thrust::make_zip_iterator(first, result); + ZipIterator end = begin; + + // get a zip_iterator pointing to the end + const thrust::detail::it_difference_t n = _THRUST_STD::distance(first, last); + _THRUST_STD::advance(end, n); + + // create a functor + using InputType = it_value_t; + using OutputType = it_value_t; + + // do the for_each + // note we use to_system to dispatch the for_each + thrust::for_each(to_system, begin, end, copy_construct_with_allocator{a}); + + // return the end of the output range + return thrust::get<1>(end.get_iterator_tuple()); + } + else + { + // the systems aren't trivially interoperable + // just call two_system_copy and hope for the best + return thrust::detail::two_system_copy(from_system, to_system, first, last, result); + } +} + +// XXX it's regrettable that this implementation is copied almost +// exactly from system::detail::generic::uninitialized_copy_n +// perhaps generic::uninitialized_copy_n could call this routine +// with a default allocator +template +THRUST_HOST_DEVICE Pointer uninitialized_copy_with_allocator_n( + Allocator& a, + const thrust::execution_policy& from_system, + const thrust::execution_policy& to_system, + InputIterator first, + Size n, + Pointer result) +{ + if constexpr (_THRUST_STD::is_convertible_v) + { + // zip up the iterators + using IteratorTuple = thrust::tuple; + using ZipIterator = thrust::zip_iterator; + + ZipIterator begin = thrust::make_zip_iterator(first, result); + + // create a functor + using InputType = it_value_t; + using OutputType = it_value_t; + + // do the for_each_n + // note we use to_system to dispatch the for_each_n + ZipIterator end = + thrust::for_each_n(to_system, begin, n, copy_construct_with_allocator(a)); + + // return the end of the output range + return thrust::get<1>(end.get_iterator_tuple()); + } + else + { + // the systems aren't trivially interoperable + // just call two_system_copy_n and hope for the best + return thrust::detail::two_system_copy_n(from_system, to_system, first, n, result); + } +} template THRUST_HOST_DEVICE Pointer copy_construct_range( - thrust::execution_policy& from_system, Allocator& a, InputIterator first, InputIterator last, Pointer result); + thrust::execution_policy& from_system, Allocator& a, InputIterator first, InputIterator last, Pointer result) +{ + if constexpr (needs_copy_construct_via_allocator::type>) + { + return uninitialized_copy_with_allocator(a, from_system, allocator_system::get(a), first, last, result); + } + else + { + // just call two_system_copy + return thrust::detail::two_system_copy(from_system, allocator_system::get(a), first, last, result); + } +} template THRUST_HOST_DEVICE Pointer copy_construct_range_n( - thrust::execution_policy& from_system, Allocator& a, InputIterator first, Size n, Pointer result); - + thrust::execution_policy& from_system, Allocator& a, InputIterator first, Size n, Pointer result) +{ + if constexpr (needs_copy_construct_via_allocator::type>) + { + return uninitialized_copy_with_allocator_n(a, from_system, allocator_system::get(a), first, n, result); + } + else + { + // just call two_system_copy_n + return thrust::detail::two_system_copy_n(from_system, allocator_system::get(a), first, n, result); + } +} } // namespace detail THRUST_NAMESPACE_END - -#include diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/copy_construct_range.inl b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/copy_construct_range.inl deleted file mode 100644 index 806c50a3e5da..000000000000 --- a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/copy_construct_range.inl +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Copyright 2008-2013 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if !_THRUST_HAS_DEVICE_SYSTEM_STD -# include -#endif - -THRUST_NAMESPACE_BEGIN -namespace detail -{ -namespace allocator_traits_detail -{ - -template -struct copy_construct_with_allocator -{ - Allocator& a; - - THRUST_HOST_DEVICE copy_construct_with_allocator(Allocator& a) - : a(a) - {} - - template - inline THRUST_HOST_DEVICE void operator()(Tuple t) - { - const InputType& in = thrust::get<0>(t); - OutputType& out = thrust::get<1>(t); - - allocator_traits::construct(a, &out, in); - } -}; - -// we need to use allocator_traits::construct() to -// copy construct a T if either: -// 1. Allocator has a 2-argument construct() member or -// 2. T has a non-trivial copy constructor -template -struct needs_copy_construct_via_allocator - : integral_constant::value - || !_THRUST_STD::is_trivially_copy_constructible::value)> -{}; - -// we know that std::allocator::construct's only effect is to call T's -// copy constructor, so we needn't consider or use its construct() member for copy construction -template -struct needs_copy_construct_via_allocator, T> - : integral_constant::value> -{}; - -// XXX it's regrettable that this implementation is copied almost -// exactly from system::detail::generic::uninitialized_copy -// perhaps generic::uninitialized_copy could call this routine -// with a default allocator -template -THRUST_HOST_DEVICE enable_if_convertible_t uninitialized_copy_with_allocator( - Allocator& a, - const thrust::execution_policy&, - const thrust::execution_policy& to_system, - InputIterator first, - InputIterator last, - Pointer result) -{ - // zip up the iterators - using IteratorTuple = thrust::tuple; - using ZipIterator = thrust::zip_iterator; - - ZipIterator begin = thrust::make_zip_iterator(first, result); - ZipIterator end = begin; - - // get a zip_iterator pointing to the end - const thrust::detail::it_difference_t n = _THRUST_STD::distance(first, last); - _THRUST_STD::advance(end, n); - - // create a functor - using InputType = it_value_t; - using OutputType = it_value_t; - - // do the for_each - // note we use to_system to dispatch the for_each - thrust::for_each(to_system, begin, end, copy_construct_with_allocator(a)); - - // return the end of the output range - return thrust::get<1>(end.get_iterator_tuple()); -} - -// XXX it's regrettable that this implementation is copied almost -// exactly from system::detail::generic::uninitialized_copy_n -// perhaps generic::uninitialized_copy_n could call this routine -// with a default allocator -template -THRUST_HOST_DEVICE enable_if_convertible_t uninitialized_copy_with_allocator_n( - Allocator& a, - const thrust::execution_policy&, - const thrust::execution_policy& to_system, - InputIterator first, - Size n, - Pointer result) -{ - // zip up the iterators - using IteratorTuple = thrust::tuple; - using ZipIterator = thrust::zip_iterator; - - ZipIterator begin = thrust::make_zip_iterator(first, result); - - // create a functor - using InputType = it_value_t; - using OutputType = it_value_t; - - // do the for_each_n - // note we use to_system to dispatch the for_each_n - ZipIterator end = - thrust::for_each_n(to_system, begin, n, copy_construct_with_allocator(a)); - - // return the end of the output range - return thrust::get<1>(end.get_iterator_tuple()); -} - -template -THRUST_HOST_DEVICE typename disable_if_convertible::type uninitialized_copy_with_allocator( - Allocator&, - const thrust::execution_policy& from_system, - const thrust::execution_policy& to_system, - InputIterator first, - InputIterator last, - Pointer result) -{ - // the systems aren't trivially interoperable - // just call two_system_copy and hope for the best - return thrust::detail::two_system_copy(from_system, to_system, first, last, result); -} // end uninitialized_copy_with_allocator() - -template -THRUST_HOST_DEVICE typename disable_if_convertible::type -uninitialized_copy_with_allocator_n( - Allocator&, - const thrust::execution_policy& from_system, - const thrust::execution_policy& to_system, - InputIterator first, - Size n, - Pointer result) -{ - // the systems aren't trivially interoperable - // just call two_system_copy_n and hope for the best - return thrust::detail::two_system_copy_n(from_system, to_system, first, n, result); -} // end uninitialized_copy_with_allocator_n() - -template -THRUST_HOST_DEVICE -typename disable_if::type>::value, - Pointer>::type -copy_construct_range(thrust::execution_policy& from_system, - Allocator& a, - InputIterator first, - InputIterator last, - Pointer result) -{ - // just call two_system_copy - return thrust::detail::two_system_copy(from_system, allocator_system::get(a), first, last, result); -} - -template -THRUST_HOST_DEVICE -typename disable_if::type>::value, - Pointer>::type -copy_construct_range_n( - thrust::execution_policy& from_system, Allocator& a, InputIterator first, Size n, Pointer result) -{ - // just call two_system_copy_n - return thrust::detail::two_system_copy_n(from_system, allocator_system::get(a), first, n, result); -} - -template -THRUST_HOST_DEVICE _THRUST_STD:: - enable_if_t::type>::value, Pointer> - copy_construct_range(thrust::execution_policy& from_system, - Allocator& a, - InputIterator first, - InputIterator last, - Pointer result) -{ - return uninitialized_copy_with_allocator(a, from_system, allocator_system::get(a), first, last, result); -} - -template -THRUST_HOST_DEVICE _THRUST_STD:: - enable_if_t::type>::value, Pointer> - copy_construct_range_n( - thrust::execution_policy& from_system, Allocator& a, InputIterator first, Size n, Pointer result) -{ - return uninitialized_copy_with_allocator_n(a, from_system, allocator_system::get(a), first, n, result); -} - -} // namespace allocator_traits_detail - -template -THRUST_HOST_DEVICE Pointer copy_construct_range( - thrust::execution_policy& from_system, Allocator& a, InputIterator first, InputIterator last, Pointer result) -{ - return allocator_traits_detail::copy_construct_range(from_system, a, first, last, result); -} - -template -THRUST_HOST_DEVICE Pointer copy_construct_range_n( - thrust::execution_policy& from_system, Allocator& a, InputIterator first, Size n, Pointer result) -{ - return allocator_traits_detail::copy_construct_range_n(from_system, a, first, n, result); -} - -} // namespace detail -THRUST_NAMESPACE_END diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/destroy_range.h b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/destroy_range.h index 9679455e899f..5c86d96bd962 100644 --- a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/destroy_range.h +++ b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/destroy_range.h @@ -26,14 +26,70 @@ # pragma system_header #endif // no system header +#include +#include +#include +#include + +#include _THRUST_STD_INCLUDE(__host_stdlib/memory) + THRUST_NAMESPACE_BEGIN namespace detail { +// destroy_range has three cases: +// if Allocator has an effectful member function destroy: +// 1. destroy via the allocator +// else +// 2. if T has a non-trivial destructor, destroy the range without using the allocator +// 3. if T has a trivial destructor, do a no-op + +template +inline constexpr bool has_effectful_member_destroy = allocator_traits_detail::has_member_destroy::value; + +// std::allocator::destroy's only effect is to invoke its argument's destructor +template +inline constexpr bool has_effectful_member_destroy, T> = false; + +template +struct destroy_via_allocator +{ + Allocator& a; + + template + THRUST_HOST_DEVICE void operator()(T& x) noexcept + { + allocator_traits::destroy(a, &x); + } +}; + +// we must prepare for His coming +struct gozer +{ + THRUST_EXEC_CHECK_DISABLE + template + inline THRUST_HOST_DEVICE void operator()(T& x) noexcept + { + x.~T(); + } +}; template -THRUST_HOST_DEVICE inline void destroy_range(Allocator& a, Pointer p, Size n) noexcept; +THRUST_HOST_DEVICE void +destroy_range([[maybe_unused]] Allocator& a, [[maybe_unused]] Pointer p, [[maybe_unused]] Size n) noexcept +{ + using pe_t = typename pointer_element::type; + // case 1: destroy via allocator + if constexpr (has_effectful_member_destroy) + { + thrust::for_each_n(allocator_system::get(a), p, n, destroy_via_allocator{a}); + } + // case 2: destroy without the allocator + else if constexpr (!_THRUST_STD::is_trivially_destructible_v) + { + thrust::for_each_n(allocator_system::get(a), p, n, gozer()); + } + // case 3: Allocator has no member function "destroy", and T has a trivial destructor, nothing to be done +} } // namespace detail THRUST_NAMESPACE_END - -#include diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/destroy_range.inl b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/destroy_range.inl deleted file mode 100644 index 31a56f0ce1d2..000000000000 --- a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/destroy_range.inl +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright 2008-2021 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include - -#if !_THRUST_HAS_DEVICE_SYSTEM_STD -# include -#endif - -THRUST_NAMESPACE_BEGIN -namespace detail -{ -namespace allocator_traits_detail -{ - -// destroy_range has three cases: -// if Allocator has an effectful member function destroy: -// 1. destroy via the allocator -// else -// 2. if T has a non-trivial destructor, destroy the range without using the allocator -// 3. if T has a trivial destructor, do a no-op - -template -struct has_effectful_member_destroy : has_member_destroy -{}; - -// std::allocator::destroy's only effect is to invoke its argument's destructor -template -struct has_effectful_member_destroy, T> : _THRUST_STD::false_type -{}; - -// case 1: Allocator has an effectful 1-argument member function "destroy" -template -struct enable_if_destroy_range_case1 - : _THRUST_STD::enable_if::type>::value> -{}; - -// case 2: Allocator has no member function "destroy", but T has a non-trivial destructor -template -struct enable_if_destroy_range_case2 - : _THRUST_STD::enable_if::type>::value - && !_THRUST_STD::is_trivially_destructible::type>::value> -{}; - -// case 3: Allocator has no member function "destroy", and T has a trivial destructor -template -struct enable_if_destroy_range_case3 - : _THRUST_STD::enable_if::type>::value - && _THRUST_STD::is_trivially_destructible::type>::value> -{}; - -template -struct destroy_via_allocator -{ - Allocator& a; - - THRUST_HOST_DEVICE destroy_via_allocator(Allocator& a) noexcept - : a(a) - {} - - template - inline THRUST_HOST_DEVICE void operator()(T& x) noexcept - { - allocator_traits::destroy(a, &x); - } -}; - -// destroy_range case 1: destroy via allocator -template -THRUST_HOST_DEVICE typename enable_if_destroy_range_case1::type -destroy_range(Allocator& a, Pointer p, Size n) noexcept -{ - thrust::for_each_n(allocator_system::get(a), p, n, destroy_via_allocator(a)); -} - -// we must prepare for His coming -struct gozer -{ - THRUST_EXEC_CHECK_DISABLE - template - inline THRUST_HOST_DEVICE void operator()(T& x) noexcept - { - x.~T(); - } -}; - -// destroy_range case 2: destroy without the allocator -template -THRUST_HOST_DEVICE typename enable_if_destroy_range_case2::type -destroy_range(Allocator& a, Pointer p, Size n) noexcept -{ - thrust::for_each_n(allocator_system::get(a), p, n, gozer()); -} - -// destroy_range case 3: no-op -template -THRUST_HOST_DEVICE typename enable_if_destroy_range_case3::type -destroy_range(Allocator&, Pointer, Size) noexcept -{ - // no op -} - -} // namespace allocator_traits_detail - -template -THRUST_HOST_DEVICE void destroy_range(Allocator& a, Pointer p, Size n) noexcept -{ - return allocator_traits_detail::destroy_range(a, p, n); -} - -} // namespace detail -THRUST_NAMESPACE_END diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/fill_construct_range.h b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/fill_construct_range.h index 3bb84f765598..62dcabe7c669 100644 --- a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/fill_construct_range.h +++ b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/fill_construct_range.h @@ -26,14 +26,54 @@ # pragma system_header #endif // no system header +#include +#include +#include +#include +#include + +#include _THRUST_STD_INCLUDE(__host_stdlib/memory) + THRUST_NAMESPACE_BEGIN namespace detail { +// fill_construct_range has 2 cases: +// if Allocator has an effectful member function construct: +// 1. construct via the allocator +// else +// 2. construct via uninitialized_fill -template -THRUST_HOST_DEVICE inline void fill_construct_range(Allocator& a, Pointer p, Size n, const T& value); +template +inline constexpr bool has_effectful_member_construct2 = + allocator_traits_detail::has_member_construct2::value; +// std::allocator::construct's only effect is to invoke placement new +template +inline constexpr bool has_effectful_member_construct2, T, Arg1> = false; + +template +struct construct2_via_allocator +{ + Allocator& a; + Arg1 arg; + + template + inline THRUST_HOST_DEVICE void operator()(T& x) + { + allocator_traits::construct(a, &x, arg); + } +}; +template +THRUST_HOST_DEVICE void fill_construct_range(Allocator& a, Pointer p, Size n, const T& value) +{ + if constexpr (has_effectful_member_construct2::type, T>) + { + thrust::for_each_n(allocator_system::get(a), p, n, construct2_via_allocator{a, value}); + } + else + { + thrust::uninitialized_fill_n(allocator_system::get(a), p, n, value); + } +} } // namespace detail THRUST_NAMESPACE_END - -#include diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/fill_construct_range.inl b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/fill_construct_range.inl deleted file mode 100644 index bf1190374b4c..000000000000 --- a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/fill_construct_range.inl +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2008-2013 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header -#include -#include -#include -#include -#include -#include - -#if !_THRUST_HAS_DEVICE_SYSTEM_STD -# include -#endif - -THRUST_NAMESPACE_BEGIN -namespace detail -{ -namespace allocator_traits_detail -{ - -// fill_construct_range has 2 cases: -// if Allocator has an effectful member function construct: -// 1. construct via the allocator -// else -// 2. construct via uninitialized_fill - -template -struct has_effectful_member_construct2 : has_member_construct2 -{}; - -// std::allocator::construct's only effect is to invoke placement new -template -struct has_effectful_member_construct2, T, Arg1> : _THRUST_STD::false_type -{}; - -template -struct construct2_via_allocator -{ - Allocator& a; - Arg1 arg; - - THRUST_HOST_DEVICE construct2_via_allocator(Allocator& a, const Arg1& arg) - : a(a) - , arg(arg) - {} - - template - inline THRUST_HOST_DEVICE void operator()(T& x) - { - allocator_traits::construct(a, &x, arg); - } -}; - -template -THRUST_HOST_DEVICE _THRUST_STD::enable_if_t< - has_effectful_member_construct2::type, T>::value> -fill_construct_range(Allocator& a, Pointer p, Size n, const T& value) -{ - thrust::for_each_n(allocator_system::get(a), p, n, construct2_via_allocator(a, value)); -} - -template -THRUST_HOST_DEVICE _THRUST_STD::enable_if_t< - !has_effectful_member_construct2::type, T>::value> -fill_construct_range(Allocator& a, Pointer p, Size n, const T& value) -{ - thrust::uninitialized_fill_n(allocator_system::get(a), p, n, value); -} - -} // namespace allocator_traits_detail - -template -THRUST_HOST_DEVICE void fill_construct_range(Alloc& a, Pointer p, Size n, const T& value) -{ - return allocator_traits_detail::fill_construct_range(a, p, n, value); -} - -} // namespace detail -THRUST_NAMESPACE_END diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/malloc_allocator.h b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/malloc_allocator.h index d0896c18dd4f..858baefeb94d 100644 --- a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/malloc_allocator.h +++ b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/malloc_allocator.h @@ -25,29 +25,52 @@ #elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) # pragma system_header #endif // no system header + +#include #include +#include +#include #include +#include +#include THRUST_NAMESPACE_BEGIN namespace detail { - template -class malloc_allocator : public thrust::detail::tagged_allocator +class malloc_allocator : public tagged_allocator { private: - using super_t = thrust::detail::tagged_allocator; + using super_t = tagged_allocator; public: using pointer = typename super_t::pointer; using size_type = typename super_t::size_type; - pointer allocate(size_type cnt); + pointer allocate(size_type cnt) + { + using thrust::system::detail::generic::select_system; - void deallocate(pointer p, size_type n) noexcept; -}; + // XXX should use a hypothetical thrust::static_pointer_cast here + System system; + + pointer result = thrust::malloc(select_system(system), cnt); + + if (result.get() == 0) + { + throw thrust::system::detail::bad_alloc("malloc_allocator::allocate: malloc failed"); + } // end if + return result; + } + + void deallocate(pointer p, size_type n) noexcept + { + using thrust::system::detail::generic::select_system; + + System system; + thrust::free(select_system(system), p); + } +}; } // namespace detail THRUST_NAMESPACE_END - -#include diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/malloc_allocator.inl b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/malloc_allocator.inl deleted file mode 100644 index 91eda7b4cc79..000000000000 --- a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/malloc_allocator.inl +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2008-2013 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header -#include -#include -#include -#include -#include - -THRUST_NAMESPACE_BEGIN -namespace detail -{ - -template -typename malloc_allocator::pointer -malloc_allocator::allocate(typename malloc_allocator::size_type cnt) -{ - using thrust::system::detail::generic::select_system; - - // XXX should use a hypothetical thrust::static_pointer_cast here - System system; - - pointer result = thrust::malloc(select_system(system), cnt); - - if (result.get() == 0) - { - throw thrust::system::detail::bad_alloc("malloc_allocator::allocate: malloc failed"); - } // end if - - return result; -} // end malloc_allocator::allocate() - -template -void malloc_allocator::deallocate(typename malloc_allocator::pointer p, - typename malloc_allocator::size_type) noexcept -{ - using thrust::system::detail::generic::select_system; - - System system; - thrust::free(select_system(system), p); -} // end malloc_allocator - -} // namespace detail -THRUST_NAMESPACE_END diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/no_throw_allocator.h b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/no_throw_allocator.h index fbc8a27796f7..630ae2db0208 100644 --- a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/no_throw_allocator.h +++ b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/no_throw_allocator.h @@ -30,7 +30,6 @@ THRUST_NAMESPACE_BEGIN namespace detail { - template struct no_throw_allocator : BaseAllocator { @@ -70,6 +69,5 @@ struct no_throw_allocator : BaseAllocator return super_t::operator!=(other); } }; // end no_throw_allocator - } // namespace detail THRUST_NAMESPACE_END diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/tagged_allocator.h b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/tagged_allocator.h index 37825b603d74..5483126fa152 100644 --- a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/tagged_allocator.h +++ b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/tagged_allocator.h @@ -25,13 +25,15 @@ #elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) # pragma system_header #endif // no system header +#include #include #include +#include _THRUST_STD_INCLUDE(limits) + THRUST_NAMESPACE_BEGIN namespace detail { - template class tagged_allocator; @@ -72,31 +74,40 @@ class tagged_allocator using other = tagged_allocator; }; // end rebind - THRUST_HOST_DEVICE inline tagged_allocator(); + tagged_allocator() = default; - THRUST_HOST_DEVICE inline tagged_allocator(const tagged_allocator&); + tagged_allocator(const tagged_allocator&) = default; template - THRUST_HOST_DEVICE inline tagged_allocator(const tagged_allocator&); - - THRUST_HOST_DEVICE inline ~tagged_allocator(); + THRUST_HOST_DEVICE tagged_allocator(const tagged_allocator&) + {} - THRUST_HOST_DEVICE pointer address(reference x) const; + ~tagged_allocator() = default; - THRUST_HOST_DEVICE const_pointer address(const_reference x) const; + THRUST_HOST_DEVICE pointer address(reference x) const + { + return &x; + } - size_type max_size() const; -}; + THRUST_HOST_DEVICE const_pointer address(const_reference x) const + { + return &x; + } -template -THRUST_HOST_DEVICE bool -operator==(const tagged_allocator&, const tagged_allocator&); + size_type max_size() const + { + return (_THRUST_STD::numeric_limits::max)() / sizeof(T); + } -template -THRUST_HOST_DEVICE bool -operator!=(const tagged_allocator&, const tagged_allocator&); + THRUST_HOST_DEVICE friend bool operator==(const tagged_allocator&, const tagged_allocator&) + { + return true; + } + THRUST_HOST_DEVICE friend bool operator!=(const tagged_allocator&, const tagged_allocator&) + { + return false; + } +}; } // namespace detail THRUST_NAMESPACE_END - -#include diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/tagged_allocator.inl b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/tagged_allocator.inl deleted file mode 100644 index dee43ce08445..000000000000 --- a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/tagged_allocator.inl +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2008-2013 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header -#include - -#include _THRUST_STD_INCLUDE(limits) - -THRUST_NAMESPACE_BEGIN -namespace detail -{ - -template -THRUST_HOST_DEVICE tagged_allocator::tagged_allocator() -{} - -template -THRUST_HOST_DEVICE tagged_allocator::tagged_allocator(const tagged_allocator&) -{} - -template -template -THRUST_HOST_DEVICE tagged_allocator::tagged_allocator(const tagged_allocator&) -{} - -template -THRUST_HOST_DEVICE tagged_allocator::~tagged_allocator() -{} - -template -THRUST_HOST_DEVICE typename tagged_allocator::pointer -tagged_allocator::address(reference x) const -{ - return &x; -} - -template -THRUST_HOST_DEVICE typename tagged_allocator::const_pointer -tagged_allocator::address(const_reference x) const -{ - return &x; -} - -template -typename tagged_allocator::size_type tagged_allocator::max_size() const -{ - return (_THRUST_STD::numeric_limits::max)() / sizeof(T); -} - -template -THRUST_HOST_DEVICE bool operator==(const tagged_allocator&, const tagged_allocator&) -{ - return true; -} - -template -THRUST_HOST_DEVICE bool operator!=(const tagged_allocator&, const tagged_allocator&) -{ - return false; -} - -} // namespace detail -THRUST_NAMESPACE_END diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/temporary_allocator.h b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/temporary_allocator.h index 3feb8842d606..4b445d54ab78 100644 --- a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/temporary_allocator.h +++ b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/temporary_allocator.h @@ -1,5 +1,6 @@ /* * Copyright 2008-2013 NVIDIA Corporation + * Modifications Copyright© 2019-2026 Advanced Micro Devices, Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,16 +26,29 @@ #elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) # pragma system_header #endif // no system header + #include #include +#include #include +#include #include -#include +#include + +#include _THRUST_STD_INCLUDE(__utility/pair.h) +#include _THRUST_STD_INCLUDE(cassert) + +#if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA +# if _CCCL_DEVICE_COMPILATION() +# include +# endif // _CCCL_DEVICE_COMPILATION() +#elif THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_HIP +# include +#endif // CUDA THRUST_NAMESPACE_BEGIN namespace detail { - // XXX the pointer parameter given to tagged_allocator should be related to // the type of the expression get_temporary_buffer(system, n).first // without decltype, compromise on pointer @@ -60,9 +74,43 @@ class temporary_allocator : public thrust::detail::tagged_allocator(system(), cnt); + + // handle failure + if (result.second < cnt) + { + // deallocate and throw + // note that we pass cnt to deallocate, not a value derived from result.second + deallocate(result.first, cnt); - THRUST_HOST_DEVICE void deallocate(pointer p, size_type n) noexcept; +#if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA + _THRUST_IF_TARGET( + _THRUST_IS_HOST, + (throw thrust::system::detail::bad_alloc("temporary_buffer::allocate: get_temporary_buffer failed");), + ( // _THRUST_IS_DEVICE + thrust::system::cuda::detail::terminate_with_message("temporary_buffer::allocate: " + "get_temporary_buffer failed");)); +#elif THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_HIP + _THRUST_IF_TARGET( + _THRUST_IS_HOST, + (throw thrust::system::detail::bad_alloc("temporary_buffer::allocate: get_temporary_buffer failed");), + ( // _THRUST_IS_DEVICE + thrust::system::hip::detail::terminate_with_message("temporary_buffer::allocate: " + "get_temporary_buffer failed");)); +#else + throw thrust::system::detail::bad_alloc("temporary_buffer::allocate: get_temporary_buffer failed"); +#endif + } // end if + + return result.first; + } + + THRUST_HOST_DEVICE void deallocate(pointer p, size_type n) noexcept + { + return thrust::return_temporary_buffer(system(), p, n); + } THRUST_HOST_DEVICE inline System& system() { @@ -70,10 +118,7 @@ class temporary_allocator : public thrust::detail::tagged_allocator; + using pointer_and_size = _THRUST_STD::pair; }; // end temporary_allocator - } // namespace detail THRUST_NAMESPACE_END - -#include diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/temporary_allocator.inl b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/temporary_allocator.inl deleted file mode 100644 index 268583120e5a..000000000000 --- a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/temporary_allocator.inl +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2008-2013 NVIDIA Corporation - * Modifications Copyright© 2019-2026 Advanced Micro Devices, Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header -#include -#include -#include - -#include _THRUST_STD_INCLUDE(cassert) - -#if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA -# if _CCCL_DEVICE_COMPILATION() -# include -# endif // _CCCL_DEVICE_COMPILATION() -#elif THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_HIP -# include -#endif // CUDA - -THRUST_NAMESPACE_BEGIN -namespace detail -{ - -template -THRUST_HOST_DEVICE typename temporary_allocator::pointer -temporary_allocator::allocate(typename temporary_allocator::size_type cnt) -{ - pointer_and_size result = thrust::get_temporary_buffer(system(), cnt); - - // handle failure - if (result.second < cnt) - { - // deallocate and throw - // note that we pass cnt to deallocate, not a value derived from result.second - deallocate(result.first, cnt); - -#if THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_CUDA - _THRUST_IF_TARGET(_THRUST_IS_HOST, - (throw thrust::system::detail::bad_alloc("temporary_buffer::allocate: get_temporary_buffer failed");), - ( // _THRUST_IS_DEVICE - thrust::system::cuda::detail::terminate_with_message("temporary_buffer::allocate: " - "get_temporary_buffer failed");)); -#elif THRUST_DEVICE_SYSTEM == THRUST_DEVICE_SYSTEM_HIP - _THRUST_IF_TARGET(_THRUST_IS_HOST, - (throw thrust::system::detail::bad_alloc("temporary_buffer::allocate: get_temporary_buffer failed");), - ( // _THRUST_IS_DEVICE - thrust::system::hip::detail::terminate_with_message("temporary_buffer::allocate: " - "get_temporary_buffer failed");)); -#else - throw thrust::system::detail::bad_alloc("temporary_buffer::allocate: get_temporary_buffer failed"); -#endif - } // end if - - return result.first; -} // end temporary_allocator::allocate() - -template -THRUST_HOST_DEVICE void temporary_allocator::deallocate( - typename temporary_allocator::pointer p, typename temporary_allocator::size_type n) noexcept -{ - return thrust::return_temporary_buffer(system(), p, n); -} // end temporary_allocator - -} // namespace detail -THRUST_NAMESPACE_END diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/value_initialize_range.h b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/value_initialize_range.h index eaf7d385674e..e959f74c17f3 100644 --- a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/value_initialize_range.h +++ b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/value_initialize_range.h @@ -26,14 +26,54 @@ # pragma system_header #endif // no system header +#include +#include +#include +#include +#include + +#include _THRUST_STD_INCLUDE(__type_traits/is_trivially_default_constructible.h) + THRUST_NAMESPACE_BEGIN namespace detail { +template +struct construct1_via_allocator +{ + Allocator& a; -template -THRUST_HOST_DEVICE inline void value_initialize_range(Allocator& a, Pointer p, Size n); + template + inline THRUST_HOST_DEVICE void operator()(T& x) + { + allocator_traits::construct(a, &x); + } +}; + +// we need to construct T via the allocator if the Allocator does something interesting or if T's default constructor +// does something interesting +template +inline constexpr bool needs_default_construct_via_allocator = + allocator_traits_detail::has_member_construct1::value + || !_THRUST_STD::is_trivially_default_constructible_v; +// we know that std::allocator::construct's only effect is to call T's +// default constructor, so we needn't use it for default construction +// unless T's constructor does something interesting +template +inline constexpr bool needs_default_construct_via_allocator, T> = + !_THRUST_STD::is_trivially_default_constructible_v; + +template +THRUST_HOST_DEVICE void value_initialize_range(Allocator& a, Pointer p, Size n) +{ + if constexpr (needs_default_construct_via_allocator::type>) + { + thrust::for_each_n(allocator_system::get(a), p, n, construct1_via_allocator{a}); + } + else + { + thrust::uninitialized_fill_n(allocator_system::get(a), p, n, typename pointer_element::type()); + } +} } // namespace detail THRUST_NAMESPACE_END - -#include diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/value_initialize_range.inl b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/value_initialize_range.inl deleted file mode 100644 index 21954b16ea53..000000000000 --- a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator/value_initialize_range.inl +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2008-2013 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header -#include -#include -#include -#include -#include - -#if !_THRUST_HAS_DEVICE_SYSTEM_STD -# include -#endif - -THRUST_NAMESPACE_BEGIN -namespace detail -{ -namespace allocator_traits_detail -{ - -template -struct construct1_via_allocator -{ - Allocator& a; - - THRUST_HOST_DEVICE construct1_via_allocator(Allocator& a) - : a(a) - {} - - template - inline THRUST_HOST_DEVICE void operator()(T& x) - { - allocator_traits::construct(a, &x); - } -}; - -// we need to construct T via the allocator if... -template -struct needs_default_construct_via_allocator - : _THRUST_STD::disjunction, // if the Allocator does something interesting - // or if T's default constructor does something - // interesting - thrust::detail::not_<_THRUST_STD::is_trivially_default_constructible>> -{}; - -// we know that std::allocator::construct's only effect is to call T's -// default constructor, so we needn't use it for default construction -// unless T's constructor does something interesting -template -struct needs_default_construct_via_allocator, T> - : thrust::detail::not_<_THRUST_STD::is_trivially_default_constructible> -{}; - -template -THRUST_HOST_DEVICE _THRUST_STD::enable_if_t< - needs_default_construct_via_allocator::type>::value> -value_initialize_range(Allocator& a, Pointer p, Size n) -{ - thrust::for_each_n(allocator_system::get(a), p, n, construct1_via_allocator(a)); -} - -template -THRUST_HOST_DEVICE -typename disable_if::type>::value>::type -value_initialize_range(Allocator& a, Pointer p, Size n) -{ - thrust::uninitialized_fill_n(allocator_system::get(a), p, n, typename pointer_element::type()); -} - -} // namespace allocator_traits_detail - -template -THRUST_HOST_DEVICE void value_initialize_range(Allocator& a, Pointer p, Size n) -{ - return allocator_traits_detail::value_initialize_range(a, p, n); -} - -} // namespace detail -THRUST_NAMESPACE_END diff --git a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator_aware_execution_policy.h b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator_aware_execution_policy.h index c6e35d7156ed..316980fa356c 100644 --- a/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator_aware_execution_policy.h +++ b/projects/hipccl/hipccl3/rocthrust/thrust/detail/allocator_aware_execution_policy.h @@ -28,24 +28,20 @@ #include #include -#include _THRUST_STD_INCLUDE(type_traits) - -#if !_THRUST_HAS_DEVICE_SYSTEM_STD -# include -#endif +#include _THRUST_STD_INCLUDE(__type_traits/enable_if.h) +#include _THRUST_STD_INCLUDE(__type_traits/is_reference.h) +#include _THRUST_STD_INCLUDE(__utility/move.h) THRUST_NAMESPACE_BEGIN namespace mr { - template class allocator; } namespace detail { - template