From 1a67be83d4db829677da7fa5aaba62cd716fec95 Mon Sep 17 00:00:00 2001 From: Jordan Patterson Date: Wed, 8 Jul 2026 20:09:05 +0000 Subject: [PATCH 1/7] Add Taskflow 4.1.0 to third-party. --- projects/hipfile/third-party/taskflow/LICENSE | 25 + .../taskflow/algorithm/algorithm.hpp | 18 + .../taskflow/algorithm/data_pipeline.hpp | 639 +++++ .../third-party/taskflow/algorithm/fill.hpp | 151 + .../third-party/taskflow/algorithm/find.hpp | 515 ++++ .../taskflow/algorithm/for_each.hpp | 251 ++ .../taskflow/algorithm/generate.hpp | 144 + .../third-party/taskflow/algorithm/merge.hpp | 214 ++ .../third-party/taskflow/algorithm/module.hpp | 81 + .../taskflow/algorithm/partitioner.hpp | 918 ++++++ .../taskflow/algorithm/pipeline.hpp | 1232 ++++++++ .../third-party/taskflow/algorithm/reduce.hpp | 544 ++++ .../third-party/taskflow/algorithm/scan.hpp | 699 +++++ .../third-party/taskflow/algorithm/sort.hpp | 649 +++++ .../taskflow/algorithm/transform.hpp | 184 ++ .../third-party/taskflow/core/async.hpp | 437 +++ .../third-party/taskflow/core/async_task.hpp | 296 ++ .../taskflow/core/atomic_notifier.hpp | 176 ++ .../taskflow/core/declarations.hpp | 65 + .../third-party/taskflow/core/environment.hpp | 8 + .../third-party/taskflow/core/error.hpp | 100 + .../third-party/taskflow/core/executor.hpp | 2546 +++++++++++++++++ .../taskflow/core/flow_builder.hpp | 1862 ++++++++++++ .../third-party/taskflow/core/graph.hpp | 1075 +++++++ .../taskflow/core/nonblocking_notifier.hpp | 641 +++++ .../third-party/taskflow/core/runtime.hpp | 984 +++++++ .../third-party/taskflow/core/semaphore.hpp | 188 ++ .../third-party/taskflow/core/task.hpp | 1814 ++++++++++++ .../third-party/taskflow/core/task_group.hpp | 884 ++++++ .../third-party/taskflow/core/taskflow.hpp | 730 +++++ .../third-party/taskflow/core/topology.hpp | 17 + .../third-party/taskflow/core/worker.hpp | 327 +++ .../hipfile/third-party/taskflow/core/wsq.hpp | 884 ++++++ .../taskflow/cuda/algorithm/find.hpp | 294 ++ .../taskflow/cuda/algorithm/for_each.hpp | 109 + .../taskflow/cuda/algorithm/matmul.hpp | 57 + .../taskflow/cuda/algorithm/merge.hpp | 585 ++++ .../taskflow/cuda/algorithm/reduce.hpp | 460 +++ .../taskflow/cuda/algorithm/scan.hpp | 488 ++++ .../taskflow/cuda/algorithm/single_task.hpp | 36 + .../taskflow/cuda/algorithm/sort.hpp | 506 ++++ .../taskflow/cuda/algorithm/transform.hpp | 126 + .../taskflow/cuda/algorithm/transpose.hpp | 41 + .../taskflow/cuda/cuda_capturer.hpp | 732 +++++ .../third-party/taskflow/cuda/cuda_device.hpp | 338 +++ .../third-party/taskflow/cuda/cuda_error.hpp | 34 + .../taskflow/cuda/cuda_execution_policy.hpp | 136 + .../third-party/taskflow/cuda/cuda_graph.hpp | 1126 ++++++++ .../taskflow/cuda/cuda_graph_exec.hpp | 384 +++ .../third-party/taskflow/cuda/cuda_memory.hpp | 839 ++++++ .../third-party/taskflow/cuda/cuda_meta.hpp | 452 +++ .../taskflow/cuda/cuda_optimizer.hpp | 404 +++ .../third-party/taskflow/cuda/cuda_stream.hpp | 345 +++ .../third-party/taskflow/cuda/cudaflow.hpp | 27 + .../third-party/taskflow/observer/chrome.hpp | 223 ++ .../taskflow/observer/interface.hpp | 111 + .../third-party/taskflow/observer/tfprof.hpp | 979 +++++++ .../hipfile/third-party/taskflow/taskflow.hpp | 95 + .../third-party/taskflow/utility/iterator.hpp | 1035 +++++++ .../taskflow/utility/lazy_string.hpp | 73 + .../third-party/taskflow/utility/macros.hpp | 41 + .../third-party/taskflow/utility/math.hpp | 505 ++++ .../third-party/taskflow/utility/mpmc.hpp | 508 ++++ .../taskflow/utility/object_pool.hpp | 684 +++++ .../third-party/taskflow/utility/os.hpp | 338 +++ .../taskflow/utility/serializer.hpp | 771 +++++ .../taskflow/utility/small_vector.hpp | 1056 +++++++ .../third-party/taskflow/utility/spsc.hpp | 242 ++ .../third-party/taskflow/utility/stream.hpp | 32 + .../third-party/taskflow/utility/traits.hpp | 316 ++ .../third-party/taskflow/utility/uuid.hpp | 235 ++ 71 files changed, 34061 insertions(+) create mode 100644 projects/hipfile/third-party/taskflow/LICENSE create mode 100644 projects/hipfile/third-party/taskflow/algorithm/algorithm.hpp create mode 100644 projects/hipfile/third-party/taskflow/algorithm/data_pipeline.hpp create mode 100644 projects/hipfile/third-party/taskflow/algorithm/fill.hpp create mode 100644 projects/hipfile/third-party/taskflow/algorithm/find.hpp create mode 100644 projects/hipfile/third-party/taskflow/algorithm/for_each.hpp create mode 100644 projects/hipfile/third-party/taskflow/algorithm/generate.hpp create mode 100644 projects/hipfile/third-party/taskflow/algorithm/merge.hpp create mode 100644 projects/hipfile/third-party/taskflow/algorithm/module.hpp create mode 100644 projects/hipfile/third-party/taskflow/algorithm/partitioner.hpp create mode 100644 projects/hipfile/third-party/taskflow/algorithm/pipeline.hpp create mode 100644 projects/hipfile/third-party/taskflow/algorithm/reduce.hpp create mode 100644 projects/hipfile/third-party/taskflow/algorithm/scan.hpp create mode 100644 projects/hipfile/third-party/taskflow/algorithm/sort.hpp create mode 100644 projects/hipfile/third-party/taskflow/algorithm/transform.hpp create mode 100644 projects/hipfile/third-party/taskflow/core/async.hpp create mode 100644 projects/hipfile/third-party/taskflow/core/async_task.hpp create mode 100644 projects/hipfile/third-party/taskflow/core/atomic_notifier.hpp create mode 100644 projects/hipfile/third-party/taskflow/core/declarations.hpp create mode 100644 projects/hipfile/third-party/taskflow/core/environment.hpp create mode 100644 projects/hipfile/third-party/taskflow/core/error.hpp create mode 100644 projects/hipfile/third-party/taskflow/core/executor.hpp create mode 100644 projects/hipfile/third-party/taskflow/core/flow_builder.hpp create mode 100644 projects/hipfile/third-party/taskflow/core/graph.hpp create mode 100644 projects/hipfile/third-party/taskflow/core/nonblocking_notifier.hpp create mode 100644 projects/hipfile/third-party/taskflow/core/runtime.hpp create mode 100644 projects/hipfile/third-party/taskflow/core/semaphore.hpp create mode 100644 projects/hipfile/third-party/taskflow/core/task.hpp create mode 100644 projects/hipfile/third-party/taskflow/core/task_group.hpp create mode 100644 projects/hipfile/third-party/taskflow/core/taskflow.hpp create mode 100644 projects/hipfile/third-party/taskflow/core/topology.hpp create mode 100644 projects/hipfile/third-party/taskflow/core/worker.hpp create mode 100644 projects/hipfile/third-party/taskflow/core/wsq.hpp create mode 100644 projects/hipfile/third-party/taskflow/cuda/algorithm/find.hpp create mode 100644 projects/hipfile/third-party/taskflow/cuda/algorithm/for_each.hpp create mode 100644 projects/hipfile/third-party/taskflow/cuda/algorithm/matmul.hpp create mode 100644 projects/hipfile/third-party/taskflow/cuda/algorithm/merge.hpp create mode 100644 projects/hipfile/third-party/taskflow/cuda/algorithm/reduce.hpp create mode 100644 projects/hipfile/third-party/taskflow/cuda/algorithm/scan.hpp create mode 100644 projects/hipfile/third-party/taskflow/cuda/algorithm/single_task.hpp create mode 100644 projects/hipfile/third-party/taskflow/cuda/algorithm/sort.hpp create mode 100644 projects/hipfile/third-party/taskflow/cuda/algorithm/transform.hpp create mode 100644 projects/hipfile/third-party/taskflow/cuda/algorithm/transpose.hpp create mode 100644 projects/hipfile/third-party/taskflow/cuda/cuda_capturer.hpp create mode 100644 projects/hipfile/third-party/taskflow/cuda/cuda_device.hpp create mode 100644 projects/hipfile/third-party/taskflow/cuda/cuda_error.hpp create mode 100644 projects/hipfile/third-party/taskflow/cuda/cuda_execution_policy.hpp create mode 100644 projects/hipfile/third-party/taskflow/cuda/cuda_graph.hpp create mode 100644 projects/hipfile/third-party/taskflow/cuda/cuda_graph_exec.hpp create mode 100644 projects/hipfile/third-party/taskflow/cuda/cuda_memory.hpp create mode 100644 projects/hipfile/third-party/taskflow/cuda/cuda_meta.hpp create mode 100644 projects/hipfile/third-party/taskflow/cuda/cuda_optimizer.hpp create mode 100644 projects/hipfile/third-party/taskflow/cuda/cuda_stream.hpp create mode 100644 projects/hipfile/third-party/taskflow/cuda/cudaflow.hpp create mode 100644 projects/hipfile/third-party/taskflow/observer/chrome.hpp create mode 100644 projects/hipfile/third-party/taskflow/observer/interface.hpp create mode 100644 projects/hipfile/third-party/taskflow/observer/tfprof.hpp create mode 100644 projects/hipfile/third-party/taskflow/taskflow.hpp create mode 100644 projects/hipfile/third-party/taskflow/utility/iterator.hpp create mode 100644 projects/hipfile/third-party/taskflow/utility/lazy_string.hpp create mode 100644 projects/hipfile/third-party/taskflow/utility/macros.hpp create mode 100644 projects/hipfile/third-party/taskflow/utility/math.hpp create mode 100644 projects/hipfile/third-party/taskflow/utility/mpmc.hpp create mode 100644 projects/hipfile/third-party/taskflow/utility/object_pool.hpp create mode 100644 projects/hipfile/third-party/taskflow/utility/os.hpp create mode 100644 projects/hipfile/third-party/taskflow/utility/serializer.hpp create mode 100644 projects/hipfile/third-party/taskflow/utility/small_vector.hpp create mode 100644 projects/hipfile/third-party/taskflow/utility/spsc.hpp create mode 100644 projects/hipfile/third-party/taskflow/utility/stream.hpp create mode 100644 projects/hipfile/third-party/taskflow/utility/traits.hpp create mode 100644 projects/hipfile/third-party/taskflow/utility/uuid.hpp diff --git a/projects/hipfile/third-party/taskflow/LICENSE b/projects/hipfile/third-party/taskflow/LICENSE new file mode 100644 index 000000000000..ea3fd94e8d2a --- /dev/null +++ b/projects/hipfile/third-party/taskflow/LICENSE @@ -0,0 +1,25 @@ +TASKFLOW MIT LICENSE + +Copyright (c) 2018-2026 Dr. Tsung-Wei Huang + +The University of Wisconsin at Madison + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + diff --git a/projects/hipfile/third-party/taskflow/algorithm/algorithm.hpp b/projects/hipfile/third-party/taskflow/algorithm/algorithm.hpp new file mode 100644 index 000000000000..8b74d995ebcc --- /dev/null +++ b/projects/hipfile/third-party/taskflow/algorithm/algorithm.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include "../core/graph.hpp" + +namespace tf { + +/** +@private +*/ +class Algorithm { + + public: + + template + static auto make_module_task(T&); +}; + +} // end of namespace tf ----------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/algorithm/data_pipeline.hpp b/projects/hipfile/third-party/taskflow/algorithm/data_pipeline.hpp new file mode 100644 index 000000000000..698d3a5fa369 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/algorithm/data_pipeline.hpp @@ -0,0 +1,639 @@ +#pragma once + +#include "pipeline.hpp" + + +namespace tf { + +// ---------------------------------------------------------------------------- +// Class Definition: DataPipe +// ---------------------------------------------------------------------------- + +/** +@class DataPipe + +@brief class to create a stage in a data-parallel pipeline + +A data pipe represents a stage of a data-parallel pipeline. +A data pipe can be either @em parallel direction or @em serial direction +(specified by tf::PipeType) and is associated with a callable to invoke +by the pipeline scheduler. + +You need to use the template function, tf::make_data_pipe, to create +a data pipe. The input and output types of a tf::DataPipe should be decayed types +(though the library will always decay them for you using `std::decay`) +to allow internal storage to work. +The data will be passed by reference to your callable, at which you can take +it by copy or reference. + +@code{.cpp} +tf::make_data_pipe( + tf::PipeType::SERIAL, + [](int& input) {return std::to_string(input + 100);} +); +@endcode + +In addition to the data, you callable can take an additional reference +of tf::Pipeflow in the second argument to probe the runtime information +for a stage task, such as its line number and token number: + +@code{.cpp} +tf::make_data_pipe( + tf::PipeType::SERIAL, + [](int& input, tf::Pipeflow& pf) { + printf("token=%lu, line=%lu\n", pf.token(), pf.line()); + return std::to_string(input + 100); + } +); +@endcode + +*/ +template +class DataPipe { + + template + friend class DataPipeline; + + public: + + /** + @brief callable type of the data pipe + */ + using callable_t = C; + + /** + @brief input type of the data pipe + */ + using input_t = Input; + + /** + @brief output type of the data pipe + */ + using output_t = Output; + + /** + @brief default constructor + */ + DataPipe() = default; + + /** + @brief constructs a data pipe + + You should use the helper function, tf::make_data_pipe, + to create a DataPipe object, especially when you need tf::DataPipe + to automatically deduct the lambda type. + */ + DataPipe(PipeType d, callable_t&& callable) : + _type{d}, _callable{std::forward(callable)} { + } + + /** + @brief queries the type of the data pipe + + A data pipe can be either parallel (tf::PipeType::PARALLEL) or serial + (tf::PipeType::SERIAL). + */ + PipeType type() const { + return _type; + } + + /** + @brief assigns a new type to the data pipe + */ + void type(PipeType type) { + _type = type; + } + + /** + @brief assigns a new callable to the data pipe + + @tparam U callable type + @param callable a callable object constructible from the callable type + of this data pipe + + Assigns a new callable to the pipe using universal forwarding. + */ + template + void callable(U&& callable) { + _callable = std::forward(callable); + } + + private: + + PipeType _type; + + callable_t _callable; +}; + +/** +@brief function to construct a data pipe (tf::DataPipe) + +@tparam Input input data type +@tparam Output output data type +@tparam C callable type + +tf::make_data_pipe is a helper function to create a data pipe (tf::DataPipe) +in a data-parallel pipeline (tf::DataPipeline). +The first argument specifies the direction of the data pipe, +either tf::PipeType::SERIAL or tf::PipeType::PARALLEL, +and the second argument is a callable to invoke by the pipeline scheduler. +Input and output data types are specified via template parameters, +which will always be decayed by the library to its original form +for storage purpose. +The callable must take the input data type in its first argument +and returns a value of the output data type. + +@code{.cpp} +tf::make_data_pipe( + tf::PipeType::SERIAL, + [](int& input) { + return std::to_string(input + 100); + } +); +@endcode + +The callable can additionally take a reference of tf::Pipeflow, +which allows you to query the runtime information of a stage task, +such as its line number and token number. + +@code{.cpp} +tf::make_data_pipe( + tf::PipeType::SERIAL, + [](int& input, tf::Pipeflow& pf) { + printf("token=%lu, line=%lu\n", pf.token(), pf.line()); + return std::to_string(input + 100); + } +); +@endcode + +*/ +template +auto make_data_pipe(PipeType d, C&& callable) { + return DataPipe(d, std::forward(callable)); +} + +// ---------------------------------------------------------------------------- +// Class Definition: DataPipeline +// ---------------------------------------------------------------------------- + +/** +@class DataPipeline + +@brief class to create a data-parallel pipeline scheduling framework + +@tparam Ps data pipe types + +Similar to tf::Pipeline, a tf::DataPipeline is a composable graph object +for users to create a data-parallel pipeline scheduling framework +using a module task in a taskflow. +The only difference is that tf::DataPipeline provides a data abstraction +for users to quickly express dataflow in a pipeline. +The following example creates a data-parallel pipeline of three stages +that generate dataflow from `void` to `int`, `std::string`, and `void`. + +@code{.cpp} +#include +#include + +int main() { + + // data flow => void -> int -> std::string -> void + tf::Taskflow taskflow("pipeline"); + tf::Executor executor; + + const size_t num_lines = 4; + + tf::DataPipeline pl(num_lines, + tf::make_data_pipe(tf::PipeType::SERIAL, [&](tf::Pipeflow& pf) -> int{ + if(pf.token() == 5) { + pf.stop(); + return 0; + } + else { + return pf.token(); + } + }), + tf::make_data_pipe(tf::PipeType::SERIAL, [](int& input) { + return std::to_string(input + 100); + }), + tf::make_data_pipe(tf::PipeType::SERIAL, [](std::string& input) { + std::cout << input << std::endl; + }) + ); + + // build the pipeline graph using composition + taskflow.composed_of(pl).name("pipeline"); + + // dump the pipeline graph structure (with composition) + taskflow.dump(std::cout); + + // run the pipeline + executor.run(taskflow).wait(); + + return 0; +} +@endcode + +The pipeline schedules five tokens over four parallel lines in a circular fashion, +as depicted below: + +@code{.bash} +o -> o -> o +| | | +v v v +o -> o -> o +| | | +v v v +o -> o -> o +| | | +v v v +o -> o -> o +@endcode +*/ +template +class DataPipeline { + + static_assert(sizeof...(Ps)>0, "must have at least one pipe"); + + /** + @private + */ + struct Line { + std::atomic join_counter; + }; + + /** + @private + */ + struct PipeMeta { + PipeType type; + }; + + + public: + + /** + @brief internal storage type for each data token (default std::variant) + */ + using data_t = unique_variant_t, + std::monostate, + std::decay_t>... + >>; + + /** + @brief constructs a data-parallel pipeline object + + @param num_lines the number of parallel lines + @param ps a list of pipes + + Constructs a data-parallel pipeline of up to @c num_lines parallel lines to schedule + tokens through the given linear chain of pipes. + The first pipe must define a serial direction (tf::PipeType::SERIAL) + or an exception will be thrown. + */ + DataPipeline(size_t num_lines, Ps&&... ps); + + /** + @brief constructs a data-parallel pipeline object + + @param num_lines the number of parallel lines + @param ps a tuple of pipes + + Constructs a data-parallel pipeline of up to @c num_lines parallel lines to schedule + tokens through the given linear chain of pipes stored in a std::tuple. + The first pipe must define a serial direction (tf::PipeType::SERIAL) + or an exception will be thrown. + */ + DataPipeline(size_t num_lines, std::tuple&& ps); + + /** + @brief queries the number of parallel lines + + The function returns the number of parallel lines given by the user + upon the construction of the pipeline. + The number of lines represents the maximum parallelism this pipeline + can achieve. + */ + size_t num_lines() const noexcept; + + /** + @brief queries the number of pipes + + The Function returns the number of pipes given by the user + upon the construction of the pipeline. + */ + constexpr size_t num_pipes() const noexcept; + + /** + @brief resets the pipeline + + Resetting the pipeline to the initial state. After resetting a pipeline, + its token identifier will start from zero as if the pipeline was just + constructed. + */ + void reset(); + + /** + @brief queries the number of generated tokens in the pipeline + + The number represents the total scheduling tokens that has been + generated by the pipeline so far. + */ + size_t num_tokens() const noexcept; + + /** + @brief obtains the graph object associated with the pipeline construct + + This method is primarily used as an opaque data structure for creating + a module task of this pipeline. + */ + Graph& graph(); + + private: + + Graph _graph; + + size_t _num_tokens; + + std::tuple _pipes; + std::array _meta; + std::vector> _lines; + std::vector _tasks; + std::vector _pipeflows; + std::vector> _buffer; + + template + auto _gen_meta(std::tuple&&, std::index_sequence); + + void _on_pipe(Pipeflow&, NonpreemptiveRuntime&); + void _build(); +}; + +// constructor +template +DataPipeline::DataPipeline(size_t num_lines, Ps&&... ps) : + _pipes {std::make_tuple(std::forward(ps)...)}, + _meta {PipeMeta{ps.type()}...}, + _lines (num_lines), + _tasks (num_lines + 1), + _pipeflows (num_lines), + _buffer (num_lines) { + + if(num_lines == 0) { + TF_THROW("must have at least one line"); + } + + if(std::get<0>(_pipes).type() != PipeType::SERIAL) { + TF_THROW("first pipe must be serial"); + } + + reset(); + _build(); +} + +// constructor +template +DataPipeline::DataPipeline(size_t num_lines, std::tuple&& ps) : + _pipes {std::forward>(ps)}, + _meta {_gen_meta( + std::forward>(ps), std::make_index_sequence{} + )}, + _lines (num_lines), + _tasks (num_lines + 1), + _pipeflows (num_lines), + _buffer (num_lines) { + + if(num_lines == 0) { + TF_THROW("must have at least one line"); + } + + if(std::get<0>(_pipes).type() != PipeType::SERIAL) { + TF_THROW("first pipe must be serial"); + } + + reset(); + _build(); +} + +// Function: _get_meta +template +template +auto DataPipeline::_gen_meta(std::tuple&& ps, std::index_sequence) { + return std::array{PipeMeta{std::get(ps).type()}...}; +} + +// Function: num_lines +template +size_t DataPipeline::num_lines() const noexcept { + return _pipeflows.size(); +} + +// Function: num_pipes +template +constexpr size_t DataPipeline::num_pipes() const noexcept { + return sizeof...(Ps); +} + +// Function: num_tokens +template +size_t DataPipeline::num_tokens() const noexcept { + return _num_tokens; +} + +// Function: graph +template +Graph& DataPipeline::graph() { + return _graph; +} + +// Function: reset +template +void DataPipeline::reset() { + + _num_tokens = 0; + + for(size_t l = 0; l(_meta[f].type), std::memory_order_relaxed + ); + } + } + + for(size_t f=1; f(_meta[0].type) - 1, std::memory_order_relaxed + ); + } +} + +// Procedure: _on_pipe +template +void DataPipeline::_on_pipe(Pipeflow& pf, NonpreemptiveRuntime&) { + + visit_tuple([&](auto&& pipe){ + + using data_pipe_t = std::decay_t; + using callable_t = typename data_pipe_t::callable_t; + using input_t = std::decay_t; + using output_t = std::decay_t; + + // first pipe + if constexpr (std::is_invocable_v) { + // [](tf::Pipeflow&) -> void {}, i.e., we only have one pipe + if constexpr (std::is_void_v) { + pipe._callable(pf); + // [](tf::Pipeflow&) -> output_t {} + } else { + _buffer[pf._line].data = pipe._callable(pf); + } + } + // other pipes without pipeflow in the second argument + else if constexpr (std::is_invocable_v >) { + // [](input_t&) -> void {}, i.e., the last pipe + if constexpr (std::is_void_v) { + pipe._callable(std::get(_buffer[pf._line].data)); + // [](input_t&) -> output_t {} + } else { + _buffer[pf._line].data = pipe._callable( + std::get(_buffer[pf._line].data) + ); + } + } + // other pipes with pipeflow in the second argument + else if constexpr (std::is_invocable_v) { + // [](input_t&, tf::Pipeflow&) -> void {} + if constexpr (std::is_void_v) { + pipe._callable(std::get(_buffer[pf._line].data), pf); + // [](input_t&, tf::Pipeflow&) -> output_t {} + } else { + _buffer[pf._line].data = pipe._callable( + std::get(_buffer[pf._line].data), pf + ); + } + } + //else if constexpr(std::is_invocable_v) { + // pipe._callable(pf, rt); + //} + else { + static_assert(dependent_false_v, "un-supported pipe callable type"); + } + }, _pipes, pf._pipe); +} + +// Procedure: _build +template +void DataPipeline::_build() { + + using namespace std::literals::string_literals; + + FlowBuilder fb(_graph); + + // init task + _tasks[0] = fb.emplace([this]() { + return static_cast(_num_tokens % num_lines()); + }).name("cond"); + + // line task + for(size_t l = 0; l < num_lines(); l++) { + + _tasks[l + 1] = fb.emplace([this, l] (tf::NonpreemptiveRuntime& rt) mutable { + + auto pf = &_pipeflows[l]; + + pipeline: + + _lines[pf->_line][pf->_pipe].join_counter.store( + static_cast(_meta[pf->_pipe].type), std::memory_order_relaxed + ); + + if (pf->_pipe == 0) { + pf->_token = _num_tokens; + if (pf->_stop = false, _on_pipe(*pf, rt); pf->_stop == true) { + // here, the pipeline is not stopped yet because other + // lines of tasks may still be running their last stages + return; + } + ++_num_tokens; + } + else { + _on_pipe(*pf, rt); + } + + size_t c_f = pf->_pipe; + size_t n_f = (pf->_pipe + 1) % num_pipes(); + size_t n_l = (pf->_line + 1) % num_lines(); + + pf->_pipe = n_f; + + // ---- scheduling starts here ---- + // Notice that the shared variable f must not be changed after this + // point because it can result in data race due to the following + // condition: + // + // a -> b + // | | + // v v + // c -> d + // + // d will be spawned by either c or b, so if c changes f but b spawns d + // then data race on f will happen + + std::array retval; + size_t n = 0; + + // downward dependency + if(_meta[c_f].type == PipeType::SERIAL && + _lines[n_l][c_f].join_counter.fetch_sub( + 1, std::memory_order_acq_rel) == 1 + ) { + retval[n++] = 1; + } + + // forward dependency + if(_lines[pf->_line][n_f].join_counter.fetch_sub( + 1, std::memory_order_acq_rel) == 1 + ) { + retval[n++] = 0; + } + + // notice that the task index starts from 1 + switch(n) { + case 2: { + rt.schedule(_tasks[n_l+1]); + goto pipeline; + } + case 1: { + // downward dependency + if (retval[0] == 1) { + pf = &_pipeflows[n_l]; + } + // forward dependency + goto pipeline; + } + } + }).name("rt-"s + std::to_string(l)); + + _tasks[0].precede(_tasks[l+1]); + } +} + + +} // end of namespace tf ----------------------------------------------------- + + + + + diff --git a/projects/hipfile/third-party/taskflow/algorithm/fill.hpp b/projects/hipfile/third-party/taskflow/algorithm/fill.hpp new file mode 100644 index 000000000000..c1bf450aa203 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/algorithm/fill.hpp @@ -0,0 +1,151 @@ +#pragma once + +#include "../taskflow.hpp" + +#include +#include + +namespace tf { + +template +auto make_fill_task(B first, E last, V value, P part = P()) { + using B_t = std::decay_t>; + using E_t = std::decay_t>; + return [=](Runtime &rt) mutable { + B_t beg = first; + E_t end = last; + + size_t W = rt.executor().num_workers(); + size_t N = std::distance(beg, end); + + if(N == 0) { + return; + } + + // the workload should be sequential + if (W <= 1 || N <= part.chunk_size()) { + part([=]() mutable { std::fill(beg, end, value); })(); + return; + } + + // use no more workers than the iteration count + if (N < W) { + W = N; + } + + // static partitioner + if constexpr (part.type() == PartitionerType::STATIC) { + for (size_t w = 0, curr_b = 0; w < W && curr_b < N;) { + auto chunk_size = part.adjusted_chunk_size(N, W, w); + auto task = part([=]() mutable { + part.loop( + N, W, curr_b, chunk_size, + [=, prev_e = size_t{0}](size_t part_b, size_t part_e) mutable { + std::advance(beg, part_b - prev_e); + for (size_t x = part_b; x < part_e; x++) { + *beg++ = value; + } + prev_e = part_e; + }); + }); + (++w == W || (curr_b += chunk_size) >= N) ? task() + : rt.silent_async(task); + } + } + // dynamic partitioner + else { + auto next = std::make_shared>(0); + for (size_t w = 0; w < W;) { + auto task = part([=]() mutable { + part.loop( + N, W, *next, + [=, prev_e = size_t{0}](size_t part_b, size_t part_e) mutable { + std::advance(beg, part_b - prev_e); + for (size_t x = part_b; x < part_e; x++) { + *beg++ = value; + } + prev_e = part_e; + }); + }); + (++w == W) ? task() : rt.silent_async(task); + } + } + }; +} + +template +auto make_fill_n_task(B first, C count, V value, P part = P()) { + using B_t = std::decay_t>; + return [=](Runtime &rt) mutable { + B_t beg = first; + + size_t W = rt.executor().num_workers(); + size_t N = count; + + if(N == 0) { + return; + } + + // the workload should be sequential + if (W <= 1 || N <= part.chunk_size()) { + part([=]() mutable { std::fill_n(beg, count, value); })(); + return; + } + + // use no more workers than the iteration count + if (N < W) { + W = N; + } + + // static partitioner + if constexpr (part.type() == PartitionerType::STATIC) { + for (size_t w = 0, curr_b = 0; w < W && curr_b < N;) { + auto chunk_size = part.adjusted_chunk_size(N, W, w); + auto task = part([=]() mutable { + part.loop( + N, W, curr_b, chunk_size, + [=, prev_e = size_t{0}](size_t part_b, size_t part_e) mutable { + std::advance(beg, part_b - prev_e); + for (size_t x = part_b; x < part_e; x++) { + *beg++ = value; + } + prev_e = part_e; + }); + }); + (++w == W || (curr_b += chunk_size) >= N) ? task() + : rt.silent_async(task); + } + } + // dynamic partitioner + else { + auto next = std::make_shared>(0); + for (size_t w = 0; w < W;) { + auto task = part([=]() mutable { + part.loop( + N, W, *next, + [=, prev_e = size_t{0}](size_t part_b, size_t part_e) mutable { + std::advance(beg, part_b - prev_e); + for (size_t x = part_b; x < part_e; x++) { + *beg++ = value; + } + prev_e = part_e; + }); + }); + (++w == W) ? task() : rt.silent_async(task); + } + } + }; +} + +template +Task FlowBuilder::fill(B first, E last, V value, P part) { + return emplace(make_fill_task(first, last, value, part)); +} + +template +Task FlowBuilder::fill_n(B first, C count, V value, P part) { + return emplace(make_fill_n_task(first, count, value, part)); +} + + +} // namespace tf --------------------------------------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/algorithm/find.hpp b/projects/hipfile/third-party/taskflow/algorithm/find.hpp new file mode 100644 index 000000000000..71496e277fa0 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/algorithm/find.hpp @@ -0,0 +1,515 @@ +#pragma once + +#include "../taskflow.hpp" + +namespace tf { + +// Function: make_find_if_task +template +auto make_find_if_task(B first, E last, T& result, UOP predicate, P part = P()) { + + using namespace std::string_literals; + + using B_t = std::decay_t>; + using E_t = std::decay_t>; + + return [=, &result] (Runtime& rt) mutable { + + // fetch the stateful values + B_t beg = first; + E_t end = last; + + size_t W = rt.executor().num_workers(); + size_t N = std::distance(beg, end); + + if(N == 0) { + result = end; + return; + } + + // only myself - no need to spawn another graph + if(W <= 1 || N <= part.chunk_size()) { + part([=, &result]() mutable { result = std::find_if(beg, end, predicate); })(); + return; + } + + // use no more workers than the iteration count + if(N < W) { + W = N; + } + + auto mutex = std::make_shared(); + const auto origin = beg; + result = std::next(origin, N); + + // static partitioner + if constexpr(part.type() == PartitionerType::STATIC) { + for(size_t w=0, curr_b=0; w lock(*mutex); + if(size_t offset = std::distance(origin, result); x < offset) { + result = std::next(origin, x); + } + return true; + } + } + prev_e = part_e; + return false; + } + ); + }); + (++w == W || (curr_b += chunk_size) >= N) ? task() : rt.silent_async(task); + } + } + // dynamic partitioner + else { + auto next = std::make_shared>(0); + for(size_t w=0; w lock(*mutex); + if(size_t offset = std::distance(origin, result); x < offset) { + result = std::next(origin, x); + } + return true; + } + } + prev_e = part_e; + return false; + } + ); + }); + (++w == W) ? task() : rt.silent_async(task); + } + } + }; +} + +// Function: make_find_if_not_task +template +auto make_find_if_not_task(B first, E last, T& result, UOP predicate, P part = P()) { + + using namespace std::string_literals; + + using B_t = std::decay_t>; + using E_t = std::decay_t>; + + return [=, &result] (Runtime& rt) mutable { + + // fetch the stateful values + B_t beg = first; + E_t end = last; + + size_t W = rt.executor().num_workers(); + size_t N = std::distance(beg, end); + + if(N == 0) { + result = end; + return; + } + + // only myself - no need to spawn another graph + if(W <= 1 || N <= part.chunk_size()) { + part([=, &result] () mutable { result = std::find_if_not(beg, end, predicate); })(); + return; + } + + if(N < W) { + W = N; + } + + auto mutex = std::make_shared(); + const auto origin = beg; + result = std::next(origin, N); + + // static partitioner + if constexpr(part.type() == PartitionerType::STATIC) { + for(size_t w=0, curr_b=0; w lock(*mutex); + if(size_t offset = std::distance(origin, result); x < offset) { + result = std::next(origin, x); + } + return true; + } + } + prev_e = part_e; + return false; + } + ); + }); + (++w == W || (curr_b += chunk_size) >= N) ? task() : rt.silent_async(task); + } + } + // dynamic partitioner + else { + auto next = std::make_shared>(0); + for(size_t w=0; w lock(*mutex); + if(size_t offset = std::distance(origin, result); x < offset) { + result = std::next(origin, x); + } + return true; + } + } + prev_e = part_e; + return false; + } + ); + }); + (++w == W) ? task() : rt.silent_async(task); + } + } + }; +} + +// Function: make_min_element_task +template +auto make_min_element_task(B first, E last, T& result, C comp, P part = P()) { + + using namespace std::string_literals; + + using B_t = std::decay_t>; + using E_t = std::decay_t>; + + return [=, &result] (Runtime& rt) mutable { + + // fetch the iterator values + B_t beg = first; + E_t end = last; + + size_t W = rt.executor().num_workers(); + size_t N = std::distance(beg, end); + + if(N == 0) { + result = end; + return; + } + + // only myself - no need to spawn another graph + if(W <= 1 || N <= part.chunk_size()) { + part([=, &result] () mutable { result = std::min_element(beg, end, comp); })(); + return; + } + + if(N < W) { + W = N; + } + + auto mutex = std::make_shared(); + + // initialize the result to the first element + result = beg++; + N--; + + // static partitioner + if constexpr(part.type() == PartitionerType::STATIC) { + + for(size_t w=0, curr_b=0; w lock(*mutex); + if(comp(*beg, *result)) { + result = beg; + } + return; + } + + auto beg1 = beg++; + auto beg2 = beg++; + T smallest = comp(*beg1, *beg2) ? beg1 : beg2; + + // loop reduce + part.loop(N, W, curr_b, chunk_size, + [=, &smallest, prev_e=curr_b+2](size_t part_b, size_t part_e) mutable { + + if(part_b > prev_e) { + std::advance(beg, part_b - prev_e); + } + else { + part_b = prev_e; + } + + for(size_t x=part_b; x lock(*mutex); + if(comp(*smallest, *result)) { + result = smallest; + } + }); + + (++w == W || (curr_b += chunk_size) >= N) ? task() : rt.silent_async(task); + } + } + // dynamic partitioner + else { + auto next = std::make_shared>(0); + + for(size_t w=0; wfetch_add(2, std::memory_order_relaxed); + + if(s0 >= N) { + return; + } + + std::advance(beg, s0); + + if(N - s0 == 1) { + std::lock_guard lock(*mutex); + if(comp(*beg, *result)) { + result = beg; + } + return; + } + + auto beg1 = beg++; + auto beg2 = beg++; + + T smallest = comp(*beg1, *beg2) ? beg1 : beg2; + + // loop reduce + part.loop(N, W, *next, + [=, &smallest, prev_e=s0+2](size_t part_b, size_t part_e) mutable { + std::advance(beg, part_b - prev_e); + for(size_t x=part_b; x lock(*mutex); + if(comp(*smallest, *result)) { + result = smallest; + } + }); + (++w == W) ? task() : rt.silent_async(task); + } + } + }; +} + +// Function: make_max_element_task +template +auto make_max_element_task(B first, E last, T& result, C comp, P part = P()) { + + using namespace std::string_literals; + + using B_t = std::decay_t>; + using E_t = std::decay_t>; + + return [=, &result] (Runtime& rt) mutable { + + // fetch the iterator values + B_t beg = first; + E_t end = last; + + size_t W = rt.executor().num_workers(); + size_t N = std::distance(beg, end); + + if(N == 0) { + result = end; + return; + } + + // only myself - no need to spawn another graph + if(W <= 1 || N <= part.chunk_size()) { + part([=, &result] () mutable { result = std::max_element(beg, end, comp); })(); + return; + } + + if(N < W) { + W = N; + } + + auto mutex = std::make_shared(); + + // initialize the result to the first element + result = beg++; + N--; + + // static partitioner + if constexpr(part.type() == PartitionerType::STATIC) { + + for(size_t w=0, curr_b=0; w lock(*mutex); + if(comp(*result, *beg)) { + result = beg; + } + return; + } + + auto beg1 = beg++; + auto beg2 = beg++; + T largest = comp(*beg1, *beg2) ? beg2 : beg1; + + // loop reduce + part.loop(N, W, curr_b, chunk_size, + [=, &largest, prev_e=curr_b+2](size_t part_b, size_t part_e) mutable { + + if(part_b > prev_e) { + std::advance(beg, part_b - prev_e); + } + else { + part_b = prev_e; + } + + for(size_t x=part_b; x lock(*mutex); + if(comp(*result, *largest)) { + result = largest; + } + }); + (++w == W || (curr_b += chunk_size) >= N) ? task() : rt.silent_async(task); + } + } + // dynamic partitioner + else { + auto next = std::make_shared>(0); + + for(size_t w=0; wfetch_add(2, std::memory_order_relaxed); + + if(s0 >= N) { + return; + } + + std::advance(beg, s0); + + if(N - s0 == 1) { + std::lock_guard lock(*mutex); + if(comp(*result, *beg)) { + result = beg; + } + return; + } + + auto beg1 = beg++; + auto beg2 = beg++; + + T largest = comp(*beg1, *beg2) ? beg2 : beg1; + + // loop reduce + part.loop(N, W, *next, + [=, &largest, prev_e=s0+2](size_t part_b, size_t part_e) mutable { + std::advance(beg, part_b - prev_e); + for(size_t x=part_b; x lock(*mutex); + if(comp(*result, *largest)) { + result = largest; + } + }); + (++w == W) ? task() : rt.silent_async(task); + } + } + }; +} + + +// Function: find_if +template +Task tf::FlowBuilder::find_if(B first, E last, T& result, UOP predicate, P part) { + return emplace(make_find_if_task(first, last, result, predicate, part)); +} + +// Function: find_if_not +template +Task tf::FlowBuilder::find_if_not(B first, E last, T& result, UOP predicate, P part) { + return emplace(make_find_if_not_task(first, last, result, predicate, part)); +} + +// ---------------------------------------------------------------------------- +// min_element +// ---------------------------------------------------------------------------- + +// Function: min_element +template +Task FlowBuilder::min_element(B first, E last, T& result, C comp, P part) { + return emplace(make_min_element_task(first, last, result, comp, part)); +} + +// ---------------------------------------------------------------------------- +// max_element +// ---------------------------------------------------------------------------- + +// Function: max_element +template +Task FlowBuilder::max_element(B first, E last, T& result, C comp, P part) { + return emplace(make_max_element_task(first, last, result, comp, part)); +} + +} // end of namespace tf ----------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/algorithm/for_each.hpp b/projects/hipfile/third-party/taskflow/algorithm/for_each.hpp new file mode 100644 index 000000000000..bed0db9f111c --- /dev/null +++ b/projects/hipfile/third-party/taskflow/algorithm/for_each.hpp @@ -0,0 +1,251 @@ +#pragma once + +#include "../taskflow.hpp" + +namespace tf { + +// Function: make_for_each_task +template +auto make_for_each_task(B b, E e, C c, P part = P()) { + + using B_t = std::decay_t>; + using E_t = std::decay_t>; + + return [=] (Runtime& rt) mutable { + + // fetch the stateful values + B_t beg = b; + E_t end = e; + + size_t W = rt.executor().num_workers(); + size_t N = std::distance(beg, end); + + if(N == 0) { + return; + } + + // the workload is sequentially doable + if(W <= 1 || N <= part.chunk_size()) { + part([=]() mutable { std::for_each(beg, end, c); })(); + return; + } + + // use no more workers than the iteration count + if(N < W) { + W = N; + } + + // static partitioner + if constexpr(part.type() == PartitionerType::STATIC) { + for(size_t w=0, curr_b=0; w= N) ? task() : rt.silent_async(task); + } + } + // dynamic partitioner + else { + auto next = std::make_shared>(0); + for(size_t w=0; w +auto make_for_each_index_task(B b, E e, S s, C c, P part = P()){ + + using B_t = std::decay_t>; + using E_t = std::decay_t>; + using S_t = std::decay_t>; + + return [=] (Runtime& rt) mutable { + + // fetch the iterator values + B_t beg = b; + E_t end = e; + S_t inc = s; + + // nothing to be done if the range is invalid + if(is_index_range_invalid(beg, end, inc)) { + return; + } + + size_t W = rt.executor().num_workers(); + size_t N = distance(beg, end, inc); + + if(N == 0) { + return; + } + + // only myself - no need to spawn another graph + if(W <= 1 || N <= part.chunk_size()) { + part([=]() mutable { + for(size_t x=0; x(part_b) * inc + beg; + for(size_t x=part_b; x= N) ? task() : rt.silent_async(task); + } + } + // dynamic partitioner + else { + auto next = std::make_shared>(0); + for(size_t w=0; w(part_b) * inc + beg; + for(size_t x=part_b; x +auto make_for_each_by_index_task(R range, C c, P part = P()){ + + using range_type = std::decay_t>; + + return [=] (Runtime& rt) mutable { + + // fetch the iterator values + range_type r = range; + + if constexpr (range_type::rank == 1) { + // nothing to be done if the range is invalid + if(is_index_range_invalid(r.begin(), r.end(), r.step_size())) { + return; + } + } + + size_t W = rt.executor().num_workers(); + size_t N = r.size(); + + // nothing to do if the active iteration space is empty + if(N == 0) { + return; + } + + // only myself - no need to spawn another graph + if(W <= 1 || N <= part.chunk_size()) { + part([=]() mutable { c(r); })(); + return; + } + + if(N < W) { + W = N; + } + + // static partitioner + if constexpr(part.type() == PartitionerType::STATIC) { + for(size_t w=0, curr_b=0; w= N) ? task() : rt.silent_async(task); + } + } + // dynamic partitioner + else { + auto next = std::make_shared>(0); + for(size_t w=0; w +Task FlowBuilder::for_each(B beg, E end, C c, P part) { + return emplace( + make_for_each_task(beg, end, c, part) + ); +} + +// ------------------------------------------------------------------------------------------------ +// for_each_index +// ------------------------------------------------------------------------------------------------ + +// Function: for_each_index +template +Task FlowBuilder::for_each_index(B beg, E end, S inc, C c, P part){ + return emplace( + make_for_each_index_task(beg, end, inc, c, part) + ); +} + +// Function: for_each_by_index +template +Task FlowBuilder::for_each_by_index(R range, C c, P part){ + return emplace( + make_for_each_by_index_task(range, c, part) + ); +} + +} // end of namespace tf ------------------------------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/algorithm/generate.hpp b/projects/hipfile/third-party/taskflow/algorithm/generate.hpp new file mode 100644 index 000000000000..04a4d33cc217 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/algorithm/generate.hpp @@ -0,0 +1,144 @@ +#pragma once + +#include "../taskflow.hpp" + +#include +#include + +namespace tf { + +template +auto make_generate_task(B first, E last, G gen, P part = P()) { + using B_t = std::decay_t>; + using E_t = std::decay_t>; + return [=](Runtime &rt) mutable { + B_t beg = first; + E_t end = last; + + size_t W = rt.executor().num_workers(); + size_t N = std::distance(beg, end); + + // the workload should be sequential + if (W <= 1 || N <= part.chunk_size()) { + part([=]() mutable { std::generate(beg, end, gen); })(); + return; + } + + // use no more workers than the iteration count + if (N < W) { + W = N; + } + + // static partitioner + if constexpr (part.type() == PartitionerType::STATIC) { + for (size_t w = 0, curr_b = 0; w < W && curr_b < N;) { + auto chunk_size = part.adjusted_chunk_size(N, W, w); + auto task = part([=]() mutable { + part.loop( + N, W, curr_b, chunk_size, + [=, prev_e = size_t{0}](size_t part_b, size_t part_e) mutable { + std::advance(beg, part_b - prev_e); + for (size_t x = part_b; x < part_e; x++) { + *beg++ = gen(); + } + prev_e = part_e; + }); + }); + (++w == W || (curr_b += chunk_size) >= N) ? task() + : rt.silent_async(task); + } + } + // dynamic partitioner + else { + auto next = std::make_shared>(0); + for (size_t w = 0; w < W;) { + auto task = part([=]() mutable { + part.loop( + N, W, *next, + [=, prev_e = size_t{0}](size_t part_b, size_t part_e) mutable { + std::advance(beg, part_b - prev_e); + for (size_t x = part_b; x < part_e; x++) { + *beg++ = gen(); + } + prev_e = part_e; + }); + }); + (++w == W) ? task() : rt.silent_async(task); + } + } + }; +} + +template +auto make_generate_n_task(B first, C count, G gen, P part = P()) { + using B_t = std::decay_t>; + + return [=](Runtime &rt) mutable { + B_t beg = first; + + size_t W = rt.executor().num_workers(); + size_t N = count; + + // the workload should be sequential + if (W <= 1 || N <= part.chunk_size()) { + part([=]() mutable { std::generate_n(beg, count, gen); })(); + return; + } + + // use no more workers than the iteration count + if (N < W) { + W = N; + } + + // static partitioner + if constexpr (part.type() == PartitionerType::STATIC) { + for (size_t w = 0, curr_b = 0; w < W && curr_b < N;) { + auto chunk_size = part.adjusted_chunk_size(N, W, w); + auto task = part([=]() mutable { + part.loop( + N, W, curr_b, chunk_size, + [=, prev_e = size_t{0}](size_t part_b, size_t part_e) mutable { + std::advance(beg, part_b - prev_e); + for (size_t x = part_b; x < part_e; x++) { + *beg++ = gen(); + } + prev_e = part_e; + }); + }); + (++w == W || (curr_b += chunk_size) >= N) ? task() + : rt.silent_async(task); + } + } + // dynamic partitioner + else { + auto next = std::make_shared>(0); + for (size_t w = 0; w < W;) { + auto task = part([=]() mutable { + part.loop( + N, W, *next, + [=, prev_e = size_t{0}](size_t part_b, size_t part_e) mutable { + std::advance(beg, part_b - prev_e); + for (size_t x = part_b; x < part_e; x++) { + *beg++ = gen(); + } + prev_e = part_e; + }); + }); + (++w == W) ? task() : rt.silent_async(task); + } + } + }; +} + +template +Task FlowBuilder::generate(B first, E last, G gen, P part) { + return emplace(make_generate_task(first, last, gen, part)); +} + +template +Task FlowBuilder::generate_n(B first, C count, G gen, P part) { + return emplace(make_generate_n_task(first, count, gen, part)); +} + + +} // namespace tf --------------------------------------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/algorithm/merge.hpp b/projects/hipfile/third-party/taskflow/algorithm/merge.hpp new file mode 100644 index 000000000000..76d31d7bab13 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/algorithm/merge.hpp @@ -0,0 +1,214 @@ +#pragma once + +#include "../taskflow.hpp" + +namespace tf { + +// ---------------------------------------------------------------------------- +// make_merge_task +// ---------------------------------------------------------------------------- + +/** +@brief creates a parallel merge task over two sorted ranges + +@tparam B1 iterator type for the beginning of the first range +@tparam E1 iterator type for the end of the first range +@tparam B2 iterator type for the beginning of the second range +@tparam E2 iterator type for the end of the second range +@tparam O output iterator type +@tparam C comparator type + +@param first1 iterator to the beginning of the first sorted range +@param last1 iterator to the end of the first sorted range +@param first2 iterator to the beginning of the second sorted range +@param last2 iterator to the end of the second sorted range +@param cmp comparator defining the sort order +@param d_first iterator to the beginning of the output range + +Returns a callable suitable for @c tf::Executor::async, +@c tf::Executor::silent_async, or @c tf::Taskflow::emplace. + +@par Algorithm + +The algorithm parallelizes @c std::merge using the @em co-rank technique, +which avoids any inter-worker synchronization during the merge itself. + +Step 1 — Flat output partitioning. +The merged output has @c N = n + m elements (where @c n and @c m are the +sizes of the two input ranges). The output is divided into @c W equal +contiguous chunks, one per worker thread. Worker @c w is responsible for +writing output positions [w*K, (w+1)*K) where @c K = N/W. + +Step 2 — Co-rank: finding the input sub-ranges. +For each output chunk boundary @c rank, we need to know: "among the first +@c rank elements of the merged output, how many came from seq1 (call it +@c i) and how many from seq2 (call it @c j = rank - i)?" + +Once @c (i, j) is known, the worker's input from seq1 is +seq1[i_start .. i_end) and from seq2 is +seq2[j_start .. j_end) — both independent slices that can be +merged in isolation. + +Co-rank finds @c i by binary search over the range +[max(0, rank-m), min(n, rank)]. For a candidate @c i with +@c j = rank - i, the partition is correct when: + +@code + seq1[i-1] <= seq2[j] and seq2[j-1] <= seq1[i] +@endcode + +i.e. the last element of each slice does not exceed the first unused element +of the other sequence. Because both sequences are sorted, only one of these +two conditions needs to be checked — as @c i increases, the correctness +condition transitions monotonically from "too small" to "correct", making +binary search applicable. + +The special case @c j==0 (all @c rank elements come from seq1, none from +seq2) skips the @c seq2[j-1] dereference entirely, which would otherwise be +an out-of-bounds access. + +Step 3 — Independent parallel merge. +Each worker calls @c std::merge on its identified input sub-slices and writes +directly to its non-overlapping portion of the output. No mutex, no atomic +cursor, no synchronization. + +Why no partitioner? +Unlike @c for_each or @c find, where dynamic/guided scheduling can adapt to +unequal per-element costs, @c std::merge on a chunk of size @c K always costs +exactly O(K) regardless of data values. There is no load imbalance to +mitigate. Static equal partitioning (W chunks, one per worker) is always +optimal. Smaller chunks would only add unnecessary @c co_rank overhead +(O(log N) per boundary) without any benefit. + +Complexity. +Each worker performs 2 co-rank calls at O(log N) each, plus one +@c std::merge at O(N/W). Total: O(N + W log N), which is effectively O(N) +for typical W << N. + +@note Both input ranges must be sorted with respect to @c cmp. + The output range must not overlap either input range. + Both input iterators must be random-access iterators. +*/ +template +auto make_merge_task(B1 first1, E1 last1, B2 first2, E2 last2, C cmp, O d_first) { + + using B1_t = std::decay_t>; + using E1_t = std::decay_t>; + using B2_t = std::decay_t>; + using E2_t = std::decay_t>; + using O_t = std::decay_t>; + + return [=](Runtime& rt) mutable { + + B1_t beg1 = first1; + E1_t end1 = last1; + B2_t beg2 = first2; + E2_t end2 = last2; + O_t d_beg = d_first; + + size_t n = static_cast(std::distance(beg1, end1)); + size_t m = static_cast(std::distance(beg2, end2)); + size_t N = n + m; + size_t W = rt.executor().num_workers(); + + if(N == 0) { + return; + } + + // sequential fallback for single worker or small N + if(W <= 1 || N <= W) { + std::merge(beg1, end1, beg2, end2, d_beg, cmp); + return; + } + + // co_rank: binary search for the unique split point i in [low, high) + // such that taking i elements from seq1 and (rank-i) from seq2 gives + // exactly the rank smallest merged elements. + // + // Invariant: seq2[j-1] <= seq1[i] means i is not too large (high = i), + // otherwise i is too small (low = i+1). + // + // Guard: when j==0, skip the seq2[j-1] dereference (would be seq2[-1]) + // and always increase low — if seq2 contributes nothing, i must grow. + // + // Captured by value so the lambda is safe to copy into async tasks. + auto co_rank = [=](size_t rank) -> std::pair { + size_t low = (rank > m) ? rank - m : 0; + size_t high = (std::min)(n, rank); + + while(low < high) { + size_t i = low + (high - low) / 2; + size_t j = rank - i; + + if(j == 0 || !cmp(*std::next(beg2, j - 1), *std::next(beg1, i))) { + low = i + 1; // seq1[i] <= seq2[j-1]: i is too small, take more from seq1 + } else { + high = i; // seq1[i] > seq2[j-1]: i is too large + } + } + return {low, rank - low}; + }; + + // merge_chunk: given an output range [part_b, part_e), use co_rank to + // find the corresponding input sub-ranges and merge them independently. + // Captured by value so it is safe to copy into rt.silent_async tasks — + // all iterators and the co_rank callable are value-copied, no dangling + // references to the outer stack frame. + auto merge_chunk = [=](size_t part_b, size_t part_e) { + auto [i_beg, j_beg] = co_rank(part_b); + auto [i_end, j_end] = co_rank(part_e); + std::merge( + std::next(beg1, i_beg), std::next(beg1, i_end), + std::next(beg2, j_beg), std::next(beg2, j_end), + std::next(d_beg, part_b), + cmp + ); + }; + + // static equal partition: W chunks, one per worker. + // Each chunk boundary costs 2 co_rank calls O(log N) — smaller chunks + // would add overhead with no load-balancing benefit since std::merge + // always costs O(K) for a chunk of size K regardless of data. + size_t chunk_size = (N + W - 1) / W; + + for(size_t w = 0; w < W; w++) { + size_t curr_b = w * chunk_size; + if(curr_b >= N) { + break; + } + size_t curr_e = (std::min)(curr_b + chunk_size, N); + if(w + 1 == W) { + merge_chunk(curr_b, curr_e); // last worker runs inline + } else { + rt.silent_async([=]() { merge_chunk(curr_b, curr_e); }); + } + } + }; +} + +// ---------------------------------------------------------------------------- +// tf::Taskflow::merge +// ---------------------------------------------------------------------------- + +/** +@private +*/ +template +Task FlowBuilder::merge(B1 first1, E1 last1, B2 first2, E2 last2, O d_first) { + return emplace(make_merge_task( + first1, last1, first2, last2, std::less<>{}, d_first + )); +} + +/** +@private +*/ +template +Task FlowBuilder::merge(B1 first1, E1 last1, B2 first2, E2 last2, O d_first, C cmp) { + return emplace(make_merge_task( + first1, last1, first2, last2, cmp, d_first + )); +} + +} // namespace tf diff --git a/projects/hipfile/third-party/taskflow/algorithm/module.hpp b/projects/hipfile/third-party/taskflow/algorithm/module.hpp new file mode 100644 index 000000000000..39befcbd6151 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/algorithm/module.hpp @@ -0,0 +1,81 @@ +#pragma once + +#include "../taskflow.hpp" + +/** +@file module.hpp +@brief module algorithm include file +*/ + +namespace tf { + +// ---------------------------------------------------------------------------- + +template +auto Algorithm::make_module_task(T& target) { + return [&graph=retrieve_graph(target)](tf::Runtime& rt){ + if(graph.empty()) { + return; + } + rt._executor._schedule_graph(rt._worker, graph, rt._node->_topology, rt._node); + }; +} + +// ---------------------------------------------------------------------------- + +/** +@brief creates a module task using the given graph + +@tparam T type satisfying tf::GraphLike +@param target the target object used to create the module task +@return a module task that can be used by %Taskflow or asynchronous tasking + +This example demonstrates how to create and launch multiple taskflows in parallel +using modules with asynchronous tasking: + +@code{.cpp} +tf::Executor executor; + +tf::Taskflow A; +tf::Taskflow B; +tf::Taskflow C; +tf::Taskflow D; + +A.emplace([](){ printf("Taskflow A\n"); }); +B.emplace([](){ printf("Taskflow B\n"); }); +C.emplace([](){ printf("Taskflow C\n"); }); +D.emplace([](){ printf("Taskflow D\n"); }); + +// launch the four taskflows using asynchronous tasking +executor.async(tf::make_module_task(A)); +executor.async(tf::make_module_task(B)); +executor.async(tf::make_module_task(C)); +executor.async(tf::make_module_task(D)); +executor.wait_for_all(); +@endcode + +The module task maker, tf::make_module_task, is basically the same as tf::Taskflow::composed_of +but provides a more generic interface that can be used beyond %Taskflow. +For instance, the following two approaches achieve the same functionality. + +@code{.cpp} +// approach 1: composition using composed_of +tf::Task m1 = taskflow1.composed_of(taskflow2); + +// approach 2: composition using make_module_task +tf::Task m1 = taskflow1.emplace(tf::make_module_task(taskflow2)); +@endcode + +@attention +Users are responsible for ensuring that the given target remains valid throughout its execution. +The executor does not assume ownership of the target object. +*/ +template +auto make_module_task(T& target) { + return Algorithm::make_module_task(target); +} + +} // end of namespact tf ----------------------------------------------------- + + + diff --git a/projects/hipfile/third-party/taskflow/algorithm/partitioner.hpp b/projects/hipfile/third-party/taskflow/algorithm/partitioner.hpp new file mode 100644 index 000000000000..bc1a671574a4 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/algorithm/partitioner.hpp @@ -0,0 +1,918 @@ +// reference: +// - gomp: https://github.com/gcc-mirror/gcc/blob/master/libgomp/iter.c +// - komp: https://github.com/llvm-mirror/openmp/blob/master/runtime/src/kmp_dispatch.cpp + +#pragma once + +/** +@file partitioner.hpp +@brief partitioner include file +*/ + +namespace tf { + +/** +@enum PartitionerType + +@brief enumeration of all partitioner types +*/ +enum class PartitionerType : int { + /** @brief static partitioner type */ + STATIC, + /** @brief dynamic partitioner type */ + DYNAMIC +}; + + +//template +//class PartitionInvoker : public PartitionerBase { +// +// protected +// +// C _closure; +// +// template +// auto operator()(ArgsT&&... args) { +// return std::invoke(closure, std::forward(args)...); +// } +// +// template +// auto operator()(ArgsT&&... args) const { +// return std::invoke(closure, std::forward(args)...); +// } +// +//}; + +/** +@class DefaultClosureWrapper + +@brief class to create a default closure wrapper +*/ +class DefaultClosureWrapper {}; + + + +// ---------------------------------------------------------------------------- +// Partitioner Base +// ---------------------------------------------------------------------------- + +/** +@class PartitionerBase + +@brief class to derive a partitioner for scheduling parallel algorithms + +@tparam C closure wrapper type + +The class provides base methods to derive a partitioner that can be used +to schedule parallel iterations (e.g., tf::Taskflow::for_each). + +An partitioner defines the scheduling method for running parallel algorithms, +such tf::Taskflow::for_each, tf::Taskflow::reduce, and so on. +By default, we provide the following partitioners: + ++ tf::GuidedPartitioner to enable guided scheduling algorithm of adaptive chunk size ++ tf::DynamicPartitioner to enable dynamic scheduling algorithm of equal chunk size ++ tf::StaticPartitioner to enable static scheduling algorithm of static chunk size ++ tf::RandomPartitioner to enable random scheduling algorithm of random chunk size + +Depending on applications, partitioning algorithms can impact the performance +a lot. +For example, if a parallel-iteration workload contains a regular work unit per +iteration, tf::StaticPartitioner can deliver the best performance. +On the other hand, if the work unit per iteration is irregular and unbalanced, +tf::GuidedPartitioner or tf::DynamicPartitioner can outperform tf::StaticPartitioner. +In most situations, tf::GuidedPartitioner can deliver decent performance and +is thus used as our default partitioner. + +@attention +Giving the partition size of 0 lets the %Taskflow runtime automatically determines +the partition size for the given partitioner. + + +In addition to partition size, the application can specify a closure wrapper +for a partitioner. +A closure wrapper allows the application to wrap a partitioned task +(i.e., closure) with a custom function object that performs additional tasks. +For example: + +@code{.cpp} +std::atomic count = 0; +tf::Taskflow taskflow; +taskflow.for_each_index(0, 100, 1, + [](){ + printf("%d\n", i); + }, + tf::StaticPartitioner(0, [](auto&& closure){ + // do something before invoking the partitioned task + // ... + + // invoke the partitioned task + closure(); + + // do something else after invoking the partitioned task + // ... + } +); +executor.run(taskflow).wait(); +@endcode + +@attention +The default closure wrapper (tf::DefaultClosureWrapper) does nothing but invoke +the partitioned task (closure). + +*/ +template +class PartitionerBase { + + public: + + /** + @brief indicating if the given closure wrapper is a default wrapper (i.e., empty) + */ + constexpr static bool is_default_wrapper_v = std::is_same_v; + + /** + @brief the closure type + */ + using closure_wrapper_type = C; + + /** + @brief default constructor + */ + PartitionerBase() = default; + + /** + @brief construct a partitioner with the given chunk size + */ + explicit PartitionerBase(size_t chunk_size) : _chunk_size {chunk_size} {} + + /** + @brief construct a partitioner with the given chunk size and closure wrapper + */ + PartitionerBase(size_t chunk_size, C&& closure_wrapper) : + _chunk_size {chunk_size}, + _closure_wrapper {std::forward(closure_wrapper)} { + } + + /** + @brief query the chunk size of this partitioner + */ + size_t chunk_size() const { return _chunk_size; } + + /** + @brief update the chunk size of this partitioner + */ + void chunk_size(size_t cz) { _chunk_size = cz; } + + /** + @brief acquire an immutable access to the closure wrapper object + */ + const C& closure_wrapper() const { return _closure_wrapper; } + + /** + @brief acquire a mutable access to the closure wrapper object + */ + C& closure_wrapper() { return _closure_wrapper; } + + /** + @brief modify the closure wrapper object + */ + template + void closure_wrapper(F&& fn) { _closure_wrapper = std::forward(fn); } + + /** + @brief wraps the given callable with the associated closure wrapper + */ + template + TF_FORCE_INLINE decltype(auto) operator () (F&& callable) { + if constexpr(is_default_wrapper_v) { + return std::forward(callable); + } + else { + // closure wrapper is stateful - capture it by reference + return [this, c=std::forward(callable)]() mutable { _closure_wrapper(c); }; + } + } + + protected: + + /** + @private + */ + size_t _chunk_size{0}; + + /** + @private + */ + C _closure_wrapper; +}; + +// ---------------------------------------------------------------------------- +// Static Partitioner +// ---------------------------------------------------------------------------- + +/** +@class StaticPartitioner + +@brief class to construct a static partitioner for scheduling parallel algorithms + +@tparam C closure wrapper type (default tf::DefaultClosureWrapper) + +The partitioner divides iterations into chunks and distributes chunks +to workers in order. +If the chunk size is not specified (default @c 0), the partitioner resorts to a chunk size +that equally distributes iterations into workers. + +@code{.cpp} +std::vector data = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10} +taskflow.for_each( + data.begin(), data.end(), [](int i){}, StaticPartitioner(0) +); +executor.run(taskflow).run(); +@endcode + +In addition to partition size, the application can specify a closure wrapper +for a static partitioner. +A closure wrapper allows the application to wrap a partitioned task +(i.e., closure) with a custom function object that performs additional tasks. +For example: + +@code{.cpp} +std::atomic count = 0; +tf::Taskflow taskflow; +taskflow.for_each_index(0, 100, 1, + [](){ + printf("%d\n", i); + }, + tf::StaticPartitioner(0, [](auto&& closure){ + // do something before invoking the partitioned task + // ... + + // invoke the partitioned task + closure(); + + // do something else after invoking the partitioned task + // ... + } +); +executor.run(taskflow).wait(); +@endcode +*/ +template +class StaticPartitioner : public PartitionerBase { + + public: + + /** + @brief queries the partition type (static) + */ + static constexpr PartitionerType type() { return PartitionerType::STATIC; } + + /** + @brief default constructor + */ + StaticPartitioner() = default; + + /** + @brief construct a static partitioner with the given chunk size + */ + explicit StaticPartitioner(size_t sz) : PartitionerBase(sz) {} + + /** + @brief construct a static partitioner with the given chunk size and the closure + */ + explicit StaticPartitioner(size_t sz, C&& closure) : + PartitionerBase(sz, std::forward(closure)) { + } + + /** + @brief queries the adjusted chunk size + + Returns the given chunk size if it is not zero, or returns + N/W + (w < N%W), where @c N is the number of iterations, + @c W is the number of workers, and @c w is the worker ID. + */ + size_t adjusted_chunk_size(size_t N, size_t W, size_t w) const { + return this->_chunk_size ? this->_chunk_size : N/W + (w < N%W); + } + + // -------------------------------------------------------------------------- + // scheduling methods + // -------------------------------------------------------------------------- + + /** + @private + */ + template + void loop( + size_t N, size_t W, size_t curr_b, size_t chunk_size, F&& func + ) { + size_t stride = W * chunk_size; + while(curr_b < N) { + size_t curr_e = (std::min)(curr_b + chunk_size, N); + if constexpr (std::is_same_v, bool>) { + if(func(curr_b, curr_e)) { + return; + } + } else { + func(curr_b, curr_e); + } + curr_b += stride; + } + } + + /** + @private + + Static partitioner loop for index ranges of any rank. + + Each worker is pre-assigned a flat quota (chunk_size) starting at curr_b, + then strides by W*chunk_size to its next partition — the classic static + strided pattern. For a 1D range (rank == 1) the quota maps directly to a + single subrange via unravel(), since there is no hyperplane alignment to + respect. For an N-D range (rank > 1) the worker repeatedly calls + lower_slice to drain its quota in box-shaped pieces; lower_slice guarantees + consumed <= remaining budget, so curr_b never overshoots curr_e and no + elements are double-processed across partition boundaries. Either way, + curr_b ends up exactly stride past where it started once the quota is + drained. + */ + template + void loop(const R& range, size_t N, size_t W, size_t curr_b, size_t chunk_size, F&& func) const { + size_t stride = W * chunk_size; + while(curr_b < N) { + size_t curr_e = (std::min)(curr_b + chunk_size, N); + if constexpr (R::rank == 1) { + if constexpr (std::is_same_v, bool>) { + if(func(range.unravel(curr_b, curr_e))) { + return; + } + } else { + func(range.unravel(curr_b, curr_e)); + } + curr_b = curr_e; + } else { + while(curr_b < curr_e) { + auto box = range.lower_slice(curr_b, curr_e - curr_b); + if constexpr (std::is_same_v, bool>) { + if(func(box)) { + return; + } + } else { + func(box); + } + curr_b += box.size(); + } + } + curr_b += stride - chunk_size; + } + } + +}; + +// ---------------------------------------------------------------------------- +// Guided Partitioner +// ---------------------------------------------------------------------------- + +/** +@class GuidedPartitioner + +@tparam C closure wrapper type (default tf::DefaultClosureWrapper) + +@brief class to create a guided partitioner for scheduling parallel algorithms + +The size of a partition is proportional to the number of unassigned iterations +divided by the number of workers, +and the size will gradually decrease to the given chunk size. +The last partition may be smaller than the chunk size. + +In addition to partition size, the application can specify a closure wrapper +for a guided partitioner. +A closure wrapper allows the application to wrap a partitioned task +(i.e., closure) with a custom function object that performs additional tasks. +For example: + +@code{.cpp} +std::atomic count = 0; +tf::Taskflow taskflow; +taskflow.for_each_index(0, 100, 1, + [](){ + printf("%d\n", i); + }, + tf::GuidedPartitioner(0, [](auto&& closure){ + // do something before invoking the partitioned task + // ... + + // invoke the partitioned task + closure(); + + // do something else after invoking the partitioned task + // ... + } +); +executor.run(taskflow).wait(); +@endcode +*/ +template +class GuidedPartitioner : public PartitionerBase { + + public: + + /** + @brief queries the partition type (dynamic) + */ + static constexpr PartitionerType type() { return PartitionerType::DYNAMIC; } + + /** + @brief default constructor + */ + GuidedPartitioner() = default; + + /** + @brief construct a guided partitioner with the given chunk size + + */ + explicit GuidedPartitioner(size_t sz) : PartitionerBase (sz) {} + + /** + @brief construct a guided partitioner with the given chunk size and the closure + */ + explicit GuidedPartitioner(size_t sz, C&& closure) : + PartitionerBase(sz, std::forward(closure)) { + } + + // -------------------------------------------------------------------------- + // scheduling methods + // -------------------------------------------------------------------------- + + /** + @private + */ + template + void loop( + size_t N, size_t W, std::atomic& next, F&& func + ) const { + + size_t chunk_size = (this->_chunk_size == 0) ? size_t{1} : this->_chunk_size; + + size_t p1 = 2 * W * (chunk_size + 1); + float p2 = 0.5f / static_cast(W); + size_t curr_b = next.load(std::memory_order_relaxed); + + while(curr_b < N) { + + size_t r = N - curr_b; + + // fine-grained + if(r < p1) { + while(1) { + curr_b = next.fetch_add(chunk_size, std::memory_order_relaxed); + if(curr_b >= N) { + return; + } + if constexpr (std::is_same_v, bool>) { + if(func(curr_b, (std::min)(curr_b + chunk_size, N))) { + return; + } + } else { + func(curr_b, (std::min)(curr_b + chunk_size, N)); + } + } + break; + } + // coarse-grained + else { + size_t q = static_cast(p2 * r); + if(q < chunk_size) { + q = chunk_size; + } + size_t curr_e = (std::min)(curr_b + q, N); + if(next.compare_exchange_strong(curr_b, curr_e, std::memory_order_relaxed, + std::memory_order_relaxed)) { + if constexpr (std::is_same_v, bool>) { + if(func(curr_b, curr_e)) { + return; + } + } else { + func(curr_b, curr_e); + } + curr_b = curr_e; + } + } + } + } + + /** + @private + */ + template + void loop(const R& range, size_t N, size_t W, std::atomic& next, F&& func) const { + + size_t chunk_size = (this->_chunk_size == 0) ? size_t{1} : this->_chunk_size; + size_t p1 = 2 * W * (chunk_size + 1); + float p2 = 0.5f / static_cast(W); + size_t curr_b = next.load(std::memory_order_relaxed); + + while(curr_b < N) { + size_t r = N - curr_b; + size_t csize = (r < p1) ? chunk_size : (std::max)(static_cast(p2 * r), chunk_size); + if constexpr (R::rank == 1) { + size_t curr_e = (std::min)(curr_b + csize, N); + if(next.compare_exchange_weak(curr_b, curr_e, + std::memory_order_relaxed, + std::memory_order_relaxed)) { + if constexpr (std::is_same_v, bool>) { + if(func(range.unravel(curr_b, curr_e))) { + return; + } + } else { + func(range.unravel(curr_b, curr_e)); + } + curr_b = curr_e; + } + } else { + auto box = range.upper_slice(curr_b, csize); + if(next.compare_exchange_weak(curr_b, curr_b + box.size(), + std::memory_order_relaxed, + std::memory_order_relaxed)) { + if constexpr (std::is_same_v, bool>) { + if(func(box)) { + return; + } + } else { + func(box); + } + curr_b += box.size(); + } + } + } + } + +}; + +// ---------------------------------------------------------------------------- +// Dynamic Partitioner +// ---------------------------------------------------------------------------- + +/** +@class DynamicPartitioner + +@brief class to create a dynamic partitioner for scheduling parallel algorithms + +@tparam C closure wrapper type (default tf::DefaultClosureWrapper) + +The partitioner splits iterations into many partitions each of size equal to +the given chunk size. +Different partitions are distributed dynamically to workers +without any specific order. + +In addition to partition size, the application can specify a closure wrapper +for a dynamic partitioner. +A closure wrapper allows the application to wrap a partitioned task +(i.e., closure) with a custom function object that performs additional tasks. +For example: + +@code{.cpp} +std::atomic count = 0; +tf::Taskflow taskflow; +taskflow.for_each_index(0, 100, 1, + [](){ + printf("%d\n", i); + }, + tf::DynamicPartitioner(0, [](auto&& closure){ + // do something before invoking the partitioned task + // ... + + // invoke the partitioned task + closure(); + + // do something else after invoking the partitioned task + // ... + } +); +executor.run(taskflow).wait(); +@endcode +*/ +template +class DynamicPartitioner : public PartitionerBase { + + public: + + /** + @brief queries the partition type (dynamic) + */ + static constexpr PartitionerType type() { return PartitionerType::DYNAMIC; } + + /** + @brief default constructor + */ + DynamicPartitioner() = default; + + /** + @brief construct a dynamic partitioner with the given chunk size + */ + explicit DynamicPartitioner(size_t sz) : PartitionerBase(sz) {} + + /** + @brief construct a dynamic partitioner with the given chunk size and the closure + */ + explicit DynamicPartitioner(size_t sz, C&& closure) : + PartitionerBase(sz, std::forward(closure)) { + } + + // -------------------------------------------------------------------------- + // scheduling methods + // -------------------------------------------------------------------------- + + /** + @private + */ + template + void loop( + size_t N, size_t, std::atomic& next, F&& func + ) const { + + size_t chunk_size = (this->_chunk_size == 0) ? size_t{1} : this->_chunk_size; + size_t curr_b = next.fetch_add(chunk_size, std::memory_order_relaxed); + + while(curr_b < N) { + if constexpr (std::is_same_v, bool>) { + if(func(curr_b, (std::min)(curr_b + chunk_size, N))) { + return; + } + } else { + func(curr_b, (std::min)(curr_b + chunk_size, N)); + } + curr_b = next.fetch_add(chunk_size, std::memory_order_relaxed); + } + } + + /** + @private + */ + template + void loop(const R& range, size_t N, size_t, std::atomic& next, F&& func) const { + size_t curr_b = next.load(std::memory_order_relaxed); + size_t chunk_size = (this->_chunk_size == 0) ? size_t{1} : this->_chunk_size; + + while(curr_b < N) { + if constexpr (R::rank == 1) { + size_t curr_e = (std::min)(curr_b + chunk_size, N); + if(next.compare_exchange_weak(curr_b, curr_e, + std::memory_order_relaxed, + std::memory_order_relaxed)) { + if constexpr (std::is_same_v, bool>) { + if(func(range.unravel(curr_b, curr_e))) { + return; + } + } else { + func(range.unravel(curr_b, curr_e)); + } + curr_b = curr_e; + } + } else { + auto box = range.upper_slice(curr_b, chunk_size); + if(next.compare_exchange_weak(curr_b, curr_b + box.size(), + std::memory_order_relaxed, + std::memory_order_relaxed)) { + if constexpr (std::is_same_v, bool>) { + if(func(box)) { + return; + } + } else { + func(box); + } + curr_b += box.size(); + } + } + } + } + +}; + +// ---------------------------------------------------------------------------- +// RandomPartitioner +// ---------------------------------------------------------------------------- + +/** +@class RandomPartitioner + +@brief class to construct a random partitioner for scheduling parallel algorithms + +@tparam C closure wrapper type (default tf::DefaultClosureWrapper) + +Similar to tf::DynamicPartitioner, +the partitioner splits iterations into many partitions but each with a random +chunk size in the range, c = [alpha * N * W, beta * N * W]. +By default, @c alpha is 0.01 and @c beta is 0.5, respectively. + +In addition to partition size, the application can specify a closure wrapper +for a random partitioner. +A closure wrapper allows the application to wrap a partitioned task +(i.e., closure) with a custom function object that performs additional tasks. +For example: + +@code{.cpp} +std::atomic count = 0; +tf::Taskflow taskflow; +taskflow.for_each_index(0, 100, 1, + [](){ + printf("%d\n", i); + }, + tf::RandomPartitioner(0, [](auto&& closure){ + // do something before invoking the partitioned task + // ... + + // invoke the partitioned task + closure(); + + // do something else after invoking the partitioned task + // ... + } +); +executor.run(taskflow).wait(); +@endcode +*/ +template +class RandomPartitioner : public PartitionerBase { + + public: + + /** + @brief queries the partition type (dynamic) + */ + static constexpr PartitionerType type() { return PartitionerType::DYNAMIC; } + + /** + @brief default constructor + */ + RandomPartitioner() = default; + + /** + @brief construct a dynamic partitioner with the given chunk size + */ + explicit RandomPartitioner(size_t sz) : PartitionerBase(sz) {} + + /** + @brief construct a random partitioner with the given chunk size and the closure + */ + explicit RandomPartitioner(size_t sz, C&& closure) : + PartitionerBase(sz, std::forward(closure)) { + } + + /** + @brief constructs a random partitioner with the given parameters + */ + RandomPartitioner(float alpha, float beta) : _alpha{alpha}, _beta{beta} {} + + /** + @brief constructs a random partitioner with the given parameters and the closure + */ + RandomPartitioner(float alpha, float beta, C&& closure) : + _alpha {alpha}, _beta {beta}, + PartitionerBase(0, std::forward(closure)) { + } + + /** + @brief queries the @c alpha value + */ + float alpha() const { return _alpha; } + + /** + @brief queries the @c beta value + */ + float beta() const { return _beta; } + + /** + @brief queries the range of chunk size + + @param N number of iterations + @param W number of workers + */ + std::pair chunk_size_range(size_t N, size_t W) const { + + size_t b1 = static_cast(_alpha * N * W); + size_t b2 = static_cast(_beta * N * W); + + if(b1 > b2) { + std::swap(b1, b2); + } + + b1 = (std::max)(b1, size_t{1}); + b2 = (std::max)(b2, b1 + 1); + + return {b1, b2}; + } + + // -------------------------------------------------------------------------- + // scheduling methods + // -------------------------------------------------------------------------- + + /** + @private + */ + template + void loop( + size_t N, size_t W, std::atomic& next, F&& func + ) const { + + auto [b1, b2] = chunk_size_range(N, W); + + std::default_random_engine engine {std::random_device{}()}; + std::uniform_int_distribution dist(b1, b2); + + size_t chunk_size = dist(engine); + size_t curr_b = next.fetch_add(chunk_size, std::memory_order_relaxed); + + while(curr_b < N) { + if constexpr (std::is_same_v, bool>) { + if(func(curr_b, (std::min)(curr_b + chunk_size, N))) { + return; + } + } else { + func(curr_b, (std::min)(curr_b + chunk_size, N)); + } + chunk_size = dist(engine); + curr_b = next.fetch_add(chunk_size, std::memory_order_relaxed); + } + } + + /** + @private + */ + template + void loop(const R& range, size_t N, size_t W, std::atomic& next, F&& func) const { + + auto [b1, b2] = chunk_size_range(N, W); + + std::default_random_engine engine{std::random_device{}()}; + std::uniform_int_distribution dist(b1, b2); + + size_t curr_b = next.load(std::memory_order_relaxed); + + while(curr_b < N) { + if constexpr (R::rank == 1) { + size_t curr_e = (std::min)(curr_b + dist(engine), N); + if(next.compare_exchange_weak(curr_b, curr_e, + std::memory_order_relaxed, + std::memory_order_relaxed)) { + if constexpr (std::is_same_v, bool>) { + if(func(range.unravel(curr_b, curr_e))) { + return; + } + } else { + func(range.unravel(curr_b, curr_e)); + } + curr_b = curr_e; + } + } else { + auto box = range.upper_slice(curr_b, dist(engine)); + if(next.compare_exchange_weak(curr_b, curr_b + box.size(), + std::memory_order_relaxed, + std::memory_order_relaxed)) { + if constexpr (std::is_same_v, bool>) { + if(func(box)) { + return; + } + } else { + func(box); + } + curr_b += box.size(); + } + } + } + } + + private: + + float _alpha {0.01f}; + float _beta {0.50f}; +}; + +/** +@brief default partitioner set to tf::GuidedPartitioner + +Guided partitioning algorithm can achieve stable and decent performance +for most parallel algorithms. +*/ +using DefaultPartitioner = GuidedPartitioner<>; + +/** +@brief determines if a type is a partitioner + +A type satisfies tf::PartitionerLike if it is derived from tf::PartitionerBase. +*/ +template +concept PartitionerLike = std::derived_from>; + +/** +@brief determines if a type is a partitioner (variable template) + +@tparam P type to check + +Equivalent to tf::PartitionerLike

. Provided for backward compatibility. +*/ +template +inline constexpr bool is_partitioner_v = PartitionerLike

; + +} // end of namespace tf ----------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/algorithm/pipeline.hpp b/projects/hipfile/third-party/taskflow/algorithm/pipeline.hpp new file mode 100644 index 000000000000..677762afe592 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/algorithm/pipeline.hpp @@ -0,0 +1,1232 @@ +#pragma once + +#include "../taskflow.hpp" + +/** +@file pipeline.hpp +@brief pipeline include file +*/ + +namespace tf { + + +// ---------------------------------------------------------------------------- +// Class Definition: Pipeflow +// ---------------------------------------------------------------------------- + +/** +@class Pipeflow + +@brief class to create a pipeflow object used by the pipe callable + +Pipeflow represents a scheduling token in the pipeline scheduling +framework. A pipeflow is created by the pipeline scheduler at runtime to +pass to the pipe callable. Users can query the present statistics +of that scheduling token, including the line identifier, pipe identifier, +and token identifier, and build their application algorithms based on +these statistics. +At the first stage, users can explicitly call the stop method +to stop the pipeline scheduler. + +@code{.cpp} +tf::Pipe{tf::PipeType::SERIAL, [](tf::Pipeflow& pf){ + std::cout << "token id=" << pf.token() + << " at line=" << pf.line() + << " at pipe=" << pf.pipe() + << '\n'; +}}; +@endcode + +Pipeflow can only be created privately by the tf::Pipeline and +be used through the pipe callable. +*/ +class Pipeflow { + + template + friend class Pipeline; + + template + friend class ScalablePipeline; + + template + friend class DataPipeline; + + public: + + /** + @brief default constructor + */ + Pipeflow() = default; + + /** + @brief queries the line identifier of the present token + */ + size_t line() const { + return _line; + } + + /** + @brief queries the pipe identifier of the present token + */ + size_t pipe() const { + return _pipe; + } + + /** + @brief queries the token identifier + */ + size_t token() const { + return _token; + } + + /** + @brief stops the pipeline scheduling + + Only the first pipe can call this method to stop the pipeline. + Calling stop from other pipes will throw exception. + */ + void stop() { + if(_pipe != 0) { + TF_THROW("only the first pipe can stop the token"); + } + _stop = true; + } + + private: + + // Regular data + size_t _line; + size_t _pipe; + size_t _token; + bool _stop; +}; + +// ---------------------------------------------------------------------------- +// Class Definition: PipeType +// ---------------------------------------------------------------------------- + +/** +@enum PipeType + +@brief enumeration of all pipe types +*/ +enum class PipeType : int { + /** @brief parallel type */ + PARALLEL = 1, + /** @brief serial type */ + SERIAL = 2 +}; + +// ---------------------------------------------------------------------------- +// Class Definition: Pipe +// ---------------------------------------------------------------------------- + +/** +@class Pipe + +@brief class to create a pipe object for a pipeline stage + +@tparam C callable type + +A pipe represents a stage of a pipeline. A pipe can be either +@em parallel direction or @em serial direction (specified by tf::PipeType) +and is coupled with a callable to invoke by the pipeline scheduler. +The callable must take a referenced tf::Pipeflow object in the first argument: + +@code{.cpp} +Pipe{PipeType::SERIAL, [](tf::Pipeflow&){}} +@endcode + +The pipeflow object is used to query the statistics of a scheduling token +in the pipeline, such as pipe, line, and token numbers. +*/ +template > +class Pipe { + + template + friend class Pipeline; + + template + friend class ScalablePipeline; + + public: + + /** + @brief alias of the callable type + */ + using callable_t = C; + + /** + @brief default constructor + */ + Pipe() = default; + + /** + @brief constructs the pipe object + + @param d pipe type (tf::PipeType) + @param callable callable type + + The constructor constructs a pipe with the given direction + (tf::PipeType::SERIAL or tf::PipeType::PARALLEL) and the given callable. + The callable must take a referenced tf::Pipeflow object in the first argument. + + @code{.cpp} + Pipe{PipeType::SERIAL, [](tf::Pipeflow&){}} + @endcode + + When creating a pipeline, the direction of the first pipe must be serial + (tf::PipeType::SERIAL). + */ + Pipe(PipeType d, C&& callable) : + _type{d}, _callable{std::forward(callable)} { + } + + /** + @brief queries the type of the pipe + + Returns the type of the callable. + */ + PipeType type() const { + return _type; + } + + /** + @brief assigns a new type to the pipe + + @param type a tf::PipeType variable + */ + void type(PipeType type) { + _type = type; + } + + /** + @brief assigns a new callable to the pipe + + @tparam U callable type + @param callable a callable object constructible from std::function + + Assigns a new callable to the pipe with universal forwarding. + */ + template + void callable(U&& callable) { + _callable = std::forward(callable); + } + + private: + + PipeType _type; + + C _callable; +}; + +// ---------------------------------------------------------------------------- +// Class Definition: Pipeline +// ---------------------------------------------------------------------------- + +/** +@class Pipeline + +@brief class to create a pipeline scheduling framework + +@tparam Ps pipe types + +A tf::Pipeline is a composable graph object that allows users to parallelize an application +using pipeline parallelism. +Unlike conventional pipeline programming frameworks (e.g., Intel TBB), +tf::Pipeline does not provide any data abstraction but a task-parallel framework +for users to customize their data layout when leveraging pipeline parallelism. +The following code creates a pipeline with four parallel lines that schedule +five tokens through three pipes of a serial-parallel-serial direction: + +@code{.cpp} +tf::Taskflow taskflow; +tf::Executor executor; + +const size_t num_lines = 4; +const size_t num_pipes = 3; + +// create a custom data buffer +std::array, num_lines> buffer; + +// create a pipeline graph of four concurrent lines and three serial pipes +tf::Pipeline pipeline(num_lines, + // first pipe must define a serial direction + tf::Pipe{tf::PipeType::SERIAL, [&buffer](tf::Pipeflow& pf) { + // generate only 5 scheduling tokens + if(pf.token() == 5) { + pf.stop(); + } + // save the token id into the buffer + else { + buffer[pf.line()][pf.pipe()] = pf.token(); + } + }}, + tf::Pipe{tf::PipeType::PARALLEL, [&buffer] (tf::Pipeflow& pf) { + // propagate the previous result to this pipe by adding one + buffer[pf.line()][pf.pipe()] = buffer[pf.line()][pf.pipe()-1] + 1; + }}, + tf::Pipe{tf::PipeType::SERIAL, [&buffer](tf::Pipeflow& pf){ + // propagate the previous result to this pipe by adding one + buffer[pf.line()][pf.pipe()] = buffer[pf.line()][pf.pipe()-1] + 1; + }} +); + +// build the pipeline graph using composition +tf::Task init = taskflow.emplace([](){ std::cout << "ready\n"; }) + .name("starting pipeline"); +tf::Task task = taskflow.composed_of(pipeline) + .name("pipeline"); +tf::Task stop = taskflow.emplace([](){ std::cout << "stopped\n"; }) + .name("pipeline stopped"); + +// create task dependency +init.precede(task); +task.precede(stop); + +// run the pipeline +executor.run(taskflow).wait(); +@endcode + +The five tokens will be scheduled across four parallel lines in a circular fashion, +as depicted below: + +@dotfile images/pipeline_our_structure.dot + +At each pipe stage, the program propagates the result to the next pipe +by adding one to the result stored in a custom data storage, @c buffer. +The pipeline scheduler will generate five scheduling tokens and then stop. + +Internally, tf::Pipeline uses std::tuple to store the given sequence of pipes. +The definition of each pipe can be different, completely decided by the compiler +to optimize the object layout. +After a pipeline is constructed, it is not possible to change its pipes. +If applications need to change these pipes, please use tf::ScalablePipeline. +*/ +template +class Pipeline { + + static_assert(sizeof...(Ps)>0, "must have at least one pipe"); + + /** + @private + */ + struct Line { + std::atomic join_counter; + }; + + /** + @private + */ + struct PipeMeta { + PipeType type; + }; + + public: + + /** + @brief constructs a pipeline object + + @param num_lines the number of parallel lines + @param ps a list of pipes + + Constructs a pipeline of up to @c num_lines parallel lines to schedule + tokens through the given linear chain of pipes. + The first pipe must define a serial direction (tf::PipeType::SERIAL) + or an exception will be thrown. + */ + Pipeline(size_t num_lines, Ps&&... ps); + + /** + @brief constructs a pipeline object + + @param num_lines the number of parallel lines + @param ps a tuple of pipes + + Constructs a pipeline of up to @c num_lines parallel lines to schedule + tokens through the given linear chain of pipes. + The first pipe must define a serial direction (tf::PipeType::SERIAL) + or an exception will be thrown. + */ + Pipeline(size_t num_lines, std::tuple&& ps); + + /** + @brief queries the number of parallel lines + + The function returns the number of parallel lines given by the user + upon the construction of the pipeline. + The number of lines represents the maximum parallelism this pipeline + can achieve. + */ + size_t num_lines() const noexcept; + + /** + @brief queries the number of pipes + + The Function returns the number of pipes given by the user + upon the construction of the pipeline. + */ + constexpr size_t num_pipes() const; + + /** + @brief resets the pipeline + + Resetting the pipeline to the initial state. After resetting a pipeline, + its token identifier will start from zero as if the pipeline was just + constructed. + */ + void reset(); + + /** + @brief queries the number of generated tokens in the pipeline + + The number represents the total scheduling tokens that has been + generated by the pipeline so far. + */ + size_t num_tokens() const noexcept; + + /** + @brief obtains the graph object associated with the pipeline construct + + This method is primarily used as an opaque data structure for creating + a module task of the this pipeline. + */ + Graph& graph(); + + + private: + + Graph _graph; + + size_t _num_tokens; + + std::tuple _pipes; + std::array _meta; + std::vector> _lines; + std::vector _tasks; + std::vector _pipeflows; + + template + auto _gen_meta(std::tuple&&, std::index_sequence); + + void _on_pipe(Pipeflow&, NonpreemptiveRuntime&); + void _build(); +}; + +// constructor +template +Pipeline::Pipeline(size_t num_lines, Ps&&... ps) : + _pipes {std::make_tuple(std::forward(ps)...)}, + _meta {PipeMeta{ps.type()}...}, + _lines (num_lines), + _tasks (num_lines + 1), + _pipeflows (num_lines) { + + if(num_lines == 0) { + TF_THROW("must have at least one line"); + } + + if(std::get<0>(_pipes).type() != PipeType::SERIAL) { + TF_THROW("first pipe must be serial"); + } + + reset(); + _build(); +} + +// constructor +template +Pipeline::Pipeline(size_t num_lines, std::tuple&& ps) : + _pipes {std::forward>(ps)}, + _meta {_gen_meta( + std::forward>(ps), std::make_index_sequence{} + )}, + _lines (num_lines), + _tasks (num_lines + 1), + _pipeflows (num_lines) { + + if(num_lines == 0) { + TF_THROW("must have at least one line"); + } + + if(std::get<0>(_pipes).type() != PipeType::SERIAL) { + TF_THROW("first pipe must be serial"); + } + + reset(); + _build(); +} + +// Function: _get_meta +template +template +auto Pipeline::_gen_meta(std::tuple&& ps, std::index_sequence) { + return std::array{PipeMeta{std::get(ps).type()}...}; +} + +// Function: num_lines +template +size_t Pipeline::num_lines() const noexcept { + return _pipeflows.size(); +} + +// Function: num_pipes +template +constexpr size_t Pipeline::num_pipes() const { + return sizeof...(Ps); +} + +// Function: num_tokens +template +size_t Pipeline::num_tokens() const noexcept { + return _num_tokens; +} + +// Function: graph +template +Graph& Pipeline::graph() { + return _graph; +} + +// Function: reset +template +void Pipeline::reset() { + + _num_tokens = 0; + + for(size_t l = 0; l(_meta[f].type), std::memory_order_relaxed + ); + } + } + + for(size_t f=1; f(_meta[0].type) - 1, std::memory_order_relaxed + ); + } +} + +// Procedure: _on_pipe +template +void Pipeline::_on_pipe(Pipeflow& pf, NonpreemptiveRuntime& rt) { + visit_tuple([&](auto&& pipe){ + using callable_t = typename std::decay_t::callable_t; + if constexpr (std::is_invocable_v) { + pipe._callable(pf); + } + else if constexpr(std::is_invocable_v) { + pipe._callable(pf, rt); + } + else { + static_assert(dependent_false_v, "un-supported pipe callable type"); + } + }, _pipes, pf._pipe); +} + +// Procedure: _build +template +void Pipeline::_build() { + + using namespace std::literals::string_literals; + + FlowBuilder fb(_graph); + + // init task + _tasks[0] = fb.emplace([this]() { + return static_cast(_num_tokens % num_lines()); + }).name("cond"); + + // line task + for(size_t l = 0; l < num_lines(); l++) { + + _tasks[l + 1] = fb.emplace([this, l] (tf::NonpreemptiveRuntime& rt) mutable { + + auto pf = &_pipeflows[l]; + + pipeline: + + _lines[pf->_line][pf->_pipe].join_counter.store( + static_cast(_meta[pf->_pipe].type), std::memory_order_relaxed + ); + + // First pipe does all jobs of initialization and token dependencies + if (pf->_pipe == 0) { + pf->_token = _num_tokens; + if (pf->_stop = false, _on_pipe(*pf, rt); pf->_stop == true) { + // here, the pipeline is not stopped yet because other + // lines of tasks may still be running their last stages + return; + } + ++_num_tokens; + } + else { + _on_pipe(*pf, rt); + } + + size_t c_f = pf->_pipe; + size_t n_f = (pf->_pipe + 1) % num_pipes(); + size_t n_l = (pf->_line + 1) % num_lines(); + + pf->_pipe = n_f; + + // ---- scheduling starts here ---- + // Notice that the shared variable f must not be changed after this + // point because it can result in data race due to the following + // condition: + // + // a -> b + // | | + // v v + // c -> d + // + // d will be spawned by either c or b, so if c changes f but b spawns d + // then data race on f will happen + + std::array retval; + size_t n = 0; + + // downward dependency + if(_meta[c_f].type == PipeType::SERIAL && + _lines[n_l][c_f].join_counter.fetch_sub( + 1, std::memory_order_acq_rel) == 1 + ) { + retval[n++] = 1; + } + + // forward dependency + if(_lines[pf->_line][n_f].join_counter.fetch_sub( + 1, std::memory_order_acq_rel) == 1 + ) { + retval[n++] = 0; + } + + // notice that the task index starts from 1 + switch(n) { + case 2: { + rt.schedule(_tasks[n_l+1]); + goto pipeline; + } + case 1: { + // downward dependency + if (retval[0] == 1) { + pf = &_pipeflows[n_l]; + } + // forward dependency + goto pipeline; + } + } + }).name("nprt-"s + std::to_string(l)); + + _tasks[0].precede(_tasks[l+1]); + } +} + +// ---------------------------------------------------------------------------- +// Class Definition: ScalablePipeline +// ---------------------------------------------------------------------------- + +/** +@class ScalablePipeline + +@brief class to create a scalable pipeline object + +@tparam P type of the iterator to a range of pipes + +A scalable pipeline is a composable graph object for users to create a +pipeline scheduling framework using a module task in a taskflow. +Unlike tf::Pipeline that instantiates all pipes upon the construction time, +tf::ScalablePipeline allows variable assignments of pipes using range iterators. +Users can also reset a scalable pipeline to a different range of pipes +between runs. The following code creates a scalable pipeline of four +parallel lines to schedule tokens through three serial pipes in a custom storage, +then resetting the pipeline to a new range of five serial pipes: + +@code{.cpp} +tf::Taskflow taskflow("pipeline"); +tf::Executor executor; + +const size_t num_lines = 4; + +// create data storage +std::array buffer; + +// define the pipe callable +auto pipe_callable = [&buffer] (tf::Pipeflow& pf) mutable { + switch(pf.pipe()) { + // first stage generates only 5 scheduling tokens and saves the + // token number into the buffer. + case 0: { + if(pf.token() == 5) { + pf.stop(); + } + else { + printf("stage 1: input token = %zu\n", pf.token()); + buffer[pf.line()] = pf.token(); + } + return; + } + break; + + // other stages propagate the previous result to this pipe and + // increment it by one + default: { + printf( + "stage %zu: input buffer[%zu] = %d\n", pf.pipe(), pf.line(), buffer[pf.line()] + ); + buffer[pf.line()] = buffer[pf.line()] + 1; + } + break; + } +}; + +// create a vector of three pipes +std::vector< tf::Pipe> > pipes; + +for(size_t i=0; i<3; i++) { + pipes.emplace_back(tf::PipeType::SERIAL, pipe_callable); +} + +// create a pipeline of four parallel lines based on the given vector of pipes +tf::ScalablePipeline pl(num_lines, pipes.begin(), pipes.end()); + +// build the pipeline graph using composition +tf::Task init = taskflow.emplace([](){ std::cout << "ready\n"; }) + .name("starting pipeline"); +tf::Task task = taskflow.composed_of(pl) + .name("pipeline"); +tf::Task stop = taskflow.emplace([](){ std::cout << "stopped\n"; }) + .name("pipeline stopped"); + +// create task dependency +init.precede(task); +task.precede(stop); + +// dump the pipeline graph structure (with composition) +taskflow.dump(std::cout); + +// run the pipeline +executor.run(taskflow).wait(); + +// reset the pipeline to a new range of five pipes and starts from +// the initial state (i.e., token counts from zero) +for(size_t i=0; i<2; i++) { + pipes.emplace_back(tf::PipeType::SERIAL, pipe_callable); +} +pl.reset(pipes.begin(), pipes.end()); + +executor.run(taskflow).wait(); +@endcode + +The above example creates a pipeline graph that schedules five tokens over +four parallel lines in a circular fashion, first going through three serial pipes +and then five serial pipes: + +@code{.bash} +# initial construction of three serial pipes +o -> o -> o +| | | +v v v +o -> o -> o +| | | +v v v +o -> o -> o +| | | +v v v +o -> o -> o + +# resetting to a new range of five serial pipes +o -> o -> o -> o -> o +| | | | | +v v v v v +o -> o -> o -> o -> o +| | | | | +v v v v v +o -> o -> o -> o -> o +| | | | | +v v v v v +o -> o -> o -> o -> o +@endcode + +Each pipe has the same type of `%tf::Pipe<%std::function>` +and is kept in a vector that is amenable to change. +We construct the scalable pipeline using two range iterators pointing to the +beginning and the end of the vector. +At each pipe stage, the program propagates the result to the next pipe +by adding one to the result stored in a custom data storage, @c buffer. +The pipeline scheduler will generate five scheduling tokens and then stop. + +A scalable pipeline is move-only. +*/ +template +class ScalablePipeline { + + /** + @private + */ + struct Line { + std::atomic join_counter; + }; + + + public: + + /** + @brief pipe type + */ + using pipe_t = typename std::iterator_traits

::value_type; + + /** + @brief default constructor + */ + ScalablePipeline() = default; + + /** + @brief constructs an empty scalable pipeline object + + @param num_lines the number of parallel lines + + An empty scalable pipeline does not have any pipes. + The pipeline needs to be reset to a valid range of pipes + before running. + */ + ScalablePipeline(size_t num_lines); + + /** + @brief constructs a scalable pipeline object + + @param num_lines the number of parallel lines + @param first iterator to the beginning of the range + @param last iterator to the end of the range + + Constructs a pipeline from the given range of pipes specified in + [first, last) using @c num_lines parallel lines. + The first pipe must define a serial direction (tf::PipeType::SERIAL) + or an exception will be thrown. + + Internally, the scalable pipeline copies the iterators + from the specified range. Those pipe callables pointed to by + these iterators must remain valid during the execution of the pipeline. + */ + ScalablePipeline(size_t num_lines, P first, P last); + + /** + @brief disabled copy constructor + */ + ScalablePipeline(const ScalablePipeline&) = delete; + + /** + @brief move constructor + + Constructs a pipeline from the given @c rhs using move semantics + (i.e. the data in @c rhs is moved into this pipeline). + After the move, @c rhs is in a state as if it is just constructed. + The behavior is undefined if @c rhs is running during the move. + */ + ScalablePipeline(ScalablePipeline&& rhs); + + /** + @brief disabled copy assignment operator + */ + ScalablePipeline& operator = (const ScalablePipeline&) = delete; + + /** + @brief move constructor + + Replaces the contents with those of @c rhs using move semantics + (i.e. the data in @c rhs is moved into this pipeline). + After the move, @c rhs is in a state as if it is just constructed. + The behavior is undefined if @c rhs is running during the move. + */ + ScalablePipeline& operator = (ScalablePipeline&& rhs); + + /** + @brief queries the number of parallel lines + + The function returns the number of parallel lines given by the user + upon the construction of the pipeline. + The number of lines represents the maximum parallelism this pipeline + can achieve. + */ + size_t num_lines() const noexcept; + + /** + @brief queries the number of pipes + + The Function returns the number of pipes given by the user + upon the construction of the pipeline. + */ + size_t num_pipes() const noexcept; + + /** + @brief resets the pipeline + + Resets the pipeline to the initial state. After resetting a pipeline, + its token identifier will start from zero. + */ + void reset(); + + /** + @brief resets the pipeline with a new range of pipes + + @param first iterator to the beginning of the range + @param last iterator to the end of the range + + The member function assigns the pipeline to a new range of pipes + specified in [first, last) and resets the pipeline to the + initial state. After resetting a pipeline, its token identifier will + start from zero. + + Internally, the scalable pipeline copies the iterators + from the specified range. Those pipe callables pointed to by + these iterators must remain valid during the execution of the pipeline. + */ + void reset(P first, P last); + + /** + @brief resets the pipeline to a new line number and a + new range of pipes + + @param num_lines number of parallel lines + @param first iterator to the beginning of the range + @param last iterator to the end of the range + + The member function resets the pipeline to a new number of + parallel lines and a new range of pipes specified in + [first, last), as if the pipeline is just constructed. + After resetting a pipeline, its token identifier will start from zero. + + Internally, the scalable pipeline copies the iterators + from the specified range. Those pipe callables pointed to by + these iterators must remain valid during the execution of the pipeline. + */ + void reset(size_t num_lines, P first, P last); + + /** + @brief queries the number of generated tokens in the pipeline + + The number represents the total scheduling tokens that has been + generated by the pipeline so far. + */ + size_t num_tokens() const noexcept; + + /** + @brief obtains the graph object associated with the pipeline construct + + This method is primarily used as an opaque data structure for creating + a module task of the this pipeline. + */ + Graph& graph(); + + private: + + Graph _graph; + + size_t _num_tokens{0}; + + std::vector

_pipes; + std::vector _tasks; + std::vector _pipeflows; + std::unique_ptr _lines; + + void _on_pipe(Pipeflow&, NonpreemptiveRuntime&); + void _build(); + + Line& _line(size_t, size_t); +}; + +// constructor +template +ScalablePipeline

::ScalablePipeline(size_t num_lines) : + _tasks (num_lines + 1), + _pipeflows (num_lines) { + + if(num_lines == 0) { + TF_THROW("must have at least one line"); + } + + _build(); +} + +// constructor +template +ScalablePipeline

::ScalablePipeline(size_t num_lines, P first, P last) : + _tasks (num_lines + 1), + _pipeflows (num_lines) { + + if(num_lines == 0) { + TF_THROW("must have at least one line"); + } + + reset(first, last); + _build(); +} + +// move constructor +template +ScalablePipeline

::ScalablePipeline(ScalablePipeline&& rhs): + _num_tokens {rhs._num_tokens}, + _pipes {std::move(rhs._pipes)}, + _pipeflows {std::move(rhs._pipeflows)}, + _lines {std::move(rhs._lines)} { + + _graph.clear(); + _tasks.resize(_pipeflows.size()+1); + rhs._num_tokens = 0; + rhs._tasks.clear(); + _build(); +} + +// move assignment operator +template +ScalablePipeline

& ScalablePipeline

::operator = (ScalablePipeline&& rhs) { + _num_tokens = rhs._num_tokens; + _pipes = std::move(rhs._pipes); + _pipeflows = std::move(rhs._pipeflows); + _lines = std::move(rhs._lines); + + _graph.clear(); + _tasks.resize(_pipeflows.size()+1); + + rhs._num_tokens = 0; + rhs._tasks.clear(); + _build(); + return *this; +} + +// Function: num_lines +template +size_t ScalablePipeline

::num_lines() const noexcept { + return _pipeflows.size(); +} + +// Function: num_pipes +template +size_t ScalablePipeline

::num_pipes() const noexcept { + return _pipes.size(); +} + +// Function: num_tokens +template +size_t ScalablePipeline

::num_tokens() const noexcept { + return _num_tokens; +} + +// Function: graph +template +Graph& ScalablePipeline

::graph() { + return _graph; +} + +// Function: _line +template +typename ScalablePipeline

::Line& ScalablePipeline

::_line(size_t l, size_t p) { + return _lines[l*num_pipes() + p]; +} + +template +void ScalablePipeline

::reset(size_t num_lines, P first, P last) { + + if(num_lines == 0) { + TF_THROW("must have at least one line"); + } + + _graph.clear(); + _tasks.resize(num_lines + 1); + _pipeflows.resize(num_lines); + + reset(first, last); + + _build(); +} + +// Function: reset +template +void ScalablePipeline

::reset(P first, P last) { + + size_t num_pipes = static_cast(std::distance(first, last)); + + if(num_pipes == 0) { + TF_THROW("pipeline cannot be empty"); + } + + if(first->type() != PipeType::SERIAL) { + TF_THROW("first pipe must be serial"); + } + + _pipes.resize(num_pipes); + + size_t i=0; + for(auto itr = first; itr != last; itr++) { + _pipes[i++] = itr; + } + + _lines = std::make_unique(num_lines() * _pipes.size()); + + reset(); +} + +// Function: reset +template +void ScalablePipeline

::reset() { + + _num_tokens = 0; + + for(size_t l = 0; l(_pipes[f]->type()), std::memory_order_relaxed + ); + } + } + + for(size_t f=1; f(_pipes[0]->type()) - 1, std::memory_order_relaxed + ); + } +} + +// Procedure: _on_pipe +template +void ScalablePipeline

::_on_pipe(Pipeflow& pf, NonpreemptiveRuntime& rt) { + + using callable_t = typename pipe_t::callable_t; + + if constexpr (std::is_invocable_v) { + _pipes[pf._pipe]->_callable(pf); + } + else if constexpr(std::is_invocable_v) { + _pipes[pf._pipe]->_callable(pf, rt); + } + else { + static_assert(dependent_false_v, "un-supported pipe callable type"); + } +} + +// Procedure: _build +template +void ScalablePipeline

::_build() { + + using namespace std::literals::string_literals; + + FlowBuilder fb(_graph); + + // init task + _tasks[0] = fb.emplace([this]() { + return static_cast(_num_tokens % num_lines()); + }).name("cond"); + + // line task + for(size_t l = 0; l < num_lines(); l++) { + + _tasks[l + 1] = fb.emplace([this, l] (tf::NonpreemptiveRuntime& rt) mutable { + + auto pf = &_pipeflows[l]; + + pipeline: + + _line(pf->_line, pf->_pipe).join_counter.store( + static_cast(_pipes[pf->_pipe]->type()), std::memory_order_relaxed + ); + + // First pipe does all jobs of initialization and token dependencies + if (pf->_pipe == 0) { + pf->_token = _num_tokens; + if (pf->_stop = false, _on_pipe(*pf, rt); pf->_stop == true) { + // here, the pipeline is not stopped yet because other + // lines of tasks may still be running their last stages + return; + } + ++_num_tokens; + } + else { + _on_pipe(*pf, rt); + } + + size_t c_f = pf->_pipe; + size_t n_f = (pf->_pipe + 1) % num_pipes(); + size_t n_l = (pf->_line + 1) % num_lines(); + + pf->_pipe = n_f; + + // ---- scheduling starts here ---- + // Notice that the shared variable f must not be changed after this + // point because it can result in data race due to the following + // condition: + // + // a -> b + // | | + // v v + // c -> d + // + // d will be spawned by either c or b, so if c changes f but b spawns d + // then data race on f will happen + + std::array retval; + size_t n = 0; + + // downward dependency + if(_pipes[c_f]->type() == PipeType::SERIAL && + _line(n_l, c_f).join_counter.fetch_sub( + 1, std::memory_order_acq_rel) == 1 + ) { + retval[n++] = 1; + } + + // forward dependency + if(_line(pf->_line, n_f).join_counter.fetch_sub( + 1, std::memory_order_acq_rel) == 1 + ) { + retval[n++] = 0; + } + + // notice that the task index starts from 1 + switch(n) { + case 2: { + rt.schedule(_tasks[n_l+1]); + goto pipeline; + } + case 1: { + if (retval[0] == 1) { + pf = &_pipeflows[n_l]; + } + goto pipeline; + } + } + }).name("nprt-"s + std::to_string(l)); + + _tasks[0].precede(_tasks[l+1]); + } +} + +} // end of namespace tf ----------------------------------------------------- + + + + + diff --git a/projects/hipfile/third-party/taskflow/algorithm/reduce.hpp b/projects/hipfile/third-party/taskflow/algorithm/reduce.hpp new file mode 100644 index 000000000000..29f2333db9d6 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/algorithm/reduce.hpp @@ -0,0 +1,544 @@ +#pragma once + +#include "../taskflow.hpp" + +namespace tf { + +// Function: make_reduce_task +template +auto make_reduce_task(B b, E e, T& init, O bop, P part = P()) { + + using namespace std::string_literals; + + using B_t = std::decay_t>; + using E_t = std::decay_t>; + + return [=, &init] (Runtime& rt) mutable { + + // fetch the iterator values + B_t beg = b; + E_t end = e; + + size_t W = rt.executor().num_workers(); + size_t N = std::distance(beg, end); + + if(N == 0) { + return; + } + + // only myself - no need to spawn another graph + if(W <= 1 || N <= part.chunk_size()) { + part([=, &init] () mutable { for(; beg!=end; init = bop(init, *beg++)); })(); + return; + } + + if(N < W) { + W = N; + } + + auto mutex = std::make_shared(); + + // static partitioner + if constexpr(part.type() == PartitionerType::STATIC) { + + for(size_t w=0, curr_b=0; w lock(*mutex); + init = bop(init, *beg); + return; + } + + auto beg1 = beg++; + auto beg2 = beg++; + T sum = bop(*beg1, *beg2); + + // loop reduce + part.loop(N, W, curr_b, chunk_size, + [=, &sum, prev_e=curr_b+2](size_t part_b, size_t part_e) mutable { + + if(part_b > prev_e) { + std::advance(beg, part_b - prev_e); + } + else { + part_b = prev_e; + } + + for(size_t x=part_b; x lock(*mutex); + init = bop(init, sum); + }); + + (++w == W || (curr_b += chunk_size) >= N) ? task() : rt.silent_async(task); + } + } + // dynamic partitioner + else { + auto next = std::make_shared>(0); + + for(size_t w=0; wfetch_add(2, std::memory_order_relaxed); + + if(s0 >= N) { + return; + } + + std::advance(beg, s0); + + if(N - s0 == 1) { + std::lock_guard lock(*mutex); + init = bop(init, *beg); + return; + } + + auto beg1 = beg++; + auto beg2 = beg++; + + T sum = bop(*beg1, *beg2); + + // loop reduce + part.loop(N, W, *next, + [=, &sum, prev_e=s0+2](size_t curr_b, size_t curr_e) mutable { + std::advance(beg, curr_b - prev_e); + for(size_t x=curr_b; x lock(*mutex); + init = bop(init, sum); + }); + (++w == W) ? task() : rt.silent_async(task); + } + } + }; +} + +// Function: make_transform_reduce_task +template < + typename B, typename E, typename T, typename BOP, typename UOP, + PartitionerLike P = DefaultPartitioner +> +auto make_transform_reduce_task(B b, E e, T& init, BOP bop, UOP uop, P part = P()) { + + using namespace std::string_literals; + using B_t = std::decay_t>; + using E_t = std::decay_t>; + + return [=, &init] (Runtime& rt) mutable { + + // fetch the iterator values + B_t beg = b; + E_t end = e; + + size_t W = rt.executor().num_workers(); + size_t N = std::distance(beg, end); + + if(N == 0) { + return; + } + + // only myself - no need to spawn another graph + if(W <= 1 || N <= part.chunk_size()) { + part([=, &init] () mutable { for(; beg!=end; init = bop(std::move(init), uop(*beg++))); })(); + return; + } + + if(N < W) { + W = N; + } + + auto mutex = std::make_shared(); + + // static partitioner + if constexpr(part.type() == PartitionerType::STATIC) { + + for(size_t w=0, curr_b=0; w lock(*mutex); + init = bop(std::move(init), uop(*beg)); + return; + } + + //auto beg1 = beg++; + //auto beg2 = beg++; + //T sum = bop(uop(*beg1), uop(*beg2)); + + T sum = (chunk_size == 1) ? uop(*beg++) : bop(uop(*beg++), uop(*beg++)); + + // loop reduce + part.loop(N, W, curr_b, chunk_size, + [=, &sum, prev_e=curr_b+(chunk_size == 1 ? 1 : 2)] + (size_t part_b, size_t part_e) mutable { + if(part_b > prev_e) { + std::advance(beg, part_b - prev_e); + } + else { + part_b = prev_e; + } + for(size_t x=part_b; x lock(*mutex); + init = bop(std::move(init), std::move(sum)); + }); + + (++w == W || (curr_b += chunk_size) >= N) ? task() : rt.silent_async(task); + } + } + // dynamic partitioner + else { + auto next = std::make_shared>(0); + for(size_t w=0; wfetch_add(2, std::memory_order_relaxed); + + if(s0 >= N) { + return; + } + + std::advance(beg, s0); + + if(N - s0 == 1) { + std::lock_guard lock(*mutex); + init = bop(std::move(init), uop(*beg)); + return; + } + + auto beg1 = beg++; + auto beg2 = beg++; + + T sum = bop(uop(*beg1), uop(*beg2)); + + // loop reduce + part.loop(N, W, *next, + [=, &sum, prev_e=s0+2](size_t curr_b, size_t curr_e) mutable { + std::advance(beg, curr_b - prev_e); + for(size_t x=curr_b; x lock(*mutex); + init = bop(std::move(init), std::move(sum)); + }); + (++w == W) ? task() : rt.silent_async(task); + } + } + }; +} + +// Function: make_transform_reduce_task with two binary operation +template +requires (!PartitionerLike>) +auto make_transform_reduce_task( + B1 b1, E1 e1, B2 b2, T& init, BOP_R bop_r, BOP_T bop_t, P part = P() +) { + + using namespace std::string_literals; + + using B1_t = std::decay_t>; + using E1_t = std::decay_t>; + using B2_t = std::decay_t>; + + return [=, &r=init] (Runtime& rt) mutable { + + // fetch the iterator values + B1_t beg1 = b1; + E1_t end1 = e1; + B2_t beg2 = b2; + + size_t W = rt.executor().num_workers(); + size_t N = std::distance(beg1, end1); + + if(N == 0) { + return; + } + + // only myself - no need to spawn another graph + if(W <= 1 || N <= part.chunk_size()) { + part([=, &r] () mutable { for(; beg1!=end1; r = bop_r(std::move(r), bop_t(*beg1++, *beg2++))); })(); + return; + } + + if(N < W) { + W = N; + } + + auto mutex = std::make_shared(); + + // static partitioner + if constexpr(part.type() == PartitionerType::STATIC) { + + for(size_t w=0, curr_b=0; w lock(*mutex); + r = bop_r(std::move(r), bop_t(*beg1, *beg2)); + return; + } + + T sum = (chunk_size == 1) ? bop_t(*beg1++, *beg2++) : + bop_r(bop_t(*beg1++, *beg2++), bop_t(*beg1++, *beg2++)); + + // loop reduce + part.loop(N, W, curr_b, chunk_size, + [=, &sum, prev_e=curr_b+(chunk_size == 1 ? 1 : 2)] + (size_t part_b, size_t part_e) mutable { + if(part_b > prev_e) { + std::advance(beg1, part_b - prev_e); + std::advance(beg2, part_b - prev_e); + } + else { + part_b = prev_e; + } + for(size_t x=part_b; x lock(*mutex); + r = bop_r(std::move(r), std::move(sum)); + }); + + (++w == W || (curr_b += chunk_size) >= N) ? task() : rt.silent_async(task); + } + } + // dynamic partitioner + else { + auto next = std::make_shared>(0); + + for(size_t w=0; wfetch_add(2, std::memory_order_relaxed); + + if(s0 >= N) { + return; + } + + std::advance(beg1, s0); + std::advance(beg2, s0); + + if(N - s0 == 1) { + std::lock_guard lock(*mutex); + r = bop_r(std::move(r), bop_t(*beg1, *beg2)); + return; + } + + auto beg11 = beg1++; + auto beg12 = beg1++; + auto beg21 = beg2++; + auto beg22 = beg2++; + + T sum = bop_r(bop_t(*beg11, *beg21), bop_t(*beg12, *beg22)); + + // loop reduce + part.loop(N, W, *next, + [=, &sum, prev_e=s0+2](size_t curr_b, size_t curr_e) mutable { + std::advance(beg1, curr_b - prev_e); + std::advance(beg2, curr_b - prev_e); + for(size_t x=curr_b; x lock(*mutex); + r = bop_r(std::move(r), std::move(sum)); + }); + + (++w == W) ? task() : rt.silent_async(task); + } + } + }; +} + + +// Function: make_reduce_by_index_task +template +auto make_reduce_by_index_task(R range, T& init, L lop, G gop, P part = P()) { + + using range_type = std::decay_t>; + + return [=, &init] (Runtime& rt) mutable { + + // fetch the iterator values + range_type r = range; + + // nothing to be done if the range is invalid + if(is_index_range_invalid(r.begin(), r.end(), r.step_size())) { + return; + } + + size_t W = rt.executor().num_workers(); + size_t N = r.size(); + + if(N == 0) { + return; + } + + // only myself - no need to spawn another graph + if(W <= 1 || N <= part.chunk_size()) { + part([=, &init] () mutable { init = lop(r, std::move(init)); })(); + return; + } + + if(N < W) { + W = N; + } + + auto mutex = std::make_shared(); + + // static partitioner + if constexpr(part.type() == PartitionerType::STATIC) { + + for(size_t w=0, curr_b=0; w tmp; + + // loop reduce + part.loop(N, W, curr_b, chunk_size, [=, &tmp](size_t part_b, size_t part_e) mutable { + tmp = lop(r.unravel(part_b, part_e), std::move(tmp)); + }); + + // final reduce - tmp is guaranteed to have value + // assert(tmp.has_value()); + std::lock_guard lock(*mutex); + init = gop(std::move(init), std::move(*tmp)); + }); + + (++w == W || (curr_b += chunk_size) >= N) ? task() : rt.silent_async(task); + } + } + // dynamic partitioner + else { + auto next = std::make_shared>(0); + + for(size_t w=0; w tmp; + + // loop reduce + part.loop(N, W, *next, [=, &tmp](size_t part_b, size_t part_e) mutable { + tmp = lop(r.unravel(part_b, part_e), std::move(tmp)); + }); + + // final reduce - need to check if the running total has value since + // this is a dynamic scheduler; the worker may not actually acquire any work + if(tmp) { + std::lock_guard lock(*mutex); + init = gop(std::move(init), std::move(*tmp)); + } + }); + (++w == W) ? task() : rt.silent_async(task); + } + } + }; +} + +// ------------------------------------------------------------------------------------------------ +// default reduction +// ------------------------------------------------------------------------------------------------ + +// Function: reduce +template +Task FlowBuilder::reduce(B beg, E end, T& init, O bop, P part) { + return emplace(make_reduce_task(beg, end, init, bop, part)); +} + +// ------------------------------------------------------------------------------------------------ +// default transform and reduction +// ------------------------------------------------------------------------------------------------ + +// Function: transform_reduce +template +Task FlowBuilder::transform_reduce( + B beg, E end, T& init, BOP bop, UOP uop, P part +) { + return emplace(make_transform_reduce_task(beg, end, init, bop, uop, part)); +} + +// Function: transform_reduce +template +requires (!PartitionerLike>) +Task FlowBuilder::transform_reduce( + B1 beg1, E1 end1, B2 beg2, T& init, BOP_R bop_r, BOP_T bop_t, P part +) { + return emplace(make_transform_reduce_task(beg1, end1, beg2, init, bop_r, bop_t, part)); +} + +// ------------------------------------------------------------------------------------------------ +// default reduce_by_key +// ------------------------------------------------------------------------------------------------ + +// Function: make_index_reduce_task +template +Task FlowBuilder::reduce_by_index(R range, T& init, L lop, G gop, P part) { + return emplace(make_reduce_by_index_task(range, init, lop, gop, part)); +} + +} // end of namespace tf ------------------------------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/algorithm/scan.hpp b/projects/hipfile/third-party/taskflow/algorithm/scan.hpp new file mode 100644 index 000000000000..d5d6abba5466 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/algorithm/scan.hpp @@ -0,0 +1,699 @@ +#pragma once + +#include "../taskflow.hpp" + +namespace tf { + +/* + +Block-parallel scan algorithm: + +----------------------------------------------------------------- +| block 1 | block 2 | block 3 | block 4 | +----------------------------------------------------------------- + + ----------------------------- + | B1 | B2 | B3 | B4 | // scan block sum to auxilinary array + ----------------------------- + | | + v v + ----------------------------- + | B1 | B2 | B3 | B4 | // scan block sums + ----------------------------- + | + | // add scanned block sum i to all + | // values of scanned block i+1 + v +----------------------------------------------------------------- +| block 1 | block 2 | block 3 | block 4 | +----------------------------------------------------------------- + +Example OpenMP implementation for inclusive scan: + +void inclusive_scan(std::vector& data) { + + int n = data.size(); + int num_threads; + + #pragma omp parallel + { + num_threads = omp_get_num_threads(); + } + + std::vector partial_sums(num_threads, 0); + + // Step 1: Up-sweep + #pragma omp parallel + { + int tid = omp_get_thread_num(); + int chunk_size = (n + num_threads - 1) / num_threads; + int start = tid * chunk_size; + int end = std::min(start + chunk_size, n); + + // Compute partial sum + for (int i = start + 1; i < end; ++i) { + data[i] += data[i - 1]; + } + partial_sums[tid] = data[end - 1]; + } + + // Step 2: Propagate partial sums + for (int i = 1; i < num_threads; ++i) { + partial_sums[i] += partial_sums[i - 1]; + } + + // Step 3: Down-sweep + #pragma omp parallel + { + int tid = omp_get_thread_num(); + int chunk_size = (n + num_threads - 1) / num_threads; + int start = tid * chunk_size; + int end = std::min(start + chunk_size, n); + + // Adjust with partial sums + if (tid > 0) { + for (int i = start; i < end; ++i) { + data[i] += partial_sums[tid - 1]; + } + } + } +} + +*/ + +namespace detail { + +template +struct ScanData { + + ScanData(size_t N, size_t c) : buf(N), counter(c) {} + + std::vector> buf; + std::atomic counter; +}; + +// down scan task +template +auto make_dscan_task( + std::shared_ptr sdata, + I d_beg, + B bop, + size_t w, + size_t block_size +) { + return [=, sdata=std::move(sdata)]() mutable { + for(size_t i=0; ibuf[w-1].data, *d_beg); + } + }; +} + +// middle scan task +template +auto make_mscan_task(std::shared_ptr sdata, B bop) { + return [=, sdata=std::move(sdata)](){ + for(size_t i=1; ibuf.size(); i++) { + sdata->buf[i].data = bop(sdata->buf[i-1].data, sdata->buf[i].data); + } + }; +} + +template +void scan_loop( + tf::Runtime& rt, + std::atomic& counter, + BufferT& buf, + B bop, + Iterator d_beg, + size_t W, + size_t w, + size_t chunk_size +){ + // whoever finishes the last performs global scan + if(counter.fetch_add(1, std::memory_order_acq_rel) == W-1) { + for(size_t i=1; i +auto make_inclusive_scan_task(B first, E last, D d_first, BOP bop) { + + using namespace std::string_literals; + + using B_t = std::decay_t>; + using E_t = std::decay_t>; + using D_t = std::decay_t>; + using value_type = typename std::iterator_traits::value_type; + + return [=] (Runtime& rt) mutable { + + // fetch the stateful values + B_t s_beg = first; + E_t s_end = last; + D_t d_beg = d_first; + + if(s_beg == s_end) { + return; + } + + size_t W = rt.executor().num_workers(); + size_t N = std::distance(s_beg, s_end); + + // only myself - no need to spawn another graph + if(W <= 1 || N <= 2) { + std::inclusive_scan(s_beg, s_end, d_beg, bop); + return; + } + + if(N < W) { + W = N; + } + + std::vector> buf(W); + std::atomic counter(0); + + size_t Q = N/W; + size_t R = N%W; + + for(size_t w=0, curr_b=0, chunk_size; w +auto make_inclusive_scan_task(B first, E last, D d_first, BOP bop, T init) { + + using namespace std::string_literals; + + using B_t = std::decay_t>; + using E_t = std::decay_t>; + using D_t = std::decay_t>; + using value_type = typename std::iterator_traits::value_type; + + return [=] (Runtime& rt) mutable { + + // fetch the stateful values + B_t s_beg = first; + E_t s_end = last; + D_t d_beg = d_first; + + if(s_beg == s_end) { + return; + } + + size_t W = rt.executor().num_workers(); + size_t N = std::distance(s_beg, s_end); + + // only myself - no need to spawn another graph + if(W <= 1 || N <= 2) { + std::inclusive_scan(s_beg, s_end, d_beg, bop, init); + return; + } + + if(N < W) { + W = N; + } + + std::vector> buf(W); + std::atomic counter(0); + + // set up the initial value for the first worker + buf[0].data = std::move(init); + + size_t Q = N/W; + size_t R = N%W; + + for(size_t w=0, curr_b=0, chunk_size; w +auto make_transform_inclusive_scan_task( + B first, E last, D d_first, BOP bop, UOP uop +) { + + using namespace std::string_literals; + + using B_t = std::decay_t>; + using E_t = std::decay_t>; + using D_t = std::decay_t>; + using value_type = typename std::iterator_traits::value_type; + + return [=] (Runtime& rt) mutable { + + // fetch the stateful values + B_t s_beg = first; + E_t s_end = last; + D_t d_beg = d_first; + + if(s_beg == s_end) { + return; + } + + size_t W = rt.executor().num_workers(); + size_t N = std::distance(s_beg, s_end); + + // only myself - no need to spawn another graph + if(W <= 1 || N <= 2) { + std::transform_inclusive_scan(s_beg, s_end, d_beg, bop, uop); + return; + } + + if(N < W) { + W = N; + } + + std::vector> buf(W); + std::atomic counter(0); + + size_t Q = N/W; + size_t R = N%W; + + for(size_t w=0, curr_b=0, chunk_size; w +auto make_transform_inclusive_scan_task( + B first, E last, D d_first, BOP bop, UOP uop, T init +) { + + using namespace std::string_literals; + + using B_t = std::decay_t>; + using E_t = std::decay_t>; + using D_t = std::decay_t>; + using value_type = typename std::iterator_traits::value_type; + + return [=] (Runtime& rt) mutable { + + // fetch the stateful values + B_t s_beg = first; + E_t s_end = last; + D_t d_beg = d_first; + + if(s_beg == s_end) { + return; + } + + size_t W = rt.executor().num_workers(); + size_t N = std::distance(s_beg, s_end); + + // only myself - no need to spawn another graph + if(W <= 1 || N <= 2) { + std::transform_inclusive_scan(s_beg, s_end, d_beg, bop, uop, init); + return; + } + + if(N < W) { + W = N; + } + std::vector> buf(W); + std::atomic counter(0); + + // set up the initial value for the first worker + buf[0].data = std::move(init); + + size_t Q = N/W; + size_t R = N%W; + + for(size_t w=0, curr_b=0, chunk_size; w +auto make_exclusive_scan_task( + B first, E last, D d_first, T init, BOP bop +) { + + using namespace std::string_literals; + + using B_t = std::decay_t>; + using E_t = std::decay_t>; + using D_t = std::decay_t>; + using value_type = typename std::iterator_traits::value_type; + + return [=] (Runtime& rt) mutable { + + // fetch the stateful values + B_t s_beg = first; + E_t s_end = last; + D_t d_beg = d_first; + + if(s_beg == s_end) { + return; + } + + size_t W = rt.executor().num_workers(); + size_t N = std::distance(s_beg, s_end); + + // only myself - no need to spawn another graph + if(W <= 1 || N <= 2) { + std::exclusive_scan(s_beg, s_end, d_beg, init, bop); + return; + } + + if(N < W) { + W = N; + } + + std::vector> buf(W); + std::atomic counter(0); + + size_t Q = N/W; + size_t R = N%W; + + // fetch the init value + auto s_beg_temp = s_beg; + for(size_t w=0, curr_b=0, chunk_size; w +auto make_transform_exclusive_scan_task( + B first, E last, D d_first, T init, BOP bop, UOP uop +) { + + using namespace std::string_literals; + + using B_t = std::decay_t>; + using E_t = std::decay_t>; + using D_t = std::decay_t>; + using value_type = typename std::iterator_traits::value_type; + + return [=] (Runtime& rt) mutable { + + // fetch the stateful values + B_t s_beg = first; + E_t s_end = last; + D_t d_beg = d_first; + + if(s_beg == s_end) { + return; + } + + size_t W = rt.executor().num_workers(); + size_t N = std::distance(s_beg, s_end); + + // only myself - no need to spawn another graph + if(W <= 1 || N <= 2) { + std::transform_exclusive_scan(s_beg, s_end, d_beg, init, bop, uop); + return; + } + + if(N < W) { + W = N; + } + + std::vector> buf(W); + std::atomic counter(0); + + size_t Q = N/W; + size_t R = N%W; + // fetch the init value + auto s_beg_temp = s_beg; + for(size_t w=0, curr_b=0, chunk_size; w +Task FlowBuilder::inclusive_scan(B first, E last, D d_first, BOP bop) { + return emplace(make_inclusive_scan_task(first, last, d_first, bop)); +} + +// Function: inclusive_scan +template +Task FlowBuilder::inclusive_scan(B first, E last, D d_first, BOP bop, T init) { + return emplace(make_inclusive_scan_task(first, last, d_first, bop, init)); +} + +// ---------------------------------------------------------------------------- +// Transform Inclusive Scan +// ---------------------------------------------------------------------------- + +// Function: transform_inclusive_scan +template +Task FlowBuilder::transform_inclusive_scan( + B first, E last, D d_first, BOP bop, UOP uop +) { + return emplace(make_transform_inclusive_scan_task( + first, last, d_first, bop, uop + )); +} + +// Function: transform_inclusive_scan +template +Task FlowBuilder::transform_inclusive_scan( + B first, E last, D d_first, BOP bop, UOP uop, T init +) { + return emplace(make_transform_inclusive_scan_task( + first, last, d_first, bop, uop, init + )); +} + +// ---------------------------------------------------------------------------- +// Exclusive Scan +// ---------------------------------------------------------------------------- + +// Function: exclusive_scan +template +Task FlowBuilder::exclusive_scan(B first, E last, D d_first, T init, BOP bop) { + return emplace(make_exclusive_scan_task(first, last, d_first, init, bop)); +} + +// ---------------------------------------------------------------------------- +// Transform Exclusive Scan +// ---------------------------------------------------------------------------- + +// Function: transform_exclusive_scan +template +Task FlowBuilder::transform_exclusive_scan( + B first, E last, D d_first, T init, BOP bop, UOP uop +) { + return emplace(make_transform_exclusive_scan_task( + first, last, d_first, init, bop, uop + )); +} + +} // end of namespace tf ----------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/algorithm/sort.hpp b/projects/hipfile/third-party/taskflow/algorithm/sort.hpp new file mode 100644 index 000000000000..d3fa8d5c50d4 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/algorithm/sort.hpp @@ -0,0 +1,649 @@ +#pragma once + +#include "../taskflow.hpp" + +namespace tf::detail { + +// threshold whether or not to perform parallel sort +template +constexpr size_t parallel_sort_cutoff() { + + //using value_type = std::decay_t())>; + using value_type = typename std::iterator_traits::value_type; + + constexpr size_t object_size = sizeof(value_type); + + if constexpr(std::is_same_v) { + return 65536 / sizeof(std::string); + } + else { + if constexpr(object_size < 16) return 4096; + else if constexpr(object_size < 32) return 2048; + else if constexpr(object_size < 64) return 1024; + else if constexpr(object_size < 128) return 768; + else if constexpr(object_size < 256) return 512; + else if constexpr(object_size < 512) return 256; + else return 128; + } +} + +// ---------------------------------------------------------------------------- +// pattern-defeating quick sort (pdqsort) +// https://github.com/orlp/pdqsort/ +// ---------------------------------------------------------------------------- + +template +inline T* align_cacheline(T* p) { +#if defined(UINTPTR_MAX) && __cplusplus >= 201103L + std::uintptr_t ip = reinterpret_cast(p); +#else + std::size_t ip = reinterpret_cast(p); +#endif + //ip = (ip + cacheline_size - 1) & -cacheline_size; + ip = (ip + cacheline_size - 1) & ~(cacheline_size - 1); + return reinterpret_cast(ip); +} + +template +inline void swap_offsets( + Iter first, Iter last, + unsigned char* offsets_l, unsigned char* offsets_r, + size_t num, bool use_swaps +) { + typedef typename std::iterator_traits::value_type T; + if (use_swaps) { + // This case is needed for the descending distribution, where we need + // to have proper swapping for pdqsort to remain O(n). + for (size_t i = 0; i < num; ++i) { + std::iter_swap(first + offsets_l[i], last - offsets_r[i]); + } + } else if (num > 0) { + Iter l = first + offsets_l[0]; Iter r = last - offsets_r[0]; + T tmp(std::move(*l)); *l = std::move(*r); + for (size_t i = 1; i < num; ++i) { + l = first + offsets_l[i]; *r = std::move(*l); + r = last - offsets_r[i]; *l = std::move(*r); + } + *r = std::move(tmp); + } +} + +// Sorts [begin, end) using insertion sort with the given comparison function. +template +void insertion_sort(RandItr begin, RandItr end, Compare comp) { + + using T = typename std::iterator_traits::value_type; + + if (begin == end) { + return; + } + + for (RandItr cur = begin + 1; cur != end; ++cur) { + + RandItr shift = cur; + RandItr shift_1 = cur - 1; + + // Compare first to avoid 2 moves for an element + // already positioned correctly. + if (comp(*shift, *shift_1)) { + T tmp = std::move(*shift); + do { + *shift-- = std::move(*shift_1); + }while (shift != begin && comp(tmp, *--shift_1)); + *shift = std::move(tmp); + } + } +} + +// Sorts [begin, end) using insertion sort with the given comparison function. +// Assumes *(begin - 1) is an element smaller than or equal to any element +// in [begin, end). +template +void unguarded_insertion_sort(RandItr begin, RandItr end, Compare comp) { + + using T = typename std::iterator_traits::value_type; + + if (begin == end) { + return; + } + + for (RandItr cur = begin + 1; cur != end; ++cur) { + RandItr shift = cur; + RandItr shift_1 = cur - 1; + + // Compare first so we can avoid 2 moves + // for an element already positioned correctly. + if (comp(*shift, *shift_1)) { + T tmp = std::move(*shift); + + do { + *shift-- = std::move(*shift_1); + }while (comp(tmp, *--shift_1)); + + *shift = std::move(tmp); + } + } +} + +// Attempts to use insertion sort on [begin, end). +// Will return false if more than +// partial_insertion_sort_limit elements were moved, +// and abort sorting. Otherwise it will successfully sort and return true. +template +bool partial_insertion_sort(RandItr begin, RandItr end, Compare comp) { + + using T = typename std::iterator_traits::value_type; + using D = typename std::iterator_traits::difference_type; + + // When we detect an already sorted partition, attempt an insertion sort + // that allows this amount of element moves before giving up. + constexpr auto partial_insertion_sort_limit = D{8}; + + if (begin == end) return true; + + auto limit = D{0}; + + for (RandItr cur = begin + 1; cur != end; ++cur) { + + if (limit > partial_insertion_sort_limit) { + return false; + } + + RandItr shift = cur; + RandItr shift_1 = cur - 1; + + // Compare first so we can avoid 2 moves + // for an element already positioned correctly. + if (comp(*shift, *shift_1)) { + T tmp = std::move(*shift); + + do { + *shift-- = std::move(*shift_1); + }while (shift != begin && comp(tmp, *--shift_1)); + + *shift = std::move(tmp); + limit += cur - shift; + } + } + + return true; +} + +// Partitions [begin, end) around pivot *begin using comparison function comp. Elements equal +// to the pivot are put in the right-hand partition. Returns the position of the pivot after +// partitioning and whether the passed sequence already was correctly partitioned. Assumes the +// pivot is a median of at least 3 elements and that [begin, end) is at least +// insertion_sort_threshold long. Uses branchless partitioning. +template +std::pair partition_right_branchless(Iter begin, Iter end, Compare comp) { + + typedef typename std::iterator_traits::value_type T; + + constexpr size_t block_size = 64; + constexpr size_t cacheline_size = 64; + + // Move pivot into local for speed. + T pivot(std::move(*begin)); + Iter first = begin; + Iter last = end; + + // Find the first element greater than or equal than the pivot (the median of 3 guarantees + // this exists). + while (comp(*++first, pivot)); + + // Find the first element strictly smaller than the pivot. We have to guard this search if + // there was no element before *first. + if (first - 1 == begin) while (first < last && !comp(*--last, pivot)); + else while ( !comp(*--last, pivot)); + + // If the first pair of elements that should be swapped to partition are the same element, + // the passed in sequence already was correctly partitioned. + bool already_partitioned = first >= last; + if (!already_partitioned) { + std::iter_swap(first, last); + ++first; + + // The following branchless partitioning is derived from "BlockQuicksort: How Branch + // Mispredictions don't affect Quicksort" by Stefan Edelkamp and Armin Weiss, but + // heavily micro-optimized. + unsigned char offsets_l_storage[block_size + cacheline_size]; + unsigned char offsets_r_storage[block_size + cacheline_size]; + unsigned char* offsets_l = align_cacheline(offsets_l_storage); + unsigned char* offsets_r = align_cacheline(offsets_r_storage); + + Iter offsets_l_base = first; + Iter offsets_r_base = last; + size_t num_l, num_r, start_l, start_r; + num_l = num_r = start_l = start_r = 0; + + while (first < last) { + // Fill up offset blocks with elements that are on the wrong side. + // First we determine how much elements are considered for each offset block. + size_t num_unknown = last - first; + size_t left_split = num_l == 0 ? (num_r == 0 ? num_unknown / 2 : num_unknown) : 0; + size_t right_split = num_r == 0 ? (num_unknown - left_split) : 0; + + // Fill the offset blocks. + if (left_split >= block_size) { + for (unsigned char i = 0; i < block_size;) { + offsets_l[num_l] = i++; num_l += !comp(*first, pivot); ++first; + offsets_l[num_l] = i++; num_l += !comp(*first, pivot); ++first; + offsets_l[num_l] = i++; num_l += !comp(*first, pivot); ++first; + offsets_l[num_l] = i++; num_l += !comp(*first, pivot); ++first; + offsets_l[num_l] = i++; num_l += !comp(*first, pivot); ++first; + offsets_l[num_l] = i++; num_l += !comp(*first, pivot); ++first; + offsets_l[num_l] = i++; num_l += !comp(*first, pivot); ++first; + offsets_l[num_l] = i++; num_l += !comp(*first, pivot); ++first; + } + } else { + for (unsigned char i = 0; i < left_split;) { + offsets_l[num_l] = i++; num_l += !comp(*first, pivot); ++first; + } + } + + if (right_split >= block_size) { + for (unsigned char i = 0; i < block_size;) { + offsets_r[num_r] = ++i; num_r += comp(*--last, pivot); + offsets_r[num_r] = ++i; num_r += comp(*--last, pivot); + offsets_r[num_r] = ++i; num_r += comp(*--last, pivot); + offsets_r[num_r] = ++i; num_r += comp(*--last, pivot); + offsets_r[num_r] = ++i; num_r += comp(*--last, pivot); + offsets_r[num_r] = ++i; num_r += comp(*--last, pivot); + offsets_r[num_r] = ++i; num_r += comp(*--last, pivot); + offsets_r[num_r] = ++i; num_r += comp(*--last, pivot); + } + } else { + for (unsigned char i = 0; i < right_split;) { + offsets_r[num_r] = ++i; num_r += comp(*--last, pivot); + } + } + + // Swap elements and update block sizes and first/last boundaries. + size_t num = std::min(num_l, num_r); + swap_offsets( + offsets_l_base, offsets_r_base, + offsets_l + start_l, offsets_r + start_r, + num, num_l == num_r + ); + num_l -= num; num_r -= num; + start_l += num; start_r += num; + + if (num_l == 0) { + start_l = 0; + offsets_l_base = first; + } + + if (num_r == 0) { + start_r = 0; + offsets_r_base = last; + } + } + + // We have now fully identified [first, last)'s proper position. Swap the last elements. + if (num_l) { + offsets_l += start_l; + while (num_l--) std::iter_swap(offsets_l_base + offsets_l[num_l], --last); + first = last; + } + if (num_r) { + offsets_r += start_r; + while (num_r--) std::iter_swap(offsets_r_base - offsets_r[num_r], first), ++first; + last = first; + } + } + + // Put the pivot in the right place. + Iter pivot_pos = first - 1; + *begin = std::move(*pivot_pos); + *pivot_pos = std::move(pivot); + + return std::make_pair(pivot_pos, already_partitioned); +} + +// Partitions [begin, end) around pivot *begin using comparison function comp. +// Elements equal to the pivot are put in the right-hand partition. +// Returns the position of the pivot after partitioning and whether the passed +// sequence already was correctly partitioned. +// Assumes the pivot is a median of at least 3 elements and that [begin, end) +// is at least insertion_sort_threshold long. +template +std::pair partition_right(Iter begin, Iter end, Compare comp) { + + using T = typename std::iterator_traits::value_type; + + // Move pivot into local for speed. + T pivot(std::move(*begin)); + + Iter first = begin; + Iter last = end; + + // Find the first element greater than or equal than the pivot + // (the median of 3 guarantees/ this exists). + while (comp(*++first, pivot)); + + // Find the first element strictly smaller than the pivot. + // We have to guard this search if there was no element before *first. + if (first - 1 == begin) while (first < last && !comp(*--last, pivot)); + else while (!comp(*--last, pivot)); + + // If the first pair of elements that should be swapped to partition + // are the same element, the passed in sequence already was correctly + // partitioned. + bool already_partitioned = first >= last; + + // Keep swapping pairs of elements that are on the wrong side of the pivot. + // Previously swapped pairs guard the searches, + // which is why the first iteration is special-cased above. + while (first < last) { + std::iter_swap(first, last); + while (comp(*++first, pivot)); + while (!comp(*--last, pivot)); + } + + // Put the pivot in the right place. + Iter pivot_pos = first - 1; + *begin = std::move(*pivot_pos); + *pivot_pos = std::move(pivot); + + return std::make_pair(pivot_pos, already_partitioned); +} + +// Similar function to the one above, except elements equal to the pivot +// are put to the left of the pivot and it doesn't check or return +// if the passed sequence already was partitioned. +// Since this is rarely used (the many equal case), +// and in that case pdqsort already has O(n) performance, +// no block quicksort is applied here for simplicity. +template +RandItr partition_left(RandItr begin, RandItr end, Compare comp) { + + using T = typename std::iterator_traits::value_type; + + T pivot(std::move(*begin)); + + RandItr first = begin; + RandItr last = end; + + while (comp(pivot, *--last)); + + if (last + 1 == end) { + while (first < last && !comp(pivot, *++first)); + } + else { + while (!comp(pivot, *++first)); + } + + while (first < last) { + std::iter_swap(first, last); + while (comp(pivot, *--last)); + while (!comp(pivot, *++first)); + } + + RandItr pivot_pos = last; + *begin = std::move(*pivot_pos); + *pivot_pos = std::move(pivot); + + return pivot_pos; +} + +template +void parallel_pdqsort( + Runtime& rt, + Iter begin, Iter end, Compare comp, + int bad_allowed, bool leftmost = true +) { + + // Partitions below this size are sorted sequentially + constexpr auto cutoff = parallel_sort_cutoff(); + + // Partitions below this size are sorted using insertion sort + constexpr auto insertion_sort_threshold = 24; + + // Partitions above this size use Tukey's ninther to select the pivot. + constexpr auto ninther_threshold = 128; + + //using diff_t = typename std::iterator_traits::difference_type; + + // Use a while loop for tail recursion elimination. + while (true) { + + //diff_t size = end - begin; + size_t size = end - begin; + + // Insertion sort is faster for small arrays. + if (size < insertion_sort_threshold) { + if (leftmost) { + insertion_sort(begin, end, comp); + } + else { + unguarded_insertion_sort(begin, end, comp); + } + return; + } + + if(size <= cutoff) { + std::sort(begin, end, comp); + return; + } + + // Choose pivot as median of 3 or pseudomedian of 9. + //diff_t s2 = size / 2; + size_t s2 = size >> 1; + if (size > ninther_threshold) { + sort3(begin, begin + s2, end - 1, comp); + sort3(begin + 1, begin + (s2 - 1), end - 2, comp); + sort3(begin + 2, begin + (s2 + 1), end - 3, comp); + sort3(begin + (s2 - 1), begin + s2, begin + (s2 + 1), comp); + std::iter_swap(begin, begin + s2); + } + else { + sort3(begin + s2, begin, end - 1, comp); + } + + // If *(begin - 1) is the end of the right partition + // of a previous partition operation, there is no element in [begin, end) + // that is smaller than *(begin - 1). + // Then if our pivot compares equal to *(begin - 1) we change strategy, + // putting equal elements in the left partition, + // greater elements in the right partition. + // We do not have to recurse on the left partition, + // since it's sorted (all equal). + if (!leftmost && !comp(*(begin - 1), *begin)) { + begin = partition_left(begin, end, comp) + 1; + continue; + } + + // Partition and get results. + const auto pair = Branchless ? partition_right_branchless(begin, end, comp) : + partition_right(begin, end, comp); + + const auto pivot_pos = pair.first; + const auto already_partitioned = pair.second; + + // Check for a highly unbalanced partition. + //diff_t l_size = pivot_pos - begin; + //diff_t r_size = end - (pivot_pos + 1); + const size_t l_size = pivot_pos - begin; + const size_t r_size = end - (pivot_pos + 1); + const bool highly_unbalanced = l_size < size / 8 || r_size < size / 8; + + // If we got a highly unbalanced partition we shuffle elements + // to break many patterns. + if (highly_unbalanced) { + // If we had too many bad partitions, switch to heapsort + // to guarantee O(n log n). + if (--bad_allowed == 0) { + std::make_heap(begin, end, comp); + std::sort_heap(begin, end, comp); + return; + } + + if (l_size >= insertion_sort_threshold) { + std::iter_swap(begin, begin + l_size / 4); + std::iter_swap(pivot_pos - 1, pivot_pos - l_size / 4); + if (l_size > ninther_threshold) { + std::iter_swap(begin + 1, begin + (l_size / 4 + 1)); + std::iter_swap(begin + 2, begin + (l_size / 4 + 2)); + std::iter_swap(pivot_pos - 2, pivot_pos - (l_size / 4 + 1)); + std::iter_swap(pivot_pos - 3, pivot_pos - (l_size / 4 + 2)); + } + } + + if (r_size >= insertion_sort_threshold) { + std::iter_swap(pivot_pos + 1, pivot_pos + (1 + r_size / 4)); + std::iter_swap(end - 1, end - r_size / 4); + if (r_size > ninther_threshold) { + std::iter_swap(pivot_pos + 2, pivot_pos + (2 + r_size / 4)); + std::iter_swap(pivot_pos + 3, pivot_pos + (3 + r_size / 4)); + std::iter_swap(end - 2, end - (1 + r_size / 4)); + std::iter_swap(end - 3, end - (2 + r_size / 4)); + } + } + } + // decently balanced + else { + // sequence try to use insertion sort. + if (already_partitioned && + partial_insertion_sort(begin, pivot_pos, comp) && + partial_insertion_sort(pivot_pos + 1, end, comp) + ) { + return; + } + } + + // Sort the left partition first using recursion and + // do tail recursion elimination for the right-hand partition. + // here we need to copy runtime so it stays alive during the sort recursion + rt.silent_async([=] () mutable { + parallel_pdqsort( + rt, begin, pivot_pos, comp, bad_allowed, leftmost + ); + }); + begin = pivot_pos + 1; + leftmost = false; + } +} + +// ---------------------------------------------------------------------------- +// 3-way quick sort +// ---------------------------------------------------------------------------- + +// 3-way quick sort +template +void parallel_3wqsort(Runtime& rt, RandItr first, RandItr last, C compare) { + + using namespace std::string_literals; + + constexpr auto cutoff = parallel_sort_cutoff(); + + sort_partition: + + if(static_cast(last - first) < cutoff) { + std::sort(first, last+1, compare); + return; + } + + auto m = pseudo_median_of_nine(first, last, compare); + + if(m != first) { + std::iter_swap(first, m); + } + + auto l = first; + auto r = last; + auto f = std::next(first, 1); + bool is_swapped_l = false; + bool is_swapped_r = false; + + while(f <= r) { + if(compare(*f, *l)) { + is_swapped_l = true; + std::iter_swap(l, f); + l++; + f++; + } + else if(compare(*l, *f)) { + is_swapped_r = true; + std::iter_swap(r, f); + r--; + } + else { + f++; + } + } + + if(l - first > 1 && is_swapped_l) { + rt.silent_async([=] () mutable { + parallel_3wqsort(rt, first, l-1, compare); + }); + } + + if(last - r > 1 && is_swapped_r) { + first = r+1; + goto sort_partition; + } +} + +} // end of namespace tf::detail --------------------------------------------- + +namespace tf { + +// Function: make_sort_task +template +auto make_sort_task(B b, E e, C cmp) { + + return [b, e, cmp] (Runtime& rt) mutable { + + using B_t = std::decay_t>; + using E_t = std::decay_t>; + + // fetch the iterator values + B_t beg = b; + E_t end = e; + + if(beg == end) { + return; + } + + size_t W = rt.executor().num_workers(); + size_t N = std::distance(beg, end); + + // only myself - no need to spawn another graph + if(W <= 1 || N <= detail::parallel_sort_cutoff()) { + std::sort(beg, end, cmp); + return; + } + + //PreemptionGuard preemption_guard(rt); + + //detail::parallel_3wqsort(rt, beg, end-1, cmp); + detail::parallel_pdqsort> && + std::is_arithmetic_v::value_type> + >(rt, beg, end, cmp, static_cast(log2(size_t(end - beg)))); + }; +} + +template +auto make_sort_task(B beg, E end) { + using value_type = std::decay_t())>; + return make_sort_task(beg, end, std::less{}); +} + +// ---------------------------------------------------------------------------- +// tf::Taskflow::sort +// ---------------------------------------------------------------------------- + +// Function: sort +template +Task FlowBuilder::sort(B beg, E end, C cmp) { + return emplace(make_sort_task(beg, end, cmp)); +} + +// Function: sort +template +Task FlowBuilder::sort(B beg, E end) { + return emplace(make_sort_task(beg, end)); +} + +} // namespace tf ------------------------------------------------------------ diff --git a/projects/hipfile/third-party/taskflow/algorithm/transform.hpp b/projects/hipfile/third-party/taskflow/algorithm/transform.hpp new file mode 100644 index 000000000000..cb74bad2cb7e --- /dev/null +++ b/projects/hipfile/third-party/taskflow/algorithm/transform.hpp @@ -0,0 +1,184 @@ +#pragma once + +#include "../taskflow.hpp" + +namespace tf { + +// Function: make_transform_task +template +auto make_transform_task(B first1, E last1, O d_first, C c, P part = P()) { + + using namespace std::string_literals; + + using B_t = std::decay_t>; + using E_t = std::decay_t>; + using O_t = std::decay_t>; + + return [=] (Runtime& rt) mutable { + + // fetch the stateful values + B_t beg = first1; + E_t end = last1; + O_t d_beg = d_first; + + size_t W = rt.executor().num_workers(); + size_t N = std::distance(beg, end); + + if(N == 0) { + return; + } + + // only myself - no need to spawn another graph + if(W <= 1 || N <= part.chunk_size()) { + part([=]() mutable { std::transform(beg, end, d_beg, c); })(); + return; + } + + if(N < W) { + W = N; + } + + // static partitioner + if constexpr(part.type() == PartitionerType::STATIC) { + for(size_t w=0, curr_b=0; w= N) ? task() : rt.silent_async(task); + } + } + // dynamic partitioner + else { + auto next = std::make_shared>(0); + for(size_t w=0; w +requires (!PartitionerLike>) +auto make_transform_task(B1 first1, E1 last1, B2 first2, O d_first, C c, P part = P()) { + + using namespace std::string_literals; + + using B1_t = std::decay_t>; + using E1_t = std::decay_t>; + using B2_t = std::decay_t>; + using O_t = std::decay_t>; + + return [=] (Runtime& rt) mutable { + + // fetch the stateful values + B1_t beg1 = first1; + E1_t end1 = last1; + B2_t beg2 = first2; + O_t d_beg = d_first; + + size_t W = rt.executor().num_workers(); + size_t N = std::distance(beg1, end1); + + if(N == 0) { + return; + } + + // only myself - no need to spawn another graph + if(W <= 1 || N <= part.chunk_size()) { + part([=]() mutable { std::transform(beg1, end1, beg2, d_beg, c); })(); + return; + } + + if(N < W) { + W = N; + } + + // static partitioner + if constexpr(part.type() == PartitionerType::STATIC) { + for(size_t w=0, curr_b=0; w= N) ? task() : rt.silent_async(task); + } + } + // dynamic partitioner + else { + auto next = std::make_shared>(0); + for(size_t w=0; w +Task FlowBuilder::transform(B first1, E last1, O d_first, C c, P part) { + return emplace( + make_transform_task(first1, last1, d_first, c, part) + ); +} + +// ---------------------------------------------------------------------------- +// transform2 +// ---------------------------------------------------------------------------- + +// Function: transform +template +requires (!PartitionerLike>) +Task FlowBuilder::transform( + B1 first1, E1 last1, B2 first2, O d_first, C c, P part +) { + return emplace(make_transform_task( + first1, last1, first2, d_first, c, part + )); +} + + +} // end of namespace tf ----------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/core/async.hpp b/projects/hipfile/third-party/taskflow/core/async.hpp new file mode 100644 index 000000000000..fab729866a39 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/core/async.hpp @@ -0,0 +1,437 @@ +#pragma once + +#include "executor.hpp" +#include "runtime.hpp" + +namespace tf { + +// ---------------------------------------------------------------------------- +// Async Helper Methods +// ---------------------------------------------------------------------------- + +// Procedure: _schedule_async_task +template +void Executor::_schedule_async_task(ArgsT&&... args) { + // caller is a worker of the executor + if(auto w = this_worker(); w) { + // We don't do per-worker cache as it can cause bugs in corun that are very difficult to + // track. For example, when a worker invokes an async task and then immediately enter corun, + // it becomes very difficult to get the task out of its cache correctly. + _schedule(*w, animate(std::forward(args)...)); + } + // caller is a freelance thread + else{ + _schedule(animate(std::forward(args)...)); + } +} + +// Procedure: _schedule_dependent_async_task +template +AsyncTask Executor::_schedule_dependent_async_task(I first, I last, size_t num_predecessors, ArgsT&&... args) { + + // We need to create an async-task first to acquire an ownership. + AsyncTask task(animate(std::forward(args)...)); + + for(; first != last; first++) { + auto&& x = *first; + + if constexpr (std::is_pointer_v>) { + _process_dependent_async(task._node, *x, num_predecessors); + } else { + _process_dependent_async(task._node, x, num_predecessors); + } + } + + if(num_predecessors == 0) { + // caller is a worker of the executor + if(auto w = this_worker(); w) { + // We don't do per-worker cache as it can cause bugs in corun that are very difficult to + // track. For example, when a worker invokes an async task and then immediately enter corun, + // it becomes very difficult to get the task out of its cache correctly. + _schedule(*w, task._node); + } + // caller is a freelance thread + else { + _schedule(task._node); + } + } + + return task; +} + +// Procedure: _tear_down_async +inline void Executor::_tear_down_async(Worker& worker, Node* node, Node*& cache) { + + /* + ------------------------------------------------------------------------------------------------- + node->_topology | node->_parent | secenario + ------------------------------------------------------------------------------------------------- + nullptr | nullptr | exe.async(); + ------------------------------------------------------------------------------------------------- + nullptr | 0x123 | exe.async([](Runtime rt){ rt.async(); }); + | | task_group.async([](){}); + ------------------------------------------------------------------------------------------------- + 0x123 | nullptr | ? + ------------------------------------------------------------------------------------------------- + 0x123 | 0x123 | tf.emplace([](Runtime& rt){ rt.async(); }); + ------------------------------------------------------------------------------------------------- + */ + + // no parent context (e.g., from executor) + if(auto parent = node->_parent; parent == nullptr) { + _decrement_topology(); + } + // has parent context (e.g., from runtime, from task group) + else { + auto state = parent->_nstate; + if(parent->_join_counter.fetch_sub(1, std::memory_order_acq_rel) == 1) { + // this async is spawned from a preempted parent, so we need to resume it + if(state & NSTATE::PREEMPTED) { + _update_cache(worker, cache, static_cast(parent)); + } + } + } + recycle(node); +} + +// ---------------------------------------------------------------------------- +// Async +// ---------------------------------------------------------------------------- + +// Function: async +template +auto Executor::async(F&& f) { + return async(DefaultTaskParams{}, std::forward(f)); +} + +// Function: async +template +auto Executor::async(P&& params, F&& f) { + _increment_topology(); + return _async(std::forward

(params), std::forward(f), nullptr, nullptr); +} + +// Function: _async +template +auto Executor::_async(P&& params, F&& f, Topology* tpg, NodeBase* parent) { + + // async task with runtime: [] (tf::Runtime&) -> void {} + if constexpr (is_runtime_task_v) { + + std::promise p; + auto fu{p.get_future()}; + + _schedule_async_task( + NSTATE::NONE, ESTATE::EXPLICITLY_ANCHORED, std::forward

(params), tpg, parent, 0, + std::in_place_type_t{}, + [p=MoC{std::move(p)}, f=std::forward(f)](Runtime& rt, bool reentered) mutable { + if(!reentered) { + f(rt); + } + else { + auto& eptr = rt._node->_exception_ptr; + eptr ? p.object.set_exception(eptr) : p.object.set_value(); + } + } + ); + return fu; + } + // async task with closure: [] () -> auto { return ... } + else if constexpr (std::is_invocable_v){ + using R = std::invoke_result_t; + std::packaged_task p(std::forward(f)); + auto fu{p.get_future()}; + _schedule_async_task( + NSTATE::NONE, ESTATE::NONE, std::forward

(params), tpg, parent, 0, + std::in_place_type_t{}, + [p=make_moc(std::move(p))]() mutable { p.object(); } + ); + return fu; + } + else { + static_assert(dependent_false_v, + "invalid async target - must be one of the following types:\n\ + (1) [] (tf::Runtime&) -> void {}\n\ + (2) [] () -> auto { ... return ... }\n" + ); + } +} + + +// ---------------------------------------------------------------------------- +// Silent Async +// ---------------------------------------------------------------------------- + +// Function: silent_async +template +void Executor::silent_async(F&& f) { + silent_async(DefaultTaskParams{}, std::forward(f)); +} + +// Function: silent_async +template +void Executor::silent_async(P&& params, F&& f) { + _increment_topology(); + _silent_async(std::forward

(params), std::forward(f), nullptr, nullptr); +} + +// Function: _silent_async +template +void Executor::_silent_async(P&& params, F&& f, Topology* tpg, NodeBase* parent) { + // silent task + if constexpr (is_runtime_task_v || is_static_task_v) { + _schedule_async_task( + NSTATE::NONE, ESTATE::NONE, std::forward

(params), tpg, parent, 0, + std::in_place_type_t{}, std::forward(f) + ); + } + // invalid silent async target + else { + static_assert(dependent_false_v, + "invalid silent_async target - must be one of the following types:\n\ + (1) [] (tf::Runtime&) -> void {}\n\ + (2) [] () -> void { ... }\n" + ); + } +} + +// ---------------------------------------------------------------------------- +// Silent Dependent Async +// ---------------------------------------------------------------------------- + +// Function: silent_dependent_async +template +requires (std::same_as, AsyncTask> && ...) +tf::AsyncTask Executor::silent_dependent_async(F&& func, Tasks&&... tasks) { + return silent_dependent_async( + DefaultTaskParams{}, std::forward(func), std::forward(tasks)... + ); +} + +// Function: silent_dependent_async +template +requires (std::same_as, AsyncTask> && ...) +tf::AsyncTask Executor::silent_dependent_async( + P&& params, F&& func, Tasks&&... tasks +){ + std::array array{ (&tasks)... }; + return silent_dependent_async( + std::forward

(params), std::forward(func), array.begin(), array.end() + ); +} + +// Function: silent_dependent_async +template +requires (!std::same_as, AsyncTask>) +tf::AsyncTask Executor::silent_dependent_async(F&& func, I first, I last) { + return silent_dependent_async(DefaultTaskParams{}, std::forward(func), first, last); +} + +// Function: silent_dependent_async +template +requires (!std::same_as, AsyncTask>) +tf::AsyncTask Executor::silent_dependent_async( + P&& params, F&& func, I first, I last +) { + _increment_topology(); + return _silent_dependent_async( + std::forward

(params), std::forward(func), first, last, nullptr, nullptr + ); +} + +// Function: _silent_dependent_async +template +requires (!std::same_as, AsyncTask>) +auto Executor::_silent_dependent_async( + P&& params, F&& func, I first, I last, Topology* tpg, NodeBase* parent +) { + size_t num_predecessors = std::distance(first, last); + return _schedule_dependent_async_task(first, last, num_predecessors, + NSTATE::NONE, ESTATE::REFCOUNT_ONE, std::forward

(params), tpg, parent, num_predecessors, + std::in_place_type_t{}, std::forward(func) + ); +} + +// ---------------------------------------------------------------------------- +// Dependent Async +// ---------------------------------------------------------------------------- + +// Function: dependent_async +template +requires (std::same_as, AsyncTask> && ...) +auto Executor::dependent_async(F&& func, Tasks&&... tasks) { + return dependent_async(DefaultTaskParams{}, std::forward(func), std::forward(tasks)...); +} + +// Function: dependent_async +template +requires (std::same_as, AsyncTask> && ...) +auto Executor::dependent_async(P&& params, F&& func, Tasks&&... tasks) { + std::array array{ (&tasks)... }; + return dependent_async( + std::forward

(params), std::forward(func), array.begin(), array.end() + ); +} + +// Function: dependent_async +template +requires (!std::same_as, AsyncTask>) +auto Executor::dependent_async(F&& func, I first, I last) { + return dependent_async(DefaultTaskParams{}, std::forward(func), first, last); +} + +// Function: dependent_async +template +requires (!std::same_as, AsyncTask>) +auto Executor::dependent_async(P&& params, F&& func, I first, I last) { + _increment_topology(); + return _dependent_async(std::forward

(params), std::forward(func), first, last, nullptr, nullptr); +} + +// Function: _dependent_async +template +requires (!std::same_as, AsyncTask>) +auto Executor::_dependent_async(P&& params, F&& func, I first, I last, Topology* tpg, NodeBase* parent) { + + size_t num_predecessors = std::distance(first, last); + + // async with runtime: [] (tf::Runtime&) -> void {} + if constexpr (is_runtime_task_v) { + + std::promise p; + auto fu{p.get_future()}; + + return std::make_pair(_schedule_dependent_async_task(first, last, num_predecessors, + NSTATE::NONE, ESTATE::EXPLICITLY_ANCHORED | ESTATE::REFCOUNT_ONE, std::forward

(params), tpg, parent, num_predecessors, + std::in_place_type_t{}, + [p=MoC{std::move(p)}, f=std::forward(func)] (tf::Runtime& rt, bool reentered) mutable { + if(!reentered) { + f(rt); + } + else { + auto& eptr = rt._node->_exception_ptr; + eptr ? p.object.set_exception(eptr) : p.object.set_value(); + } + } + ), std::move(fu)); + } + // async without runtime: [] () -> auto { return ... } + else if constexpr(std::is_invocable_v) { + + using R = std::invoke_result_t; + std::packaged_task p(std::forward(func)); + auto fu{p.get_future()}; + + return std::make_pair(_schedule_dependent_async_task(first, last, num_predecessors, + NSTATE::NONE, ESTATE::REFCOUNT_ONE, std::forward

(params), tpg, parent, num_predecessors, + std::in_place_type_t{}, + [p=make_moc(std::move(p))] () mutable { p.object(); } + ), std::move(fu)); + } + else { + static_assert(dependent_false_v, "invalid async callable"); + } +} + +// ---------------------------------------------------------------------------- +// Dependent Async Helper Functions +// ---------------------------------------------------------------------------- + +// Procedure: _process_dependent_async +inline void Executor::_process_dependent_async( + Node* node, tf::AsyncTask& task, size_t& num_predecessors +) { + + // special case: the task is not associated with any dependent-async task + if(task.empty()) { + num_predecessors = node->_join_counter.fetch_sub(1, std::memory_order_acq_rel) - 1; + return; + } + + auto& state = task._node->_estate; + auto target = state.load(std::memory_order_acquire); + + while (!(target & ESTATE::FINISHED)) { + // can only go from unlock and unfinished to locked + target &= ~ESTATE::LOCKED; + if(state.compare_exchange_weak(target, target | ESTATE::LOCKED, + std::memory_order_acq_rel, + std::memory_order_acquire)) { + task._node->_edges.push_back(node); + state.fetch_and(~ESTATE::LOCKED, std::memory_order_release); + return; + } + } + + num_predecessors = node->_join_counter.fetch_sub(1, std::memory_order_acq_rel) - 1; +} + +// Procedure: _tear_down_dependent_async +inline void Executor::_tear_down_dependent_async(Worker& worker, Node* node, Node*& cache) { + + auto target = node->_estate.load(std::memory_order_acquire); + + while(true) { + // We can only go from unlocked and unfinished to finished. + // Only this function can set the state to FINISHED. + target &= ~ESTATE::LOCKED; + if(node->_estate.compare_exchange_weak(target, target | ESTATE::FINISHED, + std::memory_order_acq_rel, + std::memory_order_acquire)) { + break; + } + } + + // spawn successors whenever their dependencies are resolved + for(size_t i=0; i_edges.size(); ++i) { + if(auto s = node->_edges[i]; + s->_join_counter.fetch_sub(1, std::memory_order_acq_rel) == 1 + ) { + _update_cache(worker, cache, s); + } + } + + /* + ------------------------------------------------------------------------------------------------- + node->_topology | node->_parent | secenario + ------------------------------------------------------------------------------------------------- + nullptr | nullptr | exe.async(); + ------------------------------------------------------------------------------------------------- + nullptr | 0x123 | exe.async([](Runtime rt){ rt.async(); }); + | | task_group.async([](){}); + ------------------------------------------------------------------------------------------------- + 0x123 | nullptr | ? + ------------------------------------------------------------------------------------------------- + 0x123 | 0x123 | tf.emplace([](Runtime& rt){ rt.async(); }); + ------------------------------------------------------------------------------------------------- + */ + + // no parent context (e.g., from executor) + if(auto parent = node->_parent; parent == nullptr) { + _decrement_topology(); + } + // has parent context (e.g., from runtime, from task group) + else { + auto state = parent->_nstate; + if(parent->_join_counter.fetch_sub(1, std::memory_order_acq_rel) == 1) { + // this async is spawned from a preempted parent, so we need to resume it + if(state & NSTATE::PREEMPTED) { + _update_cache(worker, cache, static_cast(parent)); + } + } + } + + // now the executor no longer needs to retain ownership — + // decrement the refcount packed in the lower 24 bits of _estate. + // if it reaches zero, no AsyncTask handle remains and we can recycle. + if((node->_estate.fetch_sub(ESTATE::REFCOUNT_ONE, std::memory_order_acq_rel) + & ESTATE::REFCOUNT_MASK) == ESTATE::REFCOUNT_ONE) { + recycle(node); + } +} + + + + + +} // end of namespace tf ----------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/core/async_task.hpp b/projects/hipfile/third-party/taskflow/core/async_task.hpp new file mode 100644 index 000000000000..64040515ea04 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/core/async_task.hpp @@ -0,0 +1,296 @@ +#pragma once + +#include "graph.hpp" + +/** +@file async_task.hpp +@brief asynchronous task include file +*/ + +namespace tf { + +// ---------------------------------------------------------------------------- +// AsyncTask +// ---------------------------------------------------------------------------- + +/** +@brief class to hold a dependent asynchronous task with shared ownership + +A tf::AsyncTask is a lightweight handle that retains @em shared ownership +of a dependent asynchronous (dependent-async) task created by an executor. +This shared ownership ensures that the dependent-async task remains alive when +adding it to the dependency list of another dependent-async task, +thus avoiding the classical [ABA problem](https://en.wikipedia.org/wiki/ABA_problem). + +@code{.cpp} +// main thread retains shared ownership of async task A +tf::AsyncTask A = executor.silent_dependent_async([](){}); + +// task A remains alive (i.e., at least one ref count by the main thread) +// when being added to the dependency list of async task B +tf::AsyncTask B = executor.silent_dependent_async([](){}, A); +@endcode + +tf::AsyncTask is implemented based on the logic of +C++ smart pointer std::shared_ptr and +is considered cheap to copy or move as long as only a handful of objects +own it. +When a worker completes an async task, it will remove the task from the executor, +decrementing the number of shared owners by one. +If that counter reaches zero, the task is destroyed. + +@note +To know more about dependent-async task, please refer to @ref DependentAsyncTasking. +*/ +class AsyncTask { + + friend class Executor; + + public: + + /** + @brief constructs an empty task handle + */ + AsyncTask() = default; + + /** + @brief destroys the managed dependent-async task if this is the last owner + */ + ~AsyncTask(); + + /** + @brief constructs a dependent-async task that shares ownership of @c rhs + */ + AsyncTask(const AsyncTask& rhs); + + /** + @brief move-constructs an dependent-async task from @c rhs + */ + AsyncTask(AsyncTask&& rhs); + + /** + @brief copy-assigns the dependent-async task from @c rhs + + Releases the managed object of @c this and retains a new shared ownership + of @c rhs. + */ + AsyncTask& operator = (const AsyncTask& rhs); + + /** + @brief move-assigns the dependent-async task from @c rhs + + Releases the managed object of @c this and takes over the ownership of @c rhs. + */ + AsyncTask& operator = (AsyncTask&& rhs); + + /** + @brief checks if this dependent-async task is associated with any task + + An empty dependent-async task is not associated with any task created + from the executor. + + @code{.cpp} + tf::AsyncTask task; + assert(task.empty()); + @endcode + */ + bool empty() const; + + /** + @brief release the managed object of @c this + + Releases the ownership of the managed task, if any. + After the call `*this` manages no task. + + @code{.cpp} + tf::AsyncTask task = executor.silent_dependent_async([](){}); + assert(task.empty() == false); + task.reset(); + assert(task.empty() == true); + @endcode + */ + void reset(); + + /** + @brief obtains the hashed value of this dependent-async task + + @code{.cpp} + tf::AsyncTask task = executor.silent_dependent_async([](){}); + std::cout << task.hash_value() << '\n'; + @endcode + */ + size_t hash_value() const; + + /** + @brief returns the number of shared owners that are currently managing + this dependent-async task + + In a multithreaded environment, `use_count` atomically retrieves + (with `memory_order_relaxed` load) the number of tf::AsyncTask instances that manage + the current task. + + @code{.cpp} + tf::AsyncTask task; + assert(task.use_count() == 0); + @endcode + */ + size_t use_count() const; + + /** + @brief checks if this dependent-async task finishes + + In a multithreaded environment, `is_done` atomically retrieves + (with `memory_order_acquire` load) the underlying state bit that indicates + the completion of this dependent-async task. + If the dependent-async task is empty, returns `true`. + + @code{.cpp} + tf::AsyncTask task = executor.silent_dependent_async([](){}); + while(task.is_done() == false); + std::cout << "dependent-async task finishes\n"; + + task.reset(); + assert(task.is_done() == true); + @endcode + + */ + bool is_done() const; + + /** + @brief retrieves the exception pointer of this task + + This method retrieves the exception pointer of the task, if any, that was silently caught by the executor. + For example, the code below retrieves the exception pointer of a dependent-async task that does not have + a shared state to propagate its exception. + + @code{.cpp} + tf::AsyncTask task = executor.silent_dependent_async([&](){ + throw std::runtime_error("oops"); + }); + executor.wait_for_all(); + + // propagate the exception to the outer caller + assert(task.exception_ptr() != nullptr); + std::rethrow_exception(task.exception_ptr()); + @endcode + + */ + std::exception_ptr exception_ptr() const; + + /** + @brief queries if the task has an exception pointer + + The method checks whether the task holds a pointer to a silently caught exception. + */ + bool has_exception_ptr() const; + + private: + + explicit AsyncTask(Node*); + + Node* _node {nullptr}; + + void _incref(); + void _decref(); +}; + +// Constructor +inline AsyncTask::AsyncTask(Node* ptr) : _node{ptr} { + _incref(); +} + +// Function: _incref +inline void AsyncTask::_incref() { + if(_node) { + // increment the refcount packed in the lower 24 bits of _estate — + // no std::get_if needed since we know this node is DependentAsync + _node->_estate.fetch_add(ESTATE::REFCOUNT_ONE, std::memory_order_relaxed); + } +} + +// Function: _decref +inline void AsyncTask::_decref() { + if(_node) { + // decrement the refcount packed in the lower 24 bits of _estate. + // mask out the state bits before comparing so we only check the refcount. + // if it reaches zero, no owners remain and we recycle the node. + if((_node->_estate.fetch_sub(ESTATE::REFCOUNT_ONE, std::memory_order_acq_rel) + & ESTATE::REFCOUNT_MASK) == ESTATE::REFCOUNT_ONE) { + recycle(_node); + } + } +} + +// Copy Constructor +inline AsyncTask::AsyncTask(const AsyncTask& rhs) : + _node{rhs._node} { + _incref(); +} + +// Move Constructor +inline AsyncTask::AsyncTask(AsyncTask&& rhs) : + _node {rhs._node} { + rhs._node = nullptr; +} + +// Destructor +inline AsyncTask::~AsyncTask() { + _decref(); +} + +// Copy assignment +inline AsyncTask& AsyncTask::operator = (const AsyncTask& rhs) { + _decref(); + _node = rhs._node; + _incref(); + return *this; +} + +// Move assignment +inline AsyncTask& AsyncTask::operator = (AsyncTask&& rhs) { + _decref(); + _node = rhs._node; + rhs._node = nullptr; + return *this; +} + +// Function: exception +inline std::exception_ptr AsyncTask::exception_ptr() const { + return _node ? _node->_exception_ptr : nullptr; +} + +// Function: has_exception +inline bool AsyncTask::has_exception_ptr() const { + return _node ? (_node->_exception_ptr != nullptr) : false; +} + +// Function: empty +inline bool AsyncTask::empty() const { + return _node == nullptr; +} + +// Function: reset +inline void AsyncTask::reset() { + _decref(); + _node = nullptr; +} + +// Function: hash_value +inline size_t AsyncTask::hash_value() const { + return std::hash{}(_node); +} + +// Function: use_count +inline size_t AsyncTask::use_count() const { + return _node == nullptr ? size_t{0} : + static_cast( + _node->_estate.load(std::memory_order_relaxed) & ESTATE::REFCOUNT_MASK + ); +} + +// Function: is_done +inline bool AsyncTask::is_done() const { + return _node == nullptr ? true: (_node->_estate.load(std::memory_order_acquire) & ESTATE::FINISHED); +} + +} // end of namespace tf ---------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/core/atomic_notifier.hpp b/projects/hipfile/third-party/taskflow/core/atomic_notifier.hpp new file mode 100644 index 000000000000..0bd41214f651 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/core/atomic_notifier.hpp @@ -0,0 +1,176 @@ +#pragma once + +#include +#include +#include +#include +#include "../utility/os.hpp" + +namespace tf { + +//----------------------------------------------------------------------------- + +/** + +@class AtomicNotifier + +@brief class to create a notifier for inter-thread synchronization + +A notifier is a lightweight synchronization primitive for waiting on arbitrary predicates +without traditional mutex-based blocking +The notifier enables non-blocking coordination between threads. Conceptually, it plays +a similar role to a condition variable, but the waiting condition (predicate) does not +need to be guarded by a mutex. + +**Typical usage** + +In a waiting thread: + +@code{.cpp} +if (predicate) + return act(); +Notifier::Waiter& w = waiters[my_index]; +ec.prepare_wait(&w); +if (predicate) { + ec.cancel_wait(&w); + return act(); +} +ec.commit_wait(&w); +@endcode + +In a notifying thread: + +@code{.cpp} +predicate = true; +ec.notify(true); +@endcode + +The `notify` operation is inexpensive when there are no waiting threads. +`prepare_wait` and `commit_wait` are more costly but are only executed when +the predicate check fails initially. + +**Algorithm overview** + +The design relies on two key variables: the user-managed `predicate` and an internal +`_state`. The synchronization behavior is reminiscent of Dekker’s algorithm +(see: https://en.wikipedia.org/wiki/Dekker%27s_algorithm). + +- A waiting thread first updates `_state`, then checks the predicate. +- A notifying thread updates the predicate, then checks `_state`. + +Sequentially consistent fences ensure visibility between threads. As a result, +at least one side will observe the other's update—either: + - The waiter sees the predicate become true and avoids blocking, or + - The notifier sees `_state` change and wakes the waiter. + +This protocol prevents the deadlock case where both threads miss each other’s updates. +*/ + +class AtomicNotifier { + + friend class Executor; + + public: + + struct Waiter { + alignas (2*TF_CACHELINE_SIZE) uint32_t epoch; + }; + + AtomicNotifier(size_t N) noexcept : _state(0), _waiters(N) {} + ~AtomicNotifier() { assert((_state.load() & WAITER_MASK) == 0); } + + void notify_one() noexcept; + void notify_all() noexcept; + void notify_n(size_t) noexcept; + void prepare_wait(size_t) noexcept; + void cancel_wait(size_t) noexcept; + void commit_wait(size_t) noexcept; + + size_t size() const noexcept; + size_t num_waiters() const noexcept; + + private: + + AtomicNotifier(const AtomicNotifier&) = delete; + AtomicNotifier(AtomicNotifier&&) = delete; + AtomicNotifier& operator=(const AtomicNotifier&) = delete; + AtomicNotifier& operator=(AtomicNotifier&&) = delete; + + // This requires 64-bit + static_assert(sizeof(int) == 4, "bad platform"); + static_assert(sizeof(uint32_t) == 4, "bad platform"); + static_assert(sizeof(uint64_t) == 8, "bad platform"); + static_assert(sizeof(std::atomic) == 8, "bad platform"); + + // _state stores the epoch in the most significant 32 bits and the + // waiter count in the least significant 32 bits. + std::atomic _state; + std::vector _waiters; + + static constexpr uint64_t WAITER_INC {1}; + static constexpr uint64_t EPOCH_SHIFT {32}; + static constexpr uint64_t EPOCH_INC {uint64_t(1) << EPOCH_SHIFT}; + static constexpr uint64_t WAITER_MASK {EPOCH_INC - 1}; +}; + +inline size_t AtomicNotifier::size() const noexcept { + return _waiters.size(); +} + +inline size_t AtomicNotifier::num_waiters() const noexcept { + return _state.load(std::memory_order_relaxed) & WAITER_MASK; +} + +inline void AtomicNotifier::notify_one() noexcept { + std::atomic_thread_fence(std::memory_order_seq_cst); + for(uint64_t state = _state.load(std::memory_order_relaxed); state & WAITER_MASK;) { + if(_state.compare_exchange_weak(state, state + EPOCH_INC, std::memory_order_relaxed)) { + _state.notify_one(); + break; + } + } +} + +inline void AtomicNotifier::notify_all() noexcept { + std::atomic_thread_fence(std::memory_order_seq_cst); + for(uint64_t state = _state.load(std::memory_order_relaxed); state & WAITER_MASK;) { + if(_state.compare_exchange_weak(state, state + EPOCH_INC, std::memory_order_relaxed)) { + _state.notify_all(); + break; + } + } +} + +inline void AtomicNotifier::notify_n(size_t n) noexcept { + if(n >= _waiters.size()) { + notify_all(); + } + else { + for(size_t k=0; k> EPOCH_SHIFT); + std::atomic_thread_fence(std::memory_order_seq_cst); +} + +inline void AtomicNotifier::cancel_wait(size_t) noexcept { + _state.fetch_sub(WAITER_INC, std::memory_order_relaxed); +} + +inline void AtomicNotifier::commit_wait(size_t w) noexcept { + uint64_t prev = _state.load(std::memory_order_relaxed); + while((prev >> EPOCH_SHIFT) == _waiters[w].epoch) { + _state.wait(prev, std::memory_order_relaxed); + prev = _state.load(std::memory_order_relaxed); + } + _state.fetch_sub(WAITER_INC, std::memory_order_relaxed); +} + + +} // namespace taskflow ------------------------------------------------------- + diff --git a/projects/hipfile/third-party/taskflow/core/declarations.hpp b/projects/hipfile/third-party/taskflow/core/declarations.hpp new file mode 100644 index 000000000000..a445e5e1cbe5 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/core/declarations.hpp @@ -0,0 +1,65 @@ +#pragma once + +namespace tf { + +// ---------------------------------------------------------------------------- +// taskflow +// ---------------------------------------------------------------------------- + +class Algorithm; +class Node; +class Graph; +class FlowBuilder; +class Semaphore; +class Subflow; +class Runtime; +class NonpreemptiveRuntime; +class TaskGroup; +class Task; +class TaskView; +class Taskflow; +class AsyncTask; +class Topology; +class Executor; +class Worker; +class WorkerView; +class ObserverInterface; +class ChromeTracingObserver; +class TFProfObserver; +class TFProfManager; +class ExplicitAnchorGuard; + +template +class Future; + +template +class Pipeline; + +// ---------------------------------------------------------------------------- +// cudaFlow +// ---------------------------------------------------------------------------- +class cudaFlowNode; +class cudaFlowGraph; +class cudaTask; +class cudaFlow; +class cudaFlowCapturer; +class cudaFlowOptimizerBase; +class cudaFlowLinearOptimizer; +class cudaFlowSequentialOptimizer; +class cudaFlowRoundRobinOptimizer; + +template +class cudaGraphExecBase; + +// ---------------------------------------------------------------------------- +// struct +// ---------------------------------------------------------------------------- +class TaskParams; +class DefaultTaskParams; + + +} // end of namespace tf ----------------------------------------------------- + + + + diff --git a/projects/hipfile/third-party/taskflow/core/environment.hpp b/projects/hipfile/third-party/taskflow/core/environment.hpp new file mode 100644 index 000000000000..f9013b6e55a7 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/core/environment.hpp @@ -0,0 +1,8 @@ +#pragma once + +#define TF_ENABLE_PROFILER "TF_ENABLE_PROFILER" + +namespace tf { + +} // end of namespace tf ----------------------------------------------------- + diff --git a/projects/hipfile/third-party/taskflow/core/error.hpp b/projects/hipfile/third-party/taskflow/core/error.hpp new file mode 100644 index 000000000000..ba1d3515067b --- /dev/null +++ b/projects/hipfile/third-party/taskflow/core/error.hpp @@ -0,0 +1,100 @@ +#pragma once + +#include +#include +#include + +#include "../utility/stream.hpp" + +namespace tf { + +// node-specific states +// 32-bit state encoding [xxxx|xxxx|xxxx|xxxx|xxxx|xxxx|xxxx|xxxx] +struct NSTATE { + + using underlying_type = int; + + // scheduler state bits + constexpr static underlying_type NONE = 0x00000000; + constexpr static underlying_type IMPLICITLY_ANCHORED = 0x10000000; + constexpr static underlying_type PREEMPTED = 0x20000000; + constexpr static underlying_type RETAIN_SUBFLOW = 0x40000000; + constexpr static underlying_type JOINED_SUBFLOW = 0x80000000; + + // mask to isolate state bits - non-state bits store # weak dependents + constexpr static underlying_type STRONG_DEPENDENCIES_MASK = 0x0FFFFFFF; +}; + +using nstate_t = NSTATE::underlying_type; + +// exception-specific states +struct ESTATE { + + using underlying_type = int; + + constexpr static underlying_type NONE = 0x00000000; + + // Exception state: + // Explicit anchor needs to be in estate other it can cause data race + // due to the read/write on creating an AnchorGuard in corun with the nstate read + // in tear_down_async. When calling corun, all async tasks may have already + // finished and trigger tear_down_async, causing data race on reading/writing + // the parent node's nstate. + constexpr static underlying_type EXCEPTION = 0x10000000; + constexpr static underlying_type CAUGHT = 0x20000000; + constexpr static underlying_type CANCELLED = 0x40000000; + constexpr static underlying_type EXPLICITLY_ANCHORED = 0x80000000; + + // Async task state + constexpr static underlying_type LOCKED = 0x01000000; + constexpr static underlying_type FINISHED = 0x02000000; + + // mask to isolate the 8 state bits in the upper byte + constexpr static underlying_type MASK = 0xFF000000; + + // lower 24 bits: shared reference count for AsyncTask (dependent-async only). + // supports up to 2^24 = 16,777,216 concurrent owners, which is safe in practice. + // arithmetic on these bits never interferes with the state bits above since + // normal refcount operations (fetch_add/fetch_sub by REFCOUNT_ONE) cannot carry + // into bit 24 as long as the count stays below 2^24. + constexpr static underlying_type REFCOUNT_MASK = 0x00FFFFFF; + constexpr static underlying_type REFCOUNT_ONE = 0x00000001; +}; + +using estate_t = ESTATE::underlying_type; + + +// Procedure: throw_re +// Throws runtime error under a given error code. +template +//void throw_se(const char* fname, const size_t line, Error::Code c, ArgsT&&... args) { +void throw_re(const char* fname, const size_t line, ArgsT&&... args) { + std::ostringstream oss; + oss << "[" << fname << ":" << line << "] "; + //ostreamize(oss, std::forward(args)...); + (oss << ... << args); +#ifdef TF_DISABLE_EXCEPTION_HANDLING + std::cerr << oss.str(); + std::terminate(); +#else + throw std::runtime_error(oss.str()); +#endif +} + +} // ------------------------------------------------------------------------ + +#define TF_THROW(...) tf::throw_re(__FILE__, __LINE__, __VA_ARGS__); + +// ---------------------------------------------------------------------------- + +#ifdef TF_DISABLE_EXCEPTION_HANDLING + #define TF_EXECUTOR_EXCEPTION_HANDLER(worker, node, code_block) \ + code_block; +#else + #define TF_EXECUTOR_EXCEPTION_HANDLER(worker, node, code_block) \ + try { \ + code_block; \ + } catch(...) { \ + _process_exception(worker, node); \ + } +#endif diff --git a/projects/hipfile/third-party/taskflow/core/executor.hpp b/projects/hipfile/third-party/taskflow/core/executor.hpp new file mode 100644 index 000000000000..38f3d28d0162 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/core/executor.hpp @@ -0,0 +1,2546 @@ +#pragma once + +#include "../observer/tfprof.hpp" +#include "taskflow.hpp" +#include "async_task.hpp" + +/** +@file executor.hpp +@brief executor include file +*/ + +namespace tf { + +// ---------------------------------------------------------------------------- +// Executor Definition +// ---------------------------------------------------------------------------- + +/** +@class Executor + +@brief class to create an executor + +An tf::Executor manages a set of worker threads to run tasks +using an efficient work-stealing scheduling algorithm. + +@code{.cpp} +// Declare an executor and a taskflow +tf::Executor executor; +tf::Taskflow taskflow; + +// Add three tasks into the taskflow +tf::Task A = taskflow.emplace([] () { std::cout << "This is TaskA\n"; }); +tf::Task B = taskflow.emplace([] () { std::cout << "This is TaskB\n"; }); +tf::Task C = taskflow.emplace([] () { std::cout << "This is TaskC\n"; }); + +// Build precedence between tasks +A.precede(B, C); + +tf::Future fu = executor.run(taskflow); +fu.wait(); // block until the execution completes + +executor.run(taskflow, [](){ std::cout << "end of 1 run"; }).wait(); +executor.run_n(taskflow, 4); +executor.wait_for_all(); // block until all associated executions finish +executor.run_n(taskflow, 4, [](){ std::cout << "end of 4 runs"; }).wait(); +executor.run_until(taskflow, [cnt=0] () mutable { return ++cnt == 10; }); +@endcode + +Most executor methods are @em thread-safe. +For example, you can submit multiple taskflows to an executor concurrently +from different threads, while other threads simultaneously create asynchronous tasks. + +@code{.cpp} +std::thread t1([&](){ executor.run(taskflow); }; +std::thread t2([&](){ executor.async([](){ std::cout << "async task from t2\n"; }); }); +executor.async([&](){ std::cout << "async task from the main thread\n"; }); +@endcode + +@note +To know more about tf::Executor, please refer to @ref ExecuteTaskflow. +*/ +class Executor { + + friend class FlowBuilder; + friend class Subflow; + friend class Runtime; + friend class NonpreemptiveRuntime; + friend class Algorithm; + friend class TaskGroup; + + public: + + /** + @brief constructs the executor with @c N worker threads + + @param N number of workers (default std::thread::hardware_concurrency) + @param wif interface class instance to configure workers' behaviors + + The constructor spawns @c N worker threads to run tasks in a + work-stealing loop. The number of workers must be greater than zero + or an exception will be thrown. + By default, the number of worker threads is equal to the maximum + hardware concurrency returned by std::thread::hardware_concurrency. + + Users can alter the worker behavior, such as changing thread affinity, + via deriving an instance from tf::WorkerInterface. + + @note + An exception will be thrown if executor construction fails. + */ + explicit Executor( + size_t N = std::thread::hardware_concurrency(), + std::shared_ptr wif = nullptr + ); + + /** + @brief destructs the executor + + The destructor calls Executor::wait_for_all to wait for all submitted + taskflows to complete and then notifies all worker threads to stop + and join these threads. + */ + ~Executor(); + + /** + @brief runs a taskflow once + + @param taskflow a tf::Taskflow object + + @return a tf::Future that holds the result of the execution + + This member function executes the given taskflow once and returns a tf::Future + object that eventually holds the result of the execution. + + @code{.cpp} + tf::Future future = executor.run(taskflow); + // do something else + future.wait(); + @endcode + + This member function is thread-safe. + */ + tf::Future run(Taskflow& taskflow); + + /** + @brief runs a moved taskflow once + + @param taskflow a moved tf::Taskflow object + + @return a tf::Future that holds the result of the execution + + This member function executes a moved taskflow once and returns a tf::Future + object that eventually holds the result of the execution. + The executor will take care of the lifetime of the moved taskflow. + + @code{.cpp} + tf::Future future = executor.run(std::move(taskflow)); + // do something else + future.wait(); + @endcode + + This member function is thread-safe. + */ + tf::Future run(Taskflow&& taskflow); + + /** + @brief runs a taskflow once and invoke a callback upon completion + + @param taskflow a tf::Taskflow object + @param callable a callable object to be invoked after this run + + @return a tf::Future that holds the result of the execution + + This member function executes the given taskflow once and invokes the given + callable when the execution completes. + This member function returns a tf::Future object that + eventually holds the result of the execution. + + @code{.cpp} + tf::Future future = executor.run(taskflow, [](){ std::cout << "done"; }); + // do something else + future.wait(); + @endcode + + This member function is thread-safe. + + */ + template + tf::Future run(Taskflow& taskflow, C&& callable); + + /** + @brief runs a moved taskflow once and invoke a callback upon completion + + @param taskflow a moved tf::Taskflow object + @param callable a callable object to be invoked after this run + + @return a tf::Future that holds the result of the execution + + This member function executes a moved taskflow once and invokes the given + callable when the execution completes. + This member function returns a tf::Future object that + eventually holds the result of the execution. + The executor will take care of the lifetime of the moved taskflow. + + @code{.cpp} + tf::Future future = executor.run( + std::move(taskflow), [](){ std::cout << "done"; } + ); + // do something else + future.wait(); + @endcode + + This member function is thread-safe. + */ + template + tf::Future run(Taskflow&& taskflow, C&& callable); + + /** + @brief runs a taskflow for @c N times + + @param taskflow a tf::Taskflow object + @param N number of runs + + @return a tf::Future that holds the result of the execution + + This member function executes the given taskflow @c N times and returns a tf::Future + object that eventually holds the result of the execution. + + @code{.cpp} + tf::Future future = executor.run_n(taskflow, 2); // run taskflow 2 times + // do something else + future.wait(); + @endcode + + This member function is thread-safe. + */ + tf::Future run_n(Taskflow& taskflow, size_t N); + + /** + @brief runs a moved taskflow for @c N times + + @param taskflow a moved tf::Taskflow object + @param N number of runs + + @return a tf::Future that holds the result of the execution + + This member function executes a moved taskflow @c N times and returns a tf::Future + object that eventually holds the result of the execution. + The executor will take care of the lifetime of the moved taskflow. + + @code{.cpp} + tf::Future future = executor.run_n( + std::move(taskflow), 2 // run the moved taskflow 2 times + ); + // do something else + future.wait(); + @endcode + + This member function is thread-safe. + */ + tf::Future run_n(Taskflow&& taskflow, size_t N); + + /** + @brief runs a taskflow for @c N times and then invokes a callback + + @param taskflow a tf::Taskflow + @param N number of runs + @param callable a callable object to be invoked after this run + + @return a tf::Future that holds the result of the execution + + This member function executes the given taskflow @c N times and invokes the given + callable when the execution completes. + This member function returns a tf::Future object that + eventually holds the result of the execution. + + @code{.cpp} + tf::Future future = executor.run( + taskflow, 2, [](){ std::cout << "done"; } // runs taskflow 2 times and invoke + // the lambda to print "done" + ); + // do something else + future.wait(); + @endcode + + This member function is thread-safe. + */ + template + tf::Future run_n(Taskflow& taskflow, size_t N, C&& callable); + + /** + @brief runs a moved taskflow for @c N times and then invokes a callback + + @param taskflow a moved tf::Taskflow + @param N number of runs + @param callable a callable object to be invoked after this run + + @return a tf::Future that holds the result of the execution + + This member function executes a moved taskflow @c N times and invokes the given + callable when the execution completes. + This member function returns a tf::Future object that + eventually holds the result of the execution. + + @code{.cpp} + tf::Future future = executor.run_n( + // run the moved taskflow 2 times and invoke the lambda to print "done" + std::move(taskflow), 2, [](){ std::cout << "done"; } + ); + // do something else + future.wait(); + @endcode + + This member function is thread-safe. + */ + template + tf::Future run_n(Taskflow&& taskflow, size_t N, C&& callable); + + /** + @brief runs a taskflow multiple times until the predicate becomes true + + @param taskflow a tf::Taskflow + @param pred a boolean predicate to return @c true for stop + + @return a tf::Future that holds the result of the execution + + This member function executes the given taskflow multiple times until + the predicate returns @c true. + This member function returns a tf::Future object that + eventually holds the result of the execution. + + @code{.cpp} + tf::Future future = executor.run_until( + taskflow, [](){ return rand()%10 == 0 } + ); + // do something else + future.wait(); + @endcode + + This member function is thread-safe. + */ + template + tf::Future run_until(Taskflow& taskflow, P&& pred); + + /** + @brief runs a moved taskflow and keeps running it + until the predicate becomes true + + @param taskflow a moved tf::Taskflow object + @param pred a boolean predicate to return @c true for stop + + @return a tf::Future that holds the result of the execution + + This member function executes a moved taskflow multiple times until + the predicate returns @c true. + This member function returns a tf::Future object that + eventually holds the result of the execution. + The executor will take care of the lifetime of the moved taskflow. + + @code{.cpp} + tf::Future future = executor.run_until( + std::move(taskflow), [](){ return rand()%10 == 0 } + ); + // do something else + future.wait(); + @endcode + + This member function is thread-safe. + */ + template + tf::Future run_until(Taskflow&& taskflow, P&& pred); + + /** + @brief runs a taskflow multiple times until the predicate becomes true and + then invokes the callback + + @param taskflow a tf::Taskflow + @param pred a boolean predicate to return @c true for stop + @param callable a callable object to be invoked after this run completes + + @return a tf::Future that holds the result of the execution + + This member function executes the given taskflow multiple times until + the predicate returns @c true and then invokes the given callable when + the execution completes. + This member function returns a tf::Future object that + eventually holds the result of the execution. + + @code{.cpp} + tf::Future future = executor.run_until( + taskflow, [](){ return rand()%10 == 0 }, [](){ std::cout << "done"; } + ); + // do something else + future.wait(); + @endcode + + This member function is thread-safe. + */ + template + tf::Future run_until(Taskflow& taskflow, P&& pred, C&& callable); + + /** + @brief runs a moved taskflow and keeps running + it until the predicate becomes true and then invokes the callback + + @param taskflow a moved tf::Taskflow + @param pred a boolean predicate to return @c true for stop + @param callable a callable object to be invoked after this run completes + + @return a tf::Future that holds the result of the execution + + This member function executes a moved taskflow multiple times until + the predicate returns @c true and then invokes the given callable when + the execution completes. + This member function returns a tf::Future object that + eventually holds the result of the execution. + The executor will take care of the lifetime of the moved taskflow. + + @code{.cpp} + tf::Future future = executor.run_until( + std::move(taskflow), + [](){ return rand()%10 == 0 }, [](){ std::cout << "done"; } + ); + // do something else + future.wait(); + @endcode + + This member function is thread-safe. + */ + template + tf::Future run_until(Taskflow&& taskflow, P&& pred, C&& callable); + + /** + @brief runs a target graph and waits until it completes using + an internal worker of this executor + + @tparam T target type which is either convertible to `tf::Graph` or has `tf::Graph& T::graph()` defined + @param target the target task graph object + + The method coruns a target graph cooperatively with other workers in the same executor + and block until the execution completes. + Under cooperative execution, a worker is not preempted. Instead, it continues + participating in the work-stealing loop, executing available tasks alongside + other workers. + + @code{.cpp} + tf::Executor executor(2); + tf::Taskflow taskflow; + std::array others; + + std::atomic counter{0}; + + for(size_t n=0; n<1000; n++) { + for(size_t i=0; i<1000; i++) { + others[n].emplace([&](){ counter++; }); + } + taskflow.emplace([&executor, &tf=others[n]](){ + executor.corun(tf); + //executor.run(tf).wait(); <- blocking the worker without doing anything + // will introduce deadlock + }); + } + executor.run(taskflow).wait(); + @endcode + + The method is thread-safe as long as the target is not concurrently + ran by two or more threads. + + @note + You must call tf::Executor::corun from a worker of the calling executor + or an exception will be thrown. + */ + template + void corun(T& target); + + /** + @brief keeps running the work-stealing loop until the predicate returns `true` + + @tparam P predicate type + @param predicate a boolean predicate to indicate when to stop the loop + + The method keeps the caller worker running in the work-stealing loop + until the stop predicate becomes true. + + The method keeps the calling worker running available tasks cooperatively + with other workers in the same executor and block until the predicate return `true`. + Under cooperative execution, a worker is not preempted. Instead, it continues + participating in the work-stealing loop, executing available tasks alongside + other workers. + + @code{.cpp} + taskflow.emplace([&](){ + std::future fu = std::async([](){ std::sleep(100s); }); + executor.corun_until([](){ + return fu.wait_for(std::chrono::seconds(0)) == future_status::ready; + }); + }); + @endcode + + @note + You must call tf::Executor::corun_until from a worker of the calling executor + or an exception will be thrown. + */ + template + void corun_until(P&& predicate); + + /** + @brief waits for all tasks to complete + + This member function waits until all submitted tasks + (e.g., taskflows, asynchronous tasks) to finish. + + @code{.cpp} + executor.run(taskflow1); + executor.run_n(taskflow2, 10); + executor.run_n(taskflow3, 100); + executor.wait_for_all(); // wait until the above submitted taskflows finish + @endcode + */ + void wait_for_all(); + + /** + @brief queries the number of worker threads + + Each worker represents a unique thread spawned by an executor upon its construction time. + + @code{.cpp} + tf::Executor executor(4); + std::cout << executor.num_workers(); // 4 + @endcode + */ + size_t num_workers() const noexcept; + + /** + @brief queries the number of workers that are in the waiting loop + + A worker in the waiting loop has exhausted its local queue and made enough stealing attempts, + and is now ready to be preempted and enter the waiting state. + */ + size_t num_waiters() const noexcept; + + /** + @brief queries the number of work-stealing queues used by the executor + */ + size_t num_queues() const noexcept; + + /** + @brief queries the number of running topologies at the time of this call + + When a taskflow is submitted to an executor, a topology is created to store + runtime metadata of the running taskflow. + When the execution of the submitted taskflow finishes, + its corresponding topology will be removed from the executor. + + @code{.cpp} + executor.run(taskflow); + std::cout << executor.num_topologies(); // 0 or 1 (taskflow still running) + @endcode + */ + size_t num_topologies() const; + + /** + @brief queries pointer to the calling worker if it belongs to this executor, otherwise returns `nullptr` + + Returns a pointer to the per-worker storage associated with this executor. + If the calling thread is not a worker of this executor, the function returns `nullptr`. + + @code{.cpp} + auto w = executor.this_worker(); + tf::Taskflow taskflow; + tf::Executor executor; + executor.async([&](){ + assert(executor.this_worker() != nullptr); + assert(executor.this_worker()->executor() == &executor); + }); + @endcode + */ + Worker* this_worker(); + + /** + @brief queries the id of the caller thread within this executor + + Each worker has an unique id in the range of @c 0 to @c N-1 associated with + its parent executor. + If the caller thread does not belong to the executor, @c -1 is returned. + + @code{.cpp} + tf::Executor executor(4); // 4 workers in the executor + executor.this_worker_id(); // -1 (main thread is not a worker) + + taskflow.emplace([&](){ + std::cout << executor.this_worker_id(); // 0, 1, 2, or 3 + }); + executor.run(taskflow); + @endcode + */ + int this_worker_id() const; + + // -------------------------------------------------------------------------- + // Observer methods + // -------------------------------------------------------------------------- + + /** + @brief constructs an observer to inspect the activities of worker threads + + @tparam Observer observer type derived from tf::ObserverInterface + @tparam ArgsT argument parameter pack + + @param args arguments to forward to the constructor of the observer + + @return a shared pointer to the created observer + + Each executor manages a list of observers with shared ownership with callers. + For each of these observers, the two member functions, + tf::ObserverInterface::on_entry and tf::ObserverInterface::on_exit + will be called before and after the execution of a task. + + This member function is not thread-safe. + */ + template + std::shared_ptr make_observer(ArgsT&&... args); + + /** + @brief removes an observer from the executor + + This member function is not thread-safe. + */ + template + void remove_observer(std::shared_ptr observer); + + /** + @brief queries the number of observers + */ + size_t num_observers() const noexcept; + + // -------------------------------------------------------------------------- + // Async Task Methods + // -------------------------------------------------------------------------- + + /** + @brief creates a parameterized asynchronous task to run the given function + + @tparam P task parameter type satisfying tf::TaskParamsLike + @tparam F callable type + + @param params task parameters + @param func callable object + + @return a @std_future that will hold the result of the execution + + The method creates a parameterized asynchronous task + to run the given function and return a @std_future object + that eventually will hold the result of the execution. + + @code{.cpp} + std::future future = executor.async("name", [](){ + std::cout << "create an asynchronous task with a name and returns 1\n"; + return 1; + }); + future.get(); + @endcode + + This member function is thread-safe. + */ + template + auto async(P&& params, F&& func); + + /** + @brief runs a given function asynchronously + + @tparam F callable type + + @param func callable object + + @return a @std_future that will hold the result of the execution + + The method creates an asynchronous task to run the given function + and return a @std_future object that eventually will hold the result + of the return value. + + @code{.cpp} + std::future future = executor.async([](){ + std::cout << "create an asynchronous task and returns 1\n"; + return 1; + }); + future.get(); + @endcode + + This member function is thread-safe. + */ + template + auto async(F&& func); + + /** + @brief similar to tf::Executor::async but does not return a future object + + @tparam F callable type + + @param params task parameters + @param func callable object + + The method creates a parameterized asynchronous task + to run the given function without returning any @std_future object. + This member function is more efficient than tf::Executor::async + and is encouraged to use when applications do not need a @std_future to acquire + the result or synchronize the execution. + + @code{.cpp} + executor.silent_async("name", [](){ + std::cout << "create an asynchronous task with a name and no return\n"; + }); + executor.wait_for_all(); + @endcode + + This member function is thread-safe. + */ + template + void silent_async(P&& params, F&& func); + + /** + @brief similar to tf::Executor::async but does not return a future object + + @tparam F callable type + + @param func callable object + + The method creates an asynchronous task + to run the given function without returning any @std_future object. + This member function is more efficient than tf::Executor::async + and is encouraged to use when applications do not need a @std_future to acquire + the result or synchronize the execution. + + @code{.cpp} + executor.silent_async([](){ + std::cout << "create an asynchronous task with no return\n"; + }); + executor.wait_for_all(); + @endcode + + This member function is thread-safe. + */ + template + void silent_async(F&& func); + + // -------------------------------------------------------------------------- + // Silent Dependent Async Methods + // -------------------------------------------------------------------------- + + /** + @brief runs the given function asynchronously + when the given predecessors finish + + @tparam F callable type + @tparam Tasks task types convertible to tf::AsyncTask + + @param func callable object + @param tasks asynchronous tasks on which this execution depends + + @return a tf::AsyncTask handle + + This member function is more efficient than tf::Executor::dependent_async + and is encouraged to use when you do not want a @std_future to + acquire the result or synchronize the execution. + The example below creates three asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + + @code{.cpp} + tf::AsyncTask A = executor.silent_dependent_async([](){ printf("A\n"); }); + tf::AsyncTask B = executor.silent_dependent_async([](){ printf("B\n"); }); + executor.silent_dependent_async([](){ printf("C runs after A and B\n"); }, A, B); + executor.wait_for_all(); + @endcode + + This member function is thread-safe. + */ + template +requires (std::same_as, AsyncTask> && ...) + tf::AsyncTask silent_dependent_async(F&& func, Tasks&&... tasks); + + /** + @brief runs the given function asynchronously + when the given predecessors finish + + @tparam F callable type + @tparam Tasks task types convertible to tf::AsyncTask + + @param params task parameters + @param func callable object + @param tasks asynchronous tasks on which this execution depends + + @return a tf::AsyncTask handle + + This member function is more efficient than tf::Executor::dependent_async + and is encouraged to use when you do not want a @std_future to + acquire the result or synchronize the execution. + The example below creates three asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + Assigned task names will appear in the observers of the executor. + + @code{.cpp} + tf::AsyncTask A = executor.silent_dependent_async("A", [](){ printf("A\n"); }); + tf::AsyncTask B = executor.silent_dependent_async("B", [](){ printf("B\n"); }); + executor.silent_dependent_async( + "C", [](){ printf("C runs after A and B\n"); }, A, B + ); + executor.wait_for_all(); + @endcode + + This member function is thread-safe. + */ + template + requires (std::same_as, AsyncTask> && ...) + tf::AsyncTask silent_dependent_async(P&& params, F&& func, Tasks&&... tasks); + + /** + @brief runs the given function asynchronously + when the given range of predecessors finish + + @tparam F callable type + @tparam I iterator type + + @param func callable object + @param first iterator to the beginning (inclusive) + @param last iterator to the end (exclusive) + + @return a tf::AsyncTask handle + + This member function is more efficient than tf::Executor::dependent_async + and is encouraged to use when you do not want a @std_future to + acquire the result or synchronize the execution. + The example below creates three asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + + @code{.cpp} + std::array array { + executor.silent_dependent_async([](){ printf("A\n"); }), + executor.silent_dependent_async([](){ printf("B\n"); }) + }; + executor.silent_dependent_async( + [](){ printf("C runs after A and B\n"); }, array.begin(), array.end() + ); + executor.wait_for_all(); + @endcode + + This member function is thread-safe. + */ + template +requires (!std::same_as, AsyncTask>) + tf::AsyncTask silent_dependent_async(F&& func, I first, I last); + + /** + @brief runs the given function asynchronously + when the given range of predecessors finish + + @tparam F callable type + @tparam I iterator type + + @param params tasks parameters + @param func callable object + @param first iterator to the beginning (inclusive) + @param last iterator to the end (exclusive) + + @return a tf::AsyncTask handle + + This member function is more efficient than tf::Executor::dependent_async + and is encouraged to use when you do not want a @std_future to + acquire the result or synchronize the execution. + The example below creates three asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + Assigned task names will appear in the observers of the executor. + + @code{.cpp} + std::array array { + executor.silent_dependent_async("A", [](){ printf("A\n"); }), + executor.silent_dependent_async("B", [](){ printf("B\n"); }) + }; + executor.silent_dependent_async( + "C", [](){ printf("C runs after A and B\n"); }, array.begin(), array.end() + ); + executor.wait_for_all(); + @endcode + + This member function is thread-safe. + */ + template + requires (!std::same_as, AsyncTask>) + tf::AsyncTask silent_dependent_async(P&& params, F&& func, I first, I last); + + // -------------------------------------------------------------------------- + // Dependent Async Methods + // -------------------------------------------------------------------------- + + /** + @brief runs the given function asynchronously + when the given predecessors finish + + @tparam F callable type + @tparam Tasks task types convertible to tf::AsyncTask + + @param func callable object + @param tasks asynchronous tasks on which this execution depends + + @return a pair of a tf::AsyncTask handle and + a @std_future that holds the result of the execution + + The example below creates three asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + Task @c C returns a pair of its tf::AsyncTask handle and a std::future + that eventually will hold the result of the execution. + + @code{.cpp} + tf::AsyncTask A = executor.silent_dependent_async([](){ printf("A\n"); }); + tf::AsyncTask B = executor.silent_dependent_async([](){ printf("B\n"); }); + auto [C, fuC] = executor.dependent_async( + [](){ + printf("C runs after A and B\n"); + return 1; + }, + A, B + ); + fuC.get(); // C finishes, which in turns means both A and B finish + @endcode + + You can mix the use of tf::AsyncTask handles + returned by tf::Executor::dependent_async and tf::Executor::silent_dependent_async + when specifying task dependencies. + + This member function is thread-safe. + */ + template +requires (std::same_as, AsyncTask> && ...) + auto dependent_async(F&& func, Tasks&&... tasks); + + /** + @brief runs the given function asynchronously + when the given predecessors finish + + @tparam P task parameters type satisfying tf::TaskParamsLike + @tparam F callable type + @tparam Tasks task types convertible to tf::AsyncTask + + @param params task parameters + @param func callable object + @param tasks asynchronous tasks on which this execution depends + + @return a pair of a tf::AsyncTask handle and + a @std_future that holds the result of the execution + + The example below creates three named asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + Task @c C returns a pair of its tf::AsyncTask handle and a std::future + that eventually will hold the result of the execution. + Assigned task names will appear in the observers of the executor. + + @code{.cpp} + tf::AsyncTask A = executor.silent_dependent_async("A", [](){ printf("A\n"); }); + tf::AsyncTask B = executor.silent_dependent_async("B", [](){ printf("B\n"); }); + auto [C, fuC] = executor.dependent_async( + "C", + [](){ + printf("C runs after A and B\n"); + return 1; + }, + A, B + ); + assert(fuC.get()==1); // C finishes, which in turns means both A and B finish + @endcode + + You can mix the use of tf::AsyncTask handles + returned by tf::Executor::dependent_async and tf::Executor::silent_dependent_async + when specifying task dependencies. + + This member function is thread-safe. + */ + template + requires (std::same_as, AsyncTask> && ...) + auto dependent_async(P&& params, F&& func, Tasks&&... tasks); + + /** + @brief runs the given function asynchronously + when the given range of predecessors finish + + @tparam F callable type + @tparam I iterator type + + @param func callable object + @param first iterator to the beginning (inclusive) + @param last iterator to the end (exclusive) + + @return a pair of a tf::AsyncTask handle and + a @std_future that holds the result of the execution + + The example below creates three asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + Task @c C returns a pair of its tf::AsyncTask handle and a std::future + that eventually will hold the result of the execution. + + @code{.cpp} + std::array array { + executor.silent_dependent_async([](){ printf("A\n"); }), + executor.silent_dependent_async([](){ printf("B\n"); }) + }; + auto [C, fuC] = executor.dependent_async( + [](){ + printf("C runs after A and B\n"); + return 1; + }, + array.begin(), array.end() + ); + assert(fuC.get()==1); // C finishes, which in turns means both A and B finish + @endcode + + You can mix the use of tf::AsyncTask handles + returned by tf::Executor::dependent_async and tf::Executor::silent_dependent_async + when specifying task dependencies. + + This member function is thread-safe. + */ + template +requires (!std::same_as, AsyncTask>) + auto dependent_async(F&& func, I first, I last); + + /** + @brief runs the given function asynchronously + when the given range of predecessors finish + + @tparam P task parameters type satisfying tf::TaskParamsLike + @tparam F callable type + @tparam I iterator type + + @param params task parameters + @param func callable object + @param first iterator to the beginning (inclusive) + @param last iterator to the end (exclusive) + + @return a pair of a tf::AsyncTask handle and + a @std_future that holds the result of the execution + + The example below creates three named asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + Task @c C returns a pair of its tf::AsyncTask handle and a std::future + that eventually will hold the result of the execution. + Assigned task names will appear in the observers of the executor. + + @code{.cpp} + std::array array { + executor.silent_dependent_async("A", [](){ printf("A\n"); }), + executor.silent_dependent_async("B", [](){ printf("B\n"); }) + }; + auto [C, fuC] = executor.dependent_async( + "C", + [](){ + printf("C runs after A and B\n"); + return 1; + }, + array.begin(), array.end() + ); + assert(fuC.get()==1); // C finishes, which in turns means both A and B finish + @endcode + + You can mix the use of tf::AsyncTask handles + returned by tf::Executor::dependent_async and tf::Executor::silent_dependent_async + when specifying task dependencies. + + This member function is thread-safe. + */ + template + requires (!std::same_as, AsyncTask>) + auto dependent_async(P&& params, F&& func, I first, I last); + + // ---------------------------------------------------------------------------------------------- + // Task Group + // ---------------------------------------------------------------------------------------------- + + /** + @brief creates a task group that executes a collection of asynchronous tasks + @return a tf::TaskGroup object associated with the current executor + + A TaskGroup allows submitting multiple asynchronous tasks to the executor + and waiting for their completion collectively using `corun()`. Tasks added + to the group can execute in parallel and may capture local variables by value + or reference, depending on your needs. + This can be useful for divide-and-conquer + algorithms, parallel loops, or any workflow that requires grouping related tasks. + + Example (computing Fibonacci numbers in parallel): + + @code{.cpp} + tf::Executor executor; + + size_t fibonacci(size_t N) { + + if (N < 2) return N; + + size_t res1, res2; + + // Create a task group from the current executor + tf::TaskGroup tg = get_executor().task_group(); + + // Submit asynchronous tasks to the group + tg.silent_async([N, &res1](){ res1 = fibonacci(N-1); }); + res2 = fibonacci(N-2); // compute one branch synchronously + + // Wait for all tasks in the group to complete + tg.corun(); + + return res1 + res2; + } + + int main() { + return executor.async([](){ return fibonacci(30); }).get(); + } + @endcode + + This member function is thread-safe. + + @note + Due to cooperative execution, a task group can only be created by a worker of an executor. + */ + TaskGroup task_group(); + + private: + + struct Buffer { + std::mutex mutex; + UnboundedWSQ queue; + }; + + std::vector _workers; + std::vector _buffers; + + // notifier's state variable and num_topologies should sit on different cachelines + // or the false sharing can cause serious performance drop + alignas(TF_CACHELINE_SIZE) DefaultNotifier _notifier; + alignas(TF_CACHELINE_SIZE) std::atomic _num_topologies {0}; + + std::unordered_map _t2w; + std::unordered_set> _observers; + + void _shutdown(); + void _observer_prologue(Worker&, Node*); + void _observer_epilogue(Worker&, Node*); + void _spawn(size_t, std::shared_ptr); + void _exploit_task(Worker&, Node*&); + bool _explore_task(Worker&, Node*&); + void _schedule(Worker&, Node*); + void _schedule(Node*); + void _schedule_graph(Worker&, Graph&, Topology*, NodeBase*); + void _spill(Node*); + void _set_up_topology(Worker*, Topology*); + void _tear_down_topology(Worker&, Topology*, Node*&); + void _tear_down_async(Worker&, Node*, Node*&); + void _tear_down_dependent_async(Worker&, Node*, Node*&); + void _tear_down_nonasync(Worker&, Node*, Node*&); + void _tear_down_invoke(Worker&, Node*, Node*&); + void _increment_topology(); + void _decrement_topology(); + void _invoke(Worker&, Node*); + void _invoke_static_task(Worker&, Node*); + void _invoke_nonpreemptive_runtime_task(Worker&, Node*); + void _invoke_condition_task(Worker&, Node*, SmallVector&); + void _invoke_multi_condition_task(Worker&, Node*, SmallVector&); + void _process_dependent_async(Node*, tf::AsyncTask&, size_t&); + void _process_exception(Worker&, Node*); + void _update_cache(Worker&, Node*&, Node*); + void _corun_graph(Worker&, Graph&, Topology*, NodeBase*); + + bool _wait_for_task(Worker&, Node*&); + bool _invoke_subflow_task(Worker&, Node*); + bool _invoke_module_task(Worker&, Node*); + bool _invoke_adopted_module_task(Worker&, Node*); + bool _invoke_module_task_impl(Worker&, Node*, Graph&); + bool _invoke_async_task(Worker&, Node*); + bool _invoke_dependent_async_task(Worker&, Node*); + bool _invoke_runtime_task(Worker&, Node*); + bool _invoke_runtime_task_impl(Worker&, Node*, std::function&); + bool _invoke_runtime_task_impl(Worker&, Node*, std::function&); + + size_t _set_up_graph(Graph&, Topology*, NodeBase*); + + template + void _corun_until(Worker&, P&&); + + template + void _bulk_schedule(Worker&, I, size_t); + + template + void _bulk_schedule(I, size_t); + + template + void _bulk_spill(I, size_t); + + template + void _bulk_spill_round_robin(I, size_t); + + template + void _bulk_update_cache(Worker&, Node*&, Node*, std::array&, size_t&); + + template + auto _async(P&&, F&&, Topology*, NodeBase*); + + template + void _silent_async(P&&, F&&, Topology*, NodeBase*); + + template + requires (!std::same_as, AsyncTask>) + auto _dependent_async(P&&, F&&, I, I, Topology*, NodeBase*); + + template + requires (!std::same_as, AsyncTask>) + auto _silent_dependent_async(P&&, F&&, I, I, Topology*, NodeBase*); + + template + void _schedule_async_task(ArgsT&&...); + + template + AsyncTask _schedule_dependent_async_task(I, I, size_t, ArgsT&&...); +}; + +#ifndef DOXYGEN_GENERATING_OUTPUT + +// Constructor +inline Executor::Executor(size_t N, std::shared_ptr wif) : + _workers (N), + _buffers (std::bit_width(N)), // Empirically, we find that log2(N) performs best. + _notifier (N) { + + if(N == 0) { + TF_THROW("executor must define at least one worker"); + } + + // If spawning N threads fails, shut down any created threads before + // rethrowing the exception. +#ifndef TF_DISABLE_EXCEPTION_HANDLING + try { +#endif + _spawn(N, std::move(wif)); +#ifndef TF_DISABLE_EXCEPTION_HANDLING + } + catch(...) { + _shutdown(); + std::rethrow_exception(std::current_exception()); + } +#endif + + // initialize the default observer if requested + if(has_env(TF_ENABLE_PROFILER)) { + TFProfManager::get()._manage(make_observer()); + } +} + +// Destructor +inline Executor::~Executor() { + _shutdown(); +} + +// Function: _shutdown +inline void Executor::_shutdown() { + + // wait for all topologies to complete + wait_for_all(); + + // shut down the scheduler + for(size_t i=0; i<_workers.size(); ++i) { + _workers[i]._done.test_and_set(std::memory_order_relaxed); + } + + _notifier.notify_all(); + + // Only join the thread if it is joinable, as std::thread construction + // may fail and throw an exception. + for(auto& w : _workers) { + if(w._thread.joinable()) { + w._thread.join(); + } + } +} + +// Function: num_workers +inline size_t Executor::num_workers() const noexcept { + return _workers.size(); +} + +// Function: num_waiters +inline size_t Executor::num_waiters() const noexcept { + return _notifier.num_waiters(); +} + +// Function: num_queues +inline size_t Executor::num_queues() const noexcept { + return _workers.size() + _buffers.size(); +} + +// Function: num_topologies +inline size_t Executor::num_topologies() const { + return _num_topologies.load(std::memory_order_relaxed); +} + +// Function: this_worker +inline Worker* Executor::this_worker() { + auto itr = _t2w.find(std::this_thread::get_id()); + return itr == _t2w.end() ? nullptr : itr->second; +} + +// Function: this_worker_id +inline int Executor::this_worker_id() const { + auto i = _t2w.find(std::this_thread::get_id()); + return i == _t2w.end() ? -1 : static_cast(i->second->_id); +} + +// Procedure: _spawn +inline void Executor::_spawn(size_t N, std::shared_ptr wif) { + + for(size_t id=0; id(std::hash()(std::this_thread::get_id()))); + + // before entering the work-stealing loop, call the scheduler prologue + if(wif) { + wif->scheduler_prologue(worker); + } + + Node* t = nullptr; + std::exception_ptr ptr = nullptr; + + // must use 1 as condition instead of !done because + // the previous worker may stop while the following workers + // are still preparing for entering the scheduling loop +#ifndef TF_DISABLE_EXCEPTION_HANDLING + try { +#endif + // work-stealing loop + while(1) { + + // drains out the local queue first + _exploit_task(worker, t); + + // steals and waits for tasks + if(_wait_for_task(worker, t) == false) { + break; + } + } + +#ifndef TF_DISABLE_EXCEPTION_HANDLING + } + catch(...) { + ptr = std::current_exception(); + } +#endif + + // call the user-specified epilogue function + if(wif) { + wif->scheduler_epilogue(worker, ptr); + } + + }); + + // We avoid using thread-local storage to track the mapping between a thread + // and its corresponding worker in an executor. On Windows, thread-local + // storage can be unreliable in certain situations (see issue #727). + // + // Instead, we maintain a per-executor mapping from threads to workers. + // This approach has an additional advantage: according to the C++ Standard, + // std::thread::id uniquely identifies a thread object. Therefore, once the map + // returns a valid worker, we can be certain that the worker belongs to this + // executor. This eliminates the need for additional executor validation + // required by using thread-local storage. + // + // Example: + // + // Worker* w = this_worker(); + // // Using thread-local storage, we would need additional executor validation: + // if (w == nullptr || w->_executor != this) { /* caller is not a worker of this executor */ } + // + // // Using per-executor mapping, it suffices to check: + // if (w == nullptr) { /* caller is not a worker of this executor */ } + // + _t2w.emplace(_workers[id]._thread.get_id(), &_workers[id]); + } +} + +// Function: _explore_task +inline bool Executor::_explore_task(Worker& w, Node*& t) { + + // Fast path: if no topologies are live, all queues are guaranteed empty + // by the executor's invariant (num_topologies reaches zero only after all + // nodes have been scheduled and their queues flushed). Skip the entire + // steal loop and return immediately so the caller enters _wait_for_task + // to sleep. relaxed ordering is sufficient — this is a hint, and any + // missed update is caught safely by the 2PC guard in _wait_for_task. + if(_num_topologies.load(std::memory_order_relaxed) == 0) { + return true; + } + + const size_t MAX_VICTIM = num_queues(); // guaranteed >= 2 by constructor + const size_t MAX_STEALS = ((MAX_VICTIM + 1) << 1); + + // local aliases for steal protocol sentinels — these are properties of the + // steal protocol, not of any specific queue type + size_t num_steals = 0; + size_t vtm = w._sticky_victim; + + while(true) { + + t = (vtm < _workers.size()) + ? _workers[vtm]._wsq.steal() + : _buffers[vtm - _workers.size()].queue.steal(); + + if(t) { + w._sticky_victim = vtm; + break; + } + + // EMPTY: pick a new victim excluding self since our own queue is likely empty. + // map [0, MAX_VICTIM-1) over [0, MAX_VICTIM) \ {w._id} — always safe since MAX_VICTIM >= 2. + vtm = w._rdgen() % (MAX_VICTIM - 1); + if(vtm >= w._id) vtm++; + + if(++num_steals > MAX_STEALS) { + std::this_thread::yield(); + if(num_steals > 150 + MAX_STEALS) { + break; + } + } + + if(w._done.test(std::memory_order_relaxed)) { + return false; + } + } + + return true; +} + +/* +// Function: _explore_task +inline bool Executor::_explore_task(Worker& w, Node*& t) { + + // Fast path: if no topologies are live, all queues are guaranteed empty + // by the executor's invariant (num_topologies reaches zero only after all + // nodes have been scheduled and their queues flushed). Skip the entire + // steal loop and return immediately so the caller enters _wait_for_task + // to sleep. relaxed ordering is sufficient — this is a hint, and any + // missed update is caught safely by the 2PC guard in _wait_for_task. + if(_num_topologies.load(std::memory_order_relaxed) == 0) { + return true; + } + + //assert(!t); + const size_t MAX_VICTIM = num_queues(); + const size_t MAX_STEALS = ((MAX_VICTIM + 1) << 1); + + size_t num_steals = 0; + size_t vtm = w._sticky_victim; + + // Make the worker steal immediately from the assigned victim. + while(true) { + + // If the worker's victim thread is within the worker pool, steal from the worker's queue. + // Otherwise, steal from the buffer, adjusting the victim index based on the worker pool size. + t = (vtm < _workers.size()) + ? _workers[vtm]._wsq.steal() + : _buffers[vtm - _workers.size()].queue.steal(); + + if(t) { + w._sticky_victim = vtm; + break; + } + + // Increment the steal count, and if it exceeds MAX_STEALS, yield the thread. + // If the number of empty steals reaches MAX_STEALS, exit the loop. + if (++num_steals > MAX_STEALS) { + std::this_thread::yield(); + if(num_steals > 150 + MAX_STEALS) { + break; + } + } + + if(w._done.test(std::memory_order_relaxed)) { + return false; + } + + // Randomely generate a next victim. + vtm = w._rdgen() % MAX_VICTIM; + } + return true; +} +*/ + +// Procedure: _exploit_task +inline void Executor::_exploit_task(Worker& w, Node*& t) { + while(t) { + _invoke(w, t); + t = w._wsq.pop(); + } +} + +// Function: _wait_for_task +inline bool Executor::_wait_for_task(Worker& w, Node*& t) { + + explore_task: + + if(_explore_task(w, t) == false) { + return false; + } + + // Go exploit the task if we successfully steal one. + if(t) { + return true; + } + + // Entering the 2PC guard as all queues are likely empty after many stealing attempts. + _notifier.prepare_wait(w._id); + + // Fast path: if no topologies are live, all queues are guaranteed empty. + // Skip the O(N) buffer and worker queue scans and go directly to sleep. + // This is safe because prepare_wait has already been called — any notify + // that arrives after this check but before commit_wait will be caught by + // the 2PC guarantee of the notifier. + if(_num_topologies.load(std::memory_order_relaxed) == 0) { + // still check done flag before committing to sleep + if(w._done.test(std::memory_order_relaxed)) { + _notifier.cancel_wait(w._id); + return false; + } + _notifier.commit_wait(w._id); + goto explore_task; + } + + // Condition #1: buffers should be empty + for(size_t b=0; b<_buffers.size(); ++b) { + if(!_buffers[b].queue.empty()) { + _notifier.cancel_wait(w._id); + w._sticky_victim = b + _workers.size(); + goto explore_task; + } + } + + // Condition #2: worker queues should be empty + // Note: We need to use index-based looping to avoid data race with _spawn + // which initializes other worker data structure at the same time. + // Also, due to the property of a work-stealing queue, we don't need to check + // this worker's work-stealing queue. + for(size_t k=0; k<_workers.size()-1; ++k) { + if(size_t vtm = k + (k >= w._id); !_workers[vtm]._wsq.empty()) { + _notifier.cancel_wait(w._id); + w._sticky_victim = vtm; + goto explore_task; + } + } + + // Condition #3: worker should be alive + if(w._done.test(std::memory_order_relaxed)) { + _notifier.cancel_wait(w._id); + return false; + } + + // Now I really need to relinquish myself to others. + _notifier.commit_wait(w._id); + goto explore_task; +} + +// Function: make_observer +template +std::shared_ptr Executor::make_observer(ArgsT&&... args) { + + static_assert( + std::is_base_of_v, + "Observer must be derived from ObserverInterface" + ); + + // use a local variable to mimic the constructor + auto ptr = std::make_shared(std::forward(args)...); + + ptr->set_up(_workers.size()); + + _observers.emplace(std::static_pointer_cast(ptr)); + + return ptr; +} + +// Procedure: remove_observer +template +void Executor::remove_observer(std::shared_ptr ptr) { + + static_assert( + std::is_base_of_v, + "Observer must be derived from ObserverInterface" + ); + + _observers.erase(std::static_pointer_cast(ptr)); +} + +// Function: num_observers +inline size_t Executor::num_observers() const noexcept { + return _observers.size(); +} + +// Procedure: _spill +inline void Executor::_spill(Node* item) { + // Since pointers are aligned to 8 bytes, we perform a simple hash to avoid + // contention caused by hashing to the same slot. + auto b = (reinterpret_cast(item) >> 16) % _buffers.size(); + std::scoped_lock lock(_buffers[b].mutex); + _buffers[b].queue.push(item); +} + +// Procedure: _bulk_spill (single batch to one buffer) +// Uses Knuth multiplicative hash on the first pointer to select a buffer, +// providing better bit diffusion than the shift-based approach, especially +// when the allocator returns pointers with regular low-bit patterns. +template +void Executor::_bulk_spill(I first, size_t N) { + //assert(N != 0); + auto b = ((reinterpret_cast(*first) * 2654435761ULL) >> 32) % _buffers.size(); + std::scoped_lock lock(_buffers[b].mutex); + _buffers[b].queue.bulk_push(first, N); +} + +// Procedure: _bulk_spill +// Distributes a batch of N spilled nodes across all buffers in round-robin +// order starting from a hash of the first node's pointer. Each buffer's lock +// is held only for its chunk, reducing contention compared to sending the +// entire batch to a single buffer. +template +void Executor::_bulk_spill_round_robin(I first, size_t N) { + + // assert(N != 0); + const size_t B = _buffers.size(); + const size_t start = ((reinterpret_cast(*first) * 2654435761ULL) >> 32) % B; + const size_t per_buf = (N + B - 1) / B; + size_t remaining = N; + for(size_t i = 0; i < B && remaining > 0; ++i) { + size_t b = (start + i) % B; + size_t chunk = std::min(per_buf, remaining); + { + std::scoped_lock lock(_buffers[b].mutex); + _buffers[b].queue.bulk_push(first, chunk); + } + // terminates early via remaining > 0, so we don't acquire unnecessary locks on empty chunks. + remaining -= chunk; + } +} + +// Procedure: _schedule +inline void Executor::_schedule(Worker& worker, Node* node) { + // starting at v3.5 we do not use any complicated notification mechanism + // as the experimental result has shown no significant advantage. + if(worker._wsq.try_push(node) == false) { + _spill(node); + } + _notifier.notify_one(); +} + +// Procedure: _schedule +inline void Executor::_schedule(Node* node) { + _spill(node); + _notifier.notify_one(); +} + +// Procedure: _schedule +template +void Executor::_bulk_schedule(Worker& worker, I first, size_t num_nodes) { + + if(num_nodes == 0) { + return; + } + + // NOTE: We cannot use first/last in the for-loop (e.g., for(; first != last; ++first)). + // This is because when a node v is inserted into the queue, v can run and finish + // immediately. If v is the last node in the graph, it will tear down the parent task vector + // which cause the last ++first to fail. This problem is specific to MSVC which has a stricter + // iterator implementation in std::vector than GCC/Clang. + if(auto n = worker._wsq.try_bulk_push(first, num_nodes); n != num_nodes) { + _bulk_spill(first, num_nodes - n); + } + _notifier.notify_n(num_nodes); + + // notify first before spilling to hopefully wake up workers earlier + // however, the experiment does not show any benefit for doing this. + //auto n = worker._wsq.try_bulk_push(first, num_nodes); + //_notifier.notify_n(n); + //_bulk_schedule(first + n, num_nodes - n); +} + +// Procedure: _schedule +template +inline void Executor::_bulk_schedule(I first, size_t num_nodes) { + + if(num_nodes == 0) { + return; + } + + // NOTE: We cannot use first/last in the for-loop (e.g., for(; first != last; ++first)). + // This is because when a node v is inserted into the queue, v can run and finish + // immediately. If v is the last node in the graph, it will tear down the parent task vector + // which cause the last ++first to fail. This problem is specific to MSVC which has a stricter + // iterator implementation in std::vector than GCC/Clang. + _bulk_spill(first, num_nodes); + _notifier.notify_n(num_nodes); +} + +// Function: _update_cache +TF_FORCE_INLINE void Executor::_update_cache(Worker& worker, Node*& cache, Node* node) { + if(cache) { + _schedule(worker, cache); + } + cache = node; +} + +// Function: _bulk_update_cache +template +TF_FORCE_INLINE void Executor::_bulk_update_cache( + Worker& worker, Node*& cache, Node* node, std::array& array, size_t& n +) { + // experimental results show no benefit of using bulk_update_cache + if(cache) { + array[n++] = cache; + if(n == N) { + _bulk_schedule(worker, array, n); + n = 0; + } + } + cache = node; +} + +// Procedure: _invoke +inline void Executor::_invoke(Worker& worker, Node* node) { + + #define TF_INVOKE_CONTINUATION() \ + if (cache) { \ + node = cache; \ + goto begin_invoke; \ + } + + begin_invoke: + + Node* cache {nullptr}; + + // if this is the second invoke due to preemption, directly jump to invoke task + if(node->_nstate & NSTATE::PREEMPTED) { + goto invoke_task; + } + + // If the work has been cancelled, there is no need to continue. + // Here, we do tear_down_invoke since async tasks may also get cancelled where + // we need to recycle the node. + if(node->_is_parent_cancelled()) { + _tear_down_invoke(worker, node, cache); + TF_INVOKE_CONTINUATION(); + return; + } + + // if acquiring semaphore(s) exists, acquire them first + if(node->_semaphores && !node->_semaphores->to_acquire.empty()) { + SmallVector waiters; + if(!node->_acquire_all(waiters)) { + _bulk_schedule(worker, waiters.begin(), waiters.size()); + return; + } + } + + invoke_task: + + SmallVector conds; + + // switch is faster than nested if-else due to jump table + switch(node->_handle.index()) { + // static task + case Node::STATIC:{ + _invoke_static_task(worker, node); + } + break; + + // runtime task + case Node::RUNTIME:{ + if(_invoke_runtime_task(worker, node)) { + return; + } + } + break; + + // non-preemptive runtime task + case Node::NONPREEMPTIVE_RUNTIME:{ + _invoke_nonpreemptive_runtime_task(worker, node); + } + break; + + // subflow task + case Node::SUBFLOW: { + if(_invoke_subflow_task(worker, node)) { + return; + } + } + break; + + // condition task + case Node::CONDITION: { + _invoke_condition_task(worker, node, conds); + } + break; + + // multi-condition task + case Node::MULTI_CONDITION: { + _invoke_multi_condition_task(worker, node, conds); + } + break; + + // module task + case Node::MODULE: { + if(_invoke_module_task(worker, node)) { + return; + } + } + break; + + // adopted module task + case Node::ADOPTED_MODULE: { + if(_invoke_adopted_module_task(worker, node)) { + return; + } + } + break; + + // async task + case Node::ASYNC: { + if(_invoke_async_task(worker, node)) { + return; + } + _tear_down_async(worker, node, cache); + TF_INVOKE_CONTINUATION(); + return; + } + break; + + // dependent async task + case Node::DEPENDENT_ASYNC: { + if(_invoke_dependent_async_task(worker, node)) { + return; + } + _tear_down_dependent_async(worker, node, cache); + TF_INVOKE_CONTINUATION(); + return; + } + break; + + // monostate (placeholder) + default: + break; + } + + // if releasing semaphores exist, release them + if(node->_semaphores && !node->_semaphores->to_release.empty()) { + SmallVector waiters; + node->_release_all(waiters); + _bulk_schedule(worker, waiters.begin(), waiters.size()); + } + + // Reset the join counter with strong dependencies to support cycles. + // + We must do this before scheduling the successors to avoid race + // condition on _predecessors. + // + We must use fetch_add instead of direct assigning + // because the user-level call on "invoke" may explicitly schedule + // this task again (e.g., pipeline) which can access the join_counter. + node->_join_counter.fetch_add( + node->_nstate & NSTATE::STRONG_DEPENDENCIES_MASK, std::memory_order_relaxed + ); + + // Invoke the task based on the corresponding type + switch(node->_handle.index()) { + + // condition and multi-condition tasks + case Node::CONDITION: + case Node::MULTI_CONDITION: { + for(auto cond : conds) { + if(cond >= 0 && static_cast(cond) < node->_num_successors) { + auto s = node->_edges[cond]; + // zeroing the join counter for invariant + s->_join_counter.store(0, std::memory_order_relaxed); + node->_parent->_join_counter.fetch_add(1, std::memory_order_relaxed); + _update_cache(worker, cache, s); + } + } + } + break; + + // non-condition task + default: { + for(size_t i=0; i_num_successors; ++i) { + if(auto s = node->_edges[i]; s->_join_counter.fetch_sub(1, std::memory_order_acq_rel) == 1) { + node->_parent->_join_counter.fetch_add(1, std::memory_order_relaxed); + _update_cache(worker, cache, s); + } + } + } + break; + } + + // clean up the node after execution + _tear_down_nonasync(worker, node, cache); + TF_INVOKE_CONTINUATION(); +} + +// Procedure: _tear_down_nonasync +inline void Executor::_tear_down_nonasync(Worker& worker, Node* node, Node*& cache) { + + // we must check parent first before subtracting the join counter, + // or it can introduce data race + if(auto parent = node->_parent; parent == node->_topology) { + if(parent->_join_counter.fetch_sub(1, std::memory_order_acq_rel) == 1) { + _tear_down_topology(worker, node->_topology, cache); + } + } + else { + // needs to fetch every data before join counter becomes zero at which + // the node may be deleted + auto state = parent->_nstate; + if(parent->_join_counter.fetch_sub(1, std::memory_order_acq_rel) == 1) { + // this task is spawned from a preempted parent, so we need to resume it + if(state & NSTATE::PREEMPTED) { + _update_cache(worker, cache, static_cast(parent)); + } + } + } +} + +// Procedure: _tear_down_invoke +inline void Executor::_tear_down_invoke(Worker& worker, Node* node, Node*& cache) { + switch(node->_handle.index()) { + case Node::ASYNC: + _tear_down_async(worker, node, cache); + break; + + case Node::DEPENDENT_ASYNC: + _tear_down_dependent_async(worker, node, cache); + break; + + default: + _tear_down_nonasync(worker, node, cache); + break; + } +} + +// Procedure: _observer_prologue +inline void Executor::_observer_prologue(Worker& worker, Node* node) { + for(auto& observer : _observers) { + observer->on_entry(WorkerView(worker), TaskView(*node)); + } +} + +// Procedure: _observer_epilogue +inline void Executor::_observer_epilogue(Worker& worker, Node* node) { + for(auto& observer : _observers) { + observer->on_exit(WorkerView(worker), TaskView(*node)); + } +} + +// Procedure: _process_exception +inline void Executor::_process_exception(Worker&, Node* node) { + + // Finds the anchor and mark the entire path with exception, + // so recursive tasks can be cancelled properly. + // Since exception can come from asynchronous task (with runtime), the node itself can be anchored. + NodeBase* ea = node; // explicit anchor + NodeBase* ia = nullptr; // implicit anchor + + while(ea && (ea->_estate.load(std::memory_order_relaxed) & ESTATE::EXPLICITLY_ANCHORED) == 0) { + ea->_estate.fetch_or(ESTATE::EXCEPTION, std::memory_order_relaxed); + // we only want the inner-most implicit anchor + if(ia == nullptr && (ea->_nstate & NSTATE::IMPLICITLY_ANCHORED)) { + ia = ea; + } + ea = ea->_parent; + } + + // flag used to ensure execution is caught in a thread-safe manner + constexpr static auto flag = ESTATE::EXCEPTION | ESTATE::CAUGHT; + + // The exception occurs under a blocking call (e.g., corun, join). + if(ea) { + // multiple tasks may throw, and we only take the first thrown exception + if((ea->_estate.fetch_or(flag, std::memory_order_relaxed) & ESTATE::CAUGHT) == 0) { + ea->_exception_ptr = std::current_exception(); + return; + } + } + // Implicit anchor has the lowest priority + else if(ia){ + if((ia->_estate.fetch_or(flag, std::memory_order_relaxed) & ESTATE::CAUGHT) == 0) { + ia->_exception_ptr = std::current_exception(); + return; + } + } + + // For now, we simply store the exception in this node; this can happen in an + // execution that does not have any external control to capture the exception, + // such as silent async task without any parent. + node->_exception_ptr = std::current_exception(); +} + +// Procedure: _invoke_static_task +inline void Executor::_invoke_static_task(Worker& worker, Node* node) { + _observer_prologue(worker, node); + TF_EXECUTOR_EXCEPTION_HANDLER(worker, node, { + std::get_if(&node->_handle)->work(); + }); + _observer_epilogue(worker, node); +} + +// Procedure: _invoke_subflow_task +inline bool Executor::_invoke_subflow_task(Worker& worker, Node* node) { + + auto& h = *std::get_if(&node->_handle); + auto& g = h.subgraph; + + if((node->_nstate & NSTATE::PREEMPTED) == 0) { + + // set up the subflow + Subflow sf(*this, worker, node, g); + + // invoke the subflow callable + _observer_prologue(worker, node); + TF_EXECUTOR_EXCEPTION_HANDLER(worker, node, { + h.work(sf); + }); + _observer_epilogue(worker, node); + + // spawn the subflow if it is joinable and its graph is non-empty + // implicit join is faster than Subflow::join as it does not involve corun + if(sf.joinable() && !g.empty()) { + + // signal the executor to preempt this node + node->_nstate |= NSTATE::PREEMPTED; + + // set up and schedule the graph + _schedule_graph(worker, g, node->_topology, node); + return true; + } + } + else { + node->_nstate &= ~NSTATE::PREEMPTED; + } + + // The subflow has finished or joined. + // By default, we clear the subflow storage as applications can perform recursive + // subflow tasking which accumulates a huge amount of memory overhead, hampering + // the performance. + if((node->_nstate & NSTATE::RETAIN_SUBFLOW) == 0) { + g.clear(); + } + + return false; +} + +// Procedure: _invoke_condition_task +inline void Executor::_invoke_condition_task( + Worker& worker, Node* node, SmallVector& conds +) { + _observer_prologue(worker, node); + TF_EXECUTOR_EXCEPTION_HANDLER(worker, node, { + auto& work = std::get_if(&node->_handle)->work; + conds = { work() }; + }); + _observer_epilogue(worker, node); +} + +// Procedure: _invoke_multi_condition_task +inline void Executor::_invoke_multi_condition_task( + Worker& worker, Node* node, SmallVector& conds +) { + _observer_prologue(worker, node); + TF_EXECUTOR_EXCEPTION_HANDLER(worker, node, { + conds = std::get_if(&node->_handle)->work(); + }); + _observer_epilogue(worker, node); +} + +// Procedure: _invoke_module_task +inline bool Executor::_invoke_module_task(Worker& w, Node* node) { + return _invoke_module_task_impl(w, node, std::get_if(&node->_handle)->graph); +} + +// Procedure: _invoke_adopted_module_task +inline bool Executor::_invoke_adopted_module_task(Worker& w, Node* node) { + return _invoke_module_task_impl(w, node, std::get_if(&node->_handle)->graph); +} + +// Procedure: _invoke_module_task_impl +inline bool Executor::_invoke_module_task_impl(Worker& w, Node* node, Graph& graph) { + + // No need to do anything for empty graph + if(graph.empty()) { + return false; + } + + // first entry - not spawned yet + if((node->_nstate & NSTATE::PREEMPTED) == 0) { + // signal the executor to preempt this node + node->_nstate |= NSTATE::PREEMPTED; + _schedule_graph(w, graph, node->_topology, node); + return true; + } + + // second entry - already spawned + node->_nstate &= ~NSTATE::PREEMPTED; + + return false; +} + + +// Procedure: _invoke_async_task +inline bool Executor::_invoke_async_task(Worker& worker, Node* node) { + auto& work = std::get_if(&node->_handle)->work; + switch(work.index()) { + // void() + case 0: + _observer_prologue(worker, node); + TF_EXECUTOR_EXCEPTION_HANDLER(worker, node, { + std::get_if<0>(&work)->operator()(); + }); + _observer_epilogue(worker, node); + break; + + // void(Runtime&) + case 1: + if(_invoke_runtime_task_impl(worker, node, *std::get_if<1>(&work))) { + return true; + } + break; + + // void(Runtime&, bool) + case 2: + if(_invoke_runtime_task_impl(worker, node, *std::get_if<2>(&work))) { + return true; + } + break; + } + + return false; +} + +// Procedure: _invoke_dependent_async_task +inline bool Executor::_invoke_dependent_async_task(Worker& worker, Node* node) { + auto& work = std::get_if(&node->_handle)->work; + switch(work.index()) { + // void() + case 0: + _observer_prologue(worker, node); + TF_EXECUTOR_EXCEPTION_HANDLER(worker, node, { + std::get_if<0>(&work)->operator()(); + }); + _observer_epilogue(worker, node); + break; + + // void(Runtime&) - silent async + case 1: + if(_invoke_runtime_task_impl(worker, node, *std::get_if<1>(&work))) { + return true; + } + break; + + // void(Runtime&, bool) - async + case 2: + if(_invoke_runtime_task_impl(worker, node, *std::get_if<2>(&work))) { + return true; + } + break; + } + return false; +} + +// Function: run +inline tf::Future Executor::run(Taskflow& f) { + return run_n(f, 1, [](){}); +} + +// Function: run +inline tf::Future Executor::run(Taskflow&& f) { + return run_n(std::move(f), 1, [](){}); +} + +// Function: run +template +tf::Future Executor::run(Taskflow& f, C&& c) { + return run_n(f, 1, std::forward(c)); +} + +// Function: run +template +tf::Future Executor::run(Taskflow&& f, C&& c) { + return run_n(std::move(f), 1, std::forward(c)); +} + +// Function: run_n +inline tf::Future Executor::run_n(Taskflow& f, size_t repeat) { + return run_n(f, repeat, [](){}); +} + +// Function: run_n +inline tf::Future Executor::run_n(Taskflow&& f, size_t repeat) { + return run_n(std::move(f), repeat, [](){}); +} + +// Function: run_n +template +tf::Future Executor::run_n(Taskflow& f, size_t repeat, C&& c) { + return run_until( + f, [repeat]() mutable { return repeat-- == 0; }, std::forward(c) + ); +} + +// Function: run_n +template +tf::Future Executor::run_n(Taskflow&& f, size_t repeat, C&& c) { + return run_until( + std::move(f), [repeat]() mutable { return repeat-- == 0; }, std::forward(c) + ); +} + +// Function: run_until +template +tf::Future Executor::run_until(Taskflow& f, P&& pred) { + return run_until(f, std::forward

(pred), [](){}); +} + +// Function: run_until +template +tf::Future Executor::run_until(Taskflow&& f, P&& pred) { + return run_until(std::move(f), std::forward

(pred), [](){}); +} + +// Function: run_until +template +tf::Future Executor::run_until(Taskflow& f, P&& p, C&& c) { + + // No need to create a real topology but returns an dummy future for invariant. + if(f.empty() || p()) { + c(); + std::promise promise; + promise.set_value(); + return tf::Future(promise.get_future()); + } + + _increment_topology(); + + // create a topology for this run + auto t = std::make_shared(f, std::forward

(p), std::forward(c)); + //auto t = std::make_shared>(f, std::forward

(p), std::forward(c)); + + // need to create future before the topology got torn down quickly + tf::Future future(t->_promise.get_future(), t); + + // modifying topology needs to be protected under the lock + if(f._fetch_enqueue(t) == 0) { + _set_up_topology(this_worker(), t.get()); + } + + return future; +} + + + +// Function: corun_until +template +void Executor::corun_until(P&& predicate) { + + Worker* w = this_worker(); + if(w == nullptr) { + TF_THROW("corun_until must be called by a worker of the executor"); + } + + _corun_until(*w, std::forward

(predicate)); +} + +/* +// Function: _corun_until +template +void Executor::_corun_until(Worker& w, P&& stop_predicate) { + + const size_t MAX_VICTIM = num_queues(); + const size_t MAX_STEALS = ((MAX_VICTIM + 1) << 1); + + bool stop = false; + + while(!stop && !(stop = stop_predicate())) { + + // try local queue first — only one task at a time to avoid deep + // recursive corun calls causing stack overflow + if(auto t = w._wsq.pop(); t) { + _invoke(w, t); + continue; + } + + // local queue empty: steal from others until stop_predicate or stolen. + // stop is set by the inner loop condition so when predicate becomes true + // the outer loop exits immediately without calling stop_predicate again. + size_t num_steals = 0; + size_t vtm = w._sticky_victim; + + while(!(stop = stop_predicate())) { + + auto t = (vtm < _workers.size()) + ? _workers[vtm]._wsq.steal() + : _buffers[vtm - _workers.size()].queue.steal(); + + if(t) { + // STOLEN: invoke task then return to outer loop to re-check + // local queue and stop_predicate + _invoke(w, t); + w._sticky_victim = vtm; + break; + } + + // pick a new victim excluding self + vtm = w._rdgen() % (MAX_VICTIM - 1); + if(vtm >= w._id) vtm++; + + if(++num_steals > MAX_STEALS) { + // unlike _explore_task we cannot sleep here — the calling worker + // is blocked inside a task and must keep making progress to avoid + // deadlock. yield to let other threads run and make progress. + std::this_thread::yield(); + } + } + } +}*/ + +// Function: _corun_until +template +void Executor::_corun_until(Worker& w, P&& stop_predicate) { + + const size_t MAX_VICTIM = num_queues(); + const size_t MAX_STEALS = ((MAX_VICTIM + 1) << 1); + + exploit: + + while(!stop_predicate()) { + + // here we don't do while-loop to drain out the local queue as it can + // potentially enter a very deep recursive corun, cuasing stack overflow + if(auto t = w._wsq.pop(); t) { + _invoke(w, t); + } + else { + size_t num_steals = 0; + size_t vtm = w._sticky_victim; + + explore: + + t = (vtm < _workers.size()) + ? _workers[vtm]._wsq.steal() + : _buffers[vtm-_workers.size()].queue.steal(); + + if(t) { + _invoke(w, t); + w._sticky_victim = vtm; + goto exploit; + } + else if(!stop_predicate()) { + if(++num_steals > MAX_STEALS) { + std::this_thread::yield(); + } + vtm = w._rdgen() % MAX_VICTIM; + goto explore; + } + else { + break; + } + } + } +} + +// Function: corun +template +void Executor::corun(T& target) { + + Worker* w = this_worker(); + if(w == nullptr) { + TF_THROW("corun must be called by a worker of the executor"); + } + + NodeBase anchor; + _corun_graph(*w, retrieve_graph(target), nullptr, &anchor); +} + +// Procedure: _corun_graph +inline void Executor::_corun_graph(Worker& w, Graph& g, Topology* tpg, NodeBase* p) { + + // empty graph + if(g.empty()) { + return; + } + + // anchor this parent as the blocking point + { + ExplicitAnchorGuard anchor(p); + _schedule_graph(w, g, tpg, p); + _corun_until(w, [p] () -> bool { + return p->_join_counter.load(std::memory_order_acquire) == 0; } + ); + } + + // rethrow the exception to the caller + p->_rethrow_exception(); +} + +// Procedure: _increment_topology +inline void Executor::_increment_topology() { + _num_topologies.fetch_add(1, std::memory_order_relaxed); +} + +// Procedure: _decrement_topology +inline void Executor::_decrement_topology() { + if(_num_topologies.fetch_sub(1, std::memory_order_acq_rel) == 1) { + _num_topologies.notify_all(); + } +} + +// Procedure: wait_for_all +inline void Executor::wait_for_all() { + size_t n = _num_topologies.load(std::memory_order_acquire); + while(n != 0) { + _num_topologies.wait(n, std::memory_order_acquire); + n = _num_topologies.load(std::memory_order_acquire); + } +} + +// Function: _schedule_graph +inline void Executor::_schedule_graph( + Worker& worker, Graph& graph, Topology* tpg, NodeBase* parent +) { + size_t num_srcs = _set_up_graph(graph, tpg, parent); + parent->_join_counter.fetch_add(num_srcs, std::memory_order_relaxed); + _bulk_schedule(worker, graph.begin(), num_srcs); +} + +// Function: _set_up_topology +inline void Executor::_set_up_topology(Worker* w, Topology* tpg) { + // ---- under taskflow lock ---- + auto& g = tpg->_taskflow._graph; + size_t num_srcs = _set_up_graph(g, tpg, tpg); + tpg->_join_counter.store(num_srcs, std::memory_order_relaxed); + w ? _bulk_schedule(*w, g.begin(), num_srcs) : _bulk_schedule(g.begin(), num_srcs); +} + +// Function: _set_up_graph +inline size_t Executor::_set_up_graph(Graph& graph, Topology* tpg, NodeBase* parent) { + + auto first = graph.begin(); + auto last = graph.end(); + auto send = first; + for(; first != last; ++first) { + + auto node = *first; + node->_topology = tpg; + node->_parent = parent; + node->_nstate = NSTATE::NONE; + node->_estate.store(ESTATE::NONE, std::memory_order_relaxed); + node->_set_up_join_counter(); + node->_exception_ptr = nullptr; + + // move source to the first partition + // root, root, root, v1, v2, v3, v4, ... + if(node->num_predecessors() == 0) { + std::iter_swap(send++, first); + } + } + return send - graph.begin(); +} + +// Function: _tear_down_topology +inline void Executor::_tear_down_topology(Worker& worker, Topology* tpg, Node*& cache) { + + auto &f = tpg->_taskflow; + + //assert(&tpg == &(f._topologies.front())); + + // case 1: we still need to run the topology again + //if(!tpg->_exception_ptr && !tpg->cancelled() && !tpg->predicate()) { + if(!tpg->cancelled() && !tpg->_predicate()) { + //assert(tpg->_join_counter == 0); + //std::lock_guard lock(f._mutex); + _schedule_graph(worker, tpg->_taskflow._graph, tpg, tpg); + } + // case 2: the final run of this topology + else { + + // invoke the callback after each run + tpg->_on_finish(); + + // there is another topologies to run + if(std::unique_lock lock(f._mutex); f._topologies.size()>1) { + + auto fetched_tpg {std::move(f._topologies.front())}; + //assert(fetched_tpg.get() == tpg); + + f._topologies.pop(); + tpg = f._topologies.front().get(); + + lock.unlock(); + + // Soon after we carry out the promise, the associate taskflow may got destroyed + // from the user side, and we should never tough it again. + fetched_tpg->_carry_out_promise(); + + // decrement the topology + _decrement_topology(); + + _schedule_graph(worker, tpg->_taskflow._graph, tpg, tpg); + } + else { + //assert(f._topologies.size() == 1); + + auto fetched_tpg {std::move(f._topologies.front())}; + //assert(fetched_tpg.get() == tpg); + + f._topologies.pop(); + + lock.unlock(); + + // Soon after we carry out the promise, the associate taskflow may got destroyed + // from the user side, and we should never tough it again. + fetched_tpg->_carry_out_promise(); + + _decrement_topology(); + + // remove the parent that owns the moved taskflow so the storage can be freed + if(auto parent = fetched_tpg->_parent; parent) { + //auto state = parent->_nstate; + if(parent->_join_counter.fetch_sub(1, std::memory_order_acq_rel) == 1) { + // this async is spawned from a preempted parent, so we need to resume it + //if(state & NSTATE::PREEMPTED) { + _update_cache(worker, cache, static_cast(parent)); + //} + } + } + } + } +} + +// ############################################################################ +// Forward Declaration: Subflow +// ############################################################################ + +inline void Subflow::join() { + + if(!joinable()) { + TF_THROW("subflow already joined"); + } + + _executor._corun_graph(_worker, _graph, _node->_topology, _node); + + // join here since corun graph may throw exception + _node->_nstate |= NSTATE::JOINED_SUBFLOW; +} + +#endif + + + + +} // end of namespace tf ----------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/core/flow_builder.hpp b/projects/hipfile/third-party/taskflow/core/flow_builder.hpp new file mode 100644 index 000000000000..1c81836ecdd2 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/core/flow_builder.hpp @@ -0,0 +1,1862 @@ +#pragma once + +#include "task.hpp" +#include "../algorithm/partitioner.hpp" + +/** +@file flow_builder.hpp +@brief flow builder include file +*/ + +namespace tf { + +/** +@class FlowBuilder + +@brief class to build a task dependency graph + +The class provides essential methods to construct a task dependency graph +from which tf::Taskflow and tf::Subflow are derived. + +*/ +class FlowBuilder { + + friend class Executor; + + public: + + /** + @brief constructs a flow builder with a graph + */ + FlowBuilder(Graph& graph); + + /** + @brief creates a static task + + @tparam C callable type satisfying tf::StaticTaskLike + + @param callable callable to construct a static task + + @return a tf::Task handle + + The following example creates a static task. + + @code{.cpp} + tf::Task static_task = taskflow.emplace([](){}); + @endcode + + @note + Please refer to @ref StaticTasking for details. + */ + template + Task emplace(C&& callable); + + /** + @brief creates a runtime task + + @tparam C callable type satisfying tf::RuntimeTaskLike + + @param callable callable to construct a runtime task + + @return a tf::Task handle + + The following example creates a runtime task. + + @code{.cpp} + tf::Task static_task = taskflow.emplace([](tf::Runtime&){}); + @endcode + + @note + Please refer to @ref RuntimeTasking for details. + */ + template + Task emplace(C&& callable); + + /** + @brief creates a dynamic task + + @tparam C callable type satisfying tf::SubflowTaskLike + + @param callable callable to construct a dynamic task + + @return a tf::Task handle + + The following example creates a dynamic task (tf::Subflow) + that spawns two static tasks. + + @code{.cpp} + tf::Task dynamic_task = taskflow.emplace([](tf::Subflow& sf){ + tf::Task static_task1 = sf.emplace([](){}); + tf::Task static_task2 = sf.emplace([](){}); + }); + @endcode + + @note + Please refer to @ref SubflowTasking for details. + */ + template + Task emplace(C&& callable); + + /** + @brief creates a condition task + + @tparam C callable type satisfying tf::ConditionTaskLike + + @param callable callable to construct a condition task + + @return a tf::Task handle + + The following example creates an if-else block using one condition task + and three static tasks. + + @code{.cpp} + tf::Taskflow taskflow; + + auto [init, cond, yes, no] = taskflow.emplace( + [] () { }, + [] () { return 0; }, + [] () { std::cout << "yes\n"; }, + [] () { std::cout << "no\n"; } + ); + + // executes yes if cond returns 0, or no if cond returns 1 + cond.precede(yes, no); + cond.succeed(init); + @endcode + + @note + Please refer to @ref ConditionalTasking for details. + */ + template + Task emplace(C&& callable); + + /** + @brief creates a multi-condition task + + @tparam C callable type satisfying tf::MultiConditionTaskLike + + @param callable callable to construct a multi-condition task + + @return a tf::Task handle + + The following example creates a multi-condition task that selectively + jumps to two successor tasks. + + @code{.cpp} + tf::Taskflow taskflow; + + auto [init, cond, branch1, branch2, branch3] = taskflow.emplace( + [] () { }, + [] () { return tf::SmallVector{0, 2}; }, + [] () { std::cout << "branch1\n"; }, + [] () { std::cout << "branch2\n"; }, + [] () { std::cout << "branch3\n"; } + ); + + // executes branch1 and branch3 when cond returns 0 and 2 + cond.precede(branch1, branch2, branch3); + cond.succeed(init); + @endcode + + @note + Please refer to @ref ConditionalTasking for details. + */ + template + Task emplace(C&& callable); + + /** + @brief creates multiple tasks from a list of callable objects + + @tparam C callable types + + @param callables one or multiple callable objects constructible from each task category + + @return a tf::Task handle + + The method returns a tuple of tasks each corresponding to the given + callable target. You can use structured binding to get the return tasks + one by one. + The following example creates four static tasks and assign them to + @c A, @c B, @c C, and @c D using structured binding. + + @code{.cpp} + auto [A, B, C, D] = taskflow.emplace( + [] () { std::cout << "A"; }, + [] () { std::cout << "B"; }, + [] () { std::cout << "C"; }, + [] () { std::cout << "D"; } + ); + @endcode + */ + template requires (sizeof...(C) > 1) + auto emplace(C&&... callables); + + /** + @brief removes a task from a taskflow + + @param task task to remove + + Removes a task and its input and output dependencies from the graph + associated with the flow builder. + If the task does not belong to the graph, nothing will happen. + + @code{.cpp} + tf::Task A = taskflow.emplace([](){ std::cout << "A"; }); + tf::Task B = taskflow.emplace([](){ std::cout << "B"; }); + tf::Task C = taskflow.emplace([](){ std::cout << "C"; }); + tf::Task D = taskflow.emplace([](){ std::cout << "D"; }); + A.precede(B, C, D); + + // erase A from the taskflow and its dependencies to B, C, and D + taskflow.erase(A); + @endcode + */ + void erase(Task task); + + /** + @brief creates a module task for the target object + + @tparam T type satisfying tf::GraphLike + @param object a custom object that defines the method @c T::graph() + + @return a tf::Task handle + + The example below demonstrates a taskflow composition using + the @c composed_of method. + + @code{.cpp} + tf::Taskflow t1, t2; + t1.emplace([](){ std::cout << "t1"; }); + + // t2 is partially composed of t1 + tf::Task comp = t2.composed_of(t1); + tf::Task init = t2.emplace([](){ std::cout << "t2"; }); + init.precede(comp); + @endcode + + The taskflow object @c t2 is composed of another taskflow object @c t1, + preceded by another static task @c init. + When taskflow @c t2 is submitted to an executor, + @c init will run first and then @c comp which spawns its definition + in taskflow @c t1. + + The target @c object being composed must define the method + T::graph() that returns a reference to a graph object of + type tf::Graph such that it can interact with the executor. + For example: + + @code{.cpp} + // custom struct + struct MyObj { + tf::Graph graph; + MyObj() { + tf::FlowBuilder builder(graph); + tf::Task task = builder.emplace([](){ + std::cout << "a task\n"; // static task + }); + } + Graph& graph() { return graph; } + }; + + MyObj obj; + tf::Task comp = taskflow.composed_of(obj); + @endcode + + Or, simply expose the graph object and pass it to `composed_of`: + + @code{.cpp} + tf::Graph graph; + tf::FlowBuilder builder(graph); + tf::Task task = builder.emplace([](){ + std::cout << "a task\n"; // static task + }); + tf::Task comp = taskflow.composed_of(graph); + @endcode + + @note + Please refer to @ref ComposableTasking for details. + */ + template + Task composed_of(T& object); + + /** + @brief creates a module task from a graph by taking over its ownership + + @param graph the graph to adopt (moved into the task) + + @return a Task handle to the adopted module task + + Unlike tf::FlowBuilder::composed_of, which references an externally-owned + tf::Taskflow, @c adopt transfers ownership of the given tf::Graph into + the task. The graph's lifetime is managed by the executor once adopted, + and the caller has no access to the moved-from graph afterward. + + @code{.cpp} + tf::Taskflow taskflow; + tf::Graph g; + tf::FlowBuilder{g}.emplace([]{ std::cout << "task in adopted graph\n"; }); + taskflow.adopt(std::move(g)).name("adopted"); + @endcode + + @note Please refer to @ref ComposableTasking for details. + */ + Task adopt(Graph&& graph); + + /** + @brief creates a placeholder task + + @return a tf::Task handle + + A placeholder task maps to a node in the taskflow graph, but + it does not have any callable work assigned yet. + A placeholder task is different from an empty task handle that + does not point to any node in a graph. + + @code{.cpp} + // create a placeholder task with no callable target assigned + tf::Task placeholder = taskflow.placeholder(); + assert(placeholder.empty() == false && placeholder.has_work() == false); + + // create an empty task handle + tf::Task task; + assert(task.empty() == true); + + // assign the task handle to the placeholder task + task = placeholder; + assert(task.empty() == false && task.has_work() == false); + @endcode + */ + Task placeholder(); + + /** + @brief adds adjacent dependency links to a linear list of tasks + + @param tasks a vector of tasks + + This member function creates linear dependencies over a vector of tasks. + + @code{.cpp} + tf::Task A = taskflow.emplace([](){ std::cout << "A"; }); + tf::Task B = taskflow.emplace([](){ std::cout << "B"; }); + tf::Task C = taskflow.emplace([](){ std::cout << "C"; }); + tf::Task D = taskflow.emplace([](){ std::cout << "D"; }); + std::vector tasks {A, B, C, D} + taskflow.linearize(tasks); // A->B->C->D + @endcode + + */ + void linearize(std::vector& tasks); + + /** + @brief adds adjacent dependency links to a linear list of tasks + + @param tasks an initializer list of tasks + + This member function creates linear dependencies over a list of tasks. + + @code{.cpp} + tf::Task A = taskflow.emplace([](){ std::cout << "A"; }); + tf::Task B = taskflow.emplace([](){ std::cout << "B"; }); + tf::Task C = taskflow.emplace([](){ std::cout << "C"; }); + tf::Task D = taskflow.emplace([](){ std::cout << "D"; }); + taskflow.linearize({A, B, C, D}); // A->B->C->D + @endcode + */ + void linearize(std::initializer_list tasks); + + + // ------------------------------------------------------------------------ + // parallel iterations + // ------------------------------------------------------------------------ + + /** + @brief constructs an STL-styled parallel-for task + + @tparam B beginning iterator type + @tparam E ending iterator type + @tparam C callable type + @tparam P type satisfying tf::PartitionerLike + + @param first iterator to the beginning (inclusive) + @param last iterator to the end (exclusive) + @param callable callable object to apply to the dereferenced iterator + @param part partitioning algorithm to schedule parallel iterations + + @return a tf::Task handle + + The task spawns asynchronous tasks that applies the callable object to each object + obtained by dereferencing every iterator in the range [first, last). + This method is equivalent to the parallel execution of the following loop: + + @code{.cpp} + for(auto itr=first; itr!=last; itr++) { + callable(*itr); + } + @endcode + + Iterators can be made stateful by using std::reference_wrapper + The callable needs to take a single argument of + the dereferenced iterator type. + + @note + Please refer to @ref ParallelIterations for details. + */ + template + Task for_each(B first, E last, C callable, P part = P()); + + /** + @brief constructs an index-based parallel-for task + + @tparam B beginning index type (must be integral) + @tparam E ending index type (must be integral) + @tparam S step type (must be integral) + @tparam C callable type + @tparam P type satisfying tf::PartitionerLike + + @param first index of the beginning (inclusive) + @param last index of the end (exclusive) + @param step step size + @param callable callable object to apply to each valid index + @param part partitioning algorithm to schedule parallel iterations + + @return a tf::Task handle + + The task spawns asynchronous tasks that applies the callable object to each index + in the range [first, last) with the step size. + This method is equivalent to the parallel execution of the following loop: + + @code{.cpp} + // case 1: step size is positive + for(auto i=first; ilast; i+=step) { + callable(i); + } + @endcode + + Iterators can be made stateful by using std::reference_wrapper + The callable needs to take a single argument of the integral index type. + + @note + Please refer to @ref ParallelIterations for details. + */ + template + Task for_each_index(B first, E last, S step, C callable, P part = P()); + + /** + @brief constructs a parallel-for task over a one- or multi-dimensional index range + + @tparam R type satisfying tf::IndexRangesLike (i.e., tf::IndexRanges); + for @c N == 1 (equivalently, @c R is tf::IndexRange) the engine + uses the 1D unraveling fast path described below, and for @c N > 1 + it partitions the Cartesian product as described further down + @tparam C callable type that is invocable with a single argument of type R + @tparam P type satisfying tf::PartitionerLike + + @param range index range + @param callable callable object to apply to each partitioned index range + @param part partitioning algorithm to schedule parallel iterations + + @return a tf::Task handle + + The task spawns asynchronous tasks that partition @c range and invoke + @c callable once per partition, where each partition is itself a (sub-)range + of the same type @c R. + + @par One-dimensional range (`N` == 1) + + For a 1D range tf::IndexRange, the task applies @c callable to + each index subrange of [first, last) with the given step size. This + is equivalent to the parallel execution of the following loop: + + @code{.cpp} + // case 1: step size is positive + for(auto i=first; ilast; i+=step) { + callable(i); + } + @endcode + + @code{.cpp} + // [0, 17) with a step size of 2 using tf::IndexRange + tf::IndexRange range(0, 17, 2); + + // parallelize the sequence [0, 2, 4, 6, 8, 10, 12, 14, 16] + taskflow.for_each_by_index(range, [](tf::IndexRange subrange) { + // iterate each index in the subrange + for(int i=subrange.begin(); i 1) + + For @c N > 1, the function parallelises iteration over the Cartesian product + of @c N independent 1D ranges. The total iteration space is linearized in + row-major order (last dimension varies fastest) and divided among workers + according to @c part. Each worker receives one or more orthogonal sub-boxes + and invokes @c callable once per sub-box. + + Each sub-box is guaranteed to be a valid hyper-rectangle: every dimension of + the sub-box lies entirely within the corresponding dimension of @c range and + preserves its original step size, including negative strides. Each + dimension of a tf::IndexRanges is a std::tuple of + (begin, end, step) accessible through @c dim(d), so the callable typically + destructures it via structured bindings and must iterate the sub-box using + the step sizes reported by each dimension: + + @code{.cpp} + // 3D range: depth x height x width + tf::IndexRanges range( + tf::IndexRange(0, D, 1), + tf::IndexRange(0, H, 1), + tf::IndexRange(0, W, 1) + ); + + taskflow.for_each_by_index(range, [](const tf::IndexRanges& sub) { + auto [d0, d1, ds] = sub.dim(0); + auto [h0, h1, hs] = sub.dim(1); + auto [w0, w1, ws] = sub.dim(2); + for(auto d = d0; d < d1; d += ds) { + for(auto h = h0; h < h1; h += hs) { + for(auto w = w0; w < w1; w += ws) { + // process element (d, h, w) + } + } + } + }); + @endcode + + Stateful ranges + + Ranges of any rank can be made stateful by passing them through + @c std::reference_wrapper (via @c std::ref). This is useful when the range + bounds are not known at task-graph construction time. An upstream task must + set the bounds before this task runs: + + @code{.cpp} + tf::IndexRanges range; + + auto init = taskflow.emplace([&](){ + range.dim(0) = {0, rows, 1}; + range.dim(1) = {0, cols, 1}; + }); + + auto loop = taskflow.for_each_by_index(std::ref(range), callable); + init.precede(loop); + @endcode + + The loop condition inside the callable must respect the sign of each + dimension's step size: use @c < for positive steps and @c > for negative steps. + + @note + Please refer to @ref ParallelIterations for details. + */ + template + Task for_each_by_index(R range, C callable, P part = P()); + + // ------------------------------------------------------------------------ + // transform + // ------------------------------------------------------------------------ + + /** + @brief constructs a parallel-transform task + + @tparam B beginning input iterator type + @tparam E ending input iterator type + @tparam O output iterator type + @tparam C callable type + @tparam P type satisfying tf::PartitionerLike + + @param first1 iterator to the beginning of the first range + @param last1 iterator to the end of the first range + @param d_first iterator to the beginning of the output range + @param c an unary callable to apply to dereferenced input elements + @param part partitioning algorithm to schedule parallel iterations + + @return a tf::Task handle + + The task spawns asynchronous tasks that applies the callable object to an + input range and stores the result in another output range. + This method is equivalent to the parallel execution of the following loop: + + @code{.cpp} + while (first1 != last1) { + *d_first++ = c(*first1++); + } + @endcode + + Iterators can be made stateful by using std::reference_wrapper + The callable needs to take a single argument of the dereferenced + iterator type. + + @note + Please refer to @ref ParallelTransforms for details. + */ + template + Task transform(B first1, E last1, O d_first, C c, P part = P()); + + /** + @brief constructs a parallel-transform task + + @tparam B1 beginning input iterator type for the first input range + @tparam E1 ending input iterator type for the first input range + @tparam B2 beginning input iterator type for the first second range + @tparam O output iterator type + @tparam C callable type + @tparam P type satisfying tf::PartitionerLike + + @param first1 iterator to the beginning of the first input range + @param last1 iterator to the end of the first input range + @param first2 iterator to the beginning of the second input range + @param d_first iterator to the beginning of the output range + @param c a binary operator to apply to dereferenced input elements + @param part partitioning algorithm to schedule parallel iterations + + @return a tf::Task handle + + The task spawns asynchronous tasks that applies the callable object to two + input ranges and stores the result in another output range. + This method is equivalent to the parallel execution of the following loop: + + @code{.cpp} + while (first1 != last1) { + *d_first++ = c(*first1++, *first2++); + } + @endcode + + Iterators can be made stateful by using std::reference_wrapper + The callable needs to take two arguments of dereferenced elements + from the two input ranges. + + @note + Please refer to @ref ParallelTransforms for details. + */ + template + requires (!PartitionerLike>) + Task transform(B1 first1, E1 last1, B2 first2, O d_first, C c, P part = P()); + + // ------------------------------------------------------------------------ + // reduction + // ------------------------------------------------------------------------ + + /** + @brief constructs an STL-styled parallel-reduction task + + @tparam B beginning iterator type + @tparam E ending iterator type + @tparam T result type + @tparam O binary reducer type + @tparam P type satisfying tf::PartitionerLike + + @param first iterator to the beginning (inclusive) + @param last iterator to the end (exclusive) + @param init initial value of the reduction and the storage for the reduced result + @param bop binary operator that will be applied + @param part partitioning algorithm to schedule parallel iterations + + @return a tf::Task handle + + The task spawns asynchronous tasks to perform parallel reduction over @c init + and the elements in the range [first, last). + The reduced result is store in @c init. + This method is equivalent to the parallel execution of the following loop: + + @code{.cpp} + for(auto itr=first; itr!=last; itr++) { + init = bop(init, *itr); + } + @endcode + + Iterators can be made stateful by using std::reference_wrapper + + @note + Please refer to @ref ParallelReduction for details. + */ + template + Task reduce(B first, E last, T& init, O bop, P part = P()); + + /** + @brief constructs an index range-based parallel-reduction task + + @tparam R type satisfying tf::IndexRanges1DLike + @tparam T result type + @tparam L local reducer type + @tparam G global reducer type + @tparam P type satisfying tf::PartitionerLike + + @param range index range + @param init initial value of the reduction and the storage for the reduced result + @param lop binary operator that will be applied locally per worker + @param gop binary operator that will be applied globally among worker + @param part partitioning algorithm to schedule parallel iterations + + @return a tf::Task handle + + The task spawns asynchronous tasks to perform parallel reduction over a range with @c init. + The reduced result is store in @c init. + Unlike the iterator-based reduction, + index range-based reduction is particularly useful for applications that benefit from SIMD optimizations + or other range-based processing strategies. + + @code{.cpp} + const size_t N = 1000000; + std::vector data(N); // uninitialized data vector + int res = 1; // res will participate in the reduction + + taskflow.reduce_by_index( + tf::IndexRange(0, N, 1), + // final result + res, + // local reducer + [&](tf::IndexRange subrange, std::optional running_total) -> int { + int residual = running_total ? *running_total : 0.0; + for(size_t i=subrange.begin(); i() + ); + executor.run(taskflow).wait(); + assert(res = N + 1); + @endcode + + Range can be made stateful by using std::reference_wrapper. + + @note + Please refer to @ref ParallelReduction for details. + */ + template + Task reduce_by_index(R range, T& init, L lop, G gop, P part = P()); + + // ------------------------------------------------------------------------ + // transform and reduction + // ------------------------------------------------------------------------ + + /** + @brief constructs an STL-styled parallel transform-reduce task + + @tparam B beginning iterator type + @tparam E ending iterator type + @tparam T result type + @tparam BOP binary reducer type + @tparam UOP unary transformation type + @tparam P type satisfying tf::PartitionerLike + + @param first iterator to the beginning (inclusive) + @param last iterator to the end (exclusive) + @param init initial value of the reduction and the storage for the reduced result + @param bop binary operator that will be applied in unspecified order to the results of @c uop + @param uop unary operator that will be applied to transform each element in the range to the result type + @param part partitioning algorithm to schedule parallel iterations + + @return a tf::Task handle + + The task spawns asynchronous tasks to perform parallel reduction over @c init and + the transformed elements in the range [first, last). + The reduced result is store in @c init. + This method is equivalent to the parallel execution of the following loop: + + @code{.cpp} + for(auto itr=first; itr!=last; itr++) { + init = bop(init, uop(*itr)); + } + @endcode + + Iterators can be made stateful by using std::reference_wrapper + + @note + Please refer to @ref ParallelReduction for details. + */ + template + Task transform_reduce(B first, E last, T& init, BOP bop, UOP uop, P part = P()); + + /** + @brief constructs an STL-styled parallel transform-reduce task + @tparam B1 first beginning iterator type + @tparam E1 first ending iterator type + @tparam B2 second beginning iterator type + @tparam T result type + @tparam BOP_R binary reducer type + @tparam BOP_T binary transformation type + @tparam P type satisfying tf::PartitionerLike + + @param first1 iterator to the beginning of the first range (inclusive) + @param last1 iterator to the end of the first range (exclusive) + @param first2 iterator to the beginning of the second range + @param init initial value of the reduction and the storage for the reduced result + @param bop_r binary operator that will be applied in unspecified order to the results of @c bop_t + @param bop_t binary operator that will be applied to transform each element in the range to the result type + @param part partitioning algorithm to schedule parallel iterations + + @return a tf::Task handle + + The task spawns asynchronous tasks to perform parallel reduction over @c init and + transformed elements in the range [first, last). + The reduced result is store in @c init. + This method is equivalent to the parallel execution of the following loop: + + @code{.cpp} + for(auto itr1=first1, itr2=first2; itr1!=last1; itr1++, itr2++) { + init = bop_r(init, bop_t(*itr1, *itr2)); + } + @endcode + + Iterators can be made stateful by using std::reference_wrapper + + @note + Please refer to @ref ParallelReduction for details. + */ + + template + requires (!PartitionerLike>) + Task transform_reduce( + B1 first1, E1 last1, B2 first2, T& init, BOP_R bop_r, BOP_T bop_t, P part = P() + ); + + // ------------------------------------------------------------------------ + // scan + // ------------------------------------------------------------------------ + + /** + @brief creates an STL-styled parallel inclusive-scan task + + @tparam B beginning iterator type + @tparam E ending iterator type + @tparam D destination iterator type + @tparam BOP summation operator type + + @param first start of input range + @param last end of input range + @param d_first start of output range (may be the same as input range) + @param bop function to perform summation + + Performs the cumulative sum (aka prefix sum, aka scan) of the input range + and writes the result to the output range. + Each element of the output range contains the + running total of all earlier elements using the given binary operator + for summation. + + This function generates an @em inclusive scan, meaning that the N-th element + of the output range is the sum of the first N input elements, + so the N-th input element is included. + + @code{.cpp} + std::vector input = {1, 2, 3, 4, 5}; + taskflow.inclusive_scan( + input.begin(), input.end(), input.begin(), std::plus{} + ); + executor.run(taskflow).wait(); + + // input is {1, 3, 6, 10, 15} + @endcode + + Iterators can be made stateful by using std::reference_wrapper + + @note + Please refer to @ref ParallelScan for details. + */ + template + Task inclusive_scan(B first, E last, D d_first, BOP bop); + + /** + @brief creates an STL-styled parallel inclusive-scan task with an initial value + + @tparam B beginning iterator type + @tparam E ending iterator type + @tparam D destination iterator type + @tparam BOP summation operator type + @tparam T initial value type + + @param first start of input range + @param last end of input range + @param d_first start of output range (may be the same as input range) + @param bop function to perform summation + @param init initial value + + Performs the cumulative sum (aka prefix sum, aka scan) of the input range + and writes the result to the output range. + Each element of the output range contains the + running total of all earlier elements (and the initial value) + using the given binary operator for summation. + + This function generates an @em inclusive scan, meaning the N-th element + of the output range is the sum of the first N input elements, + so the N-th input element is included. + + @code{.cpp} + std::vector input = {1, 2, 3, 4, 5}; + taskflow.inclusive_scan( + input.begin(), input.end(), input.begin(), std::plus{}, -1 + ); + executor.run(taskflow).wait(); + + // input is {0, 2, 5, 9, 14} + @endcode + + Iterators can be made stateful by using std::reference_wrapper + + @note + Please refer to @ref ParallelScan for details. + + */ + template + Task inclusive_scan(B first, E last, D d_first, BOP bop, T init); + + /** + @brief creates an STL-styled parallel exclusive-scan task + + @tparam B beginning iterator type + @tparam E ending iterator type + @tparam D destination iterator type + @tparam T initial value type + @tparam BOP summation operator type + + @param first start of input range + @param last end of input range + @param d_first start of output range (may be the same as input range) + @param init initial value + @param bop function to perform summation + + Performs the cumulative sum (aka prefix sum, aka scan) of the input range + and writes the result to the output range. + Each element of the output range contains the + running total of all earlier elements (and the initial value) + using the given binary operator for summation. + + This function generates an @em exclusive scan, meaning the N-th element + of the output range is the sum of the first N-1 input elements, + so the N-th input element is not included. + + @code{.cpp} + std::vector input = {1, 2, 3, 4, 5}; + taskflow.exclusive_scan( + input.begin(), input.end(), input.begin(), -1, std::plus{} + ); + executor.run(taskflow).wait(); + + // input is {-1, 0, 2, 5, 9} + @endcode + + Iterators can be made stateful by using std::reference_wrapper + + @note + Please refer to @ref ParallelScan for details. + */ + template + Task exclusive_scan(B first, E last, D d_first, T init, BOP bop); + + // ------------------------------------------------------------------------ + // transform scan + // ------------------------------------------------------------------------ + + /** + @brief creates an STL-styled parallel transform-inclusive scan task + + @tparam B beginning iterator type + @tparam E ending iterator type + @tparam D destination iterator type + @tparam BOP summation operator type + @tparam UOP transform operator type + + @param first start of input range + @param last end of input range + @param d_first start of output range (may be the same as input range) + @param bop function to perform summation + @param uop function to transform elements of the input range + + Write the cumulative sum (aka prefix sum, aka scan) of the input range + to the output range. Each element of the output range contains the + running total of all earlier elements + using @c uop to transform the input elements + and using @c bop for summation. + + This function generates an @em inclusive scan, meaning the Nth element + of the output range is the sum of the first N input elements, + so the Nth input element is included. + + @code{.cpp} + std::vector input = {1, 2, 3, 4, 5}; + taskflow.transform_inclusive_scan( + input.begin(), input.end(), input.begin(), std::plus{}, + [] (int item) { return -item; } + ); + executor.run(taskflow).wait(); + + // input is {-1, -3, -6, -10, -15} + @endcode + + Iterators can be made stateful by using std::reference_wrapper + + @note + Please refer to @ref ParallelScan for details. + */ + template + Task transform_inclusive_scan(B first, E last, D d_first, BOP bop, UOP uop); + + /** + @brief creates an STL-styled parallel transform-inclusive scan task + + @tparam B beginning iterator type + @tparam E ending iterator type + @tparam D destination iterator type + @tparam BOP summation operator type + @tparam UOP transform operator type + @tparam T initial value type + + @param first start of input range + @param last end of input range + @param d_first start of output range (may be the same as input range) + @param bop function to perform summation + @param uop function to transform elements of the input range + @param init initial value + + Write the cumulative sum (aka prefix sum, aka scan) of the input range + to the output range. Each element of the output range contains the + running total of all earlier elements (including an initial value) + using @c uop to transform the input elements + and using @c bop for summation. + + This function generates an @em inclusive scan, meaning the Nth element + of the output range is the sum of the first N input elements, + so the Nth input element is included. + + @code{.cpp} + std::vector input = {1, 2, 3, 4, 5}; + taskflow.transform_inclusive_scan( + input.begin(), input.end(), input.begin(), std::plus{}, + [] (int item) { return -item; }, + -1 + ); + executor.run(taskflow).wait(); + + // input is {-2, -4, -7, -11, -16} + @endcode + + Iterators can be made stateful by using std::reference_wrapper + + @note + Please refer to @ref ParallelScan for details. + */ + template + Task transform_inclusive_scan(B first, E last, D d_first, BOP bop, UOP uop, T init); + + /** + @brief creates an STL-styled parallel transform-exclusive scan task + + @tparam B beginning iterator type + @tparam E ending iterator type + @tparam D destination iterator type + @tparam BOP summation operator type + @tparam UOP transform operator type + @tparam T initial value type + + @param first start of input range + @param last end of input range + @param d_first start of output range (may be the same as input range) + @param bop function to perform summation + @param uop function to transform elements of the input range + @param init initial value + + Write the cumulative sum (aka prefix sum, aka scan) of the input range + to the output range. Each element of the output range contains the + running total of all earlier elements (including an initial value) + using @c uop to transform the input elements + and using @c bop for summation. + + This function generates an @em exclusive scan, meaning the Nth element + of the output range is the sum of the first N-1 input elements, + so the Nth input element is not included. + + @code{.cpp} + std::vector input = {1, 2, 3, 4, 5}; + taskflow.transform_exclusive_scan( + input.begin(), input.end(), input.begin(), -1, std::plus{}, + [](int item) { return -item; } + ); + executor.run(taskflow).wait(); + + // input is {-1, -2, -4, -7, -11} + @endcode + + Iterators can be made stateful by using std::reference_wrapper + + @note + Please refer to @ref ParallelScan for details. + */ + template + Task transform_exclusive_scan(B first, E last, D d_first, T init, BOP bop, UOP uop); + + // ------------------------------------------------------------------------ + // find + // ------------------------------------------------------------------------ + + /** + @brief constructs a task to perform STL-styled find-if algorithm + + @tparam B beginning iterator type + @tparam E ending iterator type + @tparam T resulting iterator type + @tparam UOP unary predicate type + @tparam P partitioner type + + @param first start of the input range + @param last end of the input range + @param result resulting iterator to the found element in the input range + @param predicate unary predicate which returns @c true for the required element + @param part partitioning algorithm (default tf::DefaultPartitioner) + + Returns an iterator to the first element in the range [first, last) + that satisfies the given criteria (or last if there is no such iterator). + This method is equivalent to the parallel execution of the following loop: + + @code{.cpp} + auto find_if(InputIt first, InputIt last, UnaryPredicate p) { + for (; first != last; ++first) { + if (predicate(*first)){ + return first; + } + } + return last; + } + @endcode + + For example, the code below find the element that satisfies the given + criteria (value plus one is equal to 23) from an input range of 10 elements: + + @code{.cpp} + std::vector input = {1, 6, 9, 10, 22, 5, 7, 8, 9, 11}; + std::vector::iterator result; + taskflow.find_if( + input.begin(), input.end(), [](int i){ return i+1 = 23; }, result + ); + executor.run(taskflow).wait(); + assert(*result == 22); + @endcode + + Iterators can be made stateful by using std::reference_wrapper + */ + template + Task find_if(B first, E last, T &result, UOP predicate, P part = P()); + + /** + @brief constructs a task to perform STL-styled find-if-not algorithm + + @tparam B beginning iterator type + @tparam E ending iterator type + @tparam T resulting iterator type + @tparam UOP unary predicate type + @tparam P partitioner type + + @param first start of the input range + @param last end of the input range + @param result resulting iterator to the found element in the input range + @param predicate unary predicate which returns @c false for the required element + @param part partitioning algorithm (default tf::DefaultPartitioner) + + Returns an iterator to the first element in the range [first, last) + that satisfies the given criteria (or last if there is no such iterator). + This method is equivalent to the parallel execution of the following loop: + + @code{.cpp} + auto find_if(InputIt first, InputIt last, UnaryPredicate p) { + for (; first != last; ++first) { + if (!predicate(*first)){ + return first; + } + } + return last; + } + @endcode + + For example, the code below find the element that satisfies the given + criteria (value is not equal to 1) from an input range of 10 elements: + + @code{.cpp} + std::vector input = {1, 1, 1, 1, 22, 1, 1, 1, 1, 1}; + std::vector::iterator result; + taskflow.find_if_not( + input.begin(), input.end(), [](int i){ return i == 1; }, result + ); + executor.run(taskflow).wait(); + assert(*result == 22); + @endcode + + Iterators can be made stateful by using std::reference_wrapper + */ + template + Task find_if_not(B first, E last, T &result, UOP predicate, P part = P()); + + /** + @brief constructs a task to perform STL-styled min-element algorithm + + @tparam B beginning iterator type + @tparam E ending iterator type + @tparam T resulting iterator type + @tparam C comparator type + @tparam P partitioner type + + @param first start of the input range + @param last end of the input range + @param result resulting iterator to the found element in the input range + @param comp comparison function object + @param part partitioning algorithm (default tf::DefaultPartitioner) + + Finds the smallest element in the [first, last) + using the given comparison function object. + The iterator to that smallest element is stored in @c result. + This method is equivalent to the parallel execution of the following loop: + + @code{.cpp} + if (first == last) { + return last; + } + auto smallest = first; + ++first; + for (; first != last; ++first) { + if (comp(*first, *smallest)) { + smallest = first; + } + } + return smallest; + @endcode + + For example, the code below find the smallest element from an input + range of 10 elements. + + @code{.cpp} + std::vector input = {1, 1, 1, 1, 1, -1, 1, 1, 1, 1}; + std::vector::iterator result; + taskflow.min_element( + input.begin(), input.end(), std::less(), result + ); + executor.run(taskflow).wait(); + assert(*result == -1); + @endcode + + Iterators can be made stateful by using std::reference_wrapper + */ + template + Task min_element(B first, E last, T& result, C comp, P part); + + /** + @brief constructs a task to perform STL-styled max-element algorithm + + @tparam B beginning iterator type + @tparam E ending iterator type + @tparam T resulting iterator type + @tparam C comparator type + @tparam P partitioner type + + @param first start of the input range + @param last end of the input range + @param result resulting iterator to the found element in the input range + @param comp comparison function object + @param part partitioning algorithm (default tf::DefaultPartitioner) + + Finds the largest element in the [first, last) + using the given comparison function object. + The iterator to that largest element is stored in @c result. + This method is equivalent to the parallel execution of the following loop: + + @code{.cpp} + if (first == last){ + return last; + } + auto largest = first; + ++first; + for (; first != last; ++first) { + if (comp(*largest, *first)) { + largest = first; + } + } + return largest; + @endcode + + For example, the code below find the largest element from an input + range of 10 elements. + + @code{.cpp} + std::vector input = {1, 1, 1, 1, 1, 2, 1, 1, 1, 1}; + std::vector::iterator result; + taskflow.max_element( + input.begin(), input.end(), std::less(), result + ); + executor.run(taskflow).wait(); + assert(*result == 2); + @endcode + + Iterators can be made stateful by using std::reference_wrapper + */ + template + Task max_element(B first, E last, T& result, C comp, P part); + + // ------------------------------------------------------------------------ + // sort + // ------------------------------------------------------------------------ + + /** + @brief constructs a dynamic task to perform STL-styled parallel sort + + @tparam B beginning iterator type (random-accessible) + @tparam E ending iterator type (random-accessible) + @tparam C comparator type + + @param first iterator to the beginning (inclusive) + @param last iterator to the end (exclusive) + @param cmp comparison operator + + The task spawns asynchronous tasks to sort elements in the range + [first, last) in parallel. + + Iterators can be made stateful by using std::reference_wrapper + + @note + Please refer to @ref ParallelSort for details. + */ + template + Task sort(B first, E last, C cmp); + + /** + @brief constructs a dynamic task to perform STL-styled parallel sort using + the @c std::less comparator, where @c T is the element type + + @tparam B beginning iterator type (random-accessible) + @tparam E ending iterator type (random-accessible) + + @param first iterator to the beginning (inclusive) + @param last iterator to the end (exclusive) + + The task spawns asynchronous tasks to parallel sort elements in the range + [first, last) using the @c std::less comparator, + where @c T is the dereferenced iterator type. + + Iterators can be made stateful by using std::reference_wrapper + + @note + Please refer to @ref ParallelSort for details. + */ + template + Task sort(B first, E last); + + /** + @brief merges two sorted ranges into a single sorted output using the + @c std::less comparator + + @tparam B1 beginning iterator type of the first range + @tparam E1 ending iterator type of the first range + @tparam B2 beginning iterator type of the second range + @tparam E2 ending iterator type of the second range + @tparam O destination iterator type + + @param first1 iterator to the beginning of the first range (inclusive) + @param last1 iterator to the end of the first range (exclusive) + @param first2 iterator to the beginning of the second range (inclusive) + @param last2 iterator to the end of the second range (exclusive) + @param d_first iterator to the beginning of the output range + + Creates a task that merges two sorted ranges [first1, last1) and + [first2, last2) into a single sorted output range beginning at + @c d_first, using @c std::less as the comparator. + + The algorithm partitions the output range into W equal chunks (one per + worker thread) and uses the co-rank technique to independently identify + each worker's corresponding sub-ranges in seq1 and seq2, then merges them + in parallel with no synchronization. + + Unlike @c for_each or @c find, parallel merge does not benefit from dynamic + or guided partitioning because @c std::merge always costs O(K) for a chunk + of size K regardless of data — there is no load imbalance to adapt to. + The algorithm therefore always uses static equal partitioning. + + @note Undefined behavior if either input range is not sorted with respect + to @c std::less. + */ + template + Task merge(B1 first1, E1 last1, B2 first2, E2 last2, O d_first); + + /** + @brief merges two sorted ranges into a single sorted output using a + custom comparator + + @tparam B1 beginning iterator type of the first range + @tparam E1 ending iterator type of the first range + @tparam B2 beginning iterator type of the second range + @tparam E2 ending iterator type of the second range + @tparam O destination iterator type + @tparam C comparator type + + @param first1 iterator to the beginning of the first range (inclusive) + @param last1 iterator to the end of the first range (exclusive) + @param first2 iterator to the beginning of the second range (inclusive) + @param last2 iterator to the end of the second range (exclusive) + @param d_first iterator to the beginning of the output range + @param cmp comparator function defining the sort order + + Creates a task that merges two sorted ranges [first1, last1) and + [first2, last2) into a single sorted output range beginning at + @c d_first, using @c cmp as the comparator. + + The algorithm partitions the output range into W equal chunks (one per + worker thread) and uses the co-rank technique to independently identify + each worker's corresponding sub-ranges in seq1 and seq2, then merges them + in parallel with no synchronization. + + Unlike @c for_each or @c find, parallel merge does not benefit from dynamic + or guided partitioning because @c std::merge always costs O(K) for a chunk + of size K regardless of data — there is no load imbalance to adapt to. + The algorithm therefore always uses static equal partitioning. + + @note Undefined behavior if either input range is not sorted with respect + to @c cmp. + */ + template + Task merge(B1 first1, E1 last1, B2 first2, E2 last2, O d_first, C cmp); + + /** + @brief fills a range with a given value in parallel + + @tparam B iterator type + @tparam E iterator type + @tparam V value type + @tparam P type satisfying tf::PartitionerLike + + @param first iterator to the beginning of the range (inclusive) + @param last iterator to the end of the range (exclusive) + @param value the value to fill the range with + @param part partitioning algorithm (default tf::DefaultPartitioner) + + @return a tf::Task handle + + The task spawns asynchronous tasks to fill the given range + [first, last) with the given value in parallel. + This is equivalent to calling std::fill(first, last, value) + but in parallel. + + @code{.cpp} + std::vector vec(1000); + tf::Task task = taskflow.fill(vec.begin(), vec.end(), 42); + @endcode + */ + template + Task fill(B first, E last, V value, P part = P()); + + /** + @brief fills N elements with a given value in parallel + + @tparam B iterator type + @tparam C count type (integral) + @tparam V value type + @tparam P type satisfying tf::PartitionerLike + + @param first iterator to the beginning of the range (inclusive) + @param count number of elements to fill + @param value the value to fill the range with + @param part partitioning algorithm (default tf::DefaultPartitioner) + + @return a tf::Task handle + + The task spawns asynchronous tasks to fill N elements starting from + @c first with the given value in parallel. + This is equivalent to calling std::fill_n(first, count, value) + but in parallel. + + @code{.cpp} + std::vector vec(1000); + tf::Task task = taskflow.fill_n(vec.begin(), 500, 42); + @endcode + */ + template + Task fill_n(B first, C count, V value, P part = P()); + + /** + @brief generates values into a range in parallel using a callable + + @tparam B iterator type + @tparam E iterator type + @tparam G generator callable type + @tparam P type satisfying tf::PartitionerLike + + @param first iterator to the beginning of the range (inclusive) + @param last iterator to the end of the range (exclusive) + @param gen generator callable that produces values + @param part partitioning algorithm (default tf::DefaultPartitioner) + + @return a tf::Task handle + + The task spawns asynchronous tasks to generate and fill the range + [first, last) with values produced by calling the + generator @c gen in parallel. + This is equivalent to calling std::generate(first, last, gen) + but in parallel. + + @code{.cpp} + std::vector vec(1000); + tf::Task task = taskflow.generate(vec.begin(), vec.end(), + [&counter]() { return 42; }); + @endcode + */ + template + Task generate(B first, E last, G gen, P part = P()); + + /** + @brief generates N values into a range in parallel using a callable + + @tparam B iterator type + @tparam C count type (integral) + @tparam G generator callable type + @tparam P type satisfying tf::PartitionerLike + + @param first iterator to the beginning of the range (inclusive) + @param count number of elements to generate + @param gen generator callable that produces values + @param part partitioning algorithm (default tf::DefaultPartitioner) + + @return a tf::Task handle + + The task spawns asynchronous tasks to generate and fill N elements + starting from @c first with values produced by calling the + generator @c gen in parallel. + This is equivalent to calling std::generate_n(first, count, gen) + but in parallel. + + @code{.cpp} + std::vector vec(1000); + tf::Task task = taskflow.generate_n(vec.begin(), 500, + [&counter]() { return 42; }); + @endcode + */ + template + Task generate_n(B first, C count, G gen, P part = P()); + + protected: + + /** + @private + */ + Graph& _graph; + + private: + + template + void _linearize(L&); +}; + +// Constructor +inline FlowBuilder::FlowBuilder(Graph& graph) : + _graph {graph} { +} + +// Function: emplace +template +Task FlowBuilder::emplace(C&& c) { + return Task(_graph._emplace_back(NSTATE::NONE, ESTATE::NONE, DefaultTaskParams{}, nullptr, nullptr, 0, + std::in_place_type_t{}, std::forward(c) + )); +} + +// Function: emplace +template +Task FlowBuilder::emplace(C&& c) { + if constexpr (std::is_invocable_v) { + return Task(_graph._emplace_back(NSTATE::NONE, ESTATE::NONE, DefaultTaskParams{}, nullptr, nullptr, 0, + std::in_place_type_t{}, std::forward(c) + )); + } + else if constexpr (std::is_invocable_v) { + return Task(_graph._emplace_back(NSTATE::NONE, ESTATE::NONE, DefaultTaskParams{}, nullptr, nullptr, 0, + std::in_place_type_t{}, std::forward(c) + )); + } + else { + static_assert(dependent_false_v, "invalid runtime task callable"); + } +} + +// Function: emplace +template +Task FlowBuilder::emplace(C&& c) { + return Task(_graph._emplace_back(NSTATE::NONE, ESTATE::NONE, DefaultTaskParams{}, nullptr, nullptr, 0, + std::in_place_type_t{}, std::forward(c) + )); +} + +// Function: emplace +template +Task FlowBuilder::emplace(C&& c) { + return Task(_graph._emplace_back(NSTATE::NONE, ESTATE::NONE, DefaultTaskParams{}, nullptr, nullptr, 0, + std::in_place_type_t{}, std::forward(c) + )); +} + +// Function: emplace +template +Task FlowBuilder::emplace(C&& c) { + return Task(_graph._emplace_back(NSTATE::NONE, ESTATE::NONE, DefaultTaskParams{}, nullptr, nullptr, 0, + std::in_place_type_t{}, std::forward(c) + )); +} + +// Function: composed_of +template +Task FlowBuilder::composed_of(T& target) { + return Task(_graph._emplace_back(NSTATE::NONE, ESTATE::NONE, DefaultTaskParams{}, nullptr, nullptr, 0, + std::in_place_type_t{}, retrieve_graph(target) + )); +} + +// Function: adopt +inline Task FlowBuilder::adopt(Graph&& graph) { + return Task(_graph._emplace_back(NSTATE::NONE, ESTATE::NONE, DefaultTaskParams{}, nullptr, nullptr, 0, + std::in_place_type_t{}, std::move(graph) + )); +} + +// Function: placeholder +inline Task FlowBuilder::placeholder() { + auto node = _graph._emplace_back(NSTATE::NONE, ESTATE::NONE, DefaultTaskParams{}, nullptr, nullptr, 0, + std::in_place_type_t{} + ); + return Task(node); +} + +// Function: emplace +template requires (sizeof...(C) > 1) +auto FlowBuilder::emplace(C&&... cs) { + return std::make_tuple(emplace(std::forward(cs))...); +} + +// Function: erase +inline void FlowBuilder::erase(Task task) { + + if (!task._node) { + return; + } + + // remove task from its successors' predecessor list + for(size_t i=0; i_num_successors; ++i) { + task._node->_edges[i]->_remove_predecessors(task._node); + } + + // remove task from its precedessors' successor list + for(size_t i=task._node->_num_successors; i_edges.size(); ++i) { + task._node->_edges[i]->_remove_successors(task._node); + } + + _graph._erase(task._node); +} + + +// Procedure: _linearize +template +void FlowBuilder::_linearize(L& keys) { + + auto itr = keys.begin(); + auto end = keys.end(); + + if(itr == end) { + return; + } + + auto nxt = itr; + + for(++nxt; nxt != end; ++nxt, ++itr) { + itr->_node->_precede(nxt->_node); + } +} + +// Procedure: linearize +inline void FlowBuilder::linearize(std::vector& keys) { + _linearize(keys); +} + +// Procedure: linearize +inline void FlowBuilder::linearize(std::initializer_list keys) { + _linearize(keys); +} + +// ---------------------------------------------------------------------------- + +/** +@class Subflow + +@brief class to construct a subflow graph from the execution of a dynamic task + +tf::Subflow is spawned from the execution of a task to dynamically manage a +child graph that may depend on runtime variables. +You can explicitly join a subflow by calling tf::Subflow::join, respectively. +By default, the %Taskflow runtime will implicitly join a subflow it is is joinable. + +The following example creates a taskflow graph that spawns a subflow from +the execution of task @c B, and the subflow contains three tasks, @c B1, +@c B2, and @c B3, where @c B3 runs after @c B1 and @c B2. + +@code{.cpp} +// create three static tasks +tf::Task A = taskflow.emplace([](){}).name("A"); +tf::Task C = taskflow.emplace([](){}).name("C"); +tf::Task D = taskflow.emplace([](){}).name("D"); + +// create a subflow graph (dynamic tasking) +tf::Task B = taskflow.emplace([] (tf::Subflow& subflow) { + tf::Task B1 = subflow.emplace([](){}).name("B1"); + tf::Task B2 = subflow.emplace([](){}).name("B2"); + tf::Task B3 = subflow.emplace([](){}).name("B3"); + B1.precede(B3); + B2.precede(B3); +}).name("B"); + +A.precede(B); // B runs after A +A.precede(C); // C runs after A +B.precede(D); // D runs after B +C.precede(D); // D runs after C +@endcode + +*/ +class Subflow : public FlowBuilder { + + friend class Executor; + friend class FlowBuilder; + + public: + + /** + @brief enables the subflow to join its parent task + + Performs an immediate action to join the subflow. Once the subflow is joined, + it is considered finished and you may not modify the subflow anymore. + + @code{.cpp} + taskflow.emplace([](tf::Subflow& sf){ + sf.emplace([](){}); + sf.join(); // join the subflow of one task + }); + @endcode + + Only the worker that spawns this subflow can join it. + */ + void join(); + + /** + @brief queries if the subflow is joinable + + This member function queries if the subflow is joinable. + When a subflow is joined, it becomes not joinable. + + @code{.cpp} + taskflow.emplace([](tf::Subflow& sf){ + sf.emplace([](){}); + std::cout << sf.joinable() << '\n'; // true + sf.join(); + std::cout << sf.joinable() << '\n'; // false + }); + @endcode + */ + bool joinable() const noexcept; + + /** + @brief acquires the associated executor + */ + Executor& executor() noexcept; + + /** + @brief acquires the associated graph + */ + Graph& graph() { return _graph; } + + /** + @brief specifies whether to keep the subflow after it is joined + + @param flag `true` to retain the subflow after it is joined; `false` to discard it + + By default, the runtime automatically clears a spawned subflow once it is joined. + Setting this flag to `true` allows the application to retain the subflow's structure + for post-execution analysis like visualization. + */ + void retain(bool flag) noexcept; + + /** + @brief queries if the subflow will be retained after it is joined + @return `true` if the subflow will be retained after it is joined; `false` otherwise + + By default, the runtime automatically clears a spawned subflow once it is joined. + Users can disable this before by explicitly calling tf::Subflow::retain. + */ + bool retain() const; + + private: + + Subflow(Executor&, Worker&, Node*, Graph&); + + Subflow() = delete; + Subflow(const Subflow&) = delete; + Subflow(Subflow&&) = delete; + + Executor& _executor; + Worker& _worker; + Node* _node; +}; + +// Constructor +inline Subflow::Subflow(Executor& executor, Worker& worker, Node* node, Graph& graph) : + FlowBuilder {graph}, + _executor {executor}, + _worker {worker}, + _node {node} { + + // need to reset since there could have iterative control flow + _node->_nstate &= ~(NSTATE::JOINED_SUBFLOW | NSTATE::RETAIN_SUBFLOW); + + // clear the graph + graph.clear(); +} + +// Function: joinable +inline bool Subflow::joinable() const noexcept { + return !(_node->_nstate & NSTATE::JOINED_SUBFLOW); +} + +// Function: executor +inline Executor& Subflow::executor() noexcept { + return _executor; +} + +// Function: retain +inline void Subflow::retain(bool flag) noexcept { + // default value is not to retain + if(flag == true) { + _node->_nstate |= NSTATE::RETAIN_SUBFLOW; + } + else { + _node->_nstate &= ~NSTATE::RETAIN_SUBFLOW; + } + + //_node->_nstate = (_node->_nstate & ~NSTATE::RETAIN_SUBFLOW) | + // (-static_cast(flag) & NSTATE::RETAIN_SUBFLOW); +} + +// Function: retain +inline bool Subflow::retain() const { + return _node->_nstate & NSTATE::RETAIN_SUBFLOW; +} + +} // end of namespace tf. --------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/core/graph.hpp b/projects/hipfile/third-party/taskflow/core/graph.hpp new file mode 100644 index 000000000000..e8c811b60389 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/core/graph.hpp @@ -0,0 +1,1075 @@ +#pragma once + +#include "../utility/macros.hpp" +#include "../utility/traits.hpp" +#include "../utility/iterator.hpp" + +#ifdef TF_ENABLE_TASK_POOL +#include "../utility/object_pool.hpp" +#endif + +#include "../utility/os.hpp" +#include "../utility/math.hpp" +#include "../utility/small_vector.hpp" +#include "../utility/serializer.hpp" +#include "../utility/lazy_string.hpp" +#include "error.hpp" +#include "declarations.hpp" +#include "semaphore.hpp" +#include "environment.hpp" +#include "topology.hpp" +#include "wsq.hpp" + + +/** +@file graph.hpp +@brief graph include file +*/ + +namespace tf { + +// ---------------------------------------------------------------------------- +// Class: Graph +// ---------------------------------------------------------------------------- + +/** +@class Graph + +@brief class to create a graph object + +A graph is the ultimate storage for a task dependency graph and is the main +gateway to interact with an executor. +This class is mainly used for creating an opaque graph object in a custom +class to interact with the executor through taskflow composition. + +A graph object is move-only. +*/ +class Graph { + + friend class Node; + friend class FlowBuilder; + friend class Subflow; + friend class Taskflow; + friend class Executor; + + public: + + /** + @brief constructs the graph object + */ + Graph() = default; + + /** + @brief destroys the graph object + */ + ~Graph(); + + /** + @brief disabled copy constructor + */ + Graph(const Graph&) = delete; + + /** + @brief constructs a graph using move semantics + */ + Graph(Graph&&); + + /** + @brief disabled copy assignment operator + */ + Graph& operator = (const Graph&) = delete; + + /** + @brief assigns a graph using move semantics + */ + Graph& operator = (Graph&&); + + /** + @brief clears the graph + */ + void clear(); + + /** + @brief returns the number of nodes in the graph + */ + size_t size() const; + + /** + @brief queries the emptiness of the graph + */ + bool empty() const; + + /** + @brief returns an iterator to the first node of this graph + */ + auto begin(); + + /** + @brief returns an iterator past the last element of this graph + */ + auto end(); + + /** + @brief returns an iterator to the first node of this graph + */ + auto begin() const; + + /** + @brief returns an iterator past the last element of this graph + */ + auto end() const; + + private: + + std::vector _nodes; + + void _erase(Node*); + + /** + @private + */ + template + Node* _emplace_back(ArgsT&&...); +}; + +// ---------------------------------------------------------------------------- +// TaskParams +// ---------------------------------------------------------------------------- + +/** +@brief concept that determines if a type is string-like + +A type satisfies tf::StringLike if it is convertible to std::string_view, including: + + std::string + + std::string_view + + const char* + + any type that provides an implicit conversion to std::string_view + +@code{.cpp} +static_assert(tf::StringLike); +static_assert(tf::StringLike); +static_assert(tf::StringLike); + +// custom type with implicit conversion +struct MyString { + operator std::string_view() const { return _data; } + std::string _data; +}; +static_assert(tf::StringLike); +@endcode +*/ +template +concept StringLike = std::convertible_to; + +/** +@class TaskParams + +@brief class to create a task parameter object + +tf::TaskParams is primarily used by asynchronous tasking. +*/ +class TaskParams { + + public: + + /** + @brief name of the task + */ + std::string name; + + /** + @brief C-styled pointer to user data + */ + void* data {nullptr}; +}; + +/** +@class DefaultTaskParams + +@brief class to create an empty task parameter for compile-time optimization +*/ +class DefaultTaskParams {}; + +/** +@brief determines if a type is a task parameter type + +A type satisfies tf::TaskParams if it is one of the following: + + tf::TaskParams + + tf::DefaultTaskParams + + tf::StringLike +*/ +template +concept TaskParamsLike = + std::same_as, TaskParams> || + std::same_as, DefaultTaskParams> || + StringLike

; + +/** +@brief determines if a type is a task parameter type (variable template) + +@tparam P type to check + +Equivalent to tf::TaskParameters

. Provided for backward compatibility. +*/ +template +constexpr bool is_task_params_v = TaskParamsLike

; + +// ---------------------------------------------------------------------------- +// NodeBase +// ---------------------------------------------------------------------------- + +/** +@private +*/ +class NodeBase { + + friend class Node; + friend class Graph; + friend class Task; + friend class AsyncTask; + friend class TaskView; + friend class Taskflow; + friend class Executor; + friend class FlowBuilder; + friend class Subflow; + friend class Runtime; + friend class NonpreemptiveRuntime; + friend class ExplicitAnchorGuard; + friend class TaskGroup; + friend class Algorithm; + + protected: + + nstate_t _nstate {NSTATE::NONE}; + std::atomic _estate {ESTATE::NONE}; + + NodeBase* _parent {nullptr}; + std::atomic _join_counter {0}; + + std::exception_ptr _exception_ptr {nullptr}; + + NodeBase() = default; + + NodeBase(nstate_t nstate, estate_t estate, NodeBase* parent, size_t join_counter) : + _nstate {nstate}, + _estate {estate}, + _parent {parent}, + _join_counter {join_counter} { + } + + void _rethrow_exception() { + if(_exception_ptr) { + auto e = _exception_ptr; + _exception_ptr = nullptr; + _estate.fetch_and(~(ESTATE::EXCEPTION | ESTATE::CAUGHT), std::memory_order_relaxed); + std::rethrow_exception(e); + } + } +}; + +// ---------------------------------------------------------------------------- +// Topology +// ---------------------------------------------------------------------------- + +/** +@private +*/ +class Topology : public NodeBase { + + friend class Executor; + friend class Subflow; + friend class Runtime; + friend class NonpreemptiveRuntime; + friend class Node; + + template + friend class Future; + + public: + + template + Topology(Taskflow&, Predicate&&, OnFinish&&); + + bool cancelled() const; + + private: + + Taskflow& _taskflow; + + std::promise _promise; + + std::function _predicate; + std::function _on_finish; + + void _carry_out_promise(); +}; + +// Constructor +template +Topology::Topology(Taskflow& tf, Predicate&& predicate, OnFinish&& on_finish): + NodeBase(NSTATE::NONE, ESTATE::EXPLICITLY_ANCHORED, nullptr, 0), + _taskflow(tf), + _predicate(std::forward(predicate)), + _on_finish(std::forward (on_finish)) { +} + +// Procedure +inline void Topology::_carry_out_promise() { + if(_exception_ptr) { + auto e = _exception_ptr; + _exception_ptr = nullptr; + _promise.set_exception(e); + } + else { + _promise.set_value(); + } +} + +// Function: cancelled +inline bool Topology::cancelled() const { + return _estate.load(std::memory_order_relaxed) & (ESTATE::CANCELLED | ESTATE::EXCEPTION); +} + + +// ---------------------------------------------------------------------------- +// Node +// ---------------------------------------------------------------------------- + +/** +@private +*/ +class Node : public NodeBase { + + friend class Graph; + friend class Task; + friend class AsyncTask; + friend class TaskView; + friend class Taskflow; + friend class Executor; + friend class FlowBuilder; + friend class Subflow; + friend class Runtime; + friend class NonpreemptiveRuntime; + friend class ExplicitAnchorGuard; + friend class TaskGroup; + friend class Algorithm; + + using Placeholder = std::monostate; + + // static work handle + struct Static { + + template + Static(C&&); + + std::function work; + }; + + // runtime work handle + struct Runtime { + + template + Runtime(C&&); + + std::function work; + }; + + struct NonpreemptiveRuntime { + + template + NonpreemptiveRuntime(C&&); + + std::function work; + }; + + // subflow work handle + struct Subflow { + + template + Subflow(C&&); + + std::function work; + Graph subgraph; + }; + + // condition work handle + struct Condition { + + template + Condition(C&&); + + std::function work; + }; + + // multi-condition work handle + struct MultiCondition { + + template + MultiCondition(C&&); + + std::function()> work; + }; + + // module work handle + struct Module { + + Module(Graph&); + + Graph& graph; + }; + + // adopted module work handle + struct AdoptedModule { + + AdoptedModule(Graph&&); + + Graph graph; + }; + + // Async work + struct Async { + + template + Async(T&&); + + std::variant< + std::function, + std::function, // silent async + std::function // async + > work; + }; + + // silent dependent async + struct DependentAsync { + + template + DependentAsync(C&&); + + std::variant< + std::function, + std::function, // silent async + std::function // async + > work; + + // use_count is packed into the lower 24 bits of NodeBase::_estate + // (ESTATE::REFCOUNT_MASK) to avoid a separate atomic and a std::get_if + // call on every AsyncTask copy/move/destroy. see ESTATE::REFCOUNT_ONE. + }; + + using handle_t = std::variant< + Placeholder, // placeholder + Static, // static tasking + Runtime, // runtime tasking + NonpreemptiveRuntime, // runtime (non-preemptive) tasking + Subflow, // subflow tasking + Condition, // conditional tasking + MultiCondition, // multi-conditional tasking + Module, // composable tasking + AdoptedModule, // composable tasking with move semantics + Async, // async tasking + DependentAsync // dependent async tasking + >; + + struct Semaphores { + SmallVector to_acquire; + SmallVector to_release; + }; + + public: + + // variant index + constexpr static auto PLACEHOLDER = get_index_v; + constexpr static auto STATIC = get_index_v; + constexpr static auto RUNTIME = get_index_v; + constexpr static auto NONPREEMPTIVE_RUNTIME = get_index_v; + constexpr static auto SUBFLOW = get_index_v; + constexpr static auto CONDITION = get_index_v; + constexpr static auto MULTI_CONDITION = get_index_v; + constexpr static auto MODULE = get_index_v; + constexpr static auto ADOPTED_MODULE = get_index_v; + constexpr static auto ASYNC = get_index_v; + constexpr static auto DEPENDENT_ASYNC = get_index_v; + + Node() = default; + + template + Node(nstate_t, estate_t, const TaskParams&, Topology*, NodeBase*, size_t, Args&&...); + + template + Node(nstate_t, estate_t, const DefaultTaskParams&, Topology*, NodeBase*, size_t, Args&&...); + + template + Node(nstate_t, estate_t, S&&, Topology*, NodeBase*, size_t, Args&&...); + + size_t num_successors() const; + size_t num_predecessors() const; + size_t num_strong_dependencies() const; + size_t num_weak_dependencies() const; + + const std::string& name() const; + + private: + + std::string _name; + + void* _data {nullptr}; + + Topology* _topology {nullptr}; + + size_t _num_successors {0}; + SmallVector _edges; + + handle_t _handle; + + std::unique_ptr _semaphores; + + bool _is_parent_cancelled() const; + bool _is_conditioner() const; + bool _acquire_all(SmallVector&); + void _release_all(SmallVector&); + void _precede(Node*); + void _set_up_join_counter(); + + void _remove_successors(Node*); + void _remove_predecessors(Node*); +}; + + +// ---------------------------------------------------------------------------- +// Definition for Node::Static +// ---------------------------------------------------------------------------- + +// Constructor +template +Node::Static::Static(C&& c) : work {std::forward(c)} { +} + +// ---------------------------------------------------------------------------- +// Definition for Node::Runtime +// ---------------------------------------------------------------------------- + +// Constructor +template +Node::Runtime::Runtime(C&& c) : work {std::forward(c)} { +} + +// Constructor +template +Node::NonpreemptiveRuntime::NonpreemptiveRuntime(C&& c) : work {std::forward(c)} { +} + +// ---------------------------------------------------------------------------- +// Definition for Node::Subflow +// ---------------------------------------------------------------------------- + +// Constructor +template +Node::Subflow::Subflow(C&& c) : work {std::forward(c)} { +} + +// ---------------------------------------------------------------------------- +// Definition for Node::Condition +// ---------------------------------------------------------------------------- + +// Constructor +template +Node::Condition::Condition(C&& c) : work {std::forward(c)} { +} + +// ---------------------------------------------------------------------------- +// Definition for Node::MultiCondition +// ---------------------------------------------------------------------------- + +// Constructor +template +Node::MultiCondition::MultiCondition(C&& c) : work {std::forward(c)} { +} + +// ---------------------------------------------------------------------------- +// Definition for Node::Module +// ---------------------------------------------------------------------------- + +// Constructor +inline Node::Module::Module(Graph& g) : graph(g){ +} + +// Constructor +inline Node::AdoptedModule::AdoptedModule(Graph&& g) : graph(std::move(g)){ +} + +// ---------------------------------------------------------------------------- +// Definition for Node::Async +// ---------------------------------------------------------------------------- + +// Constructor +template +Node::Async::Async(C&& c) : work {std::forward(c)} { +} + +// ---------------------------------------------------------------------------- +// Definition for Node::DependentAsync +// ---------------------------------------------------------------------------- + +// Constructor +template +Node::DependentAsync::DependentAsync(C&& c) : work {std::forward(c)} { +} + +// ---------------------------------------------------------------------------- +// Definition for Node +// ---------------------------------------------------------------------------- + +// Constructor +template +Node::Node( + nstate_t nstate, + estate_t estate, + const TaskParams& params, + Topology* topology, + NodeBase* parent, + size_t join_counter, + Args&&... args +) : + NodeBase(nstate, estate, parent, join_counter), + _name {params.name}, + _data {params.data}, + _topology {topology}, + _handle {std::forward(args)...} { +} + +// Constructor +template +Node::Node( + nstate_t nstate, + estate_t estate, + const DefaultTaskParams&, + Topology* topology, + NodeBase* parent, + size_t join_counter, + Args&&... args +) : + NodeBase(nstate, estate, parent, join_counter), + _topology {topology}, + _handle {std::forward(args)...} { +} + +// Constructor +template +Node::Node( + nstate_t nstate, + estate_t estate, + S&& name, + Topology* topology, + NodeBase* parent, + size_t join_counter, + Args&&... args +) : + NodeBase(nstate, estate, parent, join_counter), + _name {std::forward(name)}, + _topology {topology}, + _handle {std::forward(args)...} { +} + +//// Constructor +//template +//void Node::reset( +// nstate_t nstate, +// estate_t estate, +// const TaskParams& params, +// Topology* topology, +// NodeBase* parent, +// size_t join_counter, +// std::in_place_type_t, +// Args&&... args +//) { +// _nstate = nstate; +// _estate = estate; +// _parent = parent; +// _join_counter.store(join_counter, std::memory_order_relaxed); +// _exception_ptr = nullptr; +// _name = params.name; +// _data = params.data; +// _topology = topology; +// _handle.emplace(std::forward(args)...); +// _num_successors = 0; +// _edges.clear(); +// _semaphores.reset(); +//} +// +//// Constructor +//template +//void Node::reset( +// nstate_t nstate, +// estate_t estate, +// const DefaultTaskParams&, +// Topology* topology, +// NodeBase* parent, +// size_t join_counter, +// std::in_place_type_t, +// Args&&... args +//) { +// _nstate = nstate; +// _estate = estate; +// _parent = parent; +// _join_counter.store(join_counter, std::memory_order_relaxed); +// _exception_ptr = nullptr; +// _name.clear(); +// _data = nullptr; +// _topology = topology; +// _handle.emplace(std::forward(args)...); +// _num_successors = 0; +// _edges.clear(); +// _semaphores.reset(); +//} + +// Procedure: _precede +/* +u edges layout: s1, s2, s3, p1, p2 (num_successors = 3) +v edges layout: s1, p1, p2 + +add a new successor: u->v +u successor layout: + s1, s2, s3, p1, p2, v (push_back v) + s1, s2, s3, v, p2, p1 (swap edges[num_successors] with edges[n-1]) +v predecessor layout: + s1, p1, p2, u (push_back u) +*/ +inline void Node::_precede(Node* v) { + _edges.push_back(v); + std::swap(_edges[_num_successors++], _edges[_edges.size() - 1]); + v->_edges.push_back(this); +} + +// Function: _remove_successors +inline void Node::_remove_successors(Node* node) { + auto sit = std::remove(_edges.begin(), _edges.begin() + _num_successors, node); + size_t new_num_successors = std::distance(_edges.begin(), sit); + std::move(_edges.begin() + _num_successors, _edges.end(), sit); + _edges.resize(_edges.size() - (_num_successors - new_num_successors)); + _num_successors = new_num_successors; +} + +// Function: _remove_predecessors +inline void Node::_remove_predecessors(Node* node) { + _edges.erase( + std::remove(_edges.begin() + _num_successors, _edges.end(), node), _edges.end() + ); +} + +// Function: num_successors +inline size_t Node::num_successors() const { + return _num_successors; +} + +// Function: predecessors +inline size_t Node::num_predecessors() const { + return _edges.size() - _num_successors; +} + +// Function: num_weak_dependencies +inline size_t Node::num_weak_dependencies() const { + size_t n = 0; + for(size_t i=_num_successors; i<_edges.size(); i++) { + n += _edges[i]->_is_conditioner(); + } + return n; +} + +// Function: num_strong_dependencies +inline size_t Node::num_strong_dependencies() const { + size_t n = 0; + for(size_t i=_num_successors; i<_edges.size(); i++) { + n += !_edges[i]->_is_conditioner(); + } + return n; +} + +// Function: name +inline const std::string& Node::name() const { + return _name; +} + +// Function: _is_conditioner +inline bool Node::_is_conditioner() const { + return _handle.index() == Node::CONDITION || + _handle.index() == Node::MULTI_CONDITION; +} + +// Function: _is_parent_cancelled +inline bool Node::_is_parent_cancelled() const { + return (_topology && (_topology->_estate.load(std::memory_order_relaxed) & (ESTATE::CANCELLED | ESTATE::EXCEPTION))) + || + (_parent && (_parent->_estate.load(std::memory_order_relaxed) & (ESTATE::CANCELLED | ESTATE::EXCEPTION))); +} + +// Procedure: _set_up_join_counter +inline void Node::_set_up_join_counter() { + //assert(_nstate == NSTATE::NONE); + for(size_t i=_num_successors; i<_edges.size(); i++) { + _nstate += !_edges[i]->_is_conditioner(); + } + _join_counter.store(_nstate & NSTATE::STRONG_DEPENDENCIES_MASK, std::memory_order_relaxed); +} + + +// Function: _acquire_all +inline bool Node::_acquire_all(SmallVector& nodes) { + // assert(_semaphores != nullptr); + auto& to_acquire = _semaphores->to_acquire; + for(size_t i = 0; i < to_acquire.size(); ++i) { + if(!to_acquire[i]->_try_acquire_or_wait(this)) { + for(size_t j = 1; j <= i; ++j) { + to_acquire[i-j]->_release(nodes); + } + return false; + } + } + return true; +} + +// Function: _release_all +inline void Node::_release_all(SmallVector& nodes) { + // assert(_semaphores != nullptr); + auto& to_release = _semaphores->to_release; + for(const auto& sem : to_release) { + sem->_release(nodes); + } +} + + + +// ---------------------------------------------------------------------------- +// ExplicitAnchorGuard +// ---------------------------------------------------------------------------- + +/** +@private +*/ +class ExplicitAnchorGuard { + + public: + + // Explicit anchor must sit in estate as it may be accessed by multiple threads + // (e.g., corun's parent with tear_down_async's parent). + ExplicitAnchorGuard(NodeBase* node_base) : _node_base{node_base} { + _node_base->_estate.fetch_or(ESTATE::EXPLICITLY_ANCHORED, std::memory_order_relaxed); + } + + ~ExplicitAnchorGuard() { + _node_base->_estate.fetch_and(~ESTATE::EXPLICITLY_ANCHORED, std::memory_order_relaxed); + } + + private: + + NodeBase* _node_base; +}; + +// ---------------------------------------------------------------------------- +// Node Object Pool +// ---------------------------------------------------------------------------- + +#ifdef TF_ENABLE_TASK_POOL +/** +@private +*/ +using NodePool = std::conditional_t + std::atomic::is_always_lock_free, + ObjectPool, + ObjectPool> +>; +inline NodePool _node_pool; +#endif + +/** +@private +*/ +template +TF_FORCE_INLINE Node* animate(ArgsT&&... args) { +#ifdef TF_ENABLE_TASK_POOL + return _node_pool.animate(std::forward(args)...); +#else + return new Node(std::forward(args)...); +#endif +} + +/** +@private +*/ +TF_FORCE_INLINE void recycle(Node* ptr) { +#ifdef TF_ENABLE_TASK_POOL + _node_pool.recycle(ptr); +#else + delete ptr; +#endif +} + + +// ---------------------------------------------------------------------------- +// Graph definition +// ---------------------------------------------------------------------------- + +// Destructor +inline Graph::~Graph() { + clear(); +} + +// Move constructor +inline Graph::Graph(Graph&& other) : + _nodes {std::move(other._nodes)} { +} + +// Move assignment +inline Graph& Graph::operator = (Graph&& other) { + clear(); + _nodes = std::move(other._nodes); + return *this; +} + +// Procedure: clear +inline void Graph::clear() { + for(auto node : _nodes) { + recycle(node); + } + _nodes.clear(); +} + +// Function: size +inline size_t Graph::size() const { + return _nodes.size(); +} + +// Function: empty +inline bool Graph::empty() const { + return _nodes.empty(); +} + +// Function: begin +inline auto Graph::begin() { + return _nodes.begin(); +} + +// Function: end +inline auto Graph::end() { + return _nodes.end(); +} + +// Function: begin +inline auto Graph::begin() const { + return _nodes.begin(); +} + +// Function: end +inline auto Graph::end() const { + return _nodes.end(); +} + +// Function: erase +inline void Graph::_erase(Node* node) { + //erase( + // std::remove_if(begin(), end(), [&](auto& p){ return p.get() == node; }), + // end() + //); + _nodes.erase( + std::remove_if(_nodes.begin(), _nodes.end(), [&](auto& p){ + if(p == node) { + recycle(p); + return true; + } + return false; + }), + _nodes.end() + ); +} + +/** +@private +*/ +template +Node* Graph::_emplace_back(ArgsT&&... args) { + _nodes.push_back(animate(std::forward(args)...)); + return _nodes.back(); +} + +// ---------------------------------------------------------------------------- +// Graph checker +// ---------------------------------------------------------------------------- + + +/** +@brief concept that determines if a type owns or provides access to a tf::Graph + +A type satisfies @c tf::GraphLike if it meets one of the following two criteria: + + **Inheritance**: The type is derived from @c tf::Graph. + + **Composition**: The type provides a @c graph() method that returns a reference convertible to @c tf::Graph&. + +This concept is used by @c tf::retrieve_graph to abstract the way graphs are +accessed across different taskflow components. + +@code{.cpp} +// Satisfies via inheritance +struct CustomGraph1 : public tf::Graph {}; +static_assert(tf::GraphLike); + +// Satisfies via composition +struct CustomGraph2 { + tf::Graph& graph() { return _g; } + tf::Graph _g; +}; +static_assert(tf::GraphLike); + +// Fails: does not meet inheritance or method requirements +struct InvalidType { +void handle() {} +}; +static_assert(!tf::GraphLike); +@endcode +*/ +template +concept GraphLike = std::derived_from || + requires(T& t) { + { t.graph() } -> std::convertible_to; + }; + +/** +@brief retrieves a reference to the underlying tf::Graph from an object + +This helper function abstracts the retrieval of a graph reference. It uses +compile-time introspection to determine if the object provides a @c graph() +member function or if it should be treated as a @c tf::Graph directly. + +@tparam T type satisfying the @c tf::GraphLike concept +@param target object from which to retrieve the graph +@return a reference to the underlying @c tf::Graph + +@code{.cpp} +// Case 1: T has a .graph() member (composition) +struct CustomGraph1 { + tf::Graph& graph() { return _graph; } + tf::Graph _graph; +}; + +// Case 2: T is derived from tf::Graph (inheritance) +struct CustomGraph2 : public tf::Graph { +// ... +}; + +CustomGraph1 custom_graph1; +CustomGraph2 custom_graph2; + +tf::Graph& g1 = tf::retrieve_graph(custom_graph1); +tf::Graph& g2 = tf::retrieve_graph(custom_graph2); +@endcode + +@note This function is evaluated at compile time via @c if @c constexpr, +resulting in zero runtime overhead. +*/ +template +Graph& retrieve_graph(T& target) { + if constexpr (requires { target.graph(); }) { + return target.graph(); + } else { + return static_cast(target); + } +} + +} // end of namespace tf. ---------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/core/nonblocking_notifier.hpp b/projects/hipfile/third-party/taskflow/core/nonblocking_notifier.hpp new file mode 100644 index 000000000000..de6f1590e5d3 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/core/nonblocking_notifier.hpp @@ -0,0 +1,641 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../utility/os.hpp" + +/** +@file nonblocking_notifier.hpp +@brief non-blocking notifier include file +*/ + +namespace tf { + +/** @class NonblockingNotifier + +@brief class to create a non-blocking notifier + +A non-blocking notifier enables threads to wait for user-defined +predicates without blocking locks or protecting the predicate with +a mutex. Conceptually, it is similar to a condition variable, but the +wait predicate is evaluated optimistically and does not require +mutual exclusion. + +A waiting thread follows this pattern: + +@code{.cpp} +wid = this_waiter_id(); +if (predicate) { + return act(); +} +notifier.prepare_wait(wid); // enter the two-phase wait protocol +if (predicate) { + notifier.cancel_wait(wid); + return act(); +} +notifier.commit_wait(&w); // park (e.g., preempted by OS until notified) +@endcode + +A notifying thread performs: + +@code{.cpp} +wid = this_notifier_id; +predicate = true; +notifier.notify_one(wid); +@endcode + +The `notify` operation is inexpensive when no threads are waiting. +The `prepare_wait` and `commit_wait` operations are more costly, but +they are only executed when the initial predicate check fails. +The flow diagram for notifier and waiter is shown below: + +@dotfile images/nonblocking_notifier.dot + + +The synchronization algorithm relies on two shared variables: +a user-defined predicate and an internal state variable. +To avoid lost wake-ups, the protocol follows a two-phase @em update-then-check pattern. +A waiting thread publishes its intent to wait by updating the state before rechecking the predicate. +Conversely, a notifying thread updates the predicate before inspecting the state. +This interaction is governed by a memory barrier of sequential consistency that guarantees at least one thread will observe the other's progress. +Consequently, the waiter either detects the work and stays active, or the notifier detects the waiter and issues a wakeup. +It is impossible for both threads to miss each other's updates. + +The state has the following layout, which consists of the following three parts: + + `STACK_BITS` is a stack of committed waiters + + `PREWAITER_BITS` is the count of waiters in the pre-waiting stage + + `EPOCH_BITS` is the modification counter + +@dotfile images/nonblocking_notifier_state_layout.dot + +Reference: https://gitlab.com/libeigen/eigen/-/blob/master/Eigen/src/ThreadPool/EventCount.h +*/ +class NonblockingNotifier { + + friend class Executor; + + struct Waiter { + alignas (TF_CACHELINE_SIZE) std::atomic next; + uint64_t epoch; + + enum : unsigned { + kNotSignaled = 0, + kWaiting, + kSignaled, + }; + std::atomic state {0}; + + //mutable std::mutex mu; + //std::condition_variable cv; + //unsigned state; + }; + + public: + + // The state variable consists of the following three parts: + // - low STACK_BITS is a stack of waiters committed wait. + // - next PREWAITER_BITS is count of waiters in prewait state. + // - next EPOCH_BITS is modification counter. + // [ 32-bit epoch | 16-bit pre-waiter count | 16-bit pre-waiter stack] + + /// Number of bits used to encode the waiter stack index. + static const uint64_t STACK_BITS = 16; + + /// Bit mask for extracting the waiter stack index. + static const uint64_t STACK_MASK = (1ull << STACK_BITS) - 1; + + /// Number of bits used to encode the pre-waiter ticket. + static const uint64_t PREWAITER_BITS = 16; + + /// Bit shift of the pre-waiter ticket field. + static const uint64_t PREWAITER_SHIFT = 16; + + /// Bit mask for extracting the pre-waiter ticket field. + static const uint64_t PREWAITER_MASK = ((1ull << PREWAITER_BITS) - 1) << PREWAITER_SHIFT; + + /// Increment value for advancing the pre-waiter ticket. + static const uint64_t PREWAITER_INC = 1ull << PREWAITER_BITS; + + /// Number of bits used to encode the epoch counter. + static const uint64_t EPOCH_BITS = 32; + + /// Bit shift of the epoch field. + static const uint64_t EPOCH_SHIFT = 32; + + /// Bit mask for extracting the epoch field. + static const uint64_t EPOCH_MASK = ((1ull << EPOCH_BITS) - 1) << EPOCH_SHIFT; + + /// Increment value for advancing the epoch counter. + static const uint64_t EPOCH_INC = 1ull << EPOCH_SHIFT; + + /** + @brief constructs a notifier with `N` waiters + + @param N number of waiters + + Constructs a notifier that supports up to `N` waiters. + The maximum allowable number of waiters can be acquired by calling `capacity()`, + which is equal to 2STACK_BITS. + */ + explicit NonblockingNotifier(size_t N) : _state(STACK_MASK), _waiters(N) { + if(_waiters.size() >= ((1 << PREWAITER_BITS) - 1)) { + TF_THROW("nonblocking waiter supports only up to ", (1<STACK_BITS. + */ + size_t capacity() const { + return 1 << STACK_BITS; + } + + /** + @brief prepares the calling thread to enter the waiting set + + @param wid identifier of the calling thread in the range of `[0, N)`, + where `N` represents the number of waiters used to construct this notifier + + This function places the thread into the pre-waiting stage. After calling + `prepare_wait()`, the thread must re-check the wait predicate and then + complete the protocol by calling either `commit_wait()` or `cancel_wait()` + with the same waiter identifier. + + A thread in the pre-waiting stage is not yet considered a committed waiter, + and its waiting status is considered incomplete. + Failing to follow `prepare_wait()` with exactly one call to + `commit_wait()` or `cancel_wait()` results in undefined behavior. + */ + void prepare_wait(size_t wid) { + _waiters[wid].epoch = _state.fetch_add(PREWAITER_INC, std::memory_order_relaxed); + std::atomic_thread_fence(std::memory_order_seq_cst); + } + + /** + @brief commits a previously prepared wait operation + + @param wid identifier of the calling thread in the range of `[0, N)`, + where `N` represents the number of waiters used to construct this notifier + + This function completes the waiting protocol for a thread that has + previously called `prepare_wait()`. Upon successful completion, the + thread becomes a committed waiter and will park until being notified. + + The thread must have re-checked the wait predicate before calling + `commit_wait()`. Once committed, the thread may be awakened by + `notify_one()`, `notify_n()`, or `notify_all()`. + + Each call to `prepare_wait()` must be followed by exactly one call + to either `commit_wait()` or `cancel_wait()` using the same + thread identifier. + */ + void commit_wait(size_t wid) { + + auto w = &_waiters[wid]; + + w->state.store(Waiter::kNotSignaled, std::memory_order_relaxed); + + /* + Epoch and ticket semantics. + + `sepoch` = _state & EPOCH_MASK + `wepoch` = w->epoch & EPOCH_MASK + `ticket` = w->epoch & PREWAITER_MASK + + Each waiter entering the pre-waiting stage is assigned a monotonically + increasing ticket that determines the processing order (e.g., + cancel_wait, commit_wait, notify). Ticket 0 is processed first, followed + by ticket 1, and so on. + + The global epoch `sepoch` is incremented whenever a request is fulfilled. + Therefore, the difference `sepoch - wepoch` indicates which ticket is + currently ready to be handled: + + - `sepoch - wepoch == ticket` : this waiter's turn + - `sepoch - wepoch > ticket` : this waiter's ticket has expired + - `sepoch - wepoch < ticket` : this waiter's ticket has not yet reached + + Unsigned wraparound does not affect correctness. All epoch arithmetic is + performed using unsigned integers, which obey modulo-2^N arithmetic. + Converting the unsigned difference to a signed value yields the correct + result as long as the true difference lies within the signed range. + + In general: + - Unsigned range: [0, 2^N − 1] + - Signed range : [−2^(N−1), 2^(N−1) − 1] + + When overflow occurs, unsigned subtraction computes: + + (sepoch − wepoch) mod 2^N + + If the true value of `sepoch − wepoch` is within the signed range + [−2^(N−1), 2^(N−1) − 1], reinterpreting this result as a signed integer + produces the correct mathematical difference. + + Example (3-bit arithmetic): + + a b | true a−b | unsigned (bin / dec) | signed (dec) + ---------------------------------------------------- + 1 0 | 1 | 001 / 1 | +1 + 1 1 | 0 | 000 / 0 | 0 + 1 2 | -1 | 111 / 7 | -1 + 1 3 | -2 | 110 / 6 | -2 + 1 4 | -3 | 101 / 5 | -3 + 1 5 | -4 | 100 / 4 | -4 + 1 6 | -5 | 011 / 3 | +3 (wrap around) + 1 7 | -6 | 010 / 2 | +2 (wrap around) + + Signed interpretation is correct only when the true difference lies + within [−4, +3]. + + In this implementation, `sepoch − wepoch` is guaranteed not to exceed + 2^16 in magnitude, which is far smaller than 2^(EPOCH_BITS − 1). + Consequently, the expression: + + int64_t((state & EPOCH_MASK) - epoch) + + remains correct even if `sepoch` and `wepoch` individually overflow. + */ + uint64_t epoch = + (w->epoch & EPOCH_MASK) + + (((w->epoch & PREWAITER_MASK) >> PREWAITER_SHIFT) << EPOCH_SHIFT); + uint64_t state = _state.load(std::memory_order_seq_cst); + for (;;) { + if (int64_t((state & EPOCH_MASK) - epoch) < 0) { + // The preceding waiter has not decided on its fate. Wait until it + // calls either cancel_wait or commit_wait, or is notified. + std::this_thread::yield(); + state = _state.load(std::memory_order_seq_cst); + continue; + } + // We've already been notified. + if (int64_t((state & EPOCH_MASK) - epoch) > 0) { + return; + } + // Remove this thread from prewait counter and add it to the waiter stack. + assert((state & PREWAITER_MASK) != 0); + uint64_t newstate = state - PREWAITER_INC + EPOCH_INC; + newstate = (newstate & ~STACK_MASK) | wid; + + // stack is empty -> this waiter is at the top of the stack, pointing to nothing + if ((state & STACK_MASK) == STACK_MASK) { + w->next.store(nullptr, std::memory_order_relaxed); + } + // stack is non-empty -> this waiter is at the top of the stack, pointing to the origin top + else { + w->next.store(&_waiters[state & STACK_MASK], std::memory_order_relaxed); + } + if (_state.compare_exchange_weak(state, newstate, std::memory_order_release)) { + break; + } + } + _park(w); + } + + /** + @brief cancels a previously prepared wait operation + + @param wid identifier of the calling thread in the range of `[0, N)`, + where `N` represents the number of waiters used to construct this notifier + + This function aborts the waiting protocol for a thread that has + previously called `prepare_wait()`. After cancellation, the thread + does not become a committed waiter and will return to user-side control. + + `cancel_wait()` must be called after the wait predicate has been + re-checked and found to be false. This allows a thread to safely + abandon waiting without blocking or being notified. + + Each call to `prepare_wait()` must be followed by exactly one call + to either `commit_wait()` or `cancel_wait()` using the same + thread identifier. + */ + void cancel_wait(size_t wid) { + uint64_t epoch = + (_waiters[wid].epoch & EPOCH_MASK) + + (((_waiters[wid].epoch & PREWAITER_MASK) >> PREWAITER_SHIFT) << EPOCH_SHIFT); + uint64_t state = _state.load(std::memory_order_relaxed); + for (;;) { + if (int64_t((state & EPOCH_MASK) - epoch) < 0) { + // The preceding waiter has not decided on its fate. Wait until it + // calls either cancel_wait or commit_wait, or is notified. + std::this_thread::yield(); + state = _state.load(std::memory_order_relaxed); + continue; + } + // We've already been notified. + if (int64_t((state & EPOCH_MASK) - epoch) > 0) { + return; + } + // Remove this thread from prewait counter. + assert((state & PREWAITER_MASK) != 0); + if (_state.compare_exchange_weak(state, state - PREWAITER_INC + EPOCH_INC, + std::memory_order_relaxed)) { + return; + } + } + } + + /** + @brief notifies one waiter from the waiting set + + Wakes up one waiter from the waiting set, including those in the pre-waiting stage. + + The function is cheap when no threads are waiting. + */ + void notify_one() { + std::atomic_thread_fence(std::memory_order_seq_cst); + uint64_t state = _state.load(std::memory_order_acquire); + for (;;) { + // Easy case: no waiters. + if ((state & STACK_MASK) == STACK_MASK && (state & PREWAITER_MASK) == 0) { + return; + } + uint64_t num_prewaiters = (state & PREWAITER_MASK) >> PREWAITER_SHIFT; + uint64_t newstate; + if (num_prewaiters) { + // There is a thread in pre-wait state, unblock it. + newstate = state + EPOCH_INC - PREWAITER_INC; + } + else { + // Pop a waiter from list and unpark it. + Waiter* w = &_waiters[state & STACK_MASK]; + Waiter* wnext = w->next.load(std::memory_order_relaxed); + uint64_t next = STACK_MASK; + //if (wnext != nullptr) next = wnext - &_waiters[0]; + if (wnext != nullptr) { + next = static_cast(wnext - &_waiters[0]); + } + // Note: we don't add EPOCH_INC here. ABA problem on the lock-free stack + // can't happen because a waiter is re-pushed onto the stack only after + // it was in the pre-wait state which inevitably leads to epoch increment. + newstate = (state & EPOCH_MASK) + next; + } + if (_state.compare_exchange_weak(state, newstate, std::memory_order_acquire)) { + if(num_prewaiters) { + return; // unblocked pre-wait thread + } + // if there is no pre-waiters, the stack must have something + Waiter* w = &_waiters[state & STACK_MASK]; + w->next.store(nullptr, std::memory_order_relaxed); + _unpark(w); + return; + } + } + } + + /** + @brief notifies all waiter from the waiting set + + Wakes up all waiters from the waiting set, including those in the pre-waiting stage. + + The function is cheap when no threads are waiting. + */ + void notify_all() { + std::atomic_thread_fence(std::memory_order_seq_cst); + uint64_t state = _state.load(std::memory_order_acquire); + for (;;) { + + // Easy case: no waiters. + if ((state & STACK_MASK) == STACK_MASK && (state & PREWAITER_MASK) == 0) { + return; + } + uint64_t num_prewaiters = (state & PREWAITER_MASK) >> PREWAITER_SHIFT; + + // Reset prewait counter and empty wait list. + uint64_t newstate = (state & EPOCH_MASK) + (EPOCH_INC * num_prewaiters) + STACK_MASK; + + if (_state.compare_exchange_weak(state, newstate, std::memory_order_acquire)) { + if ((state & STACK_MASK) == STACK_MASK) { + return; + } + Waiter* w = &_waiters[state & STACK_MASK]; + _unpark(w); + return; + } + } + } + + /** + @brief notifies up to `N` waiters from the waiting set + + @param N maximum number of waiters to notify + + Wakes up at most @p N waiters from the waiting set. + If @p N is greater than or equal to the maximum number of waiters in this notifier, + this function behaves identically to `notify_all()`. + + The function is cheap when no threads are waiting. + */ + void notify_n(size_t N) { + + // trivial case + if(N == 0) { + return; + } + + // if the target N is bigger than the waiter size, notify all waiters + if(N >= _waiters.size()) { + notify_all(); + return; + } + + std::atomic_thread_fence(std::memory_order_seq_cst); + uint64_t state = _state.load(std::memory_order_acquire); + do { + // Easy case: no waiters. + if ((state & STACK_MASK) == STACK_MASK && (state & PREWAITER_MASK) == 0) { + return; + } + uint64_t num_prewaiters = (state & PREWAITER_MASK) >> PREWAITER_SHIFT; + uint64_t newstate; + size_t newN; + + // unblock waiters from pre-waiting list first. + if(num_prewaiters) { + size_t to_unblock = (N < num_prewaiters) ? N : num_prewaiters; + newstate = state + (EPOCH_INC * to_unblock) - (PREWAITER_INC * to_unblock); + newN = N - to_unblock; + } + // pop one waiter from the stack + else { + Waiter* w = &_waiters[state & STACK_MASK]; + Waiter* wnext = w->next.load(std::memory_order_relaxed); + uint64_t next = STACK_MASK; + //if (wnext != nullptr) next = wnext - &_waiters[0]; + if (wnext != nullptr) { + next = static_cast(wnext - &_waiters[0]); + } + // Note: we don't add EPOCH_INC here. ABA problem on the lock-free stack + // can't happen because a waiter is re-pushed onto the stack only after + // it was in the pre-wait state which inevitably leads to epoch increment. + newstate = (state & EPOCH_MASK) + next; + newN = N - 1; + } + + if (_state.compare_exchange_weak(state, newstate, std::memory_order_acquire)) { + N = newN; + if(num_prewaiters == 0) { + Waiter* w = &_waiters[state & STACK_MASK]; + w->next.store(nullptr, std::memory_order_relaxed); + _unpark(w); + } + } + } while(N > 0); + + //if(n >= _waiters.size()) { + // notify_all(); + //} + //else { + // for(size_t k=0; k _state; + std::vector _waiters; + + // only this waiter can park itself, with the following two possible paths: + // 1. kNotSignaled (this) -> in-stack -> kWaiting (this) -> wait + // 2. kNotSignaled (this) -> in-stack -> kSignaled -> unwait + void _park(Waiter* w) { + unsigned target = Waiter::kNotSignaled; + if(w->state.compare_exchange_strong(target, Waiter::kWaiting, std::memory_order_relaxed + , std::memory_order_relaxed)) { + w->state.wait(Waiter::kWaiting, std::memory_order_relaxed); + } + //std::unique_lock lock(w->mu); + //while (w->state != Waiter::kSignaled) { + // w->state = Waiter::kWaiting; + // w->cv.wait(lock); + //} + } + + // others can unpark + void _unpark(Waiter* waiters) { + Waiter* next = nullptr; + for (Waiter* w = waiters; w; w = next) { + next = w->next.load(std::memory_order_relaxed); + // We only notify if the other is waiting - this is why we use tri-state + // variable instead of binary-state variable (i.e., atomic_flag) + // Performance is about 0.1% faster + if(w->state.exchange(Waiter::kSignaled, std::memory_order_relaxed) == Waiter::kWaiting) { + w->state.notify_one(); + } + //unsigned state; + //{ + // std::unique_lock lock(w->mu); + // state = w->state; + // w->state = Waiter::kSignaled; + //} + //// Avoid notifying if it wasn't waiting. + //if (state == Waiter::kWaiting) w->cv.notify_one(); + } + } + + // notify wakes one or all waiting threads. + // Must be called after changing the associated wait predicate. + //void _notify(bool all) { + // std::atomic_thread_fence(std::memory_order_seq_cst); + // uint64_t state = _state.load(std::memory_order_acquire); + // for (;;) { + // // Easy case: no waiters. + // if ((state & STACK_MASK) == STACK_MASK && (state & PREWAITER_MASK) == 0) { + // return; + // } + // uint64_t num_prewaiters = (state & PREWAITER_MASK) >> PREWAITER_SHIFT; + // uint64_t newstate; + // if (all) { + // // Reset prewait counter and empty wait list. + // newstate = (state & EPOCH_MASK) + (EPOCH_INC * num_prewaiters) + STACK_MASK; + // } else if (num_prewaiters) { + // // There is a thread in pre-wait state, unblock it. + // newstate = state + EPOCH_INC - PREWAITER_INC; + // } else { + // // Pop a waiter from list and unpark it. + // Waiter* w = &_waiters[state & STACK_MASK]; + // Waiter* wnext = w->next.load(std::memory_order_relaxed); + // uint64_t next = STACK_MASK; + // //if (wnext != nullptr) next = wnext - &_waiters[0]; + // if (wnext != nullptr) { + // next = static_cast(wnext - &_waiters[0]); + // } + // // Note: we don't add EPOCH_INC here. ABA problem on the lock-free stack + // // can't happen because a waiter is re-pushed onto the stack only after + // // it was in the pre-wait state which inevitably leads to epoch increment. + // newstate = (state & EPOCH_MASK) + next; + // } + // if (_state.compare_exchange_weak(state, newstate, std::memory_order_acquire)) { + // if(!all && num_prewaiters) return; // unblocked pre-wait thread + // if ((state & STACK_MASK) == STACK_MASK) return; + // Waiter* w = &_waiters[state & STACK_MASK]; + // if(!all) { + // w->next.store(nullptr, std::memory_order_relaxed); + // } + // _unpark(w); + // return; + // } + // } + //} +}; + + +} // namespace tf ------------------------------------------------------------ + diff --git a/projects/hipfile/third-party/taskflow/core/runtime.hpp b/projects/hipfile/third-party/taskflow/core/runtime.hpp new file mode 100644 index 000000000000..6a68002ca083 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/core/runtime.hpp @@ -0,0 +1,984 @@ +#pragma once + +#include "executor.hpp" + +namespace tf { + +// ------------------------------------------------------------------------------------------------ +// class: Runtime +// ------------------------------------------------------------------------------------------------ + +/** +@class Runtime + +@brief class to create a runtime task + +A runtime object provides an interface for interacting with the scheduling system from within a task +(i.e., the parent task of this runtime). +It enables operations such as spawning asynchronous tasks, executing tasks cooperatively, +and implementing recursive parallelism. +The runtime guarantees an implicit join at the end of its scope, +so all spawned tasks will finish before the parent runtime task continues to its successors. + +@code{.cpp} +tf::Executor executor(num_threads); +tf::Taskflow taskflow; +std::atomic counter(0); + +tf::Task A = taskflow.emplace([&](tf::Runtime& rt){ + // spawn 1000 asynchronous tasks from this runtime task + for(size_t i=0; i<1000; i++) { + rt.silent_async([&](){ counter.fetch_add(1, std::memory_order_relaxed); }); + } + // implicit synchronization at the end of the runtime scope +}); +tf::Task B = taskflow.emplace([&](){ + assert(counter.load(std::memory_order_relaxed) == 1000); +}); +A.precede(B); + +executor.run(taskflow).wait(); +@endcode + +@note +To understand how %Taskflow schedules a runtime task, please refer to @ref RuntimeTasking. + +*/ +class Runtime { + + friend class Executor; + friend class FlowBuilder; + friend class PreemptionGuard; + friend class Algorithm; + + public: + + /** + @brief obtains the running executor + + The running executor of a runtime task is the executor that runs + the parent taskflow of that runtime task. + + @code{.cpp} + tf::Executor executor; + tf::Taskflow taskflow; + taskflow.emplace([&](tf::Runtime& rt){ + assert(&(rt.executor()) == &executor); + }); + executor.run(taskflow).wait(); + @endcode + */ + Executor& executor(); + + /** + @brief acquire a reference to the underlying worker + */ + inline Worker& worker(); + + /** + @brief schedules an active task immediately to the worker's queue + + @param task the given active task to schedule immediately + + This member function immediately schedules an active task to the + task queue of the associated worker in the runtime task. + An active task is a task in a running taskflow. + The task may or may not be running, and scheduling that task + will immediately put the task into the task queue of the worker + that is running the runtime task. + Consider the following example: + + @code{.cpp} + tf::Task A, B, C, D; + std::tie(A, B, C, D) = taskflow.emplace( + [] () { return 0; }, + [&C] (tf::Runtime& rt) { // C must be captured by reference + std::cout << "B\n"; + rt.schedule(C); + }, + [] () { std::cout << "C\n"; }, + [] () { std::cout << "D\n"; } + ); + A.precede(B, C, D); + executor.run(taskflow).wait(); + @endcode + + The executor will first run the condition task @c A which returns @c 0 + to inform the scheduler to go to the runtime task @c B. + During the execution of @c B, it directly schedules task @c C without + going through the normal taskflow graph scheduling process. + At this moment, task @c C is active because its parent taskflow is running. + When the taskflow finishes, we will see both @c B and @c C in the output. + + @attention + This method can only be called by the parent worker of this runtime, + or the behavior is undefined. + Furthermore, we currently do not support scheduling a runtime task. + */ + void schedule(Task task); + + // ---------------------------------------------------------------------------------------------- + // async methods + // ---------------------------------------------------------------------------------------------- + + /** + @brief runs the given callable asynchronously + + @tparam F callable type + @param f callable object + + This method creates an asynchronous task that executes the given function with the specified arguments. + Unlike tf::Executor::async, the task created here is parented to the runtime object and + is implicitly synchronized at the end of the runtime's scope. + Applications may also call tf::Runtime::corun explicitly to wait for all + asynchronous tasks spawned from the runtime to complete. + For example: + + @code{.cpp} + std::atomic counter(0); + taskflow.emplace([&](tf::Runtime& rt){ + auto fu1 = rt.async([&](){ counter++; }); + auto fu2 = rt.async([&](){ counter++; }); + fu1.get(); + fu2.get(); + assert(counter == 2); + // spawn 100 asynchronous tasks from the worker of the runtime + for(int i=0; i<100; i++) { + rt.silent_async([&](){ counter++; }); + } + // corun until the 100 asynchronous tasks have completed + rt.corun(); + assert(counter == 102); + // do something else afterwards ... + }); + @endcode + */ + template + auto async(F&& f); + + /** + @brief runs the given callable asynchronously + + @tparam F callable type + @tparam P task parameters type satisfying tf::TaskParamsLike + + @param params task parameters + @param f callable + + Similar to tf::Runtime::async, but takes a parameter of type tf::TaskParams to initialize + the asynchronous task. + + @code{.cpp} + taskflow.emplace([&](tf::Runtime& rt){ + auto future = rt.async("my task", [](){ return 10; }); + assert(future.get() == 10); + }); + @endcode + + */ + template + auto async(P&& params, F&& f); + + // ---------------------------------------------------------------------------------------------- + // silent async methods + // ---------------------------------------------------------------------------------------------- + + /** + @brief runs the given function asynchronously without returning any future object + + @tparam F callable type + @param f callable + + This function is more efficient than tf::Runtime::async and is recommended when the result + of the asynchronous task does not need to be accessed via a std::future. + + @code{.cpp} + std::atomic counter(0); + taskflow.emplace([&](tf::Runtime& rt){ + for(int i=0; i<100; i++) { + rt.silent_async([&](){ counter++; }); + } + rt.corun(); + assert(counter == 100); + }); + @endcode + */ + template + void silent_async(F&& f); + + /** + @brief runs the given function asynchronously without returning any future object + + @tparam F callable type + @param params task parameters + @param f callable + + Similar to tf::Runtime::silent_async, but takes a parameter of type tf::TaskParams to initialize + the created asynchronous task. + + @code{.cpp} + taskflow.emplace([&](tf::Runtime& rt){ + rt.silent_async("my task", [](){}); + }); + @endcode + */ + template + void silent_async(P&& params, F&& f); + + // ---------------------------------------------------------------------------------------------- + // dependent async methods + // ---------------------------------------------------------------------------------------------- + + /** + @brief runs the given function asynchronously + when the given predecessors finish + + @tparam F callable type + @tparam Tasks tasks of type tf::AsyncTask + + @param func callable object + @param tasks asynchronous tasks on which this execution depends + + @return a pair of a tf::AsyncTask handle and + a @std_future that holds the result of the execution + + The example below creates three asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + Task @c C returns a pair of its tf::AsyncTask handle and a std::future + that eventually will hold the result of the execution. + + @code{.cpp} + taskflow.emplace([](tf::Runtime& rt){ + tf::AsyncTask A = rt.silent_dependent_async([](){ printf("A\n"); }); + tf::AsyncTask B = rt.silent_dependent_async([](){ printf("B\n"); }); + auto [C, fuC] = rt.dependent_async( + [](){ + printf("C runs after A and B\n"); + return 1; + }, + A, B + ); + fuC.get(); // C finishes, which in turns means both A and B finish + }); // implicit synchronization of all tasks at the end of runtime's scope + executor.run(taskflow).wait(); + @endcode + */ + template + requires (std::same_as, AsyncTask> && ...) + auto dependent_async(F&& func, Tasks&&... tasks); + + /** + @brief runs the given function asynchronously + when the given predecessors finish + + @tparam P task parameters type satisfying tf::TaskParamsLike + @tparam F callable type + @tparam Tasks tasks of type tf::AsyncTask + + @param params task parameters + @param func callable object + @param tasks asynchronous tasks on which this execution depends + + @return a pair of a tf::AsyncTask handle and + a @std_future that holds the result of the execution + + The example below creates three named asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + Task @c C returns a pair of its tf::AsyncTask handle and a std::future + that eventually will hold the result of the execution. + Assigned task names will appear in the observers of the executor. + + @code{.cpp} + taskflow.emplace([](tf::Runtime& rt){ + tf::AsyncTask A = rt.silent_dependent_async("A", [](){ printf("A\n"); }); + tf::AsyncTask B = rt.silent_dependent_async("B", [](){ printf("B\n"); }); + auto [C, fuC] = rt.dependent_async( + "C", + [](){ + printf("C runs after A and B\n"); + return 1; + }, + A, B + ); + assert(fuC.get()==1); // C finishes, which in turns means both A and B finish + }); // implicit synchronization of all tasks at the end of runtime's scope + executor.run(taskflow).wait(); + @endcode + */ + template + requires (std::same_as, AsyncTask> && ...) + auto dependent_async(P&& params, F&& func, Tasks&&... tasks); + + /** + @brief runs the given function asynchronously + when the given range of predecessors finish + + @tparam F callable type + @tparam I iterator type + + @param func callable object + @param first iterator to the beginning (inclusive) + @param last iterator to the end (exclusive) + + @return a pair of a tf::AsyncTask handle and + a @std_future that holds the result of the execution + + The example below creates three asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + Task @c C returns a pair of its tf::AsyncTask handle and a std::future + that eventually will hold the result of the execution. + + @code{.cpp} + taskflow.emplace([](tf::Runtime& rt){ + std::array array { + rt.silent_dependent_async([](){ printf("A\n"); }), + rt.silent_dependent_async([](){ printf("B\n"); }) + }; + auto [C, fuC] = rt.dependent_async( + [](){ + printf("C runs after A and B\n"); + return 1; + }, + array.begin(), array.end() + ); + assert(fuC.get()==1); // C finishes, which in turns means both A and B finish + }); // implicit synchronization of all tasks at the end of runtime's scope + executor.run(taskflow).wait(); + @endcode + */ + template + requires (!std::same_as, AsyncTask>) + auto dependent_async(F&& func, I first, I last); + + /** + @brief runs the given function asynchronously + when the given range of predecessors finish + + @tparam P task parameters type satisfying tf::TaskParamsLike + @tparam F callable type + @tparam I iterator type + + @param params task parameters + @param func callable object + @param first iterator to the beginning (inclusive) + @param last iterator to the end (exclusive) + + @return a pair of a tf::AsyncTask handle and + a @std_future that holds the result of the execution + + The example below creates three named asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + Task @c C returns a pair of its tf::AsyncTask handle and a std::future + that eventually will hold the result of the execution. + Assigned task names will appear in the observers of the executor. + + @code{.cpp} + taskflow.emplace([](tf::Runtime& rt){ + std::array array { + rt.silent_dependent_async("A", [](){ printf("A\n"); }), + rt.silent_dependent_async("B", [](){ printf("B\n"); }) + }; + auto [C, fuC] = rt.dependent_async( + "C", + [](){ + printf("C runs after A and B\n"); + return 1; + }, + array.begin(), array.end() + ); + assert(fuC.get()==1); // C finishes, which in turns means both A and B finish + }); // implicit synchronization of all tasks at the end of runtime's scope + executor.run(taskflow).wait(); + @endcode + */ + template + requires (!std::same_as, AsyncTask>) + auto dependent_async(P&& params, F&& func, I first, I last); + + // ---------------------------------------------------------------------------------------------- + // silent dependent async methods + // ---------------------------------------------------------------------------------------------- + + /** + @brief runs the given function asynchronously + when the given predecessors finish + + @tparam F callable type + @tparam Tasks tasks of type tf::AsyncTask + + @param func callable object + @param tasks asynchronous tasks on which this execution depends + + @return a tf::AsyncTask handle + + This member function is more efficient than tf::Runtime::dependent_async + and is encouraged to use when you do not want a @std_future to + acquire the result or synchronize the execution. + The example below creates three asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + + @code{.cpp} + taskflow.emplace([](tf::Runtime& rt){ + tf::AsyncTask A = rt.silent_dependent_async([](){ printf("A\n"); }); + tf::AsyncTask B = rt.silent_dependent_async([](){ printf("B\n"); }); + rt.silent_dependent_async([](){ printf("C runs after A and B\n"); }, A, B); + }); // implicit synchronization of all tasks at the end of runtime's scope + executor.wait_for_all(); + @endcode + */ + template + requires (std::same_as, AsyncTask> && ...) + tf::AsyncTask silent_dependent_async(F&& func, Tasks&&... tasks); + + /** + @brief runs the given function asynchronously + when the given predecessors finish + + @tparam F callable type + @tparam Tasks tasks of type tf::AsyncTask + + @param params task parameters + @param func callable object + @param tasks asynchronous tasks on which this execution depends + + @return a tf::AsyncTask handle + + This member function is more efficient than tf::Runtime::dependent_async + and is encouraged to use when you do not want a @std_future to + acquire the result or synchronize the execution. + The example below creates three asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + Assigned task names will appear in the observers of the executor. + + @code{.cpp} + taskflow.emplace([](tf::Runtime& rt){ + tf::AsyncTask A = rt.silent_dependent_async("A", [](){ printf("A\n"); }); + tf::AsyncTask B = rt.silent_dependent_async("B", [](){ printf("B\n"); }); + rt.silent_dependent_async( + "C", [](){ printf("C runs after A and B\n"); }, A, B + ); + }); // implicit synchronization of all tasks at the end of runtime's scope + executor.wait_for_all(); + @endcode + */ + template + requires (std::same_as, AsyncTask> && ...) + tf::AsyncTask silent_dependent_async(P&& params, F&& func, Tasks&&... tasks); + + /** + @brief runs the given function asynchronously + when the given range of predecessors finish + + @tparam F callable type + @tparam I iterator type + + @param func callable object + @param first iterator to the beginning (inclusive) + @param last iterator to the end (exclusive) + + @return a tf::AsyncTask handle + + This member function is more efficient than tf::Runtime::dependent_async + and is encouraged to use when you do not want a @std_future to + acquire the result or synchronize the execution. + The example below creates three asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + + @code{.cpp} + Taskflow.emplace([&](tf::Runtime& rt){ + std::array array { + rt.silent_dependent_async([](){ printf("A\n"); }), + rt.silent_dependent_async([](){ printf("B\n"); }) + }; + rt.silent_dependent_async( + [](){ printf("C runs after A and B\n"); }, array.begin(), array.end() + ); + }); // implicit synchronization of all tasks at the end of runtime's scope + executor.wait_for_all(); + @endcode + */ + template + requires (!std::same_as, AsyncTask>) + tf::AsyncTask silent_dependent_async(F&& func, I first, I last); + + /** + @brief runs the given function asynchronously + when the given range of predecessors finish + + @tparam F callable type + @tparam I iterator type + + @param params tasks parameters + @param func callable object + @param first iterator to the beginning (inclusive) + @param last iterator to the end (exclusive) + + @return a tf::AsyncTask handle + + This member function is more efficient than tf::Runtime::dependent_async + and is encouraged to use when you do not want a @std_future to + acquire the result or synchronize the execution. + The example below creates three asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + Assigned task names will appear in the observers of the executor. + + @code{.cpp} + taskflow.emplace([](tf::Runtime& rt){ + std::array array { + rt.silent_dependent_async("A", [](){ printf("A\n"); }), + rt.silent_dependent_async("B", [](){ printf("B\n"); }) + }; + rt.silent_dependent_async( + "C", [](){ printf("C runs after A and B\n"); }, array.begin(), array.end() + ); + }); // implicit synchronization of all tasks at the end of runtime's scope + executor.run(taskflow).wait(); + @endcode + */ + template + requires (!std::same_as, AsyncTask>) + tf::AsyncTask silent_dependent_async(P&& params, F&& func, I first, I last); + + + + // ---------------------------------------------------------------------------------------------- + // cooperative execution methods + // ---------------------------------------------------------------------------------------------- + + /** + @brief corun all tasks spawned by this runtime with other workers + + Coruns all tasks spawned by this runtime cooperatively with other workers in + the same executor until all these tasks finish. + Under cooperative execution, a worker is not preempted. Instead, it continues + participating in the work-stealing loop, executing available tasks alongside + other workers. + + @code{.cpp} + std::atomic counter{0}; + taskflow.emplace([&](tf::Runtime& rt){ + // spawn 100 async tasks and wait + for(int i=0; i<100; i++) { + rt.silent_async([&](){ counter++; }); + } + rt.corun(); + assert(counter == 100); + + // spawn another 100 async tasks and wait + for(int i=0; i<100; i++) { + rt.silent_async([&](){ counter++; }); + } + rt.corun(); + assert(counter == 200); + }); + @endcode + + Only the parent worker of this runtime is allowed to call this corun. + */ + void corun(); + + /** + @brief equivalent to tf::Runtime::corun - just an alias for legacy purpose + */ + void corun_all(); + + /** + @brief queries if this runtime task has been cancelled + */ + bool is_cancelled(); + + private: + + /** + @private + */ + explicit Runtime(Executor&, Worker&, Node*); + + /** + @private + */ + Executor& _executor; + + /** + @private + */ + Worker& _worker; + + /** + @private + */ + Node* _node; +}; + +// constructor +inline Runtime::Runtime(Executor& executor, Worker& worker, Node* node) : + _executor {executor}, + _worker {worker}, + _node {node} { +} + +// Function: executor +inline Executor& Runtime::executor() { + return _executor; +} + +// Function: worker +inline Worker& Runtime::worker() { + return _worker; +} + +// Procedure: schedule +inline void Runtime::schedule(Task task) { + + auto node = task._node; + // need to keep the invariant: when scheduling a task, the task must have + // zero dependency (join counter is 0) + // or we can encounter bug when inserting a nested flow (e.g., module task) + node->_join_counter.store(0, std::memory_order_relaxed); + + auto& j = node->_parent ? node->_parent->_join_counter : + node->_topology->_join_counter; + j.fetch_add(1, std::memory_order_relaxed); + _executor._schedule(_worker, node); +} + +// Function: corun +inline void Runtime::corun() { + { + ExplicitAnchorGuard anchor(_node); + _executor._corun_until(_worker, [this] () -> bool { + return _node->_join_counter.load(std::memory_order_acquire) == 1; + }); + } + _node->_rethrow_exception(); +} + +// Function: corun_all +inline void Runtime::corun_all() { + corun(); +} + +inline bool Runtime::is_cancelled() { + return _node->_is_parent_cancelled(); +} + +// ------------------------------------------------------------------------------------------------ +// Runtime::silent_async +// ------------------------------------------------------------------------------------------------ + +// Function: silent_async +template +void Runtime::silent_async(F&& f) { + silent_async(DefaultTaskParams{}, std::forward(f)); +} + +// Function: silent_async +template +void Runtime::silent_async(P&& params, F&& f) { + _node->_join_counter.fetch_add(1, std::memory_order_relaxed); + _executor._silent_async( + std::forward

(params), std::forward(f), _node->_topology, _node + ); +} + +// ------------------------------------------------------------------------------------------------ +// Runtime::async +// ------------------------------------------------------------------------------------------------ + +// Function: async +template +auto Runtime::async(F&& f) { + return async(DefaultTaskParams{}, std::forward(f)); +} + +// Function: async +template +auto Runtime::async(P&& params, F&& f) { + _node->_join_counter.fetch_add(1, std::memory_order_relaxed); + return _executor._async( + std::forward

(params), std::forward(f), _node->_topology, _node + ); +} + +// ------------------------------------------------------------------------------------------------ +// silent dependent async +// ------------------------------------------------------------------------------------------------ + +// Function: silent_dependent_async +template +requires (std::same_as, AsyncTask> && ...) +tf::AsyncTask Runtime::silent_dependent_async(F&& func, Tasks&&... tasks) { + return silent_dependent_async( + DefaultTaskParams{}, std::forward(func), std::forward(tasks)... + ); +} + +// Function: silent_dependent_async +template +requires (std::same_as, AsyncTask> && ...) +tf::AsyncTask Runtime::silent_dependent_async( + P&& params, F&& func, Tasks&&... tasks +){ + std::array array = { std::forward(tasks)... }; + return silent_dependent_async( + std::forward

(params), std::forward(func), array.begin(), array.end() + ); +} + +// Function: silent_dependent_async +template +requires (!std::same_as, AsyncTask>) +tf::AsyncTask Runtime::silent_dependent_async(F&& func, I first, I last) { + return silent_dependent_async(DefaultTaskParams{}, std::forward(func), first, last); +} + +// Function: silent_dependent_async +template +requires (!std::same_as, AsyncTask>) +tf::AsyncTask Runtime::silent_dependent_async( + P&& params, F&& func, I first, I last +) { + _node->_join_counter.fetch_add(1, std::memory_order_relaxed); + return _executor._silent_dependent_async( + std::forward

(params), std::forward(func), first, last, _node->_topology, _node + ); +} + +// ------------------------------------------------------------------------------------------------ +// dependent async +// ------------------------------------------------------------------------------------------------ + +// Function: dependent_async +template +requires (std::same_as, AsyncTask> && ...) +auto Runtime::dependent_async(F&& func, Tasks&&... tasks) { + return dependent_async(DefaultTaskParams{}, std::forward(func), std::forward(tasks)...); +} + +// Function: dependent_async +template +requires (std::same_as, AsyncTask> && ...) +auto Runtime::dependent_async(P&& params, F&& func, Tasks&&... tasks) { + std::array array = { std::forward(tasks)... }; + return dependent_async( + std::forward

(params), std::forward(func), array.begin(), array.end() + ); +} + +// Function: dependent_async +template +requires (!std::same_as, AsyncTask>) +auto Runtime::dependent_async(F&& func, I first, I last) { + return dependent_async(DefaultTaskParams{}, std::forward(func), first, last); +} + +// Function: dependent_async +template +requires (!std::same_as, AsyncTask>) +auto Runtime::dependent_async(P&& params, F&& func, I first, I last) { + _node->_join_counter.fetch_add(1, std::memory_order_relaxed); + return _executor._dependent_async( + std::forward

(params), std::forward(func), first, last, _node->_topology, _node + ); +} + +// ---------------------------------------------------------------------------- +// Executor Forward Declaration +// ---------------------------------------------------------------------------- + +// Procedure: _invoke_runtime_task +inline bool Executor::_invoke_runtime_task(Worker& worker, Node* node) { + return _invoke_runtime_task_impl( + worker, node, std::get_if(&node->_handle)->work + ); +} + +// Function: _invoke_runtime_task_impl +inline bool Executor::_invoke_runtime_task_impl( + Worker& worker, Node* node, std::function& work +) { + // first time + if((node->_nstate & NSTATE::PREEMPTED) == 0) { + + Runtime rt(*this, worker, node); + + node->_nstate |= (NSTATE::PREEMPTED | NSTATE::IMPLICITLY_ANCHORED); + + node->_join_counter.fetch_add(1, std::memory_order_release); + + _observer_prologue(worker, node); + TF_EXECUTOR_EXCEPTION_HANDLER(worker, node, { + work(rt); + }); + _observer_epilogue(worker, node); + + // Last one to leave the runtime; no need to preempt this runtime. + if(node->_join_counter.fetch_sub(1, std::memory_order_acq_rel) == 1) { + node->_nstate &= ~(NSTATE::PREEMPTED | NSTATE::IMPLICITLY_ANCHORED); + } + // There are still child tasks running; need to preempt this runtime. + // Here, we cannot let caller check the state from node->_nstate due to data race, + // but return a stateless boolean to indicate preemption. + // Ex: if preempted, another task may finish real quck and insert this parent task + // again into the scheduling queue. When running this parent task, it will jump to + // else branch below and modify tne nstate, thus incuring data race. + else { + return true; + } + } + // second time - previously preempted + else { + node->_nstate &= ~(NSTATE::PREEMPTED | NSTATE::IMPLICITLY_ANCHORED); + } + return false; +} + +// Function: _invoke_runtime_task_impl +inline bool Executor::_invoke_runtime_task_impl( + Worker& worker, Node* node, std::function& work +) { + + Runtime rt(*this, worker, node); + + // first time + if((node->_nstate & NSTATE::PREEMPTED) == 0) { + + node->_nstate |= (NSTATE::PREEMPTED | NSTATE::IMPLICITLY_ANCHORED); + node->_join_counter.fetch_add(1, std::memory_order_release); + + _observer_prologue(worker, node); + TF_EXECUTOR_EXCEPTION_HANDLER(worker, node, { + work(rt, false); + }); + _observer_epilogue(worker, node); + + // Last one to leave this runtime; no need to preempt this runtime + if(node->_join_counter.fetch_sub(1, std::memory_order_acq_rel) == 1) { + node->_nstate &= ~(NSTATE::PREEMPTED | NSTATE::IMPLICITLY_ANCHORED); + } + // Here, we cannot let caller check the state from node->_nstate due to data race, + // but return a stateless boolean to indicate preemption. + // Ex: if preempted, another task may finish real quck and insert this parent task + // again into the scheduling queue. When running this parent task, it will jump to + // else branch below and modify tne nstate, thus incuring data race. + else { + return true; + } + } + // second time - previously preempted + else { + node->_nstate &= ~(NSTATE::PREEMPTED | NSTATE::IMPLICITLY_ANCHORED); + } + + // clean up outstanding work (e.g., exception) + work(rt, true); + + return false; +} + +// ------------------------------------------------------------------------------------------------ +// class: NonpreemptiveRuntime (internal use only) +// ------------------------------------------------------------------------------------------------ + +/** +@private +*/ +class NonpreemptiveRuntime { + + friend class Executor; + + public: + + /** + @private + */ + void schedule(Task task); + + private: + + /** + @private + */ + explicit NonpreemptiveRuntime(Executor& executor, Worker& worker) : + _executor {executor}, _worker {worker}{ + } + + /** + @private + */ + Executor& _executor; + + /** + @private + */ + Worker& _worker; +}; + +// Procedure: schedule +inline void NonpreemptiveRuntime::schedule(Task task) { + + auto node = task._node; + // need to keep the invariant: when scheduling a task, the task must have + // zero dependency (join counter is 0) + // or we can encounter bug when inserting a nested flow (e.g., module task) + node->_join_counter.store(0, std::memory_order_relaxed); + + auto& j = node->_parent ? node->_parent->_join_counter : + node->_topology->_join_counter; + j.fetch_add(1, std::memory_order_relaxed); + _executor._schedule(_worker, node); +} + +// ---------------------------------------------------------------------------- +// Executor Forward Declaration +// ---------------------------------------------------------------------------- + +// Procedure: _invoke_nonpreemptive_runtime_task +inline void Executor::_invoke_nonpreemptive_runtime_task(Worker& worker, Node* node) { + _observer_prologue(worker, node); + tf::NonpreemptiveRuntime nprt(*this, worker); + TF_EXECUTOR_EXCEPTION_HANDLER(worker, node, { + std::get_if(&node->_handle)->work(nprt); + }); + _observer_epilogue(worker, node); +} + + +// Function: run_until +template +tf::Future Executor::run_until(Taskflow&& f, P&& p, C&& c) { + + // No need to create a real topology but returns an dummy future for invariant. + if(f.empty() || p()) { + c(); + std::promise promise; + promise.set_value(); + return tf::Future(promise.get_future()); + } + + _increment_topology(); + + auto g = std::make_unique(std::move(f)); + + // creates a topology for this run + auto t = std::make_shared(*g, std::forward

(p), std::forward(c)); + //auto t = std::make_shared>(*g, std::forward

(p), std::forward(c)); + + // need to create future before the topology got torn down quickly + tf::Future future(t->_promise.get_future(), t); + + // creates a silent-async that holds the taskflow + silent_async([g=MoC{std::move(g)}, t](tf::Runtime& rt) mutable { + t->_parent = rt._node; + t->_parent->_join_counter.fetch_add(1, std::memory_order_release); + if(g.object->_fetch_enqueue(t) == 0) { + rt._executor._schedule_graph( + rt._worker, g.object->_graph, t.get(), t.get() + ); + } + }); + + return future; +} + +} // end of namespace tf ----------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/core/semaphore.hpp b/projects/hipfile/third-party/taskflow/core/semaphore.hpp new file mode 100644 index 000000000000..9684d13206ce --- /dev/null +++ b/projects/hipfile/third-party/taskflow/core/semaphore.hpp @@ -0,0 +1,188 @@ +#pragma once + +#include + +#include "declarations.hpp" +#include "../utility/small_vector.hpp" + +/** +@file semaphore.hpp +@brief semaphore include file +*/ + +namespace tf { + +// ---------------------------------------------------------------------------- +// Semaphore +// ---------------------------------------------------------------------------- + +/** +@class Semaphore + +@brief class to create a semophore object for building a concurrency constraint + +A semaphore creates a constraint that limits the maximum concurrency, +i.e., the number of workers, in a set of tasks. +You can let a task acquire/release one or multiple semaphores before/after +executing its work. +A task can acquire and release a semaphore, +or just acquire or just release it. +A tf::Semaphore object starts with an initial count. +As long as that count is above 0, tasks can acquire the semaphore and do +their work. +If the count is 0 or less, a task trying to acquire the semaphore will not run +but goes to a waiting list of that semaphore. +When the semaphore is released by another task, +it reschedules all tasks on that waiting list. + +@code{.cpp} +tf::Executor executor(8); // create an executor of 8 workers +tf::Taskflow taskflow; + +tf::Semaphore semaphore(1); // create a semaphore with initial count 1 + +SmallVector tasks { + taskflow.emplace([](){ std::cout << "A" << std::endl; }), + taskflow.emplace([](){ std::cout << "B" << std::endl; }), + taskflow.emplace([](){ std::cout << "C" << std::endl; }), + taskflow.emplace([](){ std::cout << "D" << std::endl; }), + taskflow.emplace([](){ std::cout << "E" << std::endl; }) +}; + +for(auto & task : tasks) { // each task acquires and release the semaphore + task.acquire(semaphore); + task.release(semaphore); +} + +executor.run(taskflow).wait(); +@endcode + +The above example creates five tasks with no dependencies between them. +Under normal circumstances, the five tasks would be executed concurrently. +However, this example has a semaphore with initial count 1, +and all tasks need to acquire that semaphore before running and release that +semaphore after they are done. +This arrangement limits the number of concurrently running tasks to only one. + +*/ +class Semaphore { + + friend class Node; + friend class Executor; + + public: + + /** + @brief constructs a default semaphore + + A default semaphore has the value of zero. Users can call tf::Semaphore::reset + to reassign a new value to the semaphore. + */ + Semaphore() = default; + + /** + @brief constructs a semaphore with the given value (i.e., counter) + + A semaphore creates a constraint that limits the maximum concurrency, + i.e., the number of workers, in a set of tasks. + + @code{.cpp} + tf::Semaphore semaphore(4); // concurrency constraint of 4 workers + @endcode + */ + explicit Semaphore(size_t max_value); + + /** + @brief queries the current counter value + */ + size_t value() const; + + /** + @brief queries the maximum allowable value of this semaphore + */ + size_t max_value() const; + + /** + @brief resets the semaphores to a clean state + */ + void reset(); + + /** + @brief resets the semaphores to a clean state with the given new maximum value + */ + void reset(size_t new_max_value); + + private: + + mutable std::mutex _mtx; + + size_t _max_value{0}; + size_t _cur_value{0}; + + SmallVector _waiters; + + bool _try_acquire_or_wait(Node*); + + void _release(SmallVector&); +}; + +inline Semaphore::Semaphore(size_t max_value) : + _max_value(max_value), + _cur_value(max_value) { +} + +inline bool Semaphore::_try_acquire_or_wait(Node* me) { + std::lock_guard lock(_mtx); + if(_cur_value > 0) { + --_cur_value; + return true; + } + else { + _waiters.push_back(me); + return false; + } +} + +inline void Semaphore::_release(SmallVector& dst) { + + std::lock_guard lock(_mtx); + + if(_cur_value >= _max_value) { + TF_THROW("can't release the semaphore more than its maximum value: ", _max_value); + } + + ++_cur_value; + + if(dst.empty()) { + dst.swap(_waiters); + } + else { + dst.reserve(dst.size() + _waiters.size()); + dst.insert(dst.end(), _waiters.begin(), _waiters.end()); + _waiters.clear(); + } +} + +inline size_t Semaphore::max_value() const { + return _max_value; +} + +inline size_t Semaphore::value() const { + std::lock_guard lock(_mtx); + return _cur_value; +} + +inline void Semaphore::reset() { + std::lock_guard lock(_mtx); + _cur_value = _max_value; + _waiters.clear(); +} + +inline void Semaphore::reset(size_t new_max_value) { + std::lock_guard lock(_mtx); + _cur_value = (_max_value = new_max_value); + _waiters.clear(); +} + +} // end of namespace tf. --------------------------------------------------- + diff --git a/projects/hipfile/third-party/taskflow/core/task.hpp b/projects/hipfile/third-party/taskflow/core/task.hpp new file mode 100644 index 000000000000..af6745bb8fbf --- /dev/null +++ b/projects/hipfile/third-party/taskflow/core/task.hpp @@ -0,0 +1,1814 @@ +#pragma once + +#include "graph.hpp" + +/** +@file task.hpp +@brief task include file +*/ + +namespace tf { + +// ---------------------------------------------------------------------------- +// Task Types +// ---------------------------------------------------------------------------- + +/** +@enum TaskType + +@brief enumeration of all task types +*/ +enum class TaskType : int { + /** @brief placeholder task type */ + PLACEHOLDER = 0, + /** @brief static task type */ + STATIC, + /** @brief runtime task type */ + RUNTIME, + /** @brief dynamic (subflow) task type */ + SUBFLOW, + /** @brief condition task type */ + CONDITION, + /** @brief module task type */ + MODULE, + /** @brief asynchronous task type */ + ASYNC, + /** @brief undefined task type (for internal use only) */ + UNDEFINED +}; + +/** +@private +@brief array of all task types (used for iterating task types) +*/ +inline constexpr std::array TASK_TYPES = { + TaskType::PLACEHOLDER, + TaskType::STATIC, + TaskType::RUNTIME, + TaskType::SUBFLOW, + TaskType::CONDITION, + TaskType::MODULE, + TaskType::ASYNC, +}; + +/** +@brief convert a task type to a human-readable string + +The name of each task type is the litte-case string of its characters. + + TaskType::PLACEHOLDER is of string `placeholder` + + TaskType::STATIC is of string `static` + + TaskType::RUNTIME is of string `runtime` + + TaskType::SUBFLOW is of string `subflow` + + TaskType::CONDITION is of string `condition` + + TaskType::MODULE is of string `module` + + TaskType::ASYNC is of string `async` +*/ +inline const char* to_string(TaskType type) { + + const char* val; + + switch(type) { + case TaskType::PLACEHOLDER: val = "placeholder"; break; + case TaskType::STATIC: val = "static"; break; + case TaskType::RUNTIME: val = "runtime"; break; + case TaskType::SUBFLOW: val = "subflow"; break; + case TaskType::CONDITION: val = "condition"; break; + case TaskType::MODULE: val = "module"; break; + case TaskType::ASYNC: val = "async"; break; + default: val = "undefined"; break; + } + + return val; +} + +// ---------------------------------------------------------------------------- +// Static Task Trait +// ---------------------------------------------------------------------------- + +/** +@brief determines if a callable is a static task + +A static task is a callable object that takes no arguments and returns void. +It is constructible from std::function. + +@par Requirements +- Must be invocable with no arguments +- Must return void + +@par Examples + +@code{.cpp} +// Valid static tasks (satisfy StaticTaskLike) + +// Simple lambda capturing nothing +auto static_task1 = []() {}; +static_assert(StaticTaskLike); + +// Lambda capturing variables +int x = 10; +auto static_task2 = [x]() { std::cout << x << '\n'; }; +static_assert(StaticTaskLike); + +// Function object (functor) +struct SimpleTask { + void operator()() const { std::cout << "task\n"; } +}; +static_assert(StaticTaskLike); + +// Free function +void my_task() { std::cout << "free function task\n"; } +static_assert(StaticTaskLike); + +// Use in taskflow +tf::Taskflow taskflow; +auto task = taskflow.emplace(static_task1).name("static"); +@endcode + +@par Invalid Examples + +@code{.cpp} +// Invalid: takes parameters +auto task_with_param = [](int a) { std::cout << a << '\n'; }; +// static_assert(StaticTaskLike); // FAILS + +// Invalid: returns non-void +auto task_returns_int = []() { return 42; }; +// static_assert(StaticTaskLike); // FAILS +@endcode +*/ +template +concept StaticTaskLike = std::invocable && + std::same_as, void>; + +/** +@brief determines if a callable is a static task (variable template) + +@tparam C callable type to check + +Equivalent to tf::StaticTaskLike. +*/ +template +constexpr bool is_static_task_v = StaticTaskLike; + +// ---------------------------------------------------------------------------- +// Subflow Task Trait +// ---------------------------------------------------------------------------- + +/** +@brief determines if a callable is a subflow task + +A subflow task is a callable object that takes a tf::Subflow& reference and returns void. +Subflow tasks allow dynamic creation of nested tasks at runtime. It is constructible +from std::function. + +@par Requirements +- Must be invocable with a tf::Subflow& argument +- Must return void + +@par Examples + +@code{.cpp} +// Valid subflow tasks (satisfy SubflowTaskLike) + +// Basic subflow task +auto subflow_task = [](tf::Subflow& sf) { + auto task1 = sf.emplace([]() { std::cout << "subtask 1\n"; }); + auto task2 = sf.emplace([]() { std::cout << "subtask 2\n"; }); + task1.precede(task2); +}; +static_assert(SubflowTaskLike); + +// Subflow with captured context +int num_tasks = 5; +auto dynamic_subflow = [num_tasks](tf::Subflow& sf) { + for(int i = 0; i < num_tasks; ++i) { + sf.emplace([i]() { std::cout << "task " << i << '\n'; }); + } +}; +static_assert(SubflowTaskLike); + +// Function object +struct SubflowTask { + void operator()(tf::Subflow& sf) const { + sf.emplace([]() { std::cout << "nested\n"; }); + } +}; +static_assert(SubflowTaskLike); + +// Use in taskflow +tf::Taskflow taskflow; +auto task = taskflow.emplace(subflow_task).name("subflow"); +@endcode + +@par Invalid Examples + +@code{.cpp} +// Invalid: takes no parameters +auto not_subflow1 = []() { std::cout << "no params\n"; }; +// static_assert(SubflowTaskLike); // FAILS + +// Invalid: takes non-Subflow parameter +auto not_subflow2 = [](int x) { std::cout << x << '\n'; }; +// static_assert(SubflowTaskLike); // FAILS + +// Invalid: returns non-void +auto not_subflow3 = [](tf::Subflow& sf) { return 42; }; +// static_assert(SubflowTaskLike); // FAILS +@endcode +*/ +template +concept SubflowTaskLike = std::invocable && + std::same_as, void>; + +/** +@brief determines if a callable is a subflow task (variable template) + +@tparam C callable type to check + +Equivalent to tf::SubflowTaskLike. +*/ +template +constexpr bool is_subflow_task_v = SubflowTaskLike; + +// ---------------------------------------------------------------------------- +// Runtime Task Trait +// ---------------------------------------------------------------------------- + +/** +@brief determines if a callable is a runtime task + +A runtime task is a callable object that accepts either tf::Runtime& or +tf::NonpreemptiveRuntime& and returns void. Runtime tasks provide access to +the executor's runtime information, allowing tasks to spawn work at runtime, +query the running state, and control execution flow. + +@par Requirements +- Must be invocable with either tf::Runtime& OR tf::NonpreemptiveRuntime& +- Must return void + +@par Examples + +@code{.cpp} +// Valid runtime tasks (satisfy RuntimeTaskLike) + +// Runtime task with tf::Runtime +auto runtime_task1 = [](tf::Runtime& rt) { + std::cout << "running with " << rt.num_workers() << " workers\n"; +}; +static_assert(RuntimeTaskLike); + +// Runtime task with tf::NonpreemptiveRuntime +auto runtime_task2 = [](tf::NonpreemptiveRuntime& rt) { + std::cout << "non-preemptive runtime\n"; +}; +static_assert(RuntimeTaskLike); + +// Runtime task spawning nested work +auto spawn_work = [](tf::Runtime& rt) { + // Can query and control execution at runtime + rt.spawn([](){ std::cout << "spawned task\n"; }); +}; +static_assert(RuntimeTaskLike); + +// Function object +struct RuntimeTask { + void operator()(tf::Runtime& rt) const { + std::cout << "runtime task\n"; + } +}; +static_assert(RuntimeTaskLike); + +// Use in taskflow +tf::Taskflow taskflow; +auto task = taskflow.emplace(runtime_task1).name("runtime"); +@endcode + +@par Invalid Examples + +@code{.cpp} +// Invalid: no parameters +auto not_runtime1 = []() { std::cout << "no params\n"; }; +// static_assert(RuntimeTaskLike); // FAILS + +// Invalid: wrong parameter type +auto not_runtime2 = [](int x) { std::cout << x << '\n'; }; +// static_assert(RuntimeTaskLike); // FAILS + +// Invalid: returns non-void +auto not_runtime3 = [](tf::Runtime& rt) { return 42; }; +// static_assert(RuntimeTaskLike); // FAILS +@endcode +*/ +template +concept RuntimeTaskLike = + (std::invocable && + std::same_as, void>) || + (std::invocable && + std::same_as, void>); + +/** +@brief determines if a callable is a runtime task (variable template) + +@tparam C callable type to check + +Equivalent to tf::RuntimeTaskLike. +*/ +template +constexpr bool is_runtime_task_v = RuntimeTaskLike; + + +// ---------------------------------------------------------------------------- +// Condition Task Trait +// ---------------------------------------------------------------------------- + +/** +@brief determines if a callable is a condition task + +A condition task is a callable object that takes no arguments and returns a +value convertible to int. The returned int value controls which successor branch +is taken in conditional execution (if-else branching). + +@par Requirements +- Must be invocable with no arguments +- Must return a value convertible to int + +@par Examples + +@code{.cpp} +// Valid condition tasks (satisfy ConditionTaskLike) + +// Simple condition returning int +auto condition_task1 = []() { return 0; }; +static_assert(ConditionTaskLike); + +// Condition with captured state +int counter = 0; +auto condition_task2 = [counter]() { + return (counter > 5) ? 1 : 0; // returns 1 (true) or 0 (false) +}; +static_assert(ConditionTaskLike); + +// Condition that evaluates runtime state +auto evaluate_path = []() { + // Return branch index (0 for if, 1 for else, etc.) + return rand() % 3; +}; +static_assert(ConditionTaskLike); + +// Function object +struct ConditionTask { + int operator()() const { return 42; } +}; +static_assert(ConditionTaskLike); + +// Returns bool (convertible to int) +auto bool_condition = []() { return true; }; +static_assert(ConditionTaskLike); + +// Use in taskflow +tf::Taskflow taskflow; +auto [init, cond, branch_a, branch_b] = taskflow.emplace( + [](){ std::cout << "init\n"; }, + condition_task1, + [](){ std::cout << "branch a\n"; }, + [](){ std::cout << "branch b\n"; } +); +init.precede(cond); +cond.precede(branch_a, branch_b); // branch_a if cond returns 0, branch_b if returns 1 +@endcode + +@par Invalid Examples + +@code{.cpp} +// Invalid: takes parameters +auto not_condition1 = [](int x) { return x; }; +// static_assert(ConditionTaskLike); // FAILS + +// Invalid: returns non-convertible type +auto not_condition2 = []() { return std::string("hello"); }; +// static_assert(ConditionTaskLike); // FAILS + +// Invalid: returns void +auto not_condition3 = []() { std::cout << "task\n"; }; +// static_assert(ConditionTaskLike); // FAILS +@endcode +*/ +template +concept ConditionTaskLike = std::invocable && + std::convertible_to, int>; + +/** +@brief determines if a callable is a condition task (variable template) + +@tparam C callable type to check + +Equivalent to tf::ConditionTaskLike. +*/ +template +constexpr bool is_condition_task_v = ConditionTaskLike; + +/** +@brief determines if a callable is a multi-condition task + +A multi-condition task is a callable object that takes no arguments and returns +a tf::SmallVector. The returned vector contains branch indices for multiple +successor branches, enabling complex control flow patterns where a single task +can activate multiple branches simultaneously. + +@par Requirements +- Must be invocable with no arguments +- Must return exactly tf::SmallVector + +@par Examples + +@code{.cpp} +// Valid multi-condition tasks (satisfy MultiConditionTaskLike) + +// Multi-condition returning indices for multiple branches +auto multi_condition1 = []() { + tf::SmallVector branches; + branches.push_back(0); // Take first branch + branches.push_back(2); // Also take third branch + return branches; +}; +static_assert(MultiConditionTaskLike); + +// Dynamic branch selection +int branch_mask = 5; // binary: 101 → take branches 0 and 2 +auto select_branches = [branch_mask]() { + tf::SmallVector result; + for(int i = 0; i < 3; ++i) { + if(branch_mask & (1 << i)) { + result.push_back(i); + } + } + return result; +}; +static_assert(MultiConditionTaskLike); + +// Function object +struct MultiConditionTask { + tf::SmallVector operator()() const { + return {0, 1}; // Execute branches 0 and 1 + } +}; +static_assert(MultiConditionTaskLike); + +// Use in taskflow +tf::Taskflow taskflow; +auto [init, mcond, branch_0, branch_1, branch_2] = taskflow.emplace( + [](){ std::cout << "init\n"; }, + select_branches, + [](){ std::cout << "branch 0\n"; }, + [](){ std::cout << "branch 1\n"; }, + [](){ std::cout << "branch 2\n"; } +); +init.precede(mcond); +// All three branches become successors (only those returned in SmallVector will execute) +mcond.precede(branch_0, branch_1, branch_2); +@endcode + +@par Invalid Examples + +@code{.cpp} +// Invalid: takes parameters +auto not_multi1 = [](int x) { + tf::SmallVector v; v.push_back(x); return v; +}; +// static_assert(MultiConditionTaskLike); // FAILS + +// Invalid: returns wrong container type +auto not_multi2 = []() { return std::vector{0, 1}; }; +// static_assert(MultiConditionTaskLike); // FAILS + +// Invalid: returns single int instead of SmallVector +auto not_multi3 = []() { return 0; }; +// static_assert(MultiConditionTaskLike); // FAILS + +// Invalid: returns void +auto not_multi4 = []() { std::cout << "task\n"; }; +// static_assert(MultiConditionTaskLike); // FAILS +@endcode +*/ +template +concept MultiConditionTaskLike = std::invocable && + std::same_as, SmallVector>; + +/** +@brief determines if a callable is a multi-condition task (variable template) + +@tparam C callable type to check + +Equivalent to tf::MultiConditionTaskLike. +*/ +template +constexpr bool is_multi_condition_task_v = MultiConditionTaskLike; + + +// ---------------------------------------------------------------------------- +// Task +// ---------------------------------------------------------------------------- + +/** +@class Task + +@brief class to create a task handle over a taskflow node + +A task points to a node in a taskflow graph and provides a set of methods for users to access and modify +attributes of the associated node, +such as dependencies, callable, names, and so on. +A task is a very lightweight object (i.e., it only stores a node pointer) and can be trivially +copied around. + +@code{.cpp} +// create two tasks with one dependency +auto task1 = taskflow.emplace([](){}).name("task1"); +auto task2 = taskflow.emplace([](){}).name("task2"); +task1.precede(task2); + +// dump the task information through std::cout +task1.dump(std::cout); +@endcode + +A task created from a taskflow can be one of the following types: + + tf::TaskType::STATIC - @ref StaticTasking + + tf::TaskType::CONDITION - @ref ConditionalTasking + + tf::TaskType::RUNTIME - @ref RuntimeTasking + + tf::TaskType::SUBFLOW - @ref SubflowTasking + + tf::TaskType::MODULE - @ref ComposableTasking + +@code{.cpp} +tf::Task task1 = taskflow.emplace([](){}).name("static task"); +tf::Task task2 = taskflow.emplace([](){ return 3; }).name("condition task"); +tf::Task task3 = taskflow.emplace([](tf::Runtime&){}).name("runtime task"); +tf::Task task4 = taskflow.emplace([](tf::Subflow& sf){ + tf::Task stask1 = sf.emplace([](){}); + tf::Task stask2 = sf.emplace([](){}); +}).name("subflow task"); +tf::Task task5 = taskflow.composed_of(taskflow2).name("module task"); +@endcode + +A tf::Task is polymorphic. +Once created, you can assign a different task type to it using tf::Task::work. +For example, the code below creates a static task and then reworks it to a subflow task: + +@code{.cpp} +tf::Task task = taskflow.emplace([](){}).name("static task"); +task.work([](tf::Subflow& sf){ + tf::Task stask1 = sf.emplace([](){}); + tf::Task stask2 = sf.emplace([](){}); +}).name("subflow task"); +@endcode + +@attention +tf::Task does not own the lifetime of the associated node. +Accessing the attributes of the associated node after the taskflow has been destroyed +can result in undefined behavior. + +*/ +class Task { + + friend class FlowBuilder; + friend class Runtime; + friend class NonpreemptiveRuntime; + friend class Taskflow; + friend class TaskView; + friend class Executor; + + public: + + /** + @brief constructs an empty task + + An empty task is not associated with any node in a taskflow. + */ + Task() = default; + + /** + @brief constructs the task with the copy of the other task + + @param other the other task to copy + + @code{.cpp} + tf::Taskflow taskflow; + tf::Task A = taskflow.emplace([](){ std::cout << "Task A\n"; }); + tf::Task B(A); + assert(B == A); // Now, B and A refer to the same underlying node + @endcode + */ + Task(const Task& other); + + /** + @brief replaces the contents with a copy of the other task + + @param other the other task to copy + + @code{.cpp} + tf::Task A = taskflow.emplace([](){ std::cout << "A\n"; }); + tf::Task B; + B = A; // B now refers to the same node as A + @endcode + */ + Task& operator = (const Task& other); + + /** + @brief replaces the contents with a null pointer + + @code{.cpp} + tf::Task A = taskflow.emplace([](){ std::cout << "A\n"; }); + A = nullptr; // A no longer refers to any node + @endcode + */ + Task& operator = (std::nullptr_t); + + /** + @brief compares if two tasks are associated with the same taskflow node + + @param rhs the other task to compare with + @return true if both tasks refer to the same node; false otherwise + + @code{.cpp} + tf::Task A = taskflow.emplace([](){ std::cout << "A\n"; }); + tf::Task B = A; + assert(A == B); // A and B refer to the same node + @endcode + */ + bool operator == (const Task& rhs) const; + + /** + @brief compares if two tasks are not associated with the same taskflow node + + @param rhs the other task to compare with + @return true if they refer to different nodes; false otherwise + + @code{.cpp} + tf::Task A = taskflow.emplace([](){ std::cout << "A\n"; }); + tf::Task B = taskflow.emplace([](){ std::cout << "B\n"; }); + assert(A != B); // A and B refer to different nodes + @endcode + */ + bool operator != (const Task& rhs) const; + + /** + @brief queries the name of the task + + @return the name of the task as a constant string reference + + @code{.cpp} + tf::Task task = taskflow.emplace([](){}); + task.name("MyTask"); + std::cout << "Task name: " << task.name() << std::endl; + @endcode + */ + const std::string& name() const; + + /** + @brief queries the number of successors of the task + + @return the number of successor tasks. + + @code{.cpp} + tf::Task A = taskflow.emplace([](){}); + tf::Task B = taskflow.emplace([](){}); + A.precede(B); // B is a successor of A + std::cout << "A has " << A.num_successors() << " successor(s)." << std::endl; + @endcode + */ + size_t num_successors() const; + + /** + @brief queries the number of predecessors of the task + + @return the number of predecessor tasks + + @code{.cpp} + tf::Task A = taskflow.emplace([](){}); + tf::Task B = taskflow.emplace([](){}); + A.precede(B); // A is a predecessor of B + std::cout << "B has " << B.num_predecessors() << " predecessor(s)." << std::endl; + @endcode + */ + size_t num_predecessors() const; + + /** + @brief queries the number of strong dependencies of the task + + @return the number of strong dependencies to this task + + A strong dependency is a preceding link from one non-condition task to another task. + For instance, task `cond` below has one strong dependency, while tasks `yes` and `no` + each have one weak dependency. + + @code{.cpp} + auto [init, cond, yes, no] = taskflow.emplace( + [] () { }, + [] () { return 0; }, + [] () { std::cout << "yes\n"; }, + [] () { std::cout << "no\n"; } + ); + cond.succeed(init) + .precede(yes, no); // executes yes if cond returns 0 + // executes no if cond returns 1 + @endcode + + @dotfile images/conditional-tasking-if-else.dot + + @note + To understand how %Taskflow schedule tasks under strong and weak dependencies, + please refer to @ref ConditionalTasking. + */ + size_t num_strong_dependencies() const; + + /** + @brief queries the number of weak dependencies of the task + + @return the number of weak dependencies to this task + + A weak dependency is a preceding link from one condition task to another task. + For instance, task `cond` below has one strong dependency, while tasks `yes` and `no` + each have one weak dependency. + + @code{.cpp} + auto [init, cond, yes, no] = taskflow.emplace( + [] () { }, + [] () { return 0; }, + [] () { std::cout << "yes\n"; }, + [] () { std::cout << "no\n"; } + ); + cond.succeed(init) + .precede(yes, no); // executes yes if cond returns 0 + // executes no if cond returns 1 + @endcode + + @dotfile images/conditional-tasking-if-else.dot + + @note + To understand how %Taskflow schedule tasks under strong and weak dependencies, + please refer to @ref ConditionalTasking. + */ + size_t num_weak_dependencies() const; + + /** + @brief assigns a name to the task + + @param name a @std_string + + @return @c *this + + @code{.cpp} + tf::Task task = taskflow.emplace([](){}).name("foo"); + assert(task.name*) == "foo"); + @endcode + */ + Task& name(const std::string& name); + + /** + @brief assigns a callable + + @tparam C callable type + + @param callable callable to construct a task + + @return @c *this + + A tf::Task is polymorphic. + Once created, you can reassign it to a different callable of a different task type + using tf::Task::work. + For example, the code below creates a static task and reworks it to a subflow task: + + @code{.cpp} + tf::Task task = taskflow.emplace([](){}).name("static task"); + task.work([](tf::Subflow& sf){ + tf::Task stask1 = sf.emplace([](){}); + tf::Task stask2 = sf.emplace([](){}); + }).name("subflow task"); + @endcode + */ + template + Task& work(C&& callable); + + /** + @brief creates a module task from a taskflow + + @tparam T type satisfying tf::GraphLike + @param object a custom object that defines @c T::graph() method + + @return @c *this + + The example below creates a module task from a taskflow: + + @code{.cpp} + task.composed_of(taskflow); + @endcode + + To understand how %Taskflow schedules a module task including how to create a schedulable graph, + pleas refer to @ref CreateACustomComposableGraph. + */ + template + Task& composed_of(T& object); + + /** + @brief creates a module task from a graph by taking over its ownership + + @param graph the graph to adopt (moved into the task) + + @return a Task handle to the adopted module task + + Unlike tf::Task::composed_of, which references an externally-owned + tf::Taskflow, @c adopt transfers ownership of the given tf::Graph into + the task. The graph's lifetime is managed by the executor once adopted, + and the caller has no access to the moved-from graph afterward. + + @code{.cpp} + tf::Taskflow taskflow; + tf::Graph g; + tf::FlowBuilder{g}.emplace([]{ std::cout << "task in adopted graph\n"; }); + taskflow.adopt(std::move(g)).name("adopted"); + @endcode + + @note Please refer to @ref ComposableTasking for details. + */ + Task& adopt(tf::Graph&& graph); + + /** + @brief adds precedence links from this to other tasks + + @tparam Ts parameter pack + + @param tasks one or multiple tasks + + @return @c *this + + The example below creates a taskflow of two tasks, where `task1` runs before `task2`. + + @code{.cpp} + auto [task1, task2] = taskflow.emplace( + [](){ std::cout << "task1\n"; }, + [](){ std::cout << "task2\n"; } + ); + task1.precede(task2); + @endcode + */ + template + Task& precede(Ts&&... tasks); + + /** + @brief adds precedence links from other tasks to this + + @tparam Ts parameter pack + + @param tasks one or multiple tasks + + @return @c *this + + The example below creates a taskflow of two tasks, where `task1` runs before `task2`. + + @code{.cpp} + auto [task1, task2] = taskflow.emplace( + [](){ std::cout << "task1\n"; }, + [](){ std::cout << "task2\n"; } + ); + task2.succeed(task1); + @endcode + */ + template + Task& succeed(Ts&&... tasks); + + /** + @brief removes predecessor links from other tasks to this + + @tparam Ts parameter pack + + @param tasks one or multiple tasks + + @return @c *this + + This method removes the dependency links where the given tasks are predecessors + of this task (i.e., tasks -> this). It ensures both sides of the dependency + are updated to maintain graph consistency. + + @code{.cpp} + tf::Task A = taskflow.emplace([](){}); + tf::Task B = taskflow.emplace([](){}); + tf::Task C = taskflow.emplace([](){}); + // create a linear chain of tasks, A->B->C + B.succeed(A) + .precede(C); + assert(B.num_successors() == 1 && C.num_predecessors() == 1); + + // remove C from B's successor list + C.remove_predecessors(B); + assert(B.num_successors() == 0 && C.num_predecessors() == 0); + @endcode + */ + template + Task& remove_predecessors(Ts&&... tasks); + + /** + @brief removes successor links from this to other tasks + + @tparam Ts parameter pack + + @param tasks one or multiple tasks + + @return @c *this + + This method removes the dependency links where this task is a predecessor + of the given tasks (i.e., this -> tasks). It ensures both sides of the dependency + are updated to maintain graph consistency. + + @code{.cpp} + tf::Task A = taskflow.emplace([](){}); + tf::Task B = taskflow.emplace([](){}); + tf::Task C = taskflow.emplace([](){}); + // create a linear chain of tasks, A->B->C + B.succeed(A) + .precede(C); + assert(B.num_successors() == 1 && C.num_predecessors() == 1); + + // remove C from B's successor list + B.remove_successors(C); + assert(B.num_successors() == 0 && C.num_predecessors() == 0); + @endcode + */ + template + Task& remove_successors(Ts&&... tasks); + + /** + @brief makes the task release the given semaphore + + @note + To know more about tf::Semaphore, please refer to @ref LimitTheMaximumConcurrency. + */ + Task& release(Semaphore& semaphore); + + /** + @brief makes the task release the given range of semaphores + + @note + To know more about tf::Semaphore, please refer to @ref LimitTheMaximumConcurrency. + */ + template + Task& release(I first, I last); + + /** + @brief makes the task acquire the given semaphore + + @note + To know more about tf::Semaphore, please refer to @ref LimitTheMaximumConcurrency. + */ + Task& acquire(Semaphore& semaphore); + + /** + @brief makes the task acquire the given range of semaphores + + @note + To know more about tf::Semaphore, please refer to @ref LimitTheMaximumConcurrency. + */ + template + Task& acquire(I first, I last); + + /** + @brief assigns pointer to user data + + @param data pointer to user data + @return @c *this + + The following example shows how to attach a user data to a task and retrieve it + during the execution of the task. + + @code{.cpp} + tf::Executor executor; + tf::Taskflow taskflow("attach data to a task"); + + int data; // user data + + // create a task and attach it a user data + auto A = taskflow.placeholder(); + A.data(&data).work([A](){ + auto d = *static_cast(A.data()); + std::cout << "data is " << d << std::endl; + }); + + // run the taskflow iteratively with changing data + for(data = 0; data<10; data++){ + executor.run(taskflow).wait(); + } + @endcode + + */ + Task& data(void* data); + + /** + @brief resets the task handle to null + + Resetting a task will remove its associated taskflow node and make it an empty task. + + @code{.cpp} + tf::Task task = taskflow.emplace([](){}); + assert(task.empty() == false); + task.reset(); + assert(task.empty() == true); + @endcode + */ + void reset(); + + /** + @brief resets the associated work to a placeholder + */ + void reset_work(); + + /** + @brief queries if the task handle is associated with a taskflow node + + @return `true` if the task is not associated with any taskflow node; otherwise `false` + + @code{.cpp} + tf::Task task; + assert(task.empty() == true); + @endcode + + Note that an empty task is not equal to a placeholder task. + A placeholder task is created from tf::Taskflow::placeholder and is associated with + a taskflow node, but its work is not assigned yet. + */ + bool empty() const; + + /** + @brief queries if the task has a work assigned + + @return `true` if the task has a work assigned (not placeholder); otherwise `false` + + @code{.cpp} + tf::Task task = taskflow.placeholder(); + assert(task.has_work() == false); + // assign a static task callable to this task + task.work([](){}); + assert(task.has_work() == true); + @endcode + */ + bool has_work() const; + + /** + @brief applies an visitor callable to each successor of the task + + @tparam V a callable type (function, lambda, etc.) that accepts a tf::Task handle + @param visitor visitor to apply to each subflow task + + This method allows you to traverse and inspect successor tasks of this task. + For instance, the code below iterates the two successors (`task2` and `task3`) of `task1`. + + @code{.cpp} + auto [task1, task2, task3] = taskflow.emplace( + [](){ std::cout << "task 1\n"; }, + [](){ std::cout << "task 2\n"; }, + [](){ std::cout << "task 3\n"; } + }); + task1.precede(task2, task3); + task1.for_each_successor([](tf::Task successor){ + std::cout << "successor task " << successor.name() << '\n'; + }); + @endcode + + */ + template + void for_each_successor(V&& visitor) const; + + /** + @brief applies an visitor callable to each predecessor of the task + + @tparam V a callable type (function, lambda, etc.) that accepts a tf::Task handle + @param visitor visitor to apply to each predecessor task + + This method allows you to traverse and inspect predecessor tasks of this task. + For instance, the code below iterates the two predecessors (`task2` and `task3`) of `task1`. + + @code{.cpp} + auto [task1, task2, task3] = taskflow.emplace( + [](){ std::cout << "task 1\n"; }, + [](){ std::cout << "task 2\n"; }, + [](){ std::cout << "task 3\n"; } + }); + task1.succeed(task2, task3); + task1.for_each_predecessor([](tf::Task predecessor){ + std::cout << "predecessor task " << predecessor.name() << '\n'; + }); + @endcode + */ + template + void for_each_predecessor(V&& visitor) const; + + /** + @brief applies an visitor callable to each subflow task + + @tparam V a callable type (function, lambda, etc.) that accepts a tf::Task handle + @param visitor visitor to apply to each subflow task + + This method allows you to traverse and inspect tasks within a subflow. + It only applies to a subflow task. + + @code{.cpp} + tf::Task task = taskflow.emplace([](tf::Subflow& sf){ + tf::Task stask1 = sf.emplace([](){}).name("stask1"); + tf::Task stask2 = sf.emplace([](){}).name("stask2"); + }); + // Iterate tasks in the subflow and print each subflow task. + task.for_each_subflow_task([](tf::Task stask){ + std::cout << "subflow task " << stask.name() << '\n'; + }); + @endcode + */ + template + void for_each_subflow_task(V&& visitor) const; + + /** + @brief obtains a hash value of the underlying node + + @return the hash value of the underlying node + + The method returns std::hash on the underlying node pointer. + + @code{.cpp} + tf::Task task = taskflow.emplace([](){}); + std::cout << "hash value of task is " << task.hash_value() << '\n'; + @endcode + */ + size_t hash_value() const; + + /** + @brief returns the task type + + A task can be one of the types defined in tf::TaskType and can be printed in + a human-readable form using tf::to_string. + + @code{.cpp} + auto task = taskflow.emplace([](){}).name("task"); + std::cout << task.name() << " type=[" << tf::to_string(task.type()) << "]\n"; + @endcode + + */ + TaskType type() const; + + /** + @brief dumps the task through an output stream + + The method dumps the name and the type of this task through the given output stream. + + @code{.cpp} + task.dump(std::cout); + @endcode + */ + void dump(std::ostream& ostream) const; + + /** + @brief queries pointer to user data + + @return C-styled pointer to the attached user data by tf::Task::data(void* data) + + The following example shows how to attach a user data to a task and retrieve it + during the execution of the task. + + @code{.cpp} + tf::Executor executor; + tf::Taskflow taskflow("attach data to a task"); + + int data; // user data + + // create a task and attach it a user data + auto A = taskflow.placeholder(); + A.data(&data).work([A](){ + auto d = *static_cast(A.data()); + std::cout << "data is " << d << std::endl; + }); + + // run the taskflow iteratively with changing data + for(data = 0; data<10; data++){ + executor.run(taskflow).wait(); + } + @endcode + */ + void* data() const; + + /** + @brief retrieves the exception pointer of this task + + This method retrieves the exception pointer of this task + that are silently caught by the executor, if any. + When multiple tasks throw exceptions concurrently, only one exception will be propagated, + while the others are silently caught and stored within their respective tasks. + For example, in the code below, both tasks `B` and `C` throw exceptions. + However, only one of them will be propagated to the try-catch block, + while the other will be silently caught and stored within its respective task. + + @code{.cpp} + tf::Executor executor(2); + tf::Taskflow taskflow; + std::atomic arrivals(0); + + auto [B, C] = taskflow.emplace( + [&]() { + // wait for two threads to arrive so we avoid premature cancellation + ++arrivals; while(arrivals != 2); + throw std::runtime_error("oops"); + }, + [&]() { + // wait for two threads to arrive so we avoid premature cancellation + ++arrivals; while(arrivals != 2); + throw std::runtime_error("oops"); + } + ); + + try { + executor.run(taskflow).get(); + } + catch (const std::runtime_error& e) { + std::cerr << e.what(); + } + + // exactly one holds an exception as another was propagated to the try-catch block + assert((B.exception_ptr() != nullptr) != (C.exception_ptr() != nullptr)); + @endcode + */ + std::exception_ptr exception_ptr() const; + + /** + @brief queries if the task has an exception pointer + + The method checks whether the task holds a pointer to a silently caught exception. + */ + bool has_exception_ptr() const; + + private: + + Task(Node*); + + Node* _node {nullptr}; +}; + +// Constructor +inline Task::Task(Node* node) : _node {node} { +} + +// Constructor +inline Task::Task(const Task& rhs) : _node {rhs._node} { +} + +// Function: precede +template +Task& Task::precede(Ts&&... tasks) { + (_node->_precede(tasks._node), ...); + //_precede(std::forward(tasks)...); + return *this; +} + +// Function: succeed +template +Task& Task::succeed(Ts&&... tasks) { + (tasks._node->_precede(_node), ...); + //_succeed(std::forward(tasks)...); + return *this; +} + +// Function: remove_predecessors +template +Task& Task::remove_predecessors(Ts&&... tasks) { + (tasks._node->_remove_successors(_node), ...); + (_node->_remove_predecessors(tasks._node), ...); + return *this; +} + +// Function: remove_successors +template +Task& Task::remove_successors(Ts&&... tasks) { + (_node->_remove_successors(tasks._node), ...); + (tasks._node->_remove_predecessors(_node), ...); + return *this; +} + +// Function: composed_of +template +Task& Task::composed_of(T& target) { + _node->_handle.emplace(retrieve_graph(target)); + return *this; +} + +// Function: adopt +inline Task& Task::adopt(Graph&& graph) { + _node->_handle.emplace(std::move(graph)); + return *this; +} + +// Operator = +inline Task& Task::operator = (const Task& rhs) { + _node = rhs._node; + return *this; +} + +// Operator = +inline Task& Task::operator = (std::nullptr_t ptr) { + _node = ptr; + return *this; +} + +// Operator == +inline bool Task::operator == (const Task& rhs) const { + return _node == rhs._node; +} + +// Operator != +inline bool Task::operator != (const Task& rhs) const { + return _node != rhs._node; +} + +// Function: name +inline Task& Task::name(const std::string& name) { + _node->_name = name; + return *this; +} + +// Function: acquire +inline Task& Task::acquire(Semaphore& s) { + if(!_node->_semaphores) { + _node->_semaphores = std::make_unique(); + } + _node->_semaphores->to_acquire.push_back(&s); + return *this; +} + +// Function: acquire +template +Task& Task::acquire(I first, I last) { + if(!_node->_semaphores) { + _node->_semaphores = std::make_unique(); + } + _node->_semaphores->to_acquire.reserve( + _node->_semaphores->to_acquire.size() + std::distance(first, last) + ); + for(auto s = first; s != last; ++s){ + _node->_semaphores->to_acquire.push_back(&(*s)); + } + return *this; +} + +// Function: release +inline Task& Task::release(Semaphore& s) { + if(!_node->_semaphores) { + _node->_semaphores = std::make_unique(); + } + _node->_semaphores->to_release.push_back(&s); + return *this; +} + +// Function: release +template +Task& Task::release(I first, I last) { + if(!_node->_semaphores) { + _node->_semaphores = std::make_unique(); + } + _node->_semaphores->to_release.reserve( + _node->_semaphores->to_release.size() + std::distance(first, last) + ); + for(auto s = first; s != last; ++s) { + _node->_semaphores->to_release.push_back(&(*s)); + } + return *this; +} + +// Procedure: reset +inline void Task::reset() { + _node = nullptr; +} + +// Procedure: reset_work +inline void Task::reset_work() { + _node->_handle.emplace(); +} + +// Function: name +inline const std::string& Task::name() const { + return _node->_name; +} + +// Function: num_predecessors +inline size_t Task::num_predecessors() const { + return _node->num_predecessors(); +} + +// Function: num_strong_dependencies +inline size_t Task::num_strong_dependencies() const { + return _node->num_strong_dependencies(); +} + +// Function: num_weak_dependencies +inline size_t Task::num_weak_dependencies() const { + return _node->num_weak_dependencies(); +} + +// Function: num_successors +inline size_t Task::num_successors() const { + return _node->num_successors(); +} + +// Function: empty +inline bool Task::empty() const { + return _node == nullptr; +} + +// Function: has_work +inline bool Task::has_work() const { + return _node ? _node->_handle.index() != 0 : false; +} + +// Function: exception +inline std::exception_ptr Task::exception_ptr() const { + return _node ? _node->_exception_ptr : nullptr; +} + +// Function: has_exception +inline bool Task::has_exception_ptr() const { + return _node ? (_node->_exception_ptr != nullptr) : false; +} + +// Function: task_type +inline TaskType Task::type() const { + switch(_node->_handle.index()) { + case Node::PLACEHOLDER: return TaskType::PLACEHOLDER; + case Node::STATIC: return TaskType::STATIC; + case Node::RUNTIME: return TaskType::RUNTIME; + case Node::NONPREEMPTIVE_RUNTIME: return TaskType::RUNTIME; + case Node::SUBFLOW: return TaskType::SUBFLOW; + case Node::CONDITION: return TaskType::CONDITION; + case Node::MULTI_CONDITION: return TaskType::CONDITION; + case Node::MODULE: return TaskType::MODULE; + case Node::ADOPTED_MODULE: return TaskType::MODULE; + case Node::ASYNC: return TaskType::ASYNC; + case Node::DEPENDENT_ASYNC: return TaskType::ASYNC; + default: return TaskType::UNDEFINED; + } +} + +// Function: for_each_successor +template +void Task::for_each_successor(V&& visitor) const { + for(size_t i=0; i<_node->_num_successors; ++i) { + visitor(Task(_node->_edges[i])); + } +} + +// Function: for_each_predecessor +template +void Task::for_each_predecessor(V&& visitor) const { + for(size_t i=_node->_num_successors; i<_node->_edges.size(); ++i) { + visitor(Task(_node->_edges[i])); + } +} + +// Function: for_each_subflow_task +template +void Task::for_each_subflow_task(V&& visitor) const { + if(auto ptr = std::get_if(&_node->_handle); ptr) { + for(auto itr = ptr->subgraph.begin(); itr != ptr->subgraph.end(); ++itr) { + visitor(Task(*itr)); + } + } +} + +// Function: hash_value +inline size_t Task::hash_value() const { + return std::hash{}(_node); +} + +// Procedure: dump +inline void Task::dump(std::ostream& os) const { + os << "task "; + if(name().empty()) os << _node; + else os << name(); + os << " [type=" << to_string(type()) << ']'; +} + +// Function: work +template +Task& Task::work(C&& c) { + + if constexpr(is_static_task_v) { + _node->_handle.emplace(std::forward(c)); + } + else if constexpr(is_runtime_task_v) { + _node->_handle.emplace(std::forward(c)); + } + else if constexpr(is_subflow_task_v) { + _node->_handle.emplace(std::forward(c)); + } + else if constexpr(is_condition_task_v) { + _node->_handle.emplace(std::forward(c)); + } + else if constexpr(is_multi_condition_task_v) { + _node->_handle.emplace(std::forward(c)); + } + else { + static_assert(dependent_false_v, "invalid task callable"); + } + return *this; +} + +// Function: data +inline void* Task::data() const { + return _node->_data; +} + +// Function: data +inline Task& Task::data(void* data) { + _node->_data = data; + return *this; +} + +// ---------------------------------------------------------------------------- +// global ostream +// ---------------------------------------------------------------------------- + +/** +@brief overload of ostream inserter operator for Task +*/ +inline std::ostream& operator << (std::ostream& os, const Task& task) { + task.dump(os); + return os; +} + +// ---------------------------------------------------------------------------- +// Task View +// ---------------------------------------------------------------------------- + +/** +@class TaskView + +@brief class to access task information from the observer interface +*/ +class TaskView { + + friend class Executor; + + public: + + /** + @brief queries the name of the task + */ + const std::string& name() const; + + /** + @brief queries the number of successors of the task + */ + size_t num_successors() const; + + /** + @brief queries the number of predecessors of the task + */ + size_t num_predecessors() const; + + /** + @brief queries the number of strong dependencies of the task + */ + size_t num_strong_dependencies() const; + + /** + @brief queries the number of weak dependencies of the task + */ + size_t num_weak_dependencies() const; + + /** + @brief applies an visitor callable to each successor of the task + + @tparam V a callable type (function, lambda, etc.) that accepts a tf::Task handle + @param visitor visitor to apply to each subflow task + + This method allows you to traverse and inspect successor tasks of this task. + */ + template + void for_each_successor(V&& visitor) const; + + /** + @brief applies an visitor callable to each predecessor of the task + + @tparam V a callable type (function, lambda, etc.) that accepts a tf::Task handle + @param visitor visitor to apply to each predecessor task + + This method allows you to traverse and inspect predecessor tasks of this task. + */ + template + void for_each_predecessor(V&& visitor) const; + + /** + @brief queries the task type + */ + TaskType type() const; + + /** + @brief obtains a hash value of the underlying node + */ + size_t hash_value() const; + + private: + + TaskView(const Node&); + TaskView(const TaskView&) = default; + + const Node& _node; +}; + +// Constructor +inline TaskView::TaskView(const Node& node) : _node {node} { +} + +// Function: name +inline const std::string& TaskView::name() const { + return _node._name; +} + +// Function: num_predecessors +inline size_t TaskView::num_predecessors() const { + return _node.num_predecessors(); +} + +// Function: num_strong_dependencies +inline size_t TaskView::num_strong_dependencies() const { + return _node.num_strong_dependencies(); +} + +// Function: num_weak_dependencies +inline size_t TaskView::num_weak_dependencies() const { + return _node.num_weak_dependencies(); +} + +// Function: num_successors +inline size_t TaskView::num_successors() const { + return _node.num_successors(); +} + +// Function: type +inline TaskType TaskView::type() const { + switch(_node._handle.index()) { + case Node::PLACEHOLDER: return TaskType::PLACEHOLDER; + case Node::STATIC: return TaskType::STATIC; + case Node::RUNTIME: return TaskType::RUNTIME; + case Node::NONPREEMPTIVE_RUNTIME: return TaskType::RUNTIME; + case Node::SUBFLOW: return TaskType::SUBFLOW; + case Node::CONDITION: return TaskType::CONDITION; + case Node::MULTI_CONDITION: return TaskType::CONDITION; + case Node::MODULE: return TaskType::MODULE; + case Node::ADOPTED_MODULE: return TaskType::MODULE; + case Node::ASYNC: return TaskType::ASYNC; + case Node::DEPENDENT_ASYNC: return TaskType::ASYNC; + default: return TaskType::UNDEFINED; + } +} + +// Function: hash_value +inline size_t TaskView::hash_value() const { + return std::hash{}(&_node); +} + +// Function: for_each_successor +template +void TaskView::for_each_successor(V&& visitor) const { + for(size_t i=0; i<_node._num_successors; ++i) { + visitor(TaskView(*_node._edges[i])); + } + //for(size_t i=0; i<_node._successors.size(); ++i) { + // visitor(TaskView(*_node._successors[i])); + //} +} + +// Function: for_each_predecessor +template +void TaskView::for_each_predecessor(V&& visitor) const { + for(size_t i=_node._num_successors; i<_node._edges.size(); ++i) { + visitor(TaskView(*_node._edges[i])); + } + //for(size_t i=0; i<_node._predecessors.size(); ++i) { + // visitor(TaskView(*_node._predecessors[i])); + //} +} + +} // end of namespace tf. ---------------------------------------------------- + +namespace std { + +/** +@struct hash + +@brief hash specialization for std::hash + +The hash value of a tf::Task is computed based on the underlying task node pointer. +Two tf::Task objects have the same hash value if and only if they reference the +same underlying node in the taskflow graph. + +@par Equality and Inequality + +Two tf::Task objects are: + - **Equal in hash**: when they reference the same task node (same pointer) + - **Unequal in hash**: when they reference different task nodes + +tf::Task objects created from different taskflow calls or different task +insertions will have different hash values, even if they represent logically +similar work. +The hash function is based on the associated object identity +(i.e., underlying node pointer), not semantic equivalence. + +@code{.cpp} +tf::Taskflow taskflow; + +// Create two tasks +tf::Task task1 = taskflow.emplace([]() { std::cout << "Task 1\n"; }); +tf::Task task2 = taskflow.emplace([]() { std::cout << "Task 2\n"; }); + +// Create a copy reference to task1 +tf::Task task1_copy = task1; + +// Hash values +assert(std::hash{}(task1) == std::hash{}(task1_copy)); + // task1 and task1_copy reference the same node → same hash + +assert(std::hash{}(task1) != std::hash{}(task2)); + // task1 and task2 are different nodes → different hashes + +// Use in hash-based containers +std::unordered_set task_set; +task_set.insert(task1); +task_set.insert(task2); +assert(task_set.size() == 2); // Both tasks stored + +task_set.insert(task1_copy); +assert(task_set.size() == 2); // task1_copy is same node as task1, no new insertion +@endcode + +@see tf::Task::hash_value() for obtaining the hash value directly. +*/ +template <> +struct hash { + auto operator() (const tf::Task& task) const noexcept { + return task.hash_value(); + } +}; + + + +/** +@struct hash + +@brief hash specialization for std::hash + +The hash value of a tf::TaskView is computed based on the underlying task node +address. Two tf::TaskView objects have the same hash value if and only if they +reference the same underlying node in the taskflow graph. +The hash function is based on the associated object identity +(i.e., underlying node pointer), not semantic equivalence. + +@par Equality and Inequality + +Two tf::TaskView objects are: + - **Equal in hash**: when they reference the same task node + - **Unequal in hash**: when they reference different task nodes + +Since tf::TaskView is a lightweight read-only view (typically passed by reference +or returned by graph traversal functions), hash equality follows the same node +identity principle as tf::Task: it depends on which task node is being viewed, +not on any semantic property of the task. + +@code{.cpp} +tf::Taskflow taskflow; + +tf::Task task1 = taskflow.emplace([]() {}); +tf::Task task2 = taskflow.emplace([]() {}); + +// TaskView can be created from Task (implicit conversion) +tf::TaskView view1 = task1; +tf::TaskView view2 = task2; +tf::TaskView view1_copy = task1; // Another view of the same node + +// Hash values +assert(std::hash{}(view1) == std::hash{}(view1_copy)); + // Both views reference the same node → same hash + +assert(std::hash{}(view1) != std::hash{}(view2)); + // Different nodes → different hashes + +// TaskView and Task hash to the same value for the same node +assert(std::hash{}(view1) == std::hash{}(task1)); + // view1 and task1 reference the same node → same hash + +// Use in hash-based containers +std::unordered_map task_map; +task_map[view1] = 10; +task_map[view2] = 20; +assert(task_map[view1_copy] == 10); // view1_copy views same node as view1 +@endcode + +@see tf::TaskView::hash_value() for obtaining the hash value directly. +*/ +template <> +struct hash { + auto operator() (const tf::TaskView& task_view) const noexcept { + return task_view.hash_value(); + } +}; + +} // end of namespace std ---------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/core/task_group.hpp b/projects/hipfile/third-party/taskflow/core/task_group.hpp new file mode 100644 index 000000000000..be752b3b9963 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/core/task_group.hpp @@ -0,0 +1,884 @@ +#pragma once + +#include "executor.hpp" + +namespace tf { + +// ------------------------------------------------------------------------------------------------ +// class: TaskGroup +// ------------------------------------------------------------------------------------------------ + +/** +@class TaskGroup + +@brief class to create a task group from a task + +A task group executes a group of asynchronous tasks. +It enables asynchronous task spawning, cooperative execution among worker threads, +and naturally supports recursive parallelism. +Due to cooperative execution, a task group can only be created by an executor worker; +otherwise an exception will be thrown. +The code below demonstrates how to use task groups to implement recursive Fibonacci parallelism. + +@code{.cpp} +tf::Executor executor; + +size_t fibonacci(size_t N) { + + if (N < 2) return N; + + size_t res1, res2; + + // Create a task group from the current executor + tf::TaskGroup tg = get_executor().task_group(); + + // Submit asynchronous tasks to the group + tg.silent_async([N, &res1](){ res1 = fibonacci(N-1); }); + res2 = fibonacci(N-2); // compute one branch synchronously + + // Wait for all tasks in the group to complete + tg.corun(); + + return res1 + res2; +} + +int main() { + size_t N = 30, res; + res = executor.async([](){ return fibonacci(30); }).get(); + std::cout << N << "-th Fibonacci number is " << res << '\n'; + return 0; +} +@endcode + +Users must explicitly call tf::TaskGroup::corun() to ensure that all tasks have completed or +been properly canceled before leaving the scope of a task group. +Failing to do so results in undefined behavior. + +@note +To understand how %Taskflow schedules a task group, please refer to @ref TaskGroup. + +*/ +class TaskGroup { + + friend class Executor; + + public: + + // ---------------------------------------------------------------------------------------------- + // deleted members + // ---------------------------------------------------------------------------------------------- + + /** + @brief disabled copy constructor + */ + TaskGroup(const TaskGroup&) = delete; + + /** + @brief disabled move constructor + */ + TaskGroup(TaskGroup&&) = delete; + + /** + @brief disabled copy assignment + */ + TaskGroup& operator = (TaskGroup&&) = delete; + + /** + @brief disabled move assignment + */ + TaskGroup& operator = (const TaskGroup&) = delete; + + /** + @brief obtains the executor that creates this task group + + The running executor of a task group is the executor that creates the task group. + + @code{.cpp} + executor.silent_async([&](){ + tf::TaskGroup tg = executor.task_group(); + assert(&(tg.executor()) == &executor); + }); + @endcode + */ + Executor& executor(); + + // ---------------------------------------------------------------------------------------------- + // async methods + // ---------------------------------------------------------------------------------------------- + + /** + @brief runs the given callable asynchronously + + @tparam F callable type + @param f callable object + + This method creates an asynchronous task that executes the given function with the specified arguments. + Unlike tf::Executor::async, the task created here is parented to the task group, + where applications can issue tf::TaskGroup::corun to explicitly wait for all + asynchronous tasks spawned from the task group to complete. + For example: + + @code{.cpp} + executor.silent_async([&](){ + std::atomic counter(0); + auto tg = executor.task_group(); + auto fu1 = tg.async([&](){ counter++; }); + auto fu2 = tg.async([&](){ counter++; }); + fu1.get(); + fu2.get(); + assert(counter == 2); + // spawn 100 asynchronous tasks from the task group + for(int i=0; i<100; i++) { + tg.silent_async([&](){ counter++; }); + } + // corun until the 100 asynchronous tasks have completed + tg.corun(); + assert(counter == 102); + // do something else afterwards ... + }); + @endcode + */ + template + auto async(F&& f); + + /** + @brief runs the given callable asynchronously + + @tparam F callable type + @tparam P task parameters type satisfying tf::TaskParamsLike + + @param params task parameters + @param f callable + + Similar to tf::TaskGroup::async, but takes a parameter of type tf::TaskParams to initialize + the asynchronous task. + + @code{.cpp} + executor.silent_async([&](){ + auto tg = executor.task_group(); + auto future = tg.async("my task", [](){ return 10; }); + assert(future.get() == 10); + }); + @endcode + + */ + template + auto async(P&& params, F&& f); + + // ---------------------------------------------------------------------------------------------- + // silent async methods + // ---------------------------------------------------------------------------------------------- + + /** + @brief runs the given function asynchronously without returning any future object + + @tparam F callable type + @param f callable + + This function is more efficient than tf::TaskGroup::async and is recommended when the result + of the asynchronous task does not need to be accessed via a std::future. + + @code{.cpp} + executor.silent_async([&](){ + std::atomic counter(0); + auto tg = executor.task_group(); + for(int i=0; i<100; i++) { + tg.silent_async([&](){ counter++; }); + } + tg.corun(); + assert(counter == 100); + }); + @endcode + + */ + template + void silent_async(F&& f); + + /** + @brief runs the given function asynchronously without returning any future object + + @tparam F callable type + @param params task parameters + @param f callable + + Similar to tf::TaskGroup::silent_async, but takes a parameter of type tf::TaskParams to initialize + the created asynchronous task. + + @code{.cpp} + executor.silent_async([&](){ + auto tg = executor.task_group(); + tg.silent_async("my task", [](){}); + tg.corun(); + }); + @endcode + */ + template + void silent_async(P&& params, F&& f); + + // ---------------------------------------------------------------------------------------------- + // dependent async methods + // ---------------------------------------------------------------------------------------------- + + /** + @brief runs the given function asynchronously + when the given predecessors finish + + @tparam F callable type + @tparam Tasks tasks of type tf::AsyncTask + + @param func callable object + @param tasks asynchronous tasks on which this execution depends + + @return a pair of a tf::AsyncTask handle and + a @std_future that holds the result of the execution + + The example below creates three asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + %Task @c C returns a pair of its tf::AsyncTask handle and a std::future + that eventually will hold the result of the execution. + + @code{.cpp} + executor.silent_async([&](){ + auto tg = executor.task_group(); + tf::AsyncTask A = tg.silent_dependent_async([](){ printf("A\n"); }); + tf::AsyncTask B = tg.silent_dependent_async([](){ printf("B\n"); }); + auto [C, fuC] = tg.dependent_async( + [](){ + printf("C runs after A and B\n"); + return 1; + }, + A, B + ); + fuC.get(); // C finishes, which in turns means both A and B finish, + // so we don't need tg.corun() + }); + @endcode + */ + template + requires (std::same_as, AsyncTask> && ...) + auto dependent_async(F&& func, Tasks&&... tasks); + + /** + @brief runs the given function asynchronously + when the given predecessors finish + + @tparam P task parameters type satisfying tf::TaskParamsLike + @tparam F callable type + @tparam Tasks tasks of type tf::AsyncTask + + @param params task parameters + @param func callable object + @param tasks asynchronous tasks on which this execution depends + + @return a pair of a tf::AsyncTask handle and + a @std_future that holds the result of the execution + + The example below creates three named asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + %Task @c C returns a pair of its tf::AsyncTask handle and a std::future + that eventually will hold the result of the execution. + Assigned task names will appear in the observers of the executor. + + @code{.cpp} + executor.silent_async([&](){ + auto tg = executor.task_group(); + tf::AsyncTask A = tg.silent_dependent_async("A", [](){ printf("A\n"); }); + tf::AsyncTask B = tg.silent_dependent_async("B", [](){ printf("B\n"); }); + auto [C, fuC] = tg.dependent_async( + "C", + [](){ + printf("C runs after A and B\n"); + return 1; + }, + A, B + ); + fuC.get(); // C finishes, which in turns means both A and B finish, + // so we don't need tg.corun() + }); + @endcode + */ + template + requires (std::same_as, AsyncTask> && ...) + auto dependent_async(P&& params, F&& func, Tasks&&... tasks); + + /** + @brief runs the given function asynchronously + when the given range of predecessors finish + + @tparam F callable type + @tparam I iterator type + + @param func callable object + @param first iterator to the beginning (inclusive) + @param last iterator to the end (exclusive) + + @return a pair of a tf::AsyncTask handle and + a @std_future that holds the result of the execution + + The example below creates three asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + %Task @c C returns a pair of its tf::AsyncTask handle and a std::future + that eventually will hold the result of the execution. + + @code{.cpp} + executor.silent_async([](){ + auto tg = executor.task_group(); + std::array array { + tg.silent_dependent_async([](){ printf("A\n"); }), + tg.silent_dependent_async([](){ printf("B\n"); }) + }; + auto [C, fuC] = tg.dependent_async( + [](){ + printf("C runs after A and B\n"); + return 1; + }, + array.begin(), array.end() + ); + fuC.get(); // C finishes, which in turns means both A and B finish, + // so we don't need tg.corun() + }); + @endcode + */ + template + requires (!std::same_as, AsyncTask>) + auto dependent_async(F&& func, I first, I last); + + /** + @brief runs the given function asynchronously + when the given range of predecessors finish + + @tparam P task parameters type satisfying tf::TaskParamsLike + @tparam F callable type + @tparam I iterator type + + @param params task parameters + @param func callable object + @param first iterator to the beginning (inclusive) + @param last iterator to the end (exclusive) + + @return a pair of a tf::AsyncTask handle and + a @std_future that holds the result of the execution + + The example below creates three named asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + %Task @c C returns a pair of its tf::AsyncTask handle and a std::future + that eventually will hold the result of the execution. + Assigned task names will appear in the observers of the executor. + + @code{.cpp} + executor.silent_async([&](){ + auto tg = executor.task_group(); + std::array array { + tg.silent_dependent_async("A", [](){ printf("A\n"); }), + tg.silent_dependent_async("B", [](){ printf("B\n"); }) + }; + auto [C, fuC] = tg.dependent_async( + "C", + [](){ + printf("C runs after A and B\n"); + return 1; + }, + array.begin(), array.end() + ); + fuC.get(); // C finishes, which in turns means both A and B finish, + // so we don't need tg.corun() + }); + @endcode + */ + template + requires (!std::same_as, AsyncTask>) + auto dependent_async(P&& params, F&& func, I first, I last); + + + // ---------------------------------------------------------------------------------------------- + // silent dependent async methods + // ---------------------------------------------------------------------------------------------- + + /** + @brief runs the given function asynchronously + when the given predecessors finish + + @tparam F callable type + @tparam Tasks tasks of type tf::AsyncTask + + @param func callable object + @param tasks asynchronous tasks on which this execution depends + + @return a tf::AsyncTask handle + + This member function is more efficient than tf::TaskGroup::dependent_async + and is encouraged to use when you do not want a @std_future to + acquire the result or synchronize the execution. + The example below creates three asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + + @code{.cpp} + executor.silent_async([&](){ + auto tg = executor.task_group(); + tf::AsyncTask A = tg.silent_dependent_async([](){ printf("A\n"); }); + tf::AsyncTask B = tg.silent_dependent_async([](){ printf("B\n"); }); + tg.silent_dependent_async([](){ printf("C runs after A and B\n"); }, A, B); + tg.corun(); // corun until all dependent-async tasks finish + }); + @endcode + */ + template + requires (std::same_as, AsyncTask> && ...) + tf::AsyncTask silent_dependent_async(F&& func, Tasks&&... tasks); + + /** + @brief runs the given function asynchronously + when the given predecessors finish + + @tparam F callable type + @tparam Tasks tasks of type tf::AsyncTask + + @param params task parameters + @param func callable object + @param tasks asynchronous tasks on which this execution depends + + @return a tf::AsyncTask handle + + This member function is more efficient than tf::TaskGroup::dependent_async + and is encouraged to use when you do not want a @std_future to + acquire the result or synchronize the execution. + The example below creates three asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + Assigned task names will appear in the observers of the executor. + + @code{.cpp} + executor.silent_async([&](){ + auto tg = executor.task_group(); + tf::AsyncTask A = tg.silent_dependent_async("A", [](){ printf("A\n"); }); + tf::AsyncTask B = tg.silent_dependent_async("B", [](){ printf("B\n"); }); + tg.silent_dependent_async( + "C", [](){ printf("C runs after A and B\n"); }, A, B + ); + tg.corun(); // corun until all dependent-async tasks finish + }); + @endcode + */ + template + requires (std::same_as, AsyncTask> && ...) + tf::AsyncTask silent_dependent_async(P&& params, F&& func, Tasks&&... tasks); + + /** + @brief runs the given function asynchronously + when the given range of predecessors finish + + @tparam F callable type + @tparam I iterator type + + @param func callable object + @param first iterator to the beginning (inclusive) + @param last iterator to the end (exclusive) + + @return a tf::AsyncTask handle + + This member function is more efficient than tf::TaskGroup::dependent_async + and is encouraged to use when you do not want a @std_future to + acquire the result or synchronize the execution. + The example below creates three asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + + @code{.cpp} + executor.silent_async([&](){ + auto tg = executor.task_group(); + std::array array { + tg.silent_dependent_async([](){ printf("A\n"); }), + tg.silent_dependent_async([](){ printf("B\n"); }) + }; + tg.silent_dependent_async( + [](){ printf("C runs after A and B\n"); }, array.begin(), array.end() + ); + tg.corun(); // corun until all dependent-async tasks finish + }); + @endcode + */ + template + requires (!std::same_as, AsyncTask>) + tf::AsyncTask silent_dependent_async(F&& func, I first, I last); + + /** + @brief runs the given function asynchronously + when the given range of predecessors finish + + @tparam F callable type + @tparam I iterator type + + @param params tasks parameters + @param func callable object + @param first iterator to the beginning (inclusive) + @param last iterator to the end (exclusive) + + @return a tf::AsyncTask handle + + This member function is more efficient than tf::TaskGroup::dependent_async + and is encouraged to use when you do not want a @std_future to + acquire the result or synchronize the execution. + The example below creates three asynchronous tasks, @c A, @c B, and @c C, + in which task @c C runs after task @c A and task @c B. + Assigned task names will appear in the observers of the executor. + + @code{.cpp} + executor.silent_async([&](){ + auto tg = executor.task_group(); + std::array array { + tg.silent_dependent_async("A", [](){ printf("A\n"); }), + tg.silent_dependent_async("B", [](){ printf("B\n"); }) + }; + tg.silent_dependent_async( + "C", [](){ printf("C runs after A and B\n"); }, array.begin(), array.end() + ); + tg.corun(); // corun until all dependent-async tasks finish + }); + @endcode + */ + template + requires (!std::same_as, AsyncTask>) + tf::AsyncTask silent_dependent_async(P&& params, F&& func, I first, I last); + + + // ---------------------------------------------------------------------------------------------- + // cooperative execution methods + // ---------------------------------------------------------------------------------------------- + + /** + @brief corun all tasks spawned by this task group with other workers + + Coruns all tasks spawned by this task group cooperatively with other workers in + the same executor until all these tasks finish. + Under cooperative execution, a worker is not preempted. Instead, it continues + participating in the work-stealing loop, executing available tasks alongside + other workers. + + @code{.cpp} + executor.silent_async([&](){ + auto tg = executor.task_group(); + std::atomic counter{0}; + // spawn 100 async tasks and wait + for(int i=0; i<100; i++) { + tg.silent_async([&](){ counter++; }); + } + tg.corun(); + assert(counter == 100); + + // spawn another 100 async tasks and wait + for(int i=0; i<100; i++) { + tg.silent_async([&](){ counter++; }); + } + tg.corun(); + assert(counter == 200); + }); + @endcode + + Note that only the parent worker of this task group (the worker who creates it) + can call this corun. + */ + void corun(); + + /** + @brief cancel all tasks in this task group + + Marks the task group as cancelled to stop any not-yet-started tasks in the group from running. + Tasks that are already running will continue to completion, but no new tasks belonging to the + task group will be scheduled after cancellation. + + This example below demonstrates how tf::TaskGroup::cancel() prevents pending tasks in a task group from executing, + while allowing already running tasks to complete cooperatively. + The first set of tasks deliberately occupies all but one worker thread, + ensuring that subsequently spawned tasks remain pending. + After invoking tf::TaskGroup::cancel(), these pending tasks are never scheduled, + even after the blocked workers are released. + A final call to tf::TaskGroup::corun() synchronizes with all tasks in the group, + guaranteeing safe completion and verifying that cancellation successfully suppresses task execution. + + @code{.cpp} + const size_t W = 12; // must be >1 for this example to work + tf::Executor executor(W); + + executor.async([&executor, W](){ + + auto tg = executor.task_group(); + + // deliberately block the other W-1 workers + std::atomic latch(0); + for(size_t i=0; i= 0); + tg.corun(); + }); + @endcode + */ + size_t size() const; + + private: + + /** + @private + */ + explicit TaskGroup(Executor&, Worker&); + + /** + @private + */ + Executor& _executor; + + /** + @private + */ + Worker& _worker; + + /** + @private + */ + NodeBase _node_base; +}; + +// constructor +inline TaskGroup::TaskGroup(Executor& executor, Worker& worker) : + _executor {executor}, + _worker {worker}, + _node_base {NSTATE::IMPLICITLY_ANCHORED, ESTATE::NONE, nullptr, 0} { +} + +// Function: executor +inline Executor& TaskGroup::executor() { + return _executor; +} + +// Function: corun +inline void TaskGroup::corun() { + { + ExplicitAnchorGuard anchor(&_node_base); + _executor._corun_until(_worker, [this] () -> bool { + return _node_base._join_counter.load(std::memory_order_acquire) == 0; + }); + } + _node_base._rethrow_exception(); +} + +// Function: cancel +inline void TaskGroup::cancel() { + _node_base._estate.fetch_or(ESTATE::CANCELLED, std::memory_order_relaxed); +} + +// Function: is_cancelled +inline bool TaskGroup::is_cancelled() { + return _node_base._estate.load(std::memory_order_relaxed) & ESTATE::CANCELLED; +} + +// Function: size +inline size_t TaskGroup::size() const { + return _node_base._join_counter.load(std::memory_order_relaxed); +} + +// ------------------------------------------------------------------------------------------------ +// TaskGroup::silent_async +// ------------------------------------------------------------------------------------------------ + +// Function: silent_async +template +void TaskGroup::silent_async(F&& f) { + silent_async(DefaultTaskParams{}, std::forward(f)); +} + +// Function: silent_async +template +void TaskGroup::silent_async(P&& params, F&& f) { + _node_base._join_counter.fetch_add(1, std::memory_order_relaxed); + _executor._silent_async( + std::forward

(params), std::forward(f), nullptr, &_node_base + ); +} + +// ------------------------------------------------------------------------------------------------ +// TaskGroup::async +// ------------------------------------------------------------------------------------------------ + +// Function: async +template +auto TaskGroup::async(F&& f) { + return async(DefaultTaskParams{}, std::forward(f)); +} + +// Function: async +template +auto TaskGroup::async(P&& params, F&& f) { + _node_base._join_counter.fetch_add(1, std::memory_order_relaxed); + return _executor._async( + std::forward

(params), std::forward(f), nullptr, &_node_base + ); +} + +// ------------------------------------------------------------------------------------------------ +// silent dependent async +// ------------------------------------------------------------------------------------------------ + +// Function: silent_dependent_async +template +requires (std::same_as, AsyncTask> && ...) +tf::AsyncTask TaskGroup::silent_dependent_async(F&& func, Tasks&&... tasks) { + return silent_dependent_async( + DefaultTaskParams{}, std::forward(func), std::forward(tasks)... + ); +} + +// Function: silent_dependent_async +template +requires (std::same_as, AsyncTask> && ...) +tf::AsyncTask TaskGroup::silent_dependent_async( + P&& params, F&& func, Tasks&&... tasks +){ + std::array array = { std::forward(tasks)... }; + return silent_dependent_async( + std::forward

(params), std::forward(func), array.begin(), array.end() + ); +} + +// Function: silent_dependent_async +template +requires (!std::same_as, AsyncTask>) +tf::AsyncTask TaskGroup::silent_dependent_async(F&& func, I first, I last) { + return silent_dependent_async(DefaultTaskParams{}, std::forward(func), first, last); +} + +// Function: silent_dependent_async +template +requires (!std::same_as, AsyncTask>) +tf::AsyncTask TaskGroup::silent_dependent_async( + P&& params, F&& func, I first, I last +) { + _node_base._join_counter.fetch_add(1, std::memory_order_relaxed); + return _executor._silent_dependent_async( + std::forward

(params), std::forward(func), first, last, nullptr, &_node_base + ); +} + +// ------------------------------------------------------------------------------------------------ +// dependent async +// ------------------------------------------------------------------------------------------------ + +// Function: dependent_async +template +requires (std::same_as, AsyncTask> && ...) +auto TaskGroup::dependent_async(F&& func, Tasks&&... tasks) { + return dependent_async(DefaultTaskParams{}, std::forward(func), std::forward(tasks)...); +} + +// Function: dependent_async +template +requires (std::same_as, AsyncTask> && ...) +auto TaskGroup::dependent_async(P&& params, F&& func, Tasks&&... tasks) { + std::array array = { std::forward(tasks)... }; + return dependent_async( + std::forward

(params), std::forward(func), array.begin(), array.end() + ); +} + +// Function: dependent_async +template +requires (!std::same_as, AsyncTask>) +auto TaskGroup::dependent_async(F&& func, I first, I last) { + return dependent_async(DefaultTaskParams{}, std::forward(func), first, last); +} + +// Function: dependent_async +template +requires (!std::same_as, AsyncTask>) +auto TaskGroup::dependent_async(P&& params, F&& func, I first, I last) { + _node_base._join_counter.fetch_add(1, std::memory_order_relaxed); + return _executor._dependent_async( + std::forward

(params), std::forward(func), first, last, nullptr, &_node_base + ); +} + +// ---------------------------------------------------------------------------- +// Executor Forward Declaration +// ---------------------------------------------------------------------------- + +// Procedure: task_group +inline TaskGroup Executor::task_group() { + Worker* w = this_worker(); + if(w == nullptr) { + TF_THROW("task_group can only created by a worker of the executor"); + } + return TaskGroup(*this, *w); +} + + +} // end of namespace tf ----------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/core/taskflow.hpp b/projects/hipfile/third-party/taskflow/core/taskflow.hpp new file mode 100644 index 000000000000..01953397790d --- /dev/null +++ b/projects/hipfile/third-party/taskflow/core/taskflow.hpp @@ -0,0 +1,730 @@ +#pragma once + +#include "flow_builder.hpp" + +/** +@file taskflow/core/taskflow.hpp +@brief taskflow include file +*/ + +namespace tf { + +// ---------------------------------------------------------------------------- + +/** +@class Taskflow + +@brief class to create a taskflow object + +A %taskflow manages a task dependency graph where each task represents a +callable object (e.g., @std_lambda, @std_function) and an edge represents a +dependency between two tasks. A task is one of the following types: + + 1. static task : the callable constructible from + @c std::function + 2. subflow task : the callable constructible from + @c std::function + 3. condition task : the callable constructible from + @c std::function + 4. multi-condition task: the callable constructible from + @c %std::function()> + 5. module task : the task constructed from tf::Taskflow::composed_of + @c std::function + +Each task is a basic computation unit and is run by one worker thread +from an executor. +The following example creates a simple taskflow graph of four static tasks, +@c A, @c B, @c C, and @c D, where +@c A runs before @c B and @c C and +@c D runs after @c B and @c C. + +@code{.cpp} +tf::Executor executor; +tf::Taskflow taskflow("simple"); + +tf::Task A = taskflow.emplace([](){ std::cout << "TaskA\n"; }); +tf::Task B = taskflow.emplace([](){ std::cout << "TaskB\n"; }); +tf::Task C = taskflow.emplace([](){ std::cout << "TaskC\n"; }); +tf::Task D = taskflow.emplace([](){ std::cout << "TaskD\n"; }); + +A.precede(B, C); // A runs before B and C +D.succeed(B, C); // D runs after B and C + +executor.run(taskflow).wait(); +@endcode + +The taskflow object itself is NOT thread-safe. You should not +modifying the graph while it is running, +such as adding new tasks, adding new dependencies, and moving +the taskflow to another. + +Please refer to @ref Cookbook to learn more about each task type +and how to submit a taskflow to an executor. +*/ +class Taskflow : public FlowBuilder { + + friend class Topology; + friend class Executor; + friend class FlowBuilder; + friend class Subflow; + + struct Dumper { + size_t id; + std::stack> stack; // added depth + std::unordered_map visited; + }; + + public: + + /** + @brief constructs a taskflow with the given name + + @code{.cpp} + tf::Taskflow taskflow("My Taskflow"); + std::cout << taskflow.name(); // "My Taskflow" + @endcode + */ + Taskflow(const std::string& name); + + /** + @brief constructs a taskflow + */ + Taskflow(); + + /** + @brief constructs a taskflow from a moved taskflow + + Constructing a taskflow @c taskflow1 from a moved taskflow @c taskflow2 will + migrate the graph of @c taskflow2 to @c taskflow1. + After the move, @c taskflow2 will become empty. + + @code{.cpp} + tf::Taskflow taskflow1(std::move(taskflow2)); + assert(taskflow2.empty()); + @endcode + + @attention You should avoid moving a taskflow that is currently running on an executor. + Doing so results in undefined behavior. + */ + Taskflow(Taskflow&& rhs); + + /** + @brief move assignment operator + + Moving a taskflow @c taskflow2 to another taskflow @c taskflow1 will destroy + the existing graph of @c taskflow1 and assign it the graph of @c taskflow2. + After the move, @c taskflow2 will become empty. + + @code{.cpp} + taskflow1 = std::move(taskflow2); + assert(taskflow2.empty()); + @endcode + + @attention You should avoid moving a taskflow that is currently running on an executor. + Doing so results in undefined behavior. + */ + Taskflow& operator = (Taskflow&& rhs); + + /** + @brief default destructor + + When the destructor is called, all tasks and their associated data + (e.g., captured data) will be destroyed. + It is your responsibility to ensure all submitted execution of this + taskflow have completed before destroying it. + For instance, the following code results in undefined behavior + since the executor may still be running the taskflow while + it is destroyed after the block. + + @code{.cpp} + { + tf::Taskflow taskflow; + executor.run(taskflow); + } + @endcode + + To fix the problem, we must wait for the execution to complete + before destroying the taskflow. + + @code{.cpp} + { + tf::Taskflow taskflow; + executor.run(taskflow).wait(); + } + @endcode + */ + ~Taskflow() = default; + + /** + @brief dumps the taskflow to a DOT format through a std::ostream target + + @code{.cpp} + taskflow.dump(std::cout); // dump the graph to the standard output + + std::ofstream ofs("output.dot"); + taskflow.dump(ofs); // dump the graph to the file output.dot + @endcode + + For dynamically spawned tasks, such as module tasks, subflow tasks, + and GPU tasks, you need to run the taskflow first before you can + dump the entire graph. For subflow tasks the subflow must be retained, + otherwise it is cleared when joined and the child tasks not shown in the + DOT graph. + + @code{.cpp} + tf::Task parent = taskflow.emplace([](tf::Subflow sf){ + sf.emplace([](){ std::cout << "child\n"; }); + sf.retain(true); + }); + taskflow.dump(std::cout); // this dumps only the parent tasks + executor.run(taskflow).wait(); + taskflow.dump(std::cout); // this dumps both parent and child tasks + @endcode + */ + void dump(std::ostream& ostream) const; + + /** + @brief dumps the taskflow to a std::string of DOT format + + This method is similar to tf::Taskflow::dump(std::ostream& ostream), + but returning a string of the graph in DOT format. + */ + std::string dump() const; + + /** + @brief queries the number of tasks in this taskflow + + The number of tasks in this taskflow is defined at the first level of hierarchy. + Tasks that are created dynamically, such as those via tf::Subflow, are not counted. + + @code{.cpp} + tf::Taskflow taskflow; + auto my_task = taskflow.emplace([](){}); + assert(taskflow.num_tasks() == 1); + + // reassign my_task to a subflow of four tasks + my_task.work([](tf::Subflow& sf){ + sf.emplace( + [](){ std::cout << "Task A\n"; }, + [](){ std::cout << "Task B\n"; }, + [](){ std::cout << "Task C\n"; }, + [](){ std::cout << "Task D\n"; } + ); + }); + + // subflow tasks will not be counted + assert(taskflow.num_tasks() == 1); + @endcode + */ + size_t num_tasks() const; + + /** + @brief queries if this taskflow is empty (has no tasks) + + An empty taskflow has no tasks, i.e., the return of tf::Taskflow::num_tasks is `0`. + + @code{.cpp} + tf::Taskflow taskflow; + assert(taskflow.empty() == true); + taskflow.emplace([](){}); + assert(taskflow.empty() == false); + @endcode + */ + bool empty() const; + + /** + @brief assigns a new name to this taskflow + + @code{.cpp} + taskflow.name("foo"); + assert(taskflow.name() == "foo"); + @endcode + */ + void name(const std::string&); + + /** + @brief queries the name of this taskflow + + @code{.cpp} + tf::Taskflow taskflow("foo"); + assert(taskflow.name() == "foo"); + @endcode + */ + const std::string& name() const; + + /** + @brief clears the associated task dependency graph + + When you clear a taskflow, all tasks and their associated data + (e.g., captured data in task callables) will be destroyed. + The behavior of clearing a running taskflow is undefined. + */ + void clear(); + + /** + @brief applies a visitor to each task in this taskflow + + A visitor is a callable that takes an argument of type tf::Task + and returns nothing. The following example iterates each task in a + taskflow and prints its name: + + @code{.cpp} + taskflow.for_each_task([](tf::Task task){ + std::cout << task.name() << '\n'; + }); + @endcode + */ + template + void for_each_task(V&& visitor) const; + + /** + @brief removes dependencies that go from task @c from to task @c to + + @param from from task (dependent) + @param to to task (successor) + + Removing the depencency from task `from` to task `to` is equivalent to + removing `to` from the succcessor list of `from` and + removing `from` from the predecessor list of `to`. + + @code{.cpp} + tf::Taskflow taskflow; + auto a = taskflow.placeholder().name("a"); + auto b = taskflow.placeholder().name("b"); + auto c = taskflow.placeholder().name("c"); + auto d = taskflow.placeholder().name("d"); + + a.precede(b, c, d); + assert(a.num_successors() == 3); + assert(b.num_predecessors() == 1); + assert(c.num_predecessors() == 1); + assert(d.num_predecessors() == 1); + + taskflow.remove_dependency(a, b); + assert(a.num_successors() == 2); + assert(b.num_predecessors() == 0); + @endcode + + @attention For performance reason, %Taskflow does not store the graph using linked lists but + vectors with contiguous space. + Therefore, removing tasks or dependencies incurs linear time complexity proportional + to the size of the graph and the dependency count of a task. + */ + void remove_dependency(Task from, Task to); + + /** + @brief returns a reference to the underlying graph object + + A graph object is of type tf::Graph and stores a task dependency graph that can be executed + by an tf::Executor. + + */ + Graph& graph(); + + private: + + mutable std::mutex _mutex; + + std::string _name; + + Graph _graph; + + std::queue> _topologies; + + void _dump(std::ostream&, const Graph*) const; + void _dump(std::ostream&, const Node*, Dumper&, size_t level) const; + void _dump(std::ostream&, const Graph*, Dumper&, size_t level) const; + + size_t _fetch_enqueue(std::shared_ptr); +}; + +// Constructor +inline Taskflow::Taskflow(const std::string& name) : + FlowBuilder {_graph}, + _name {name} { +} + +// Constructor +inline Taskflow::Taskflow() : FlowBuilder{_graph} { +} + +// Move constructor +inline Taskflow::Taskflow(Taskflow&& rhs) : FlowBuilder{_graph} { + std::scoped_lock lock(rhs._mutex); + _name = std::move(rhs._name); + _graph = std::move(rhs._graph); + _topologies = std::move(rhs._topologies); +} + +// Move assignment +inline Taskflow& Taskflow::operator = (Taskflow&& rhs) { + if(this != &rhs) { + std::scoped_lock lock(_mutex, rhs._mutex); + _name = std::move(rhs._name); + _graph = std::move(rhs._graph); + _topologies = std::move(rhs._topologies); + } + return *this; +} + +// Function: +inline void Taskflow::clear() { + _graph.clear(); +} + +// Function: num_tasks +inline size_t Taskflow::num_tasks() const { + return _graph.size(); +} + +// Function: empty +inline bool Taskflow::empty() const { + return _graph.empty(); +} + +// Function: name +inline void Taskflow::name(const std::string &name) { + _name = name; +} + +// Function: name +inline const std::string& Taskflow::name() const { + return _name; +} + +// Function: graph +inline Graph& Taskflow::graph() { + return _graph; +} + +// Function: for_each_task +template +void Taskflow::for_each_task(V&& visitor) const { + for(auto itr = _graph.begin(); itr != _graph.end(); ++itr) { + visitor(Task(*itr)); + } +} + +// Function: remove_dependency +inline void Taskflow::remove_dependency(Task from, Task to) { + // remove "to" from the succcessor list of "from" + from._node->_remove_successors(to._node); + + // remove "from" from the predecessor list of "to" + to._node->_remove_predecessors(from._node); +} + +// Function: _fetch_enqueue +inline size_t Taskflow::_fetch_enqueue(std::shared_ptr tpg) { + std::lock_guard lock(_mutex); + auto pre_size = _topologies.size(); + _topologies.emplace(std::move(tpg)); + return pre_size; +} + +// Function: dump +inline std::string Taskflow::dump() const { + std::ostringstream oss; + dump(oss); + return oss.str(); +} + +// Function: dump +inline void Taskflow::dump(std::ostream& os) const { + os << "digraph Taskflow {\n"; + os << " compound=true;\n"; // required for lhead/ltail cluster edges + _dump(os, &_graph); + os << "}\n"; +} + +// Function: _dump (top-level — iterates module stack) +inline void Taskflow::_dump(std::ostream& os, const Graph* top) const { + + Dumper dumper; + + dumper.id = 0; + dumper.stack.push({nullptr, top, 1}); + dumper.visited[top] = dumper.id++; + + while(!dumper.stack.empty()) { + + auto [p, f, depth] = dumper.stack.top(); + dumper.stack.pop(); + + std::string ind(depth * 2, ' '); + std::string ind2((depth + 1) * 2, ' '); + + os << ind << "subgraph cluster_p" << f << " {\n"; + os << ind2 << "label=\""; + + // n-level module + if(p) { + if (p->_name.empty()) os << 'm' << dumper.visited[f]; + else os << p->name(); + } + // top-level taskflow graph + else { + os << "Taskflow: "; + if(_name.empty()) os << 'p' << this; + else os << _name; + } + + os << "\";\n"; + + _dump(os, f, dumper, depth + 1); + os << ind << "}\n"; + } +} + +// Function: _dump (single node) +inline void Taskflow::_dump( + std::ostream& os, const Node* node, Dumper& dumper, size_t level +) const { + + std::string ind(level * 2, ' '); + + // label of the node + os << ind << 'p' << node << "[label=\""; + if(node->_name.empty()) os << 'p' << node; + else os << node->_name; + os << "\" "; + + // shape of the node + switch(node->_handle.index()) { + + case Node::CONDITION: + case Node::MULTI_CONDITION: + os << "shape=diamond color=black fillcolor=aquamarine style=filled"; + break; + + default: + break; + } + + os << "];\n"; + + for(size_t s=0; s_num_successors; ++s) { + if(node->_is_conditioner()) { + os << ind << 'p' << node << " -> p" << node->_edges[s] + << " [style=dashed label=\"" << s << "\"];\n"; + } else { + os << ind << 'p' << node << " -> p" << node->_edges[s] << ";\n"; + } + } + + // node info + switch(auto hid = node->_handle.index(); hid) { + + case Node::SUBFLOW: + case Node::ADOPTED_MODULE: { + + auto& g = (hid == Node::SUBFLOW) ? + std::get_if(&node->_handle)->subgraph : + std::get_if(&node->_handle)->graph; + + if(!g.empty()) { + std::string ind2((level + 1) * 2, ' '); + + os << ind << "subgraph cluster_p" << node << " {\n"; + os << ind2 << ((hid == Node::SUBFLOW) ? "label=\"Subflow: " + : "label=\"AdoptedModule: "); + if(node->_name.empty()) os << 'p' << node; + else os << node->_name; + os << "\";\n"; + os << ind2 << "color=blue;\n"; + + _dump(os, &g, dumper, level + 1); + os << ind << "}\n"; + + // Single cluster-level join edge: subflow cluster → parent node. + // ltail clips the arrow tail at the cluster boundary. + auto first = *g.begin(); + os << ind << 'p' << first << " -> p" << node + << " [ltail=cluster_p" << node + << " style=dashed color=blue];\n"; + } + } + break; + + default: + break; + } +} + +// Function: _dump (graph — iterates nodes) +inline void Taskflow::_dump( + std::ostream& os, const Graph* graph, Dumper& dumper, size_t level +) const { + + std::string ind(level * 2, ' '); + + for(auto itr = graph->begin(); itr != graph->end(); ++itr) { + + Node* n = *itr; + + // regular task + if(n->_handle.index() != Node::MODULE) { + _dump(os, n, dumper, level); + } + // module task + else { + auto mgraph = &(std::get_if(&n->_handle)->graph); + + os << ind << 'p' << n << "[shape=box3d, color=blue, label=\""; + if(n->_name.empty()) os << 'p' << n; + else os << n->_name; + + if(dumper.visited.find(mgraph) == dumper.visited.end()) { + dumper.visited[mgraph] = dumper.id++; + dumper.stack.push({n, mgraph, level}); + } + + if(n->_name.empty()) os << " [m" << dumper.visited[mgraph] << "]"; + os << "\"];\n"; + + for(size_t i=0; i_num_successors; ++i) { + os << ind << 'p' << n << " -> " << 'p' << n->_edges[i] << ";\n"; + } + } + } +} + +// ---------------------------------------------------------------------------- +// class definition: Future +// ---------------------------------------------------------------------------- + +/** +@class Future + +@brief class to access the result of an execution + +tf::Future is a derived class from std::future that will eventually hold the +execution result of a submitted taskflow (tf::Executor::run series). +In addition to the base methods inherited from std::future, +you can call tf::Future::cancel to cancel the execution of the running taskflow +associated with this future object. +The following example cancels a submission of a taskflow that contains +1000 tasks each running one second. + +@code{.cpp} +tf::Executor executor; +tf::Taskflow taskflow; + +for(int i=0; i<1000; i++) { + taskflow.emplace([](){ + std::this_thread::sleep_for(std::chrono::seconds(1)); + }); +} + +// submit the taskflow +tf::Future fu = executor.run(taskflow); + +// request to cancel the submitted execution above +fu.cancel(); + +// wait until the cancellation finishes +fu.get(); +@endcode +*/ +template +class Future : public std::future { + + friend class Executor; + friend class Subflow; + friend class Runtime; + + public: + + /** + @brief default constructor + */ + Future() = default; + + /** + @brief disabled copy constructor + */ + Future(const Future&) = delete; + + /** + @brief default move constructor + */ + Future(Future&&) = default; + + /** + @brief constructs `*this` from a `std::future` + */ + Future(std::future&&); + + /** + @brief disabled copy assignment + */ + Future& operator = (const Future&) = delete; + + /** + @brief default move assignment + */ + Future& operator = (Future&&) = default; + + /** + @brief cancels the execution of the running taskflow associated with + this future object + + @return @c true if the execution can be cancelled or + @c false if the execution has already completed + + When you request a cancellation, the executor will stop scheduling any tasks onwards. + Tasks that are already running will continue to finish as their executions are non-preemptive. + You can call tf::Future::wait to wait for the cancellation to complete. + + @code{.cpp} + // create a taskflow of four tasks and submit it to an executor + taskflow.emplace( + [](){ std::cout << "Task A\n"; }, + [](){ std::cout << "Task B\n"; }, + [](){ std::cout << "Task C\n"; }, + [](){ std::cout << "Task D\n"; } + ); + auto future = executor.run(taskflow); + + // cancel the execution of the taskflow and wait until it finishes all running tasks + future.cancel(); + future.wait(); + @endcode + + In the above example, we submit a taskflow of four tasks to the executor and then + issue a cancellation to stop its execution. + Since the cancellation is non-deterministic with the executor runtime, + we may still see some tasks complete their executions or none. + + */ + bool cancel(); + + private: + + std::weak_ptr _topology; + + Future(std::future&&, std::weak_ptr); +}; + +template +Future::Future(std::future&& f, std::weak_ptr p) : + std::future {std::move(f)}, + _topology {std::move(p)} { +} + +template +Future::Future(std::future&& f) : std::future {std::move(f)} { +} + +// Function: cancel +template +bool Future::cancel() { + if(auto ptr = _topology.lock(); ptr) { + ptr->_estate.fetch_or(ESTATE::CANCELLED, std::memory_order_relaxed); + return true; + } + return false; +} + + +} // end of namespace tf. --------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/core/topology.hpp b/projects/hipfile/third-party/taskflow/core/topology.hpp new file mode 100644 index 000000000000..f5dcb6e437e2 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/core/topology.hpp @@ -0,0 +1,17 @@ +#pragma once + +namespace tf { + +// ---------------------------------------------------------------------------- + + + +} // end of namespace tf. ---------------------------------------------------- + + + + + + + + diff --git a/projects/hipfile/third-party/taskflow/core/worker.hpp b/projects/hipfile/third-party/taskflow/core/worker.hpp new file mode 100644 index 000000000000..1002c4b744a3 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/core/worker.hpp @@ -0,0 +1,327 @@ +#pragma once + +#include "declarations.hpp" +#include "wsq.hpp" +#include "nonblocking_notifier.hpp" +#include "atomic_notifier.hpp" + +/** +@file worker.hpp +@brief worker include file +*/ + +namespace tf { + +// ---------------------------------------------------------------------------- +// Default Notifier +// Our experiments show that NonblockingNotifier has the most stable performance +// ---------------------------------------------------------------------------- + +/** +@typedef DefaultNotifier + +@brief the default notifier type used by %Taskflow + +By default, %Taskflow uses tf::NonblockingNotifier due to its stable performance on most platforms. +We do not use tf::AtomicNotifier since on some platforms and compiler versions, +the atomic notification may exhibit suboptimal performance due to buggy wake-up mechanisms. +These issues have been discussed in GCC bug reports and patch threads related to atomic wait/notify +implementations. + +See also: + + [GCC Bugzilla report on atomic wait/notify behavior](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=106772) + + [GCC patch discussions on refactoring and fixing atomic notify/race issues](https://gcc.gnu.org/pipermail/gcc-patches/2025-May/685050.html) +*/ +#ifdef TF_ENABLE_ATOMIC_NOTIFIER + using DefaultNotifier = AtomicNotifier; +#else + using DefaultNotifier = NonblockingNotifier; +#endif + +// ---------------------------------------------------------------------------- +// Class Definition: Worker +// ---------------------------------------------------------------------------- + +/** +@class Worker + +@brief class to create a worker in an executor + +The class is primarily used by the executor to perform work-stealing algorithm. +Users can access a worker object and alter its property +(e.g., changing the thread affinity in a POSIX-like system) +using tf::WorkerInterface. +*/ +class Worker { + + friend class Executor; + friend class Runtime; + friend class WorkerView; + + using wsq_type = BoundedWSQ; + + public: + + /** + @brief queries the worker id associated with its parent executor + + A worker id is a unsigned integer in the range [0, N), + where @c N is the number of workers spawned at the construction + time of the executor. + */ + inline size_t id() const { return _id; } + + /** + @brief queries the size of the queue (i.e., number of enqueued tasks to + run) associated with the worker + */ + inline size_t queue_size() const { return _wsq.size(); } + + /** + @brief queries the current capacity of the queue + */ + inline size_t queue_capacity() const { return static_cast(_wsq.capacity()); } + + /** + @brief acquires the associated thread + */ + std::thread& thread() { return _thread; } + + private: + + alignas(TF_CACHELINE_SIZE) std::atomic_flag _done = ATOMIC_FLAG_INIT; + + size_t _id; + size_t _sticky_victim; + Xorshift _rdgen; + std::thread _thread; + wsq_type _wsq; + //std::vector _pool; +}; + +// ---------------------------------------------------------------------------- +// Class Definition: WorkerView +// ---------------------------------------------------------------------------- + +/** +@class WorkerView + +@brief class to create an immutable view of a worker + +An executor keeps a set of internal worker threads to run tasks. +A worker view provides users an immutable interface to observe +when a worker runs a task, and the view object is only accessible +from an observer derived from tf::ObserverInterface. +*/ +class WorkerView { + + friend class Executor; + + public: + + /** + @brief queries the worker id associated with its parent executor + + A worker id is a unsigned integer in the range [0, N), + where @c N is the number of workers spawned at the construction + time of the executor. + */ + size_t id() const; + + /** + @brief queries the size of the queue (i.e., number of pending tasks to + run) associated with the worker + */ + size_t queue_size() const; + + /** + @brief queries the current capacity of the queue + */ + size_t queue_capacity() const; + + private: + + WorkerView(const Worker&); + WorkerView(const WorkerView&) = default; + + const Worker& _worker; + +}; + +// Constructor +inline WorkerView::WorkerView(const Worker& w) : _worker{w} { +} + +// function: id +inline size_t WorkerView::id() const { + return _worker._id; +} + +// Function: queue_size +inline size_t WorkerView::queue_size() const { + return _worker._wsq.size(); +} + +// Function: queue_capacity +inline size_t WorkerView::queue_capacity() const { + return static_cast(_worker._wsq.capacity()); +} + +// ---------------------------------------------------------------------------- +// Class Definition: WorkerInterface +// ---------------------------------------------------------------------------- + +/** +@class WorkerInterface + +@brief class to configure worker behavior in an executor + +The tf::WorkerInterface class allows users to customize worker properties when creating an executor. +Examples include binding workers to specific CPU cores or +invoking custom methods before and after a worker enters or leaves the work-stealing loop. +When you create an executor, it spawns a set of workers to execute tasks +with the following logic: + +@code{.cpp} +for(size_t n=0; nscheduler_prologue(worker); + + try { + while(1) { + perform_work_stealing_algorithm(); + if(stop) { + break; + } + } + } catch(...) { + exception_ptr = std::current_exception(); + } + + // leaves the scheduling loop and joins this worker thread + // Here, WorkerInterface::scheduler_epilogue is invoked, if any + worker_interface->scheduler_epilogue(worker, exception_ptr); + ); +} +@endcode + +The example below demonstrates the usage of tf::WorkerInterface to affine +a worker to a specific CPU core equal to its id on a Linux platform: + +@code{.cpp} +// affine the given thread to the given core index (linux-specific) +bool affine(std::thread& thread, unsigned int core_id) { + cpu_set_t cpuset; + CPU_ZERO(&cpuset); + CPU_SET(core_id, &cpuset); + pthread_t native_handle = thread.native_handle(); + return pthread_setaffinity_np(native_handle, sizeof(cpu_set_t), &cpuset) == 0; +} + +class CustomWorkerBehavior : public tf::WorkerInterface { + + public: + + // to call before the worker enters the scheduling loop + void scheduler_prologue(tf::Worker& w) override { + printf("worker %lu prepares to enter the work-stealing loop\n", w.id()); + + // now affine the worker to a particular CPU core equal to its id + if(affine(w.thread(), w.id())) { + printf("successfully affines worker %lu to CPU core %lu\n", w.id(), w.id()); + } + else { + printf("failed to affine worker %lu to CPU core %lu\n", w.id(), w.id()); + } + } + + // to call after the worker leaves the scheduling loop + void scheduler_epilogue(tf::Worker& w, std::exception_ptr) override { + printf("worker %lu left the work-stealing loop\n", w.id()); + } +}; + +int main() { + tf::Executor executor(4, tf::make_worker_interface()); + return 0; +} +@endcode + +When running the program, we see the following one possible output: + +@code{.bash} +worker 3 prepares to enter the work-stealing loop +successfully affines worker 3 to CPU core 3 +worker 3 left the work-stealing loop +worker 0 prepares to enter the work-stealing loop +successfully affines worker 0 to CPU core 0 +worker 0 left the work-stealing loop +worker 1 prepares to enter the work-stealing loop +worker 2 prepares to enter the work-stealing loop +successfully affines worker 1 to CPU core 1 +worker 1 left the work-stealing loop +successfully affines worker 2 to CPU core 2 +worker 2 left the work-stealing loop +@endcode + +@attention +tf::WorkerInterface::scheduler_prologue and tf::WorkerInterface::scheduler_epologue +are invoked by each worker simultaneously. + +*/ +class WorkerInterface { + + public: + + /** + @brief default destructor + */ + virtual ~WorkerInterface() = default; + + /** + @brief method to call before a worker enters the scheduling loop + @param worker a reference to the worker + + The method is called by the scheduler before entering the work-stealing loop. + */ + virtual void scheduler_prologue(Worker& worker) = 0; + + /** + @brief method to call after a worker leaves the scheduling loop + @param worker a reference to the worker + @param ptr an pointer to the exception thrown by the scheduling loop + + The method is called by the scheduler after leaving the work-stealing loop. + Any uncaught exception during the worker's execution will be propagated through + the given exception pointer. + */ + virtual void scheduler_epilogue(Worker& worker, std::exception_ptr ptr) = 0; + +}; + +/** +@brief helper function to create an instance derived from tf::WorkerInterface + +@tparam T type derived from tf::WorkerInterface +@tparam ArgsT argument types to construct @c T + +@param args arguments to forward to the constructor of @c T +*/ +template +std::shared_ptr make_worker_interface(ArgsT&&... args) { + static_assert( + std::is_base_of_v, + "T must be derived from WorkerInterface" + ); + return std::make_shared(std::forward(args)...); +} + + + + +} // end of namespact tf ------------------------------------------------------ + + diff --git a/projects/hipfile/third-party/taskflow/core/wsq.hpp b/projects/hipfile/third-party/taskflow/core/wsq.hpp new file mode 100644 index 000000000000..473ef9abf2fe --- /dev/null +++ b/projects/hipfile/third-party/taskflow/core/wsq.hpp @@ -0,0 +1,884 @@ +#pragma once + +#include + +#include "../utility/macros.hpp" +#include "../utility/traits.hpp" + +/** +@file wsq.hpp +@brief work-stealing queue (wsq) include file +*/ + +#ifndef TF_DEFAULT_BOUNDED_TASK_QUEUE_LOG_SIZE + /** + @def TF_DEFAULT_BOUNDED_TASK_QUEUE_LOG_SIZE + + This macro defines the default size of the bounded task queue in Log2. + Bounded task queue is used by each worker. + By default, the value is set to 8, allowing the queue to hold 256 tasks. + */ + #define TF_DEFAULT_BOUNDED_TASK_QUEUE_LOG_SIZE 8 +#endif + +#ifndef TF_DEFAULT_UNBOUNDED_TASK_QUEUE_LOG_SIZE + /** + @def TF_DEFAULT_UNBOUNDED_TASK_QUEUE_LOG_SIZE + + This macro defines the default size of the unbounded task queue in Log2. + Unbounded task queue is used by the executor as an overflow region. + By default, the value is set to 10, allowing the queue to hold 1024 tasks initially. + */ + #define TF_DEFAULT_UNBOUNDED_TASK_QUEUE_LOG_SIZE 10 +#endif + +namespace tf { + +// ---------------------------------------------------------------------------- +// Work-stealing queue steal protocol sentinels +// +// These free functions define the sentinel value used by steal operations +// across all work-stealing queue types (BoundedWSQ, UnboundedWSQ). They encode +// the result of a steal attempt into the return value itself, avoiding any +// out-parameter or separate status type. +// +// For pointer types T: +// wsq_empty_value() = nullptr — queue was empty (or CAS was lost) +// +// The sentinel nullptr is returned whenever steal() cannot deliver a task, +// whether due to the queue being genuinely empty or losing a concurrent CAS +// race to another thief. +// +// For non-pointer types T, std::nullopt is returned to indicate the absence +// of a value. +// +// Queue classes expose this as a static member function (empty_value) that +// delegates here, so callers can use either form. +// ---------------------------------------------------------------------------- + +/** +@brief returns the empty sentinel for work-stealing steal operations + +For pointer types @c T, returns @c nullptr. For non-pointer types, returns +@c std::nullopt. A steal operation returning this value means no task could +be stolen (either the queue was empty or the CAS was lost to another thief). +*/ +template +constexpr auto wsq_empty_value() { + if constexpr (std::is_pointer_v) { + return T{nullptr}; + } else { + return std::optional{std::nullopt}; + } +} + +// ---------------------------------------------------------------------------- +// Unbounded Work-stealing Queue (WSQ) +// ---------------------------------------------------------------------------- + + +/** +@class: UnboundedWSQ + +@tparam T data type (must be a pointer type) + +@brief class to create a lock-free unbounded work-stealing queue + +This class implements the work-stealing queue described in the paper, + +Correct and Efficient Work-Stealing for Weak Memory Models. + +A work-stealing queue supports a single owner thread that performs push and pop +operations, while multiple concurrent thief threads may steal tasks +from the opposite end of the queue. The implementation is designed to +operate correctly under weak memory models and uses atomic operations +with carefully chosen memory orderings to ensure correctness and +scalability. + +@dotfile images/unbounded_wsq.dot + +Unlike bounded queues, this queue automatically grows its internal +storage as needed, allowing it to accommodate an arbitrary number of +tasks without a fixed capacity limit. +*/ +template +class UnboundedWSQ { + + struct Array { + + size_t C; + size_t M; + std::atomic* S; + + explicit Array(size_t c) : + C {c}, + M {c-1}, + S {new std::atomic[C]} { + } + + ~Array() { + delete [] S; + } + + size_t capacity() const noexcept { + return C; + } + + void push(int64_t i, T o) noexcept { + S[i & M].store(o, std::memory_order_relaxed); + } + + T pop(int64_t i) noexcept { + return S[i & M].load(std::memory_order_relaxed); + } + + Array* resize(int64_t b, int64_t t) { + Array* ptr = new Array(2*C); + for(int64_t i=t; i!=b; ++i) { + ptr->push(i, pop(i)); + } + return ptr; + } + + Array* resize(int64_t b, int64_t t, size_t N) { + // assert(N>0); + Array* ptr = new Array(std::bit_ceil(C + N)); + for(int64_t i=t; i!=b; ++i) { + ptr->push(i, pop(i)); + } + return ptr; + } + + }; + + alignas(TF_CACHELINE_SIZE) std::atomic _top; + alignas(TF_CACHELINE_SIZE) std::atomic _bottom; + + // Owner-private cached upper bound on _top. Never read by thieves. + // Because _top is never decremented, the real occupancy can only be + // smaller than what is computed using this cached value, so using it + // for the overflow check is always safe. + int64_t _cached_top {0}; + + // _array on its own cache line: avoids false-sharing with _bottom when + // thieves load _array (consume) after reading _bottom (acquire). + alignas(TF_CACHELINE_SIZE) std::atomic _array; + std::vector _garbage; + + public: + + /** + @brief the return type of queue operations + + `value_type` represents the type returned by `pop` and `steal` operations. + For pointer element types `T`, it is `T` itself and uses `nullptr` to + indicate an empty result. For non-pointer types, it is `std::optional`, + where `std::nullopt` denotes the absence of a value. + + @code{.cpp} + static_assert(std::is_same_v::value_type, std::optional>); + static_assert(std::is_same_v::value_type, nullptr); + @endcode + + This design avoids the overhead of `std::optional` for pointer types + while providing a uniform empty-result semantics. + */ + using value_type = std::conditional_t, T, std::optional>; + + /** + @brief constructs the queue with the given size in the base-2 logarithm + + @param LogSize the base-2 logarithm of the queue size + + @code{.cpp} + tf::UnboundedWSQ wsq(10); + assert(wsq.capacity() == 1024); + @endcode + */ + explicit UnboundedWSQ(int64_t LogSize = TF_DEFAULT_UNBOUNDED_TASK_QUEUE_LOG_SIZE); + + /** + @brief destructs the queue + */ + ~UnboundedWSQ(); + + /** + @brief queries if the queue is empty at the time of this call + + @code{.cpp} + tf::UnboundedWSQ wsq(10); + assert(wsq.empty() == true); + wsq.push(1); + assert(wsq.empty() == false); + @endcode + */ + bool empty() const noexcept; + + /** + @brief queries the number of items at the time of this call + + @code{.cpp} + tf::UnboundedWSQ wsq(10); + assert(wsq.size() == 0); + wsq.push(1); + assert(wsq.size() == 1); + @endcode + */ + size_t size() const noexcept; + + /** + @brief queries the capacity of the queue + + @code{.cpp} + tf::UnboundedWSQ wsq(10); + assert(wsq.capacity() == 1024); + for(int i=0; i<1025; i++){ // insert more than 1024 ints to trigger resizing + wsq.push(i); + } + assert(wsq.capacity() == 2048); + assert(wsq.size() == 1025); + @endcode + */ + size_t capacity() const noexcept; + + /** + @brief inserts an item to the queue + + @param item the item to push to the queue + + This method pushes one item into the queue. + The operation can trigger the queue to resize its capacity if more space is required. + + @code{.cpp} + tf::UnboundedWSQ wsq(10); + assert(wsq.capacity() == 1024); + for(int i=0; i<1025; i++) { // insert more than 1024 items to trigger resizing + wsq.push(i); + } + assert(wsq.capacity() == 2048); + assert(wsq.size() == 1025); + @endcode + + Only the owner thread can insert an item to the queue. + */ + void push(T item); + + /** + @brief tries to insert a batch of items into the queue + + @tparam I input iterator type + @param first iterator to the first item in the batch + @param N number of items to insert beginning at @p first + + This method pushes up to @p N items from the range `[first, first + N)` into the queue. + The operation can trigger the queue to resize its capacity + if more space is required. + The iterator `first` is updated in place and will point to the next uninserted element after the call. + + Bulk insertion is often faster than inserting elements one by one because it requires fewer atomic operations. + + @code{.cpp} + tf::UnboundedWSQ wsq(10); + assert(wsq.capacity() == 1024); + std::vector vec(1025, 1); + auto first = vec.data(); + wsq.bulk_push(first, vec.size()); + assert(wsq.capacity() == 2048); + assert(wsq.size() == vec.size()); + assert(std::distance(vec.begin(), first) == 1025); + @endcode + + Only the owner thread can insert an item to the queue. + */ + template + void bulk_push(I& first, size_t N); + + /** + @brief pops out an item from the queue + + This method pops an item from the queue. + If the queue is empty, empty_value() is returned. + The elements popped out from the queue follow a last-in-first-out (LIFO) order. + + @code{.cpp} + tf::UnboundedWSQ wsq(10); + wsq.push(1); + wsq.push(2); + wsq.push(3); + assert(wsq.pop().value() = 3); + assert(wsq.pop().value() = 2); + assert(wsq.pop().value() = 1); + assert(wsq.pop() == std::nullopt); + @endcode + + Only the owner thread can pop out an item from the queue. + */ + value_type pop(); + + /** + @brief steals an item from the queue + + Any threads can try to steal an item from the queue. + The return can be an empty_value() if this operation failed. + The elements stolen from the queue follow a first-in-first-out (FIFO) order. + + @code{.cpp} + tf::UnboundedWSQ wsq(10); + wsq.push(1); + wsq.push(2); + wsq.push(3); + assert(wsq.steal().value() = 1); + assert(wsq.steal().value() = 2); + assert(wsq.steal().value() = 3); + assert(wsq.steal() == std::nullopt); + @endcode + + Multiple threads can simultaneously steal items from the queue. + */ + value_type steal(); + + /** + @brief returns the empty sentinel value for the queue element type + + This function provides a type-appropriate empty value used to indicate + that a pop or steal operation failed. For pointer types, the empty value + is `nullptr` of type `T`; for non-pointer types, it is `std::nullopt` of type `std::optional`. + + The function is implemented as a `constexpr` helper to avoid additional + storage, runtime overhead, or code duplication across queue operations. + + @return an empty `value_type` representing the absence of an element. + */ + static constexpr auto empty_value() { return wsq_empty_value(); } + + private: + + Array* _resize_array(Array* a, int64_t b, int64_t t); + Array* _resize_array(Array* a, int64_t b, int64_t t, size_t N); +}; + +// Constructor +template +UnboundedWSQ::UnboundedWSQ(int64_t LogSize) { + _top.store(0, std::memory_order_relaxed); + _bottom.store(0, std::memory_order_relaxed); + _array.store(new Array{(size_t{1} << LogSize)}, std::memory_order_relaxed); + _garbage.reserve(32); +} + +// Destructor +template +UnboundedWSQ::~UnboundedWSQ() { + for(auto a : _garbage) { + delete a; + } + delete _array.load(); +} + +// Function: empty +template +bool UnboundedWSQ::empty() const noexcept { + int64_t t = _top.load(std::memory_order_relaxed); + int64_t b = _bottom.load(std::memory_order_relaxed); + return (b <= t); +} + +// Function: size +template +size_t UnboundedWSQ::size() const noexcept { + int64_t t = _top.load(std::memory_order_relaxed); + int64_t b = _bottom.load(std::memory_order_relaxed); + return static_cast(b >= t ? b - t : 0); +} + +// Function: push +template +void UnboundedWSQ::push(T o) { + + int64_t b = _bottom.load(std::memory_order_relaxed); + Array* a = _array.load(std::memory_order_relaxed); + + // queue is full with one additional item (b-t+1) + if(a->capacity() < static_cast(b - _cached_top + 1)) [[unlikely]] { + _cached_top = _top.load(std::memory_order_acquire); + if(a->capacity() < static_cast(b - _cached_top + 1)) [[unlikely]] { + a = _resize_array(a, b, _cached_top); + } + } + + a->push(b, o); + std::atomic_thread_fence(std::memory_order_release); + + // original paper uses relaxed here but tsa complains + _bottom.store(b + 1, std::memory_order_release); +} + +// Function: bulk_push +template +template +void UnboundedWSQ::bulk_push(I& first, size_t N) { + + if(N == 0) return; + + int64_t b = _bottom.load(std::memory_order_relaxed); + Array* a = _array.load(std::memory_order_relaxed); + + // queue is full with N additional items + if((b - _cached_top + N) > a->capacity()) [[unlikely]] { + _cached_top = _top.load(std::memory_order_acquire); + if((b - _cached_top + N) > a->capacity()) [[unlikely]] { + a = _resize_array(a, b, _cached_top, N); + } + } + + for(size_t i=0; ipush(b++, *first++); + } + std::atomic_thread_fence(std::memory_order_release); + + // original paper uses relaxed here but tsa complains + _bottom.store(b, std::memory_order_release); +} + +// Function: pop +template +typename UnboundedWSQ::value_type +UnboundedWSQ::pop() { + + int64_t b = _bottom.load(std::memory_order_relaxed) - 1; + Array* a = _array.load(std::memory_order_relaxed); + _bottom.store(b, std::memory_order_relaxed); + std::atomic_thread_fence(std::memory_order_seq_cst); + int64_t t = _top.load(std::memory_order_relaxed); + + //T item {nullptr}; + auto item = empty_value(); + + if(t <= b) { + item = a->pop(b); + if(t == b) { + // the last item just got stolen + if(!_top.compare_exchange_strong(t, t+1, std::memory_order_seq_cst, + std::memory_order_relaxed)) { + //item = nullptr; + item = empty_value(); + } + _bottom.store(b + 1, std::memory_order_relaxed); + } + } + else { + _bottom.store(b + 1, std::memory_order_relaxed); + } + + return item; +} + +// Function: steal +template +typename UnboundedWSQ::value_type +UnboundedWSQ::steal() { + + int64_t t = _top.load(std::memory_order_acquire); + std::atomic_thread_fence(std::memory_order_seq_cst); + int64_t b = _bottom.load(std::memory_order_acquire); + + //T item {nullptr}; + auto item = empty_value(); + + if(t < b) { + Array* a = _array.load(std::memory_order_consume); + item = a->pop(t); + if(!_top.compare_exchange_strong(t, t+1, + std::memory_order_seq_cst, + std::memory_order_relaxed)) { + //return nullptr; + return empty_value(); + } + } + + return item; +} + + +// Function: capacity +template +size_t UnboundedWSQ::capacity() const noexcept { + return _array.load(std::memory_order_relaxed)->capacity(); +} + +template +typename UnboundedWSQ::Array* +UnboundedWSQ::_resize_array(Array* a, int64_t b, int64_t t) { + Array* tmp = a->resize(b, t); + _garbage.push_back(a); + // Note: the original paper using relaxed causes t-san to complain + _array.store(tmp, std::memory_order_release); + return tmp; +} + +template +typename UnboundedWSQ::Array* +UnboundedWSQ::_resize_array(Array* a, int64_t b, int64_t t, size_t N) { + Array* tmp = a->resize(b, t, N); + _garbage.push_back(a); + // Note: the original paper using relaxed causes t-san to complain + _array.store(tmp, std::memory_order_release); + return tmp; +} + +// ---------------------------------------------------------------------------- +// Bounded Work-stealing Queue (WSQ) +// ---------------------------------------------------------------------------- + +/** +@class: BoundedWSQ + +@tparam T data type +@tparam LogSize the base-2 logarithm of the queue size + +@brief class to create a lock-free bounded work-stealing queue + +This class implements the work-stealing queue described in the paper, + +Correct and Efficient Work-Stealing for Weak Memory Models. + +A work-stealing queue supports a single owner thread that performs push and pop +operations, while multiple concurrent thief threads may steal tasks +from the opposite end of the queue. The implementation is designed to +operate correctly under weak memory models and uses atomic operations +with carefully chosen memory orderings to ensure correctness and +scalability. + +@dotfile images/bounded_wsq.dot + +The queue has a fixed capacity determined at construction time and +does not grow dynamically. When the queue is full, push operations +may fail or require external handling. +*/ +template +class BoundedWSQ { + + constexpr static size_t BufferSize = size_t{1} << LogSize; + constexpr static size_t BufferMask = (BufferSize - 1); + + static_assert((BufferSize >= 2) && ((BufferSize & (BufferSize - 1)) == 0)); + + alignas(TF_CACHELINE_SIZE) std::atomic _top {0}; + alignas(TF_CACHELINE_SIZE) std::atomic _bottom {0}; + alignas(TF_CACHELINE_SIZE) std::atomic _buffer[BufferSize]; + + public: + + /** + @brief the return type of queue operations + + `value_type` represents the type returned by `pop` and `steal` operations. + For pointer element types `T`, it is `T` itself and uses `nullptr` to + indicate an empty result. For non-pointer types, it is `std::optional`, + where `std::nullopt` denotes the absence of a value. + + @code{.cpp} + static_assert(std::is_same_v::value_type, std::optional>); + static_assert(std::is_same_v::value_type, nullptr); + @endcode + + This design avoids the overhead of `std::optional` for pointer types + while providing a uniform empty-result semantics. + */ + using value_type = std::conditional_t, T, std::optional>; + + /** + @brief constructs the queue with a given capacity + + @code{.cpp} + tf::BoundedWSQ wsq; + static_assert(wsq.capacity() == 1024); + @endcode + */ + BoundedWSQ() = default; + + /** + @brief destructs the queue + */ + ~BoundedWSQ() = default; + + /** + @brief queries if the queue is empty at the time of this call + + @code{.cpp} + tf::BoundedWSQ wsq; + assert(wsq.empty() == true); + wsq.push(1); + assert(wsq.empty() == false); + @endcode + */ + bool empty() const noexcept; + + /** + @brief queries the number of items at the time of this call + + @code{.cpp} + tf::BoundedWSQ wsq; + assert(wsq.size() == 0); + wsq.push(1); + assert(wsq.size() == 1); + @endcode + */ + size_t size() const noexcept; + + /** + @brief queries the capacity of the queue + + The capacity of a bounded work-stealing queue is decided at compile time. + + @code{.cpp} + tf::BoundedWSQ wsq; + static_assert(wsq.capacity() == 1024); + @endcode + */ + constexpr size_t capacity() const; + + /** + @brief tries to insert an item to the queue + + @tparam O data type + @param item the item to perfect-forward to the queue + @return `true` if the insertion succeed or `false` (queue is full) + + This method attempts to push one item into the queue. + If the operation succeed, it returns `true` or `false` otherwise. + + @code{.cpp} + tf::BoundedWSQ wsq; + static_assert(wsq.capacity() == 1024); + for(int i=0; i<1024; i++) { + assert(wsq.try_push(i) == true); + } + assert(wsq.size() == 1024); + assert(wsq.try_push(0) == false); + @endcode + + Only the owner thread can insert an item to the queue. + */ + template + bool try_push(O&& item); + + /** + @brief tries to insert a batch of items into the queue + + @tparam I input iterator type + @param first iterator to the first item in the batch + @param N number of items to insert beginning at @p first + @return the number of items successfully inserted + + This method attempts to push up to @p N items from the range + `[first, first + N)` into the queue. Insertion stops early if the + queue becomes full. + The iterator `first` is updated in place and will point to the next uninserted element after the call. + + Bulk insertion is often faster than inserting elements one by one because it requires fewer atomic operations. + + @code{.cpp} + tf::BoundedWSQ wsq; + static_assert(wsq.capacity() == 1024); + std::vector vec(1030, 1); + auto first = vec.begin(); + assert(wsq.try_bulk_push(first, vec.size()) == wsq.capacity()); + assert(std::distance(vec.begin(), first) == wsq.capacity()); + @endcode + + Only the owner thread can insert items into the queue. + */ + template + size_t try_bulk_push(I& first, size_t N); + + /** + @brief pops out an item from the queue + + The method pops an item out of the queue based on a last-in-first-out (LIFO) order. + The return can be an empty_value() if this operation failed (empty queue). + + @code{.cpp} + tf::BoundedWSQ wsq; + wsq.push(1); + wsq.push(2); + wsq.push(3); + assert(wsq.pop().value() = 3); + assert(wsq.pop().value() = 2); + assert(wsq.pop().value() = 1); + assert(wsq.pop() == std::nullopt); + @endcode + + Only the owner thread can pop out an item from the queue. + */ + value_type pop(); + + /** + @brief steals an item from the queue + + Any threads can try to steal an item from the queue. + The return can be an empty_value() if this operation failed. + The elements stolen from the queue follow a first-in-first-out (FIFO) order. + + @code{.cpp} + tf::BoundedWSQ wsq; + wsq.push(1); + wsq.push(2); + wsq.push(3); + assert(wsq.steal().value() = 1); + assert(wsq.steal().value() = 2); + assert(wsq.steal().value() = 3); + assert(wsq.steal() == std::nullopt); + @endcode + + Multiple threads can simultaneously steal items from the queue. + */ + value_type steal(); + + + /** + @brief returns the empty sentinel value for the queue element type + + This function provides a type-appropriate empty value used to indicate + that a pop or steal operation failed. For pointer types, the empty value + is `nullptr` of type `T`; for non-pointer types, it is `std::nullopt` of type `std::optional`. + + The function is implemented as a `constexpr` helper to avoid additional + storage, runtime overhead, or code duplication across queue operations. + + @return an empty `value_type` representing the absence of an element. + */ + static constexpr auto empty_value() { return wsq_empty_value(); } +}; + +// Function: empty +template +bool BoundedWSQ::empty() const noexcept { + int64_t t = _top.load(std::memory_order_relaxed); + int64_t b = _bottom.load(std::memory_order_relaxed); + return b <= t; +} + +// Function: size +template +size_t BoundedWSQ::size() const noexcept { + int64_t t = _top.load(std::memory_order_relaxed); + int64_t b = _bottom.load(std::memory_order_relaxed); + return static_cast(b >= t ? b - t : 0); +} + +// Function: try_push +template +template +bool BoundedWSQ::try_push(O&& o) { + + int64_t b = _bottom.load(std::memory_order_relaxed); + int64_t t = _top.load(std::memory_order_acquire); + + // queue is full with one additional item (b-t+1) + if(static_cast(b - t + 1) > BufferSize) [[unlikely]] { + return false; + } + + _buffer[b & BufferMask].store(std::forward(o), std::memory_order_relaxed); + + std::atomic_thread_fence(std::memory_order_release); + + // original paper uses relaxed here but tsa complains + _bottom.store(b + 1, std::memory_order_release); + + return true; +} + +// Function: try_bulk_push +template +template +size_t BoundedWSQ::try_bulk_push(I& first, size_t N) { + + if(N == 0) return 0; + + int64_t b = _bottom.load(std::memory_order_relaxed); + int64_t t = _top.load(std::memory_order_acquire); + + size_t r = BufferSize - (b - t); // remaining capacity + size_t n = std::min(N, r); // number of pushable elements + + if(n > 0) { + // push n elements into the queue + for(size_t i=0; i +typename BoundedWSQ::value_type +BoundedWSQ::pop() { + + int64_t b = _bottom.load(std::memory_order_relaxed) - 1; + _bottom.store(b, std::memory_order_relaxed); + std::atomic_thread_fence(std::memory_order_seq_cst); + int64_t t = _top.load(std::memory_order_relaxed); + + //T item {nullptr}; + auto item = empty_value(); + + if(t <= b) { + item = _buffer[b & BufferMask].load(std::memory_order_relaxed); + if(t == b) { + // the last item just got stolen + if(!_top.compare_exchange_strong(t, t+1, + std::memory_order_seq_cst, + std::memory_order_relaxed)) { + //item = nullptr; + item = empty_value(); + } + _bottom.store(b + 1, std::memory_order_relaxed); + } + } + else { + _bottom.store(b + 1, std::memory_order_relaxed); + } + + return item; +} + +// Function: steal +template +typename BoundedWSQ::value_type +BoundedWSQ::steal() { + int64_t t = _top.load(std::memory_order_acquire); + std::atomic_thread_fence(std::memory_order_seq_cst); + int64_t b = _bottom.load(std::memory_order_acquire); + + //T item{nullptr}; + auto item = empty_value(); + + if(t < b) { + item = _buffer[t & BufferMask].load(std::memory_order_relaxed); + if(!_top.compare_exchange_strong(t, t+1, + std::memory_order_seq_cst, + std::memory_order_relaxed)) { + //return nullptr; + return empty_value(); + } + } + + return item; +} + + +// Function: capacity +template +constexpr size_t BoundedWSQ::capacity() const { + return BufferSize; +} + + +} // end of namespace tf ----------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/cuda/algorithm/find.hpp b/projects/hipfile/third-party/taskflow/cuda/algorithm/find.hpp new file mode 100644 index 000000000000..f3446669509d --- /dev/null +++ b/projects/hipfile/third-party/taskflow/cuda/algorithm/find.hpp @@ -0,0 +1,294 @@ +#pragma once + +#include "for_each.hpp" +#include "reduce.hpp" + +/** +@file taskflow/cuda/algorithm/find.hpp +@brief cuda find algorithms include file +*/ + +namespace tf::detail { + +/** @private */ +template +struct cudaFindPair { + + T key; + unsigned index; + + __device__ operator unsigned () const { return index; } +}; + +/** @private */ +template +void cuda_find_if_loop(P&& p, I input, unsigned count, unsigned* idx, U pred) { + + if(count == 0) { + cuda_single_task(p, [=] __device__ () { *idx = 0; }); + return; + } + + using E = std::decay_t

; + + auto B = (count + E::nv - 1) / E::nv; + + // set the index to the maximum + cuda_single_task(p, [=] __device__ () { *idx = count; }); + + // launch the kernel to atomic-find the minimum + cuda_kernel<<>>([=] __device__ (auto tid, auto bid) { + + __shared__ unsigned shm_id; + + if(!tid) { + shm_id = count; + } + + __syncthreads(); + + auto tile = cuda_get_tile(bid, E::nv, count); + + auto x = cuda_mem_to_reg_strided( + input + tile.begin, tid, tile.count() + ); + + auto id = count; + + for(unsigned i=0; i::Storage shm; + + //id = cudaBlockReduce()( + // tid, + // id, + // shm, + // (tile.count() < E::nt ? tile.count() : E::nt), + // cuda_minimum{}, + // false + //); + + // only need the minimum id + atomicMin(&shm_id, id); + __syncthreads(); + + // reduce all to the global memory + if(!tid) { + atomicMin(idx, shm_id); + //atomicMin(idx, id); + } + }); +} + +/** @private */ +template +void cuda_min_element_loop( + P&& p, I input, unsigned count, unsigned* idx, O op, void* ptr +) { + + if(count == 0) { + cuda_single_task(p, [=] __device__ () { *idx = 0; }); + return; + } + + using T = cudaFindPair::value_type>; + + cuda_uninitialized_reduce_loop(p, + cuda_make_load_iterator([=]__device__(auto i){ + return T{*(input+i), i}; + }), + count, + idx, + [=] __device__ (const auto& a, const auto& b) { + return op(a.key, b.key) ? a : b; + }, + ptr + ); +} + +/** @private */ +template +void cuda_max_element_loop( + P&& p, I input, unsigned count, unsigned* idx, O op, void* ptr +) { + + if(count == 0) { + cuda_single_task(p, [=] __device__ () { *idx = 0; }); + return; + } + + using T = cudaFindPair::value_type>; + + cuda_uninitialized_reduce_loop(p, + cuda_make_load_iterator([=]__device__(auto i){ + return T{*(input+i), i}; + }), + count, + idx, + [=] __device__ (const auto& a, const auto& b) { + return op(a.key, b.key) ? b : a; + }, + ptr + ); +} + +} // end of namespace tf::detail --------------------------------------------- + +namespace tf { + + +// ---------------------------------------------------------------------------- +// cuda_find_if +// ---------------------------------------------------------------------------- + +/** +@brief finds the index of the first element that satisfies the given criteria + +@tparam P execution policy type +@tparam I input iterator type +@tparam U unary operator type + +@param p execution policy +@param first iterator to the beginning of the range +@param last iterator to the end of the range +@param idx pointer to the index of the found element +@param op unary operator which returns @c true for the required element + +The function launches kernels asynchronously to find the index @c idx of the +first element in the range [first, last) +such that op(*(first+idx)) is true. +This is equivalent to the parallel execution of the following loop: + +@code{.cpp} +unsigned idx = 0; +for(; first != last; ++first, ++idx) { + if (p(*first)) { + return idx; + } +} +return idx; +@endcode +*/ +template +void cuda_find_if( + P&& p, I first, I last, unsigned* idx, U op +) { + detail::cuda_find_if_loop(p, first, std::distance(first, last), idx, op); +} + +// ---------------------------------------------------------------------------- +// cuda_min_element +// ---------------------------------------------------------------------------- + +// Function: min-element_bufsz +template +template +unsigned cudaExecutionPolicy::min_element_bufsz(unsigned count) { + return reduce_bufsz>(count); +} + +/** +@brief finds the index of the minimum element in a range + +@tparam P execution policy type +@tparam I input iterator type +@tparam O comparator type + +@param p execution policy object +@param first iterator to the beginning of the range +@param last iterator to the end of the range +@param idx solution index of the minimum element +@param op comparison function object +@param buf pointer to the buffer + +The function launches kernels asynchronously to find +the smallest element in the range [first, last) +using the given comparator @c op. +You need to provide a buffer that holds at least +tf::cuda_min_element_bufsz bytes for internal use. +The function is equivalent to a parallel execution of the following loop: + +@code{.cpp} +if(first == last) { + return 0; +} +auto smallest = first; +for (++first; first != last; ++first) { + if (op(*first, *smallest)) { + smallest = first; + } +} +return std::distance(first, smallest); +@endcode +*/ +template +void cuda_min_element(P&& p, I first, I last, unsigned* idx, O op, void* buf) { + detail::cuda_min_element_loop( + p, first, std::distance(first, last), idx, op, buf + ); +} + +// ---------------------------------------------------------------------------- +// cuda_max_element +// ---------------------------------------------------------------------------- + +// Function: max_element_bufsz +template +template +unsigned cudaExecutionPolicy::max_element_bufsz(unsigned count) { + return reduce_bufsz>(count); +} + +/** +@brief finds the index of the maximum element in a range + +@tparam P execution policy type +@tparam I input iterator type +@tparam O comparator type + +@param p execution policy object +@param first iterator to the beginning of the range +@param last iterator to the end of the range +@param idx solution index of the maximum element +@param op comparison function object +@param buf pointer to the buffer + +The function launches kernels asynchronously to find +the largest element in the range [first, last) +using the given comparator @c op. +You need to provide a buffer that holds at least +tf::cuda_max_element_bufsz bytes for internal use. +The function is equivalent to a parallel execution of the following loop: + +@code{.cpp} +if(first == last) { + return 0; +} +auto largest = first; +for (++first; first != last; ++first) { + if (op(*largest, *first)) { + largest = first; + } +} +return std::distance(first, largest); +@endcode +*/ +template +void cuda_max_element(P&& p, I first, I last, unsigned* idx, O op, void* buf) { + detail::cuda_max_element_loop( + p, first, std::distance(first, last), idx, op, buf + ); +} + + +} // end of namespace tf ----------------------------------------------------- + + diff --git a/projects/hipfile/third-party/taskflow/cuda/algorithm/for_each.hpp b/projects/hipfile/third-party/taskflow/cuda/algorithm/for_each.hpp new file mode 100644 index 000000000000..551cca178d69 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/cuda/algorithm/for_each.hpp @@ -0,0 +1,109 @@ +#pragma once + +#include "../cudaflow.hpp" + +/** +@file taskflow/cuda/algorithm/for_each.hpp +@brief cuda parallel-iteration algorithms include file +*/ + +namespace tf { + +namespace detail { + +/** +@private +*/ +template +__global__ void cuda_for_each_kernel(I first, unsigned count, C c) { + auto tid = threadIdx.x; + auto bid = blockIdx.x; + auto tile = cuda_get_tile(bid, E::nv, count); + cuda_strided_iterate( + [=](auto, auto j) { + c(*(first + tile.begin + j)); + }, + tid, tile.count() + ); +} + +/** @private */ +template +__global__ void cuda_for_each_index_kernel(I first, I inc, unsigned count, C c) { + auto tid = threadIdx.x; + auto bid = blockIdx.x; + auto tile = cuda_get_tile(bid, E::nv, count); + cuda_strided_iterate( + [=]__device__(auto, auto j) { + c(first + inc*(tile.begin+j)); + }, + tid, tile.count() + ); +} + +} // end of namespace detail ------------------------------------------------- + +// ---------------------------------------------------------------------------- +// cudaFlow: for_each, for_each_index +// ---------------------------------------------------------------------------- + +// Function: for_each +template +template +cudaTask cudaGraphBase::for_each(I first, I last, C c) { + + unsigned count = std::distance(first, last); + + return kernel( + E::num_blocks(count), E::nt, 0, + detail::cuda_for_each_kernel, first, count, c + ); +} + +// Function: for_each +template +template +void cudaGraphExecBase::for_each(cudaTask task, I first, I last, C c) { + + unsigned count = std::distance(first, last); + + kernel(task, + E::num_blocks(count), E::nt, 0, + detail::cuda_for_each_kernel, first, count, c + ); +} + +// Function: for_each_index +template +template +cudaTask cudaGraphBase::for_each_index(I first, I last, I inc, C c) { + + unsigned count = distance(first, last, inc); + + return kernel( + E::num_blocks(count), E::nt, 0, + detail::cuda_for_each_index_kernel, first, inc, count, c + ); +} + +// Function: for_each_index +template +template +void cudaGraphExecBase::for_each_index(cudaTask task, I first, I last, I inc, C c) { + + unsigned count = distance(first, last, inc); + + return kernel(task, + E::num_blocks(count), E::nt, 0, + detail::cuda_for_each_index_kernel, first, inc, count, c + ); +} + + +} // end of namespace tf ----------------------------------------------------- + + + + + + diff --git a/projects/hipfile/third-party/taskflow/cuda/algorithm/matmul.hpp b/projects/hipfile/third-party/taskflow/cuda/algorithm/matmul.hpp new file mode 100644 index 000000000000..d0f6620a1997 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/cuda/algorithm/matmul.hpp @@ -0,0 +1,57 @@ +#pragma once + +#include "../cudaflow.hpp" + +namespace tf { + +// ---------------------------------------------------------------------------- +// row-major matrix multiplication +// ---------------------------------------------------------------------------- + +template +__global__ void cuda_matmul( + const T* A, + const T* B, + T* C, + size_t M, + size_t K, + size_t N +) { + __shared__ T A_tile[32][32]; + __shared__ T B_tile[32][32]; + + size_t x = blockIdx.x * blockDim.x + threadIdx.x; + size_t y = blockIdx.y * blockDim.y + threadIdx.y; + + T res = 0; + + for(size_t k = 0; k < K; k += 32) { + if((threadIdx.x + k) < K && y < M) { + A_tile[threadIdx.y][threadIdx.x] = A[y * K + threadIdx.x + k]; + } + else{ + A_tile[threadIdx.y][threadIdx.x] = 0; + } + + if((threadIdx.y + k) < K && x < N) { + B_tile[threadIdx.y][threadIdx.x] = B[(threadIdx.y + k) * N + x]; + } + else{ + B_tile[threadIdx.y][threadIdx.x] = 0; + } + + __syncthreads(); + + for(size_t i = 0; i < 32; ++i) { + res += A_tile[threadIdx.y][i] * B_tile[i][threadIdx.x]; + } + __syncthreads(); + } + + if(x < N && y < M) { + C[y * N + x] = res; + } + +} + +} // end of namespace tf --------------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/cuda/algorithm/merge.hpp b/projects/hipfile/third-party/taskflow/cuda/algorithm/merge.hpp new file mode 100644 index 000000000000..d325491c8b6a --- /dev/null +++ b/projects/hipfile/third-party/taskflow/cuda/algorithm/merge.hpp @@ -0,0 +1,585 @@ +#pragma once + +#include "../cudaflow.hpp" + +/** +@file taskflow/cuda/algorithm/merge.hpp +@brief CUDA merge algorithm include file +*/ + +namespace tf::detail { + +/** +@private +@brief merge bound type +*/ +enum class cudaMergeBoundType { + LOWER, + UPPER +}; + +/** @private */ +template +struct cudaMergePair { + cudaArray keys; + cudaArray indices; +}; + +/** @private */ +struct cudaMergeRange { + unsigned a_begin, a_end, b_begin, b_end; + + __device__ unsigned a_count() const { return a_end - a_begin; } + __device__ unsigned b_count() const { return b_end - b_begin; } + __device__ unsigned total() const { return a_count() + b_count(); } + + __device__ cudaRange a_range() const { + return cudaRange { a_begin, a_end }; + } + __device__ cudaRange b_range() const { + return cudaRange { b_begin, b_end }; + } + + __device__ cudaMergeRange to_local() const { + return cudaMergeRange { 0, a_count(), a_count(), total() }; + } + + // Partition from mp to the end. + __device__ cudaMergeRange partition(unsigned mp0, unsigned diag) const { + return cudaMergeRange { a_begin + mp0, a_end, b_begin + diag - mp0, b_end }; + } + + // Partition from mp0 to mp1. + __device__ cudaMergeRange partition(unsigned mp0, unsigned diag0, + unsigned mp1, unsigned diag1) const { + return cudaMergeRange { + a_begin + mp0, + a_begin + mp1, + b_begin + diag0 - mp0, + b_begin + diag1 - mp1 + }; + } + + __device__ bool a_valid() const { + return a_begin < a_end; + } + + __device__ bool b_valid() const { + return b_begin < b_end; + } +}; + +/** @private */ +template< + cudaMergeBoundType bounds = cudaMergeBoundType::LOWER, + typename a_keys_it, typename b_keys_it, typename comp_t +> +__device__ auto cuda_merge_path( + a_keys_it a_keys, unsigned a_count, + b_keys_it b_keys, unsigned b_count, + unsigned diag, comp_t comp +) { + + unsigned beg = (diag > b_count) ? diag - b_count : 0; + unsigned end = diag < a_count ? diag : a_count; + + while(beg < end) { + auto mid = (beg + end) / 2; + auto a_key = a_keys[mid]; + auto b_key = b_keys[diag - 1 - mid]; + bool pred = (cudaMergeBoundType::UPPER == bounds) ? + comp(a_key, b_key) : + !comp(b_key, a_key); + + if(pred) beg = mid + 1; + else end = mid; + } + return beg; +} + +/** @private */ +template +__device__ auto cuda_merge_path( + keys_it keys, cudaMergeRange range, unsigned diag, comp_t comp +) { + + return cuda_merge_path( + keys + range.a_begin, range.a_count(), + keys + range.b_begin, range.b_count(), + diag, comp); +} + +/** @private */ +template +__device__ bool cuda_merge_predicate( + T a_key, T b_key, cudaMergeRange range, comp_t comp +) { + + bool p; + if(range_check && !range.a_valid()) { + p = false; + } + else if(range_check && !range.b_valid()) { + p = true; + } + else { + p = (cudaMergeBoundType::UPPER == bounds) ? comp(a_key, b_key) : + !comp(b_key, a_key); + } + return p; +} + +/** @private */ +inline __device__ auto cuda_compute_merge_range( + unsigned a_count, unsigned b_count, + unsigned partition, unsigned spacing, + unsigned mp0, unsigned mp1 +) { + + auto diag0 = spacing * partition; + auto diag1 = min(a_count + b_count, diag0 + spacing); + + return cudaMergeRange { mp0, mp1, diag0 - mp0, diag1 - mp1 }; +} + +/** +@private + +Specialization that emits just one LD instruction. Can only reliably used +with raw pointer types. Fixed not to use pointer arithmetic so that +we don't get undefined behaviors with unaligned types. +*/ +template +__device__ auto cuda_load_two_streams_reg( + const T* a, unsigned a_count, const T* b, unsigned b_count, unsigned tid +) { + + b -= a_count; + cudaArray x; + cuda_strided_iterate([&](auto i, auto index) { + const T* p = (index >= a_count) ? b : a; + x[i] = p[index]; + }, tid, a_count + b_count); + + return x; +} + +/** @private */ +template +__device__ +std::enable_if_t< + !(std::is_pointer::value && std::is_pointer::value), + cudaArray +> load_two_streams_reg(a_it a, unsigned a_count, b_it b, unsigned b_count, unsigned tid) { + b -= a_count; + cudaArray x; + cuda_strided_iterate([&](auto i, auto index) { + x[i] = (index < a_count) ? a[index] : b[index]; + }, tid, a_count + b_count); + return x; +} + +/** @private */ +template +__device__ void cuda_load_two_streams_shared(A a, unsigned a_count, + B b, unsigned b_count, unsigned tid, T (&shared)[S], bool sync = true +) { + // Load into register then make an unconditional strided store into memory. + auto x = cuda_load_two_streams_reg(a, a_count, b, b_count, tid); + cuda_reg_to_shared_strided(x, tid, shared, sync); +} + +/** @private */ +template +__device__ auto cuda_gather_two_streams_strided(const T* a, + unsigned a_count, const T* b, unsigned b_count, cudaArray indices, + unsigned tid) { + + ptrdiff_t b_offset = b - a - a_count; + auto count = a_count + b_count; + + cudaArray x; + cuda_strided_iterate([&](auto i, auto j) { + ptrdiff_t gather = indices[i]; + if(gather >= a_count) gather += b_offset; + x[i] = a[gather]; + }, tid, count); + + return x; +} + +/** @private */ +template +__device__ +std::enable_if_t< + !(std::is_pointer::value && std::is_pointer::value), + cudaArray +> cuda_gather_two_streams_strided(a_it a, + unsigned a_count, b_it b, unsigned b_count, cudaArray indices, unsigned tid) { + + b -= a_count; + cudaArray x; + cuda_strided_iterate([&](auto i, auto j) { + x[i] = (indices[i] < a_count) ? a[indices[i]] : b[indices[i]]; + }, tid, a_count + b_count); + + return x; +} + +/** @private */ +template +__device__ void cuda_transfer_two_streams_strided( + a_it a, unsigned a_count, b_it b, unsigned b_count, + cudaArray indices, unsigned tid, c_it c +) { + + using T = typename std::iterator_traits::value_type; + auto x = cuda_gather_two_streams_strided( + a, a_count, b, b_count, indices, tid + ); + + cuda_reg_to_mem_strided(x, tid, a_count + b_count, c); +} + + +/** +@private + +This function must be able to dereference keys[a_begin] and keys[b_begin], +no matter the indices for each. The caller should allocate at least +nt * vt + 1 elements for +*/ +template +__device__ auto cuda_serial_merge( + const T* keys_shared, cudaMergeRange range, comp_t comp, bool sync = true +) { + + auto a_key = keys_shared[range.a_begin]; + auto b_key = keys_shared[range.b_begin]; + + cudaMergePair merge_pair; + cuda_iterate([&](auto i) { + bool p = cuda_merge_predicate(a_key, b_key, range, comp); + auto index = p ? range.a_begin : range.b_begin; + + merge_pair.keys[i] = p ? a_key : b_key; + merge_pair.indices[i] = index; + + T c_key = keys_shared[++index]; + if(p) a_key = c_key, range.a_begin = index; + else b_key = c_key, range.b_begin = index; + }); + + if(sync) __syncthreads(); + return merge_pair; +} + +/** +@private + +Load arrays a and b from global memory and merge unsignedo register. +*/ +template +__device__ auto block_merge_from_mem( + a_it a, b_it b, cudaMergeRange range_mem, unsigned tid, comp_t comp, T (&keys_shared)[S] +) { + + static_assert(S >= nt * vt + 1, + "block_merge_from_mem requires temporary storage of at " + "least nt * vt + 1 items"); + + // Load the data into shared memory. + cuda_load_two_streams_shared( + a + range_mem.a_begin, range_mem.a_count(), + b + range_mem.b_begin, range_mem.b_count(), + tid, keys_shared, true + ); + + // Run a merge path to find the start of the serial merge for each thread. + auto range_local = range_mem.to_local(); + auto diag = vt * tid; + auto mp = cuda_merge_path(keys_shared, range_local, diag, comp); + + // Compute the ranges of the sources in shared memory. The end iterators + // of the range are inaccurate, but still facilitate exact merging, because + // only vt elements will be merged. + auto merged = cuda_serial_merge( + keys_shared, range_local.partition(mp, diag), comp + ); + + return merged; +}; + +/** @private */ +template +void cuda_merge_path_partitions( + P&& p, + a_keys_it a, unsigned a_count, + b_keys_it b, unsigned b_count, + unsigned spacing, + comp_t comp, + unsigned* buf +) { + + //int num_partitions = (int)div_up(a_count + b_count, spacing) + 1; + + unsigned num_partitions = (a_count + b_count + spacing - 1) / spacing + 1; + + const unsigned nt = 128; + const unsigned vt = 1; + const unsigned nv = nt * vt; + + unsigned B = (num_partitions + nv - 1) / nv; // nt = 128, vt = 1 + + cuda_kernel<<>>([=]__device__(auto tid, auto bid) { + auto range = cuda_get_tile(bid, nt * vt, num_partitions); + cuda_strided_iterate([=](auto, auto j) { + auto index = range.begin + j; + auto diag = min(spacing * index, a_count + b_count); + buf[index] = cuda_merge_path(a, a_count, b, b_count, diag, comp); + }, tid, range.count()); + }); +} + +//template +//auto load_balance_partitions(int64_t dest_count, segments_it segments, +// int num_segments, int spacing, context_t& context) -> +// mem_t::value_type> { +// +// typedef typename std::iterator_traits::value_type int_t; +// return merge_path_partitions(counting_iterator_t(0), +// dest_count, segments, num_segments, spacing, less_t(), context); +//} + +//template +//mem_t binary_search_partitions(keys_it keys, int count, int num_items, +// int spacing, context_t& context) { +// +// int num_partitions = div_up(count, spacing) + 1; +// mem_t mem(num_partitions, context); +// int* p = mem.data(); +// transform([=]MGPU_DEVICE(int index) { +// int key = min(spacing * index, count); +// p[index] = binary_search(keys, num_items, key, less_t()); +// }, num_partitions, context); +// return mem; +//} + +/** @private */ +template< + typename P, + typename a_keys_it, typename a_vals_it, + typename b_keys_it, typename b_vals_it, + typename c_keys_it, typename c_vals_it, + typename comp_t +> +void cuda_merge_loop( + P&& p, + a_keys_it a_keys, a_vals_it a_vals, unsigned a_count, + b_keys_it b_keys, b_vals_it b_vals, unsigned b_count, + c_keys_it c_keys, c_vals_it c_vals, + comp_t comp, + void* ptr +) { + + using E = std::decay_t

; + using T = typename std::iterator_traits::value_type; + using V = typename std::iterator_traits::value_type; + + auto buf = static_cast(ptr); + + auto has_values = !std::is_same::value; + + cuda_merge_path_partitions( + p, a_keys, a_count, b_keys, b_count, E::nv, comp, buf + ); + + unsigned B = p.num_blocks(a_count + b_count); + + // we use small kernel + cuda_kernel<<>>([=] __device__ (auto tid, auto bid) { + + __shared__ union { + T keys[E::nv + 1]; + unsigned indices[E::nv]; + } shared; + + // Load the range for this CTA and merge the values into register. + auto mp0 = buf[bid + 0]; + auto mp1 = buf[bid + 1]; + auto range = cuda_compute_merge_range(a_count, b_count, bid, E::nv, mp0, mp1); + + auto merge = block_merge_from_mem( + a_keys, b_keys, range, tid, comp, shared.keys + ); + + auto dest_offset = E::nv * bid; + cuda_reg_to_mem_thread( + merge.keys, tid, range.total(), c_keys + dest_offset, shared.keys + ); + + if(has_values) { + // Transpose the indices from thread order to strided order. + auto indices = cuda_reg_thread_to_strided( + merge.indices, tid, shared.indices + ); + + // Gather the input values and merge into the output values. + cuda_transfer_two_streams_strided( + a_vals + range.a_begin, range.a_count(), + b_vals + range.b_begin, range.b_count(), indices, tid, + c_vals + dest_offset + ); + } + }); +} + +} // end of namespace tf::detail --------------------------------------------- + +namespace tf { + +// ---------------------------------------------------------------------------- +// standalone merge algorithms +// ---------------------------------------------------------------------------- + +// Function: merge_bufsz +template +unsigned cudaExecutionPolicy::merge_bufsz(unsigned a_count, unsigned b_count) { + return sizeof(unsigned) * (num_blocks(a_count + b_count + nv) + 1); +} + + +// ---------------------------------------------------------------------------- +// key-value merge +// ---------------------------------------------------------------------------- + +/** +@brief performs asynchronous key-value merge over a range of keys and values + +@tparam P execution policy type +@tparam a_keys_it first key iterator type +@tparam a_vals_it first value iterator type +@tparam b_keys_it second key iterator type +@tparam b_vals_it second value iterator type +@tparam c_keys_it output key iterator type +@tparam c_vals_it output value iterator type +@tparam C comparator type + +@param p execution policy +@param a_keys_first iterator to the beginning of the first key range +@param a_keys_last iterator to the end of the first key range +@param a_vals_first iterator to the beginning of the first value range +@param b_keys_first iterator to the beginning of the second key range +@param b_keys_last iterator to the end of the second key range +@param b_vals_first iterator to the beginning of the second value range +@param c_keys_first iterator to the beginning of the output key range +@param c_vals_first iterator to the beginning of the output value range +@param comp comparator +@param buf pointer to the temporary buffer + +Performs a key-value merge that copies elements from +[a_keys_first, a_keys_last) and [b_keys_first, b_keys_last) +into a single range, [c_keys_first, c_keys_last + (a_keys_last - a_keys_first) + (b_keys_last - b_keys_first)) +such that the resulting range is in ascending key order. + +At the same time, the merge copies elements from the two associated ranges +[a_vals_first + (a_keys_last - a_keys_first)) and +[b_vals_first + (b_keys_last - b_keys_first)) into a single range, +[c_vals_first, c_vals_first + (a_keys_last - a_keys_first) + (b_keys_last - b_keys_first)) +such that the resulting range is in ascending order +implied by each input element's associated key. + +For example, assume: + + @c a_keys = {1, 8}; + + @c a_vals = {2, 1}; + + @c b_keys = {3, 7}; + + @c b_vals = {3, 4}; + +After the merge, we have: + + @c c_keys = {1, 3, 7, 8} + + @c c_vals = {2, 3, 4, 1} + +*/ +template< + typename P, + typename a_keys_it, typename a_vals_it, + typename b_keys_it, typename b_vals_it, + typename c_keys_it, typename c_vals_it, + typename C +> +void cuda_merge_by_key( + P&& p, + a_keys_it a_keys_first, a_keys_it a_keys_last, a_vals_it a_vals_first, + b_keys_it b_keys_first, b_keys_it b_keys_last, b_vals_it b_vals_first, + c_keys_it c_keys_first, c_vals_it c_vals_first, + C comp, + void* buf +) { + + unsigned a_count = std::distance(a_keys_first, a_keys_last); + unsigned b_count = std::distance(b_keys_first, b_keys_last); + + if(a_count + b_count == 0) { + return; + } + + detail::cuda_merge_loop(p, + a_keys_first, a_vals_first, a_count, + b_keys_first, b_vals_first, b_count, + c_keys_first, c_vals_first, comp, + buf + ); +} + +// ---------------------------------------------------------------------------- +// key-only merge +// ---------------------------------------------------------------------------- + +/** +@brief performs asynchronous key-only merge over a range of keys + +@tparam P execution policy type +@tparam a_keys_it first key iterator type +@tparam b_keys_it second key iterator type +@tparam c_keys_it output key iterator type +@tparam C comparator type + +@param p execution policy +@param a_keys_first iterator to the beginning of the first key range +@param a_keys_last iterator to the end of the first key range +@param b_keys_first iterator to the beginning of the second key range +@param b_keys_last iterator to the end of the second key range +@param c_keys_first iterator to the beginning of the output key range +@param comp comparator +@param buf pointer to the temporary buffer + +This function is equivalent to tf::cuda_merge_by_key without values. + +*/ +template +void cuda_merge( + P&& p, + a_keys_it a_keys_first, a_keys_it a_keys_last, + b_keys_it b_keys_first, b_keys_it b_keys_last, + c_keys_it c_keys_first, + C comp, + void* buf +) { + cuda_merge_by_key( + p, + a_keys_first, a_keys_last, (const cudaEmpty*)nullptr, + b_keys_first, b_keys_last, (const cudaEmpty*)nullptr, + c_keys_first, (cudaEmpty*)nullptr, comp, + buf + ); +} + + +} // end of namespace tf ----------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/cuda/algorithm/reduce.hpp b/projects/hipfile/third-party/taskflow/cuda/algorithm/reduce.hpp new file mode 100644 index 000000000000..5a5de0a80062 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/cuda/algorithm/reduce.hpp @@ -0,0 +1,460 @@ +#pragma once + +#include "../cudaflow.hpp" + +/** +@file taskflow/cuda/algorithm/reduce.hpp +@brief cuda reduce algorithms include file +*/ + +namespace tf::detail { + +// ---------------------------------------------------------------------------- +// reduction helper functions +// ---------------------------------------------------------------------------- + +/** @private */ +template +struct cudaBlockReduce { + + static constexpr unsigned group_size = (std::min)(nt, CUDA_WARP_SIZE); + static constexpr unsigned num_passes = static_floor_log2(); + static constexpr unsigned num_items = nt / group_size; + + static_assert( + nt && (0 == nt % CUDA_WARP_SIZE), + "cudaBlockReduce requires num threads to be a multiple of warp_size (32)" + ); + + /** @private */ + struct Storage { + T data[std::max(nt, 2 * group_size)]; + }; + + template + __device__ T operator()(unsigned, T, Storage&, unsigned, op_t, bool = true) const; +}; + +// function: reduce to be called from a block +template +template +__device__ T cudaBlockReduce::operator ()( + unsigned tid, T x, Storage& storage, unsigned count, op_t op, bool ret +) const { + + // Store your data into shared memory. + storage.data[tid] = x; + __syncthreads(); + + if(tid < group_size) { + // Each thread scans within its lane. + cuda_strided_iterate([&](auto i, auto j) { + if(i > 0) { + x = op(x, storage.data[j]); + } + }, tid, count); + storage.data[tid] = x; + } + __syncthreads(); + + auto count2 = count < group_size ? count : group_size; + auto first = (1 & num_passes) ? group_size : 0; + if(tid < group_size) { + storage.data[first + tid] = x; + } + __syncthreads(); + + cuda_iterate([&](auto pass) { + if(tid < group_size) { + if(auto offset = 1 << pass; tid + offset < count2) { + x = op(x, storage.data[first + offset + tid]); + } + first = group_size - first; + storage.data[first + tid] = x; + } + __syncthreads(); + }); + + if(ret) { + x = storage.data[0]; + __syncthreads(); + } + return x; +} + +// ---------------------------------------------------------------------------- +// cuda_reduce +// ---------------------------------------------------------------------------- + +/** +@private +*/ +template +__global__ void cuda_reduce_kernel( + I input, unsigned count, T* res, O op, void* ptr +) { + + using U = typename std::iterator_traits::value_type; + + __shared__ typename cudaBlockReduce::Storage shm; + + auto tid = threadIdx.x; + auto bid = blockIdx.x; + auto tile = cuda_get_tile(bid, nt*vt, count); + auto x = cuda_mem_to_reg_strided( + input + tile.begin, tid, tile.count() + ); + + // reduce multiple values per thread into a scalar. + U s; + cuda_strided_iterate( + [&] (auto i, auto) { s = i ? op(s, x[i]) : x[0]; }, tid, tile.count() + ); + // reduce to a scalar per block. + s = cudaBlockReduce()( + tid, s, shm, (tile.count() < nt ? tile.count() : nt), op, false + ); + + if(!tid) { + auto buf = static_cast(ptr); + (count <= nt*vt) ? *res = op(*res, s) : buf[bid] = s; + } +} + +/** @private */ +template +void cuda_reduce_loop( + P&& p, I input, unsigned count, T* res, O op, void* ptr +) { + + using U = typename std::iterator_traits::value_type; + using E = std::decay_t

; + + auto buf = static_cast(ptr); + auto B = E::num_blocks(count); + + cuda_reduce_kernel<<>>( + input, count, res, op, ptr + ); + + if(B > 1) { + cuda_reduce_loop(p, buf, B, res, op, buf+B); + } +} + +// ---------------------------------------------------------------------------- +// cuda_uninitialized_reduce +// ---------------------------------------------------------------------------- + +/** +@private +*/ +template +__global__ void cuda_uninitialized_reduce_kernel( + I input, unsigned count, T* res, O op, void* ptr +) { + + using U = typename std::iterator_traits::value_type; + + __shared__ typename cudaBlockReduce::Storage shm; + + auto tid = threadIdx.x; + auto bid = blockIdx.x; + auto tile = cuda_get_tile(bid, nt*vt, count); + auto x = cuda_mem_to_reg_strided( + input + tile.begin, tid, tile.count() + ); + + // reduce multiple values per thread into a scalar. + U s; + cuda_strided_iterate( + [&] (auto i, auto) { s = i ? op(s, x[i]) : x[0]; }, tid, tile.count() + ); + + // reduce to a scalar per block. + s = cudaBlockReduce()( + tid, s, shm, (tile.count() < nt ? tile.count() : nt), op, false + ); + + if(!tid) { + auto buf = static_cast(ptr); + (count <= nt*vt) ? *res = s : buf[bid] = s; + } +} + +/** +@private +*/ +template +void cuda_uninitialized_reduce_loop( + P&& p, I input, unsigned count, T* res, O op, void* ptr +) { + + using U = typename std::iterator_traits::value_type; + using E = std::decay_t

; + + auto buf = static_cast(ptr); + auto B = (count + E::nv - 1) / E::nv; + + cuda_uninitialized_reduce_kernel<<>>( + input, count, res, op, buf + ); + + if(B > 1) { + cuda_uninitialized_reduce_loop(p, buf, B, res, op, buf+B); + } +} + +} // namespace tf::detail ---------------------------------------------------- + +namespace tf { + +// Function: reduce_bufsz +template +template +unsigned cudaExecutionPolicy::reduce_bufsz(unsigned count) { + unsigned B = num_blocks(count); + unsigned n = 0; + while(B > 1) { + n += B; + B = num_blocks(B); + } + return n*sizeof(T); +} + +// ---------------------------------------------------------------------------- +// cuda_reduce +// ---------------------------------------------------------------------------- + +/** +@brief performs asynchronous parallel reduction over a range of items + +@tparam P execution policy type +@tparam I input iterator type +@tparam T value type +@tparam O binary operator type + +@param p execution policy +@param first iterator to the beginning of the range +@param last iterator to the end of the range +@param res pointer to the result +@param op binary operator to apply to reduce elements +@param buf pointer to the temporary buffer + +This method is equivalent to the parallel execution of the following loop on a GPU: + +@code{.cpp} +while (first != last) { + *result = op(*result, *first++); +} +@endcode + */ +template +void cuda_reduce( + P&& p, I first, I last, T* res, O op, void* buf +) { + unsigned count = std::distance(first, last); + if(count == 0) { + return; + } + detail::cuda_reduce_loop(p, first, count, res, op, buf); +} + +// ---------------------------------------------------------------------------- +// cuda_uninitialized_reduce +// ---------------------------------------------------------------------------- + +/** +@brief performs asynchronous parallel reduction over a range of items without + an initial value + +@tparam P execution policy type +@tparam I input iterator type +@tparam T value type +@tparam O binary operator type + +@param p execution policy +@param first iterator to the beginning of the range +@param last iterator to the end of the range +@param res pointer to the result +@param op binary operator to apply to reduce elements +@param buf pointer to the temporary buffer + +This method is equivalent to the parallel execution of the following loop +on a GPU: + +@code{.cpp} +*result = *first++; // no initial values partitipcate in the loop +while (first != last) { + *result = op(*result, *first++); +} +@endcode +*/ +template +void cuda_uninitialized_reduce( + P&& p, I first, I last, T* res, O op, void* buf +) { + unsigned count = std::distance(first, last); + if(count == 0) { + return; + } + detail::cuda_uninitialized_reduce_loop(p, first, count, res, op, buf); +} + +// ---------------------------------------------------------------------------- +// transform_reduce +// ---------------------------------------------------------------------------- + +/** +@brief performs asynchronous parallel reduction over a range of transformed items + without an initial value + +@tparam P execution policy type +@tparam I input iterator type +@tparam T value type +@tparam O binary operator type +@tparam U unary operator type + +@param p execution policy +@param first iterator to the beginning of the range +@param last iterator to the end of the range +@param res pointer to the result +@param bop binary operator to apply to reduce elements +@param uop unary operator to apply to transform elements +@param buf pointer to the temporary buffer + +This method is equivalent to the parallel execution of the following loop on a GPU: + +@code{.cpp} +while (first != last) { + *result = bop(*result, uop(*first++)); +} +@endcode +*/ +template +void cuda_transform_reduce( + P&& p, I first, I last, T* res, O bop, U uop, void* buf +) { + + unsigned count = std::distance(first, last); + + if(count == 0) { + return; + } + + // reduction loop + detail::cuda_reduce_loop(p, + cuda_make_load_iterator([=]__device__(auto i){ + return uop(*(first+i)); + }), + count, res, bop, buf + ); +} + +// ---------------------------------------------------------------------------- +// transform_uninitialized_reduce +// ---------------------------------------------------------------------------- + +/** +@brief performs asynchronous parallel reduction over a range of transformed items + with an initial value + +@tparam P execution policy type +@tparam I input iterator type +@tparam T value type +@tparam O binary operator type +@tparam U unary operator type + +@param p execution policy +@param first iterator to the beginning of the range +@param last iterator to the end of the range +@param res pointer to the result +@param bop binary operator to apply to reduce elements +@param uop unary operator to apply to transform elements +@param buf pointer to the temporary buffer + +This method is equivalent to the parallel execution of the following loop +on a GPU: + +@code{.cpp} +*result = uop(*first++); // no initial values partitipcate in the loop +while (first != last) { + *result = bop(*result, uop(*first++)); +} +@endcode +*/ +template +void cuda_uninitialized_transform_reduce( + P&& p, I first, I last, T* res, O bop, U uop, void* buf +) { + + unsigned count = std::distance(first, last); + + if(count == 0) { + return; + } + + detail::cuda_uninitialized_reduce_loop(p, + cuda_make_load_iterator([=]__device__(auto i){ return uop(*(first+i)); }), + count, res, bop, buf + ); +} + +// ---------------------------------------------------------------------------- + +//template +//__device__ void cuda_warp_reduce( +// volatile T* shm, size_t N, size_t tid, C op +//) { +// if(tid + 32 < N) shm[tid] = op(shm[tid], shm[tid+32]); +// if(tid + 16 < N) shm[tid] = op(shm[tid], shm[tid+16]); +// if(tid + 8 < N) shm[tid] = op(shm[tid], shm[tid+8]); +// if(tid + 4 < N) shm[tid] = op(shm[tid], shm[tid+4]); +// if(tid + 2 < N) shm[tid] = op(shm[tid], shm[tid+2]); +// if(tid + 1 < N) shm[tid] = op(shm[tid], shm[tid+1]); +//} +// +//template +//__global__ void cuda_reduce(I first, size_t N, T* res, C op) { +// +// size_t tid = threadIdx.x; +// +// if(tid >= N) { +// return; +// } +// +// cudaSharedMemory shared_memory; +// T* shm = shared_memory.get(); +// +// shm[tid] = *(first+tid); +// +// for(size_t i=tid+blockDim.x; i 32; s >>= 1) { +// if(tid < s && tid + s < N) { +// shm[tid] = op(shm[tid], shm[tid+s]); +// } +// __syncthreads(); +// } +// +// if(tid < 32) { +// cuda_warp_reduce(shm, N, tid, op); +// } +// +// if(tid == 0) { +// if constexpr (uninitialized) { +// *res = shm[0]; +// } +// else { +// *res = op(*res, shm[0]); +// } +// } +//} + + +} // end of namespace tf ----------------------------------------------------- + diff --git a/projects/hipfile/third-party/taskflow/cuda/algorithm/scan.hpp b/projects/hipfile/third-party/taskflow/cuda/algorithm/scan.hpp new file mode 100644 index 000000000000..223d683cf8bf --- /dev/null +++ b/projects/hipfile/third-party/taskflow/cuda/algorithm/scan.hpp @@ -0,0 +1,488 @@ +#pragma once + +#include "reduce.hpp" + +/** +@file taskflow/cuda/algorithm/scan.hpp +@brief CUDA scan algorithm include file +*/ + +namespace tf::detail { + +// ---------------------------------------------------------------------------- +// scan +// ---------------------------------------------------------------------------- + +/** @private */ +inline constexpr unsigned cudaScanRecursionThreshold = 8; + +/** @private */ +enum class cudaScanType : int { + EXCLUSIVE = 1, + INCLUSIVE +}; + +/** @private */ +template 0)> +struct cudaScanResult { + T scan; + T reduction; +}; + +/** @private */ +template +struct cudaScanResult { + cudaArray scan; + T reduction; +}; + +//----------------------------------------------------------------------------- + +/** @private */ +template +struct cudaBlockScan { + + static constexpr unsigned num_warps = nt / CUDA_WARP_SIZE; + static constexpr unsigned num_passes = static_floor_log2(); + static constexpr unsigned capacity = nt + num_warps; + + /** @private */ + union storage_t { + T data[2 * nt]; + struct { T threads[nt], warps[num_warps]; }; + }; + + // standard scan + template + __device__ cudaScanResult operator ()( + unsigned tid, + T x, + storage_t& storage, + unsigned count = nt, + op_t op = op_t(), + T init = T(), + cudaScanType type = cudaScanType::EXCLUSIVE + ) const; + + // vectorized scan. accepts multiple values per thread and adds in + // optional global carry-in + template + __device__ cudaScanResult operator()( + unsigned tid, + cudaArray x, + storage_t& storage, + T carry_in = T(), + bool use_carry_in = false, + unsigned count = nt, + op_t op = op_t(), + T init = T(), + cudaScanType type = cudaScanType::EXCLUSIVE + ) const; +}; + +// standard scan +template +template +__device__ cudaScanResult cudaBlockScan::operator () ( + unsigned tid, T x, storage_t& storage, unsigned count, op_t op, + T init, cudaScanType type +) const { + + unsigned first = 0; + storage.data[first + tid] = x; + __syncthreads(); + + cuda_iterate([&](auto pass) { + if(auto offset = 1<= offset) { + x = op(storage.data[first + tid - offset], x); + } + first = nt - first; + storage.data[first + tid] = x; + __syncthreads(); + }); + + cudaScanResult result; + result.reduction = storage.data[first + count - 1]; + result.scan = (tid < count) ? + (cudaScanType::INCLUSIVE == type ? x : + (tid ? storage.data[first + tid - 1] : init)) : + result.reduction; + __syncthreads(); + + return result; +} + +// vectorized scan block +template +template +__device__ cudaScanResult cudaBlockScan::operator()( + unsigned tid, + cudaArray x, + storage_t& storage, + T carry_in, + bool use_carry_in, + unsigned count, op_t op, + T init, + cudaScanType type +) const { + + // Start with an inclusive scan of the in-range elements. + if(count >= nt * vt) { + cuda_iterate([&](auto i) { + x[i] = i ? op(x[i], x[i - 1]) : x[i]; + }); + } else { + cuda_iterate([&](auto i) { + auto index = vt * tid + i; + x[i] = i ? + ((index < count) ? op(x[i], x[i - 1]) : x[i - 1]) : + (x[i] = (index < count) ? x[i] : init); + }); + } + + // Scan the thread-local reductions for a carry-in for each thread. + auto result = operator()( + tid, x[vt - 1], storage, + (count + vt - 1) / vt, op, init, cudaScanType::EXCLUSIVE + ); + + // Perform the scan downsweep and add both the global carry-in and the + // thread carry-in to the values. + if(use_carry_in) { + result.reduction = op(carry_in, result.reduction); + result.scan = tid ? op(carry_in, result.scan) : carry_in; + } else { + use_carry_in = tid > 0; + } + + cudaArray y; + cuda_iterate([&](auto i) { + if(cudaScanType::EXCLUSIVE == type) { + y[i] = i ? x[i - 1] : result.scan; + if(use_carry_in && i > 0) y[i] = op(result.scan, y[i]); + } else { + y[i] = use_carry_in ? op(x[i], result.scan) : x[i]; + } + }); + + return cudaScanResult { y, result.reduction }; +} + +/** +@private +@brief single-pass scan for small input + */ +template +void cuda_single_pass_scan( + P&& p, + cudaScanType scan_type, + I input, + unsigned count, + O output, + C op + //reduction_it reduction, +) { + + using T = typename std::iterator_traits::value_type; + using E = std::decay_t

; + + // Small input specialization. This is the non-recursive branch. + cuda_kernel<<<1, E::nt, 0, p.stream()>>>([=] __device__ (auto tid, auto bid) { + + using scan_t = cudaBlockScan; + + __shared__ union { + typename scan_t::storage_t scan; + T values[E::nv]; + } shared; + + auto carry_in = T(); + for(unsigned cur = 0; cur < count; cur += E::nv) { + // Cooperatively load values into register. + auto count2 = min(count - cur, E::nv); + + auto x = cuda_mem_to_reg_thread(input + cur, + tid, count2, shared.values); + + auto result = scan_t()(tid, x, shared.scan, + carry_in, cur > 0, count2, op, T(), scan_type); + + // Store the scanned values back to global memory. + cuda_reg_to_mem_thread(result.scan, tid, count2, + output + cur, shared.values); + + // Roll the reduction into carry_in. + carry_in = result.reduction; + } + + // Store the carry-out to the reduction pointer. This may be a + // discard_iterator_t if no reduction is wanted. + //if(!tid) *reduction = carry_in; + }); +} + +/** +@private + +@brief main scan loop +*/ +template +void cuda_scan_loop( + P&& p, + cudaScanType scan_type, + I input, + unsigned count, + O output, + C op, + //reduction_it reduction, + void* ptr +) { + + using E = std::decay_t

; + using T = typename std::iterator_traits::value_type; + + T* buffer = static_cast(ptr); + + //launch_t::cta_dim(context).B(count); + unsigned B = (count + E::nv - 1) / E::nv; + + if(B > cudaScanRecursionThreshold) { + + //cudaDeviceVector partials(B); + //auto buffer = partials.data(); + + // upsweep phase + cuda_kernel<<>>([=] __device__ (auto tid, auto bid) { + + __shared__ typename cudaBlockReduce::Storage shm; + + // Load the tile's data into register. + auto tile = cuda_get_tile(bid, E::nv, count); + auto x = cuda_mem_to_reg_strided( + input + tile.begin, tid, tile.count() + ); + + // Reduce the thread's values into a scalar. + T scalar; + cuda_strided_iterate( + [&] (auto i, auto j) { scalar = i ? op(scalar, x[i]) : x[0]; }, + tid, tile.count() + ); + + // Reduce across all threads. + auto all_reduce = cudaBlockReduce()( + tid, scalar, shm, tile.count(), op + ); + + // Store the final reduction to the partials. + if(!tid) { + buffer[bid] = all_reduce; + } + }); + + // recursively call scan + //cuda_scan_loop(p, cudaScanType::EXCLUSIVE, buffer, B, buffer, op, S); + cuda_scan_loop( + p, cudaScanType::EXCLUSIVE, buffer, B, buffer, op, buffer+B + ); + + // downsweep: perform an intra-tile scan and add the scan of the partials + // as carry-in + cuda_kernel<<>>([=] __device__ (auto tid, auto bid) { + + using scan_t = cudaBlockScan; + + __shared__ union { + typename scan_t::storage_t scan; + T values[E::nv]; + } shared; + + // Load a tile to register in thread order. + auto tile = cuda_get_tile(bid, E::nv, count); + auto x = cuda_mem_to_reg_thread( + input + tile.begin, tid, tile.count(), shared.values + ); + + // Scan the array with carry-in from the partials. + auto y = scan_t()(tid, x, shared.scan, + buffer[bid], bid > 0, tile.count(), op, T(), + scan_type).scan; + + // Store the scanned values to the output. + cuda_reg_to_mem_thread( + y, tid, tile.count(), output + tile.begin, shared.values + ); + }); + } + // Small input specialization. This is the non-recursive branch. + else { + cuda_single_pass_scan(p, scan_type, input, count, output, op); + } +} + +} // namespace tf::detail ---------------------------------------------------- + +namespace tf { + +// Function: scan_bufsz +template +template +unsigned cudaExecutionPolicy::scan_bufsz(unsigned count) { + unsigned B = num_blocks(count); + unsigned n = 0; + for(auto b=B; b>detail::cudaScanRecursionThreshold; b=num_blocks(b)) { + n += b; + } + return n*sizeof(T); +} + + +/** +@brief performs asynchronous inclusive scan over a range of items + +@tparam P execution policy type +@tparam I input iterator +@tparam O output iterator +@tparam C binary operator type + +@param p execution policy +@param first iterator to the beginning of the input range +@param last iterator to the end of the input range +@param output iterator to the beginning of the output range +@param op binary operator to apply to scan +@param buf pointer to the temporary buffer + +*/ +template +void cuda_inclusive_scan( + P&& p, I first, I last, O output, C op, void* buf +) { + + unsigned count = std::distance(first, last); + + if(count == 0) { + return; + } + + // launch the scan loop + detail::cuda_scan_loop( + p, detail::cudaScanType::INCLUSIVE, first, count, output, op, buf + ); +} + +/** +@brief performs asynchronous inclusive scan over a range of transformed items + +@tparam P execution policy type +@tparam I input iterator +@tparam O output iterator +@tparam C binary operator type +@tparam U unary operator type + +@param p execution policy +@param first iterator to the beginning of the input range +@param last iterator to the end of the input range +@param output iterator to the beginning of the output range +@param bop binary operator to apply to scan +@param uop unary operator to apply to transform each item before scan +@param buf pointer to the temporary buffer + +*/ +template +void cuda_transform_inclusive_scan( + P&& p, I first, I last, O output, C bop, U uop, void* buf +) { + + using T = typename std::iterator_traits::value_type; + + unsigned count = std::distance(first, last); + + if(count == 0) { + return; + } + + // launch the scan loop + detail::cuda_scan_loop( + p, detail::cudaScanType::INCLUSIVE, + cuda_make_load_iterator([=]__device__(auto i){ return uop(*(first+i)); }), + count, output, bop, buf + ); +} + +/** +@brief performs asynchronous exclusive scan over a range of items + +@tparam P execution policy type +@tparam I input iterator +@tparam O output iterator +@tparam C binary operator type + +@param p execution policy +@param first iterator to the beginning of the input range +@param last iterator to the end of the input range +@param output iterator to the beginning of the output range +@param op binary operator to apply to scan +@param buf pointer to the temporary buffer + +*/ +template +void cuda_exclusive_scan( + P&& p, I first, I last, O output, C op, void* buf +) { + + unsigned count = std::distance(first, last); + + if(count == 0) { + return; + } + + // launch the scan loop + detail::cuda_scan_loop( + p, detail::cudaScanType::EXCLUSIVE, first, count, output, op, buf + ); +} + +/** +@brief performs asynchronous exclusive scan over a range of items + +@tparam P execution policy type +@tparam I input iterator +@tparam O output iterator +@tparam C binary operator type +@tparam U unary operator type + +@param p execution policy +@param first iterator to the beginning of the input range +@param last iterator to the end of the input range +@param output iterator to the beginning of the output range +@param bop binary operator to apply to scan +@param uop unary operator to apply to transform each item before scan +@param buf pointer to the temporary buffer + +*/ +template +void cuda_transform_exclusive_scan( + P&& p, I first, I last, O output, C bop, U uop, void* buf +) { + + using T = typename std::iterator_traits::value_type; + + unsigned count = std::distance(first, last); + + if(count == 0) { + return; + } + + // launch the scan loop + detail::cuda_scan_loop( + p, detail::cudaScanType::EXCLUSIVE, + cuda_make_load_iterator([=]__device__(auto i){ return uop(*(first+i)); }), + count, output, bop, buf + ); +} + + +} // end of namespace tf ----------------------------------------------------- + + + diff --git a/projects/hipfile/third-party/taskflow/cuda/algorithm/single_task.hpp b/projects/hipfile/third-party/taskflow/cuda/algorithm/single_task.hpp new file mode 100644 index 000000000000..4177ff38efe1 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/cuda/algorithm/single_task.hpp @@ -0,0 +1,36 @@ +#pragma once + +/** +@file taskflow/cuda/algorithm/single_task.hpp +@brief cuda single-task algorithms include file +*/ + +namespace tf { + +/** @private */ +template +__global__ void cuda_single_task(C callable) { + callable(); +} + +// Function: single_task +template +template +cudaTask cudaGraphBase::single_task(C c) { + return kernel(1, 1, 0, cuda_single_task, c); +} + +// Function: single_task +template +template +void cudaGraphExecBase::single_task(cudaTask task, C c) { + return kernel(task, 1, 1, 0, cuda_single_task, c); +} + +} // end of namespace tf ----------------------------------------------------- + + + + + + diff --git a/projects/hipfile/third-party/taskflow/cuda/algorithm/sort.hpp b/projects/hipfile/third-party/taskflow/cuda/algorithm/sort.hpp new file mode 100644 index 000000000000..97695f877813 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/cuda/algorithm/sort.hpp @@ -0,0 +1,506 @@ +#pragma once + +#include "merge.hpp" + +/** +@file taskflow/cuda/algorithm/sort.hpp +@brief CUDA sort algorithm include file +*/ + +namespace tf::detail { + +// ---------------------------------------------------------------------------- +// odd-even sort in register +// ---------------------------------------------------------------------------- + +/** +@private +@brief counts the number of leading zeros starting from the most significant bit +*/ +constexpr int cuda_clz(int x) { + for(int i = 31; i >= 0; --i) { + if((1<< i) & x) { + return 31 - i; + } + } + return 32; +} + +/** +@private +@brief finds log2(x) and optionally round up to the next integer logarithm. +*/ +constexpr int cuda_find_log2(int x, bool round_up = false) { + int a = 31 - cuda_clz(x); + if(round_up) { + a += !is_pow2(x); + } + return a; +} + +/** @private */ +template +__device__ auto cuda_odd_even_sort( + cudaArray x, C comp, int flags = 0 +) { + cuda_iterate([&](auto I) { + #pragma unroll + for(auto i = 1 & I; i < vt - 1; i += 2) { + if((0 == ((2<< i) & flags)) && comp(x[i + 1], x[i])) + cuda_swap(x[i], x[i + 1]); + } + }); + return x; +} + +/** @private */ +template +__device__ auto cuda_odd_even_sort( + cudaKVArray x, C comp, int flags = 0 +) { + cuda_iterate([&](auto I) { + #pragma unroll + for(auto i = 1 & I; i < vt - 1; i += 2) { + if((0 == ((2<< i) & flags)) && comp(x.keys[i + 1], x.keys[i])) { + cuda_swap(x.keys[i], x.keys[i + 1]); + cuda_swap(x.vals[i], x.vals[i + 1]); + } + } + }); + return x; +} + +// ---------------------------------------------------------------------------- +// range check +// ---------------------------------------------------------------------------- + +/** @private */ +__device__ inline int cuda_out_of_range_flags(int first, int vt, int count) { + int out_of_range = min(vt, first + vt - count); + int head_flags = 0; + if(out_of_range > 0) { + const int mask = (1<< vt) - 1; + head_flags = mask & (~mask>> out_of_range); + } + return head_flags; +} + +/** @private */ +__device__ inline auto cuda_compute_merge_sort_frame( + unsigned partition, unsigned coop, unsigned spacing +) { + + unsigned size = spacing * (coop / 2); + unsigned start = ~(coop - 1) & partition; + unsigned a_begin = spacing * start; + unsigned b_begin = spacing * start + size; + + return cudaMergeRange { + a_begin, + a_begin + size, + b_begin, + b_begin + size + }; +} + +/** @private */ +__device__ inline auto cuda_compute_merge_sort_range( + unsigned count, unsigned partition, unsigned coop, unsigned spacing +) { + + auto frame = cuda_compute_merge_sort_frame(partition, coop, spacing); + + return cudaMergeRange { + frame.a_begin, + min(count, frame.a_end), + min(count, frame.b_begin), + min(count, frame.b_end) + }; +} + +/** @private */ +__device__ inline auto cuda_compute_merge_sort_range( + unsigned count, unsigned partition, unsigned coop, unsigned spacing, + unsigned mp0, unsigned mp1 +) { + + auto range = cuda_compute_merge_sort_range(count, partition, coop, spacing); + + // Locate the diagonal from the start of the A sublist. + unsigned diag = spacing * partition - range.a_begin; + + // The end partition of the last cta for each merge operation is computed + // and stored as the begin partition for the subsequent merge. i.e. it is + // the same partition but in the wrong coordinate system, so its 0 when it + // should be listSize. Correct that by checking if this is the last cta + // in this merge operation. + if(coop - 1 != ((coop - 1) & partition)) { + range.a_end = range.a_begin + mp1; + range.b_end = min(count, range.b_begin + diag + spacing - mp1); + } + + range.a_begin = range.a_begin + mp0; + range.b_begin = min(count, range.b_begin + diag - mp0); + + return range; +} + +/** @private */ +template +struct cudaBlockSort { + + static constexpr bool has_values = !std::is_same::value; + static constexpr unsigned num_passes = static_floor_log2(); + + /** @private */ + union Storage { + K keys[nt * vt + 1]; + V vals[nt * vt]; + }; + + static_assert(is_pow2(nt), "cudaBlockSort requires pow2 number of threads"); + + template + __device__ auto merge_pass( + cudaKVArray x, + unsigned tid, unsigned count, unsigned pass, + C comp, Storage& storage + ) const { + + // Divide the CTA's keys into lists. + unsigned coop = 2 << pass; + auto range = cuda_compute_merge_sort_range(count, tid, coop, vt); + unsigned diag = vt * tid - range.a_begin; + + // Store the keys into shared memory for searching. + cuda_reg_to_shared_thread(x.keys, tid, storage.keys); + + // Search for the merge path for this thread within its list. + auto mp = cuda_merge_path( + storage.keys, range, diag, comp + ); + + // Run a serial merge and return. + auto merge = cuda_serial_merge( + storage.keys, range.partition(mp, diag), comp + ); + x.keys = merge.keys; + + if(has_values) { + // Reorder values through shared memory. + cuda_reg_to_shared_thread(x.vals, tid, storage.vals); + x.vals = cuda_shared_gather(storage.vals, merge.indices); + } + + return x; + } + + template + __device__ auto block_sort(cudaKVArray x, + unsigned tid, unsigned count, C comp, Storage& storage + ) const { + + // Sort the inputs within each thread. If any threads have fewer than + // vt items, use the segmented sort network to prevent out-of-range + // elements from contaminating the sort. + if(count < nt * vt) { + auto head_flags = cuda_out_of_range_flags(vt * tid, vt, count); + x = cuda_odd_even_sort(x, comp, head_flags); + } else { + x = cuda_odd_even_sort(x, comp); + } + + // Merge threads starting with a pair until all values are merged. + for(unsigned pass = 0; pass < num_passes; ++pass) { + x = merge_pass(x, tid, count, pass, comp, storage); + } + + return x; + } +}; + +/** @private */ +template +void cuda_merge_sort_partitions( + P&& p, K keys, unsigned count, + unsigned coop, unsigned spacing, C comp, unsigned* buf +) { + + // buffer size is num_partitions + 1 + unsigned num_partitions = (count + spacing - 1) / spacing + 1; + + const unsigned nt = 128; + const unsigned vt = 1; + const unsigned nv = nt * vt; + + unsigned B = (num_partitions + nv - 1) / nv; // nt = 128, vt = 1 + + cuda_kernel<<>>([=] __device__ (auto tid, auto bid) { + auto range = cuda_get_tile(bid, nt * vt, num_partitions); + cuda_strided_iterate([=](auto, auto j) { + auto index = j + range.begin; + auto range = cuda_compute_merge_sort_range(count, index, coop, spacing); + auto diag = min(spacing * index, count) - range.a_begin; + buf[index] = cuda_merge_path( + keys + range.a_begin, range.a_count(), + keys + range.b_begin, range.b_count(), + diag, comp + ); + }, tid, range.count()); + }); +} + +/** @private */ +template +void merge_sort_loop( + P&& p, K_it keys_input, V_it vals_input, unsigned count, C comp, void* buf +) { + + using K = typename std::iterator_traits::value_type; + using V = typename std::iterator_traits::value_type; + using E = std::decay_t

; + + const bool has_values = !std::is_same::value; + + unsigned B = (count + E::nv - 1) / E::nv; + unsigned R = cuda_find_log2(B, true); + + K* keys_output {nullptr}; + V* vals_output {nullptr}; + unsigned *mp_data {nullptr}; + + if(R) { + keys_output = (K*)(buf); + if(has_values) { + vals_output = (V*)(keys_output + count); + mp_data = (unsigned*)(vals_output + count); + } + else { + mp_data = (unsigned*)(keys_output + count); + } + } + + //cudaDeviceVector keys_temp(R ? count : 0); + //auto keys_output = keys_temp.data(); + ////std::cout << "keys_output = " << keys_temp.size()*sizeof(K) << std::endl; + + //cudaDeviceVector vals_temp((has_values && R) ? count : 0); + //auto vals_output = vals_temp.data(); + //std::cout << "vals_output = " << vals_temp.size()*sizeof(V) << std::endl; + + auto keys_blocksort = (1 & R) ? keys_output : keys_input; + auto vals_blocksort = (1 & R) ? vals_output : vals_input; + + //printf("B=%u, R=%u\n", B, R); + + cuda_kernel<<>>([=] __device__ (auto tid, auto bid) { + + using sort_t = cudaBlockSort; + + __shared__ union { + typename sort_t::Storage sort; + K keys[E::nv]; + V vals[E::nv]; + } shared; + + auto tile = cuda_get_tile(bid, E::nv, count); + + // Load the keys and values. + cudaKVArray unsorted; + unsorted.keys = cuda_mem_to_reg_thread( + keys_input + tile.begin, tid, tile.count(), shared.keys + ); + + if(has_values) { + unsorted.vals = cuda_mem_to_reg_thread( + vals_input + tile.begin, tid, tile.count(), shared.vals + ); + } + + // Blocksort. + auto sorted = sort_t().block_sort(unsorted, tid, tile.count(), comp, shared.sort); + + // Store the keys and values. + cuda_reg_to_mem_thread( + sorted.keys, tid, tile.count(), keys_blocksort + tile.begin, shared.keys + ); + + if(has_values) { + cuda_reg_to_mem_thread( + sorted.vals, tid, tile.count(), vals_blocksort + tile.begin, shared.vals + ); + } + }); + + // merge passes + + if(1 & R) { + std::swap(keys_input, keys_output); + std::swap(vals_input, vals_output); + } + + // number of partitions + //unsigned num_partitions = B + 1; + //cudaDeviceVector mem(num_partitions); + //auto mp_data = mem.data(); + //std::cout << "num_partitions = " << (B+1)*sizeof(unsigned) << std::endl; + + for(unsigned pass = 0; pass < R; ++pass) { + + unsigned coop = 2 << pass; + + cuda_merge_sort_partitions( + p, keys_input, count, coop, E::nv, comp, mp_data + ); + + cuda_kernel<<>>([=]__device__(auto tid, auto bid) { + + __shared__ union { + K keys[E::nv + 1]; + unsigned indices[E::nv]; + } shared; + + auto tile = cuda_get_tile(bid, E::nv, count); + + // Load the range for this CTA and merge the values into register. + auto range = cuda_compute_merge_sort_range( + count, bid, coop, E::nv, mp_data[bid + 0], mp_data[bid + 1] + ); + + auto merge = block_merge_from_mem( + keys_input, keys_input, range, tid, comp, shared.keys + ); + + // Store merged values back out. + cuda_reg_to_mem_thread( + merge.keys, tid, tile.count(), keys_output + tile.begin, shared.keys + ); + + if(has_values) { + // Transpose the indices from thread order to strided order. + auto indices = cuda_reg_thread_to_strided( + merge.indices, tid, shared.indices + ); + + // Gather the input values and merge into the output values. + cuda_transfer_two_streams_strided( + vals_input + range.a_begin, range.a_count(), + vals_input + range.b_begin, range.b_count(), + indices, tid, vals_output + tile.begin + ); + } + }); + + std::swap(keys_input, keys_output); + std::swap(vals_input, vals_output); + } +} + +} // end of namespace tf::detail --------------------------------------------- + +namespace tf { + +/** +@brief queries the buffer size in bytes needed to call sort kernels + for the given number of elements + +@tparam P execution policy type +@tparam K key type +@tparam V value type (default tf::cudaEmpty) + +@param count number of keys/values to sort + +The function is used to allocate a buffer for calling tf::cuda_sort. + +*/ +template +unsigned cuda_sort_buffer_size(unsigned count) { + + using E = std::decay_t

; + + const bool has_values = !std::is_same::value; + + unsigned B = (count + E::nv - 1) / E::nv; + unsigned R = detail::cuda_find_log2(B, true); + + return R ? (count * sizeof(K) + (has_values ? count*sizeof(V) : 0) + + (B+1)*sizeof(unsigned)) : 0; +} + +// ---------------------------------------------------------------------------- +// key-value sort +// ---------------------------------------------------------------------------- + +/** +@brief performs asynchronous key-value sort on a range of items + +@tparam P execution policy type +@tparam K_it key iterator type +@tparam V_it value iterator type +@tparam C comparator type + +@param p execution policy +@param k_first iterator to the beginning of the key range +@param k_last iterator to the end of the key range +@param v_first iterator to the beginning of the value range +@param comp binary comparator +@param buf pointer to the temporary buffer + +Sorts key-value elements in [k_first, k_last) and +[v_first, v_first + (k_last - k_first)) into ascending key order +using the given comparator @c comp. +If @c i and @c j are any two valid iterators in [k_first, k_last) +such that @c i precedes @c j, and @c p and @c q are iterators in +[v_first, v_first + (k_last - k_first)) corresponding to +@c i and @c j respectively, then comp(*j, *i) evaluates to @c false. + +For example, assume: + + @c keys are {1, 4, 2, 8, 5, 7} + + @c values are {'a', 'b', 'c', 'd', 'e', 'f'} + +After sort: + + @c keys are {1, 2, 4, 5, 7, 8} + + @c values are {'a', 'c', 'b', 'e', 'f', 'd'} + +*/ +template +void cuda_sort_by_key( + P&& p, K_it k_first, K_it k_last, V_it v_first, C comp, void* buf +) { + + unsigned N = std::distance(k_first, k_last); + + if(N <= 1) { + return; + } + + detail::merge_sort_loop(p, k_first, v_first, N, comp, buf); +} + +// ---------------------------------------------------------------------------- +// key sort +// ---------------------------------------------------------------------------- + +/** +@brief performs asynchronous key-only sort on a range of items + +@tparam P execution policy type +@tparam K_it key iterator type +@tparam C comparator type + +@param p execution policy +@param k_first iterator to the beginning of the key range +@param k_last iterator to the end of the key range +@param comp binary comparator +@param buf pointer to the temporary buffer + +This method is equivalent to tf::cuda_sort_by_key without values. + +*/ +template +void cuda_sort(P&& p, K_it k_first, K_it k_last, C comp, void* buf) { + cuda_sort_by_key(p, k_first, k_last, (cudaEmpty*)nullptr, comp, buf); +} + +} // end of namespace tf ----------------------------------------------------- + diff --git a/projects/hipfile/third-party/taskflow/cuda/algorithm/transform.hpp b/projects/hipfile/third-party/taskflow/cuda/algorithm/transform.hpp new file mode 100644 index 000000000000..e8fc386e45c1 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/cuda/algorithm/transform.hpp @@ -0,0 +1,126 @@ +#pragma once + +#include "../cudaflow.hpp" + +/** +@file taskflow/cuda/algorithm/transform.hpp +@brief cuda parallel-transform algorithms include file +*/ + +namespace tf { + +// ---------------------------------------------------------------------------- +// transform +// ---------------------------------------------------------------------------- + +namespace detail { + +/** +@private +*/ +template +__global__ void cuda_transform_kernel(I first, unsigned count, O output, C op) { + auto tid = threadIdx.x; + auto bid = blockIdx.x; + auto tile = cuda_get_tile(bid, E::nv, count); + cuda_strided_iterate( + [=]__device__(auto, auto j) { + auto offset = j + tile.begin; + *(output + offset) = op(*(first+offset)); + }, + tid, + tile.count() + ); +} + +/** +@private +*/ +template +__global__ void cuda_transform_kernel( + I1 first1, I2 first2, unsigned count, O output, C op +) { + auto tid = threadIdx.x; + auto bid = blockIdx.x; + auto tile = cuda_get_tile(bid, E::nv, count); + cuda_strided_iterate( + [=]__device__(auto, auto j) { + auto offset = j + tile.begin; + *(output + offset) = op(*(first1+offset), *(first2+offset)); + }, + tid, + tile.count() + ); +} + +} // end of namespace detail ------------------------------------------------- + +// ---------------------------------------------------------------------------- +// cudaFlow +// ---------------------------------------------------------------------------- + +// Function: transform +template +template +cudaTask cudaGraphBase::transform(I first, I last, O output, C c) { + + unsigned count = std::distance(first, last); + + return kernel( + E::num_blocks(count), E::nt, 0, + detail::cuda_transform_kernel, + first, count, output, c + ); +} + +// Function: transform +template +template +cudaTask cudaGraphBase::transform(I1 first1, I1 last1, I2 first2, O output, C c) { + + unsigned count = std::distance(first1, last1); + + return kernel( + E::num_blocks(count), E::nt, 0, + detail::cuda_transform_kernel, + first1, first2, count, output, c + ); +} + + +// Function: update transform +template +template +void cudaGraphExecBase::transform(cudaTask task, I first, I last, O output, C c) { + + unsigned count = std::distance(first, last); + + kernel(task, + E::num_blocks(count), E::nt, 0, + detail::cuda_transform_kernel, + first, count, output, c + ); +} + +// Function: update transform +template +template +void cudaGraphExecBase::transform( + cudaTask task, I1 first1, I1 last1, I2 first2, O output, C c +) { + unsigned count = std::distance(first1, last1); + + kernel(task, + E::num_blocks(count), E::nt, 0, + detail::cuda_transform_kernel, + first1, first2, count, output, c + ); +} + +} // end of namespace tf ----------------------------------------------------- + + + + + + diff --git a/projects/hipfile/third-party/taskflow/cuda/algorithm/transpose.hpp b/projects/hipfile/third-party/taskflow/cuda/algorithm/transpose.hpp new file mode 100644 index 000000000000..3b02a7f1a8e5 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/cuda/algorithm/transpose.hpp @@ -0,0 +1,41 @@ +#pragma once + +#include "../cuda_error.hpp" + +namespace tf { + +// ---------------------------------------------------------------------------- +// row-wise matrix transpose +// ---------------------------------------------------------------------------- +// +template +__global__ void cuda_transpose( + const T* d_in, + T* d_out, + size_t rows, + size_t cols +) { + __shared__ T tile[32][32]; + size_t x = blockIdx.x * 32 + threadIdx.x; + size_t y = blockIdx.y * 32 + threadIdx.y; + + for(size_t i = 0; i < 32; i += 8) { + if(x < cols && (y + i) < rows) { + tile[threadIdx.y + i][threadIdx.x] = d_in[(y + i) * cols + x]; + } + } + + __syncthreads(); + + x = blockIdx.y * 32 + threadIdx.x; + y = blockIdx.x * 32 + threadIdx.y; + + for(size_t i = 0; i < 32; i += 8) { + if(x < rows && (y + i) < cols) { + d_out[(y + i) * rows + x] = tile[threadIdx.x][threadIdx.y + i]; + } + } +} + +} // end of namespace -------------------------------------------------------- + diff --git a/projects/hipfile/third-party/taskflow/cuda/cuda_capturer.hpp b/projects/hipfile/third-party/taskflow/cuda/cuda_capturer.hpp new file mode 100644 index 000000000000..f0a431b8c2a8 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/cuda/cuda_capturer.hpp @@ -0,0 +1,732 @@ +#pragma once + +#include "cuda_optimizer.hpp" + +/** +@file cuda_capturer.hpp +@brief %cudaFlow capturer include file +*/ + +namespace tf { + +// ---------------------------------------------------------------------------- +// class definition: cudaFlowCapturer +// ---------------------------------------------------------------------------- + +/** +@class cudaFlowCapturer + +@brief class to create a %cudaFlow graph using stream capture + +The usage of tf::cudaFlowCapturer is similar to tf::cudaFlow, except users can +call the method tf::cudaFlowCapturer::on to capture a sequence of asynchronous +CUDA operations through the given stream. +The following example creates a CUDA graph that captures two kernel tasks, +@c task_1 and @c task_2, where @c task_1 runs before @c task_2. + +@code{.cpp} +taskflow.emplace([](tf::cudaFlowCapturer& capturer){ + + // capture my_kernel_1 through the given stream managed by the capturer + auto task_1 = capturer.on([&](cudaStream_t stream){ + my_kernel_1<<>>(my_parameters_1); + }); + + // capture my_kernel_2 through the given stream managed by the capturer + auto task_2 = capturer.on([&](cudaStream_t stream){ + my_kernel_2<<>>(my_parameters_2); + }); + + task_1.precede(task_2); +}); +@endcode + +Similar to tf::cudaFlow, a %cudaFlowCapturer is a task (tf::Task) +created from tf::Taskflow +and will be run by @em one worker thread in the executor. +That is, the callable that describes a %cudaFlowCapturer +will be executed sequentially. +Inside a %cudaFlow capturer task, different GPU tasks (tf::cudaTask) may run +in parallel depending on the selected optimization algorithm. +By default, we use tf::cudaFlowRoundRobinOptimizer to transform a user-level +graph into a native CUDA graph. + +Please refer to @ref GPUTaskingcudaFlowCapturer for details. +*/ +class cudaFlowCapturer { + + friend class cudaFlow; + friend class Executor; + + // created by user + struct External { + cudaFlowGraph graph; + }; + + // created from cudaFlow + struct Internal { + }; + + using handle_t = std::variant; + + using Optimizer = std::variant< + cudaFlowRoundRobinOptimizer, + cudaFlowSequentialOptimizer, + cudaFlowLinearOptimizer + >; + + public: + + /** + @brief constructs a standalone cudaFlowCapturer + + A standalone %cudaFlow capturer does not go through any taskflow and + can be run by the caller thread using tf::cudaFlowCapturer::run. + */ + cudaFlowCapturer() = default; + + /** + @brief destructs the cudaFlowCapturer + */ + ~cudaFlowCapturer() = default; + + /** + @brief default move constructor + */ + cudaFlowCapturer(cudaFlowCapturer&&) = default; + + /** + @brief default move assignment operator + */ + cudaFlowCapturer& operator = (cudaFlowCapturer&&) = default; + + /** + @brief queries the emptiness of the graph + */ + bool empty() const; + + /** + @brief queries the number of tasks + */ + size_t num_tasks() const; + + /** + @brief clear this %cudaFlow capturer + */ + void clear(); + + /** + @brief dumps the %cudaFlow graph into a DOT format through an + output stream + */ + void dump(std::ostream& os) const; + + /** + @brief dumps the native captured graph into a DOT format through + an output stream + */ + void dump_native_graph(std::ostream& os) const; + + // ------------------------------------------------------------------------ + // basic methods + // ------------------------------------------------------------------------ + + /** + @brief captures a sequential CUDA operations from the given callable + + @tparam C callable type constructible with @c std::function + @param callable a callable to capture CUDA operations with the stream + + This methods applies a stream created by the flow to capture + a sequence of CUDA operations defined in the callable. + */ + template , void>* = nullptr + > + cudaTask on(C&& callable); + + /** + @brief updates a capture task to another sequential CUDA operations + + The method is similar to cudaFlowCapturer::on but operates + on an existing task. + */ + template , void>* = nullptr + > + void on(cudaTask task, C&& callable); + + /** + @brief captures a no-operation task + + @return a tf::cudaTask handle + + An empty node performs no operation during execution, + but can be used for transitive ordering. + For example, a phased execution graph with 2 groups of @c n nodes + with a barrier between them can be represented using an empty node + and @c 2*n dependency edges, + rather than no empty node and @c n^2 dependency edges. + */ + cudaTask noop(); + + /** + @brief updates a task to a no-operation task + + The method is similar to tf::cudaFlowCapturer::noop but + operates on an existing task. + */ + void noop(cudaTask task); + + /** + @brief copies data between host and device asynchronously through a stream + + @param dst destination memory address + @param src source memory address + @param count size in bytes to copy + + The method captures a @c cudaMemcpyAsync operation through an + internal stream. + */ + cudaTask memcpy(void* dst, const void* src, size_t count); + + /** + @brief updates a capture task to a memcpy operation + + The method is similar to cudaFlowCapturer::memcpy but operates on an + existing task. + */ + void memcpy(cudaTask task, void* dst, const void* src, size_t count); + + /** + @brief captures a copy task of typed data + + @tparam T element type (non-void) + + @param tgt pointer to the target memory block + @param src pointer to the source memory block + @param num number of elements to copy + + @return cudaTask handle + + A copy task transfers num*sizeof(T) bytes of data from a source location + to a target location. Direction can be arbitrary among CPUs and GPUs. + */ + template , void>* = nullptr + > + cudaTask copy(T* tgt, const T* src, size_t num); + + /** + @brief updates a capture task to a copy operation + + The method is similar to cudaFlowCapturer::copy but operates on + an existing task. + */ + template , void>* = nullptr + > + void copy(cudaTask task, T* tgt, const T* src, size_t num); + + /** + @brief initializes or sets GPU memory to the given value byte by byte + + @param ptr pointer to GPU memory + @param v value to set for each byte of the specified memory + @param n size in bytes to set + + The method captures a @c cudaMemsetAsync operation through an + internal stream to fill the first @c count bytes of the memory area + pointed to by @c devPtr with the constant byte value @c value. + */ + cudaTask memset(void* ptr, int v, size_t n); + + /** + @brief updates a capture task to a memset operation + + The method is similar to cudaFlowCapturer::memset but operates on + an existing task. + */ + void memset(cudaTask task, void* ptr, int value, size_t n); + + /** + @brief captures a kernel + + @tparam F kernel function type + @tparam ArgsT kernel function parameters type + + @param g configured grid + @param b configured block + @param s configured shared memory size in bytes + @param f kernel function + @param args arguments to forward to the kernel function by copy + + @return cudaTask handle + */ + template + cudaTask kernel(dim3 g, dim3 b, size_t s, F f, ArgsT&&... args); + + /** + @brief updates a capture task to a kernel operation + + The method is similar to cudaFlowCapturer::kernel but operates on + an existing task. + */ + template + void kernel( + cudaTask task, dim3 g, dim3 b, size_t s, F f, ArgsT&&... args + ); + + // ------------------------------------------------------------------------ + // generic algorithms + // ------------------------------------------------------------------------ + + /** + @brief capturers a kernel to runs the given callable with only one thread + + @tparam C callable type + + @param c callable to run by a single kernel thread + */ + template + cudaTask single_task(C c); + + /** + @brief updates a capture task to a single-threaded kernel + + This method is similar to cudaFlowCapturer::single_task but operates + on an existing task. + */ + template + void single_task(cudaTask task, C c); + + /** + @brief captures a kernel that applies a callable to each dereferenced element + of the data array + + @tparam I iterator type + @tparam C callable type + + @param first iterator to the beginning + @param last iterator to the end + @param callable a callable object to apply to the dereferenced iterator + + @return cudaTask handle + + This method is equivalent to the parallel execution of the following loop on a GPU: + + @code{.cpp} + for(auto itr = first; itr != last; i++) { + callable(*itr); + } + @endcode + */ + template + cudaTask for_each(I first, I last, C callable); + + /** + @brief updates a capture task to a for-each kernel task + + This method is similar to cudaFlowCapturer::for_each but operates + on an existing task. + */ + template + void for_each(cudaTask task, I first, I last, C callable); + + /** + @brief captures a kernel that applies a callable to each index in the range + with the step size + + @tparam I index type + @tparam C callable type + + @param first beginning index + @param last last index + @param step step size + @param callable the callable to apply to each element in the data array + + @return cudaTask handle + + This method is equivalent to the parallel execution of the following loop on a GPU: + + @code{.cpp} + // step is positive [first, last) + for(auto i=first; ilast; i+=step) { + callable(i); + } + @endcode + */ + template + cudaTask for_each_index(I first, I last, I step, C callable); + + /** + @brief updates a capture task to a for-each-index kernel task + + This method is similar to cudaFlowCapturer::for_each_index but operates + on an existing task. + */ + template + void for_each_index( + cudaTask task, I first, I last, I step, C callable + ); + + /** + @brief captures a kernel that transforms an input range to an output range + + @tparam I input iterator type + @tparam O output iterator type + @tparam C unary operator type + + @param first iterator to the beginning of the input range + @param last iterator to the end of the input range + @param output iterator to the beginning of the output range + @param op unary operator to apply to transform each item in the range + + @return cudaTask handle + + This method is equivalent to the parallel execution of the following loop on a GPU: + + @code{.cpp} + while (first != last) { + *output++ = op(*first++); + } + @endcode + */ + template + cudaTask transform(I first, I last, O output, C op); + + /** + @brief updates a capture task to a transform kernel task + + This method is similar to cudaFlowCapturer::transform but operates + on an existing task. + */ + template + void transform(cudaTask task, I first, I last, O output, C op); + + /** + @brief captures a kernel that transforms two input ranges to an output range + + @tparam I1 first input iterator type + @tparam I2 second input iterator type + @tparam O output iterator type + @tparam C unary operator type + + @param first1 iterator to the beginning of the input range + @param last1 iterator to the end of the input range + @param first2 iterato + @param output iterator to the beginning of the output range + @param op binary operator to apply to transform each pair of items in the + two input ranges + + @return cudaTask handle + + This method is equivalent to the parallel execution of the following loop on a GPU: + + @code{.cpp} + while (first1 != last1) { + *output++ = op(*first1++, *first2++); + } + @endcode + */ + template + cudaTask transform(I1 first1, I1 last1, I2 first2, O output, C op); + + /** + @brief updates a capture task to a transform kernel task + + This method is similar to cudaFlowCapturer::transform but operates + on an existing task. + */ + template + void transform( + cudaTask task, I1 first1, I1 last1, I2 first2, O output, C op + ); + + // ------------------------------------------------------------------------ + // Capturing methods + // ------------------------------------------------------------------------ + + /** + @brief selects a different optimization algorithm + + @tparam OPT optimizer type + @tparam ArgsT arguments types + + @param args arguments to forward to construct the optimizer + + @return a reference to the optimizer + + We currently supports the following optimization algorithms to capture + a user-described %cudaFlow: + + tf::cudaFlowSequentialOptimizer + + tf::cudaFlowRoundRobinOptimizer + + tf::cudaFlowLinearOptimizer + + By default, tf::cudaFlowCapturer uses the round-robin optimization + algorithm with four streams to transform a user-level graph into + a native CUDA graph. + */ + template + void make_optimizer(ArgsT&&... args); + + /** + @brief captures the cudaFlow and turns it into a CUDA Graph + */ + cudaGraph_t capture(); + + // ------------------------------------------------------------------------ + // offload methods + // ------------------------------------------------------------------------ + + /** + @brief offloads the %cudaFlowCapturer onto a GPU asynchronously via a stream + + @param stream stream for performing this operation + + Offloads the present %cudaFlowCapturer onto a GPU asynchronously via + the given stream. + + An offloaded %cudaFlowCapturer forces the underlying graph to be instantiated. + After the instantiation, you should not modify the graph topology + but update node parameters. + */ + void run(cudaStream_t stream); + + /** + @brief acquires a reference to the underlying CUDA graph + */ + cudaGraph_t native_graph(); + + /** + @brief instantiates an executable graph from this cudaflow capturer + */ + cudaGraphExec instantiate(); + + private: + + cudaFlowGraph _cfg; + + Optimizer _optimizer; +}; + +// Function: empty +inline bool cudaFlowCapturer::empty() const { + return _cfg.empty(); +} + +// Function: num_tasks +inline size_t cudaFlowCapturer::num_tasks() const { + return _cfg._nodes.size(); +} + +// Procedure: clear +inline void cudaFlowCapturer::clear() { + _cfg.clear(); +} + +// Procedure: dump +inline void cudaFlowCapturer::dump(std::ostream& os) const { + _cfg.dump(os, nullptr, ""); +} + +// Procedure: dump_native_graph +inline void cudaFlowCapturer::dump_native_graph(std::ostream& os) const { + cuda_dump_graph(os, _cfg._native_handle); +} + +// Function: capture +template , void>* +> +cudaTask cudaFlowCapturer::on(C&& callable) { + auto node = _cfg.emplace_back(_cfg, + std::in_place_type_t{}, std::forward(callable) + ); + return cudaTask(node); +} + +// Function: noop +inline cudaTask cudaFlowCapturer::noop() { + return on([](cudaStream_t){}); +} + + +// Function: memcpy +inline cudaTask cudaFlowCapturer::memcpy( + void* dst, const void* src, size_t count +) { + return on([dst, src, count] (cudaStream_t stream) mutable { + TF_CHECK_CUDA( + cudaMemcpyAsync(dst, src, count, cudaMemcpyDefault, stream), + "failed to capture memcpy" + ); + }); +} + +// Function: copy +template , void>*> +cudaTask cudaFlowCapturer::copy(T* tgt, const T* src, size_t num) { + return on([tgt, src, num] (cudaStream_t stream) mutable { + TF_CHECK_CUDA( + cudaMemcpyAsync(tgt, src, sizeof(T)*num, cudaMemcpyDefault, stream), + "failed to capture copy" + ); + }); +} + +// Function: memset +inline cudaTask cudaFlowCapturer::memset(void* ptr, int v, size_t n) { + return on([ptr, v, n] (cudaStream_t stream) mutable { + TF_CHECK_CUDA( + cudaMemsetAsync(ptr, v, n, stream), "failed to capture memset" + ); + }); +} + +// Function: kernel +template +cudaTask cudaFlowCapturer::kernel( + dim3 g, dim3 b, size_t s, F f, ArgsT&&... args +) { + return on([g, b, s, f, args...] (cudaStream_t stream) mutable { + f<<>>(args...); + }); +} + +// Function: make_optimizer +template +void cudaFlowCapturer::make_optimizer(ArgsT&&... args) { + return _optimizer.emplace(std::forward(args)...); +} + +// Function: capture +inline cudaGraph_t cudaFlowCapturer::capture() { + return std::visit( + [this](auto&& opt){ return opt._optimize(_cfg); }, _optimizer + ); +} + +// Function: instantiate +inline cudaGraphExec cudaFlowCapturer::instantiate() { + + _cfg._native_handle.reset(capture()); + + cudaGraphExec_t exec; + TF_CHECK_CUDA( + cudaGraphInstantiate(&exec, _cfg._native_handle, nullptr, nullptr, 0), + "failed to create an executable graph" + ); + + return cudaGraphExec(exec); +} + +//// Procedure: run +//inline void cudaFlowCapturer::run(cudaStream_t stream) { +// +// // If the topology got changed, we need to destroy the executable +// // and create a new one +// if(_cfg._state & cudaFlowGraph::CHANGED) { +// _cfg._native_handle.reset(capture()); +// _exe.instantiate(_cfg._native_handle); +// } +// // if the graph is just updated (i.e., topology does not change), +// // we can skip part of the optimization and just update the executable +// // with the new captured graph +// else if(_cfg._state & cudaFlowGraph::UPDATED) { +// // TODO: skip part of the optimization (e.g., levelization) +// _cfg._native_handle.reset(capture()); +// if(_exe.update(_cfg._native_handle) != cudaGraphExecUpdateSuccess) { +// _exe.instantiate(_cfg._native_handle); +// } +// } +// +// // run the executable (should exist) +// _exe.run(stream); +// +// _cfg._state = cudaFlowGraph::OFFLOADED; +//} + +// Function: native_graph +inline cudaGraph_t cudaFlowCapturer::native_graph() { + return _cfg._native_handle; +} + +//// Function: on +//template , void>* +//> +//void cudaFlowCapturer::on(cudaTask task, C&& callable) { +// +// if(task.type() != cudaTaskType::CAPTURE) { +// TF_THROW("invalid cudaTask type (must be CAPTURE)"); +// } +// +// _cfg._state |= cudaFlowGraph::UPDATED; +// +// std::get_if(&task._node->_handle)->work = +// std::forward(callable); +//} +// +//// Function: noop +//inline void cudaFlowCapturer::noop(cudaTask task) { +// on(task, [](cudaStream_t){}); +//} +//// +//// Function: memcpy +//inline void cudaFlowCapturer::memcpy( +// cudaTask task, void* dst, const void* src, size_t count +//) { +// on(task, [dst, src, count](cudaStream_t stream) mutable { +// TF_CHECK_CUDA( +// cudaMemcpyAsync(dst, src, count, cudaMemcpyDefault, stream), +// "failed to capture memcpy" +// ); +// }); +//} +// +//// Function: copy +//template , void>* +//> +//void cudaFlowCapturer::copy( +// cudaTask task, T* tgt, const T* src, size_t num +//) { +// on(task, [tgt, src, num] (cudaStream_t stream) mutable { +// TF_CHECK_CUDA( +// cudaMemcpyAsync(tgt, src, sizeof(T)*num, cudaMemcpyDefault, stream), +// "failed to capture copy" +// ); +// }); +//} +// +//// Function: memset +//inline void cudaFlowCapturer::memset( +// cudaTask task, void* ptr, int v, size_t n +//) { +// on(task, [ptr, v, n] (cudaStream_t stream) mutable { +// TF_CHECK_CUDA( +// cudaMemsetAsync(ptr, v, n, stream), "failed to capture memset" +// ); +// }); +//} +// +//// Function: kernel +//template +//void cudaFlowCapturer::kernel( +// cudaTask task, dim3 g, dim3 b, size_t s, F f, ArgsT&&... args +//) { +// on(task, [g, b, s, f, args...] (cudaStream_t stream) mutable { +// f<<>>(args...); +// }); +//} +// + + +} // end of namespace tf ----------------------------------------------------- + diff --git a/projects/hipfile/third-party/taskflow/cuda/cuda_device.hpp b/projects/hipfile/third-party/taskflow/cuda/cuda_device.hpp new file mode 100644 index 000000000000..0bf541d6a8a9 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/cuda/cuda_device.hpp @@ -0,0 +1,338 @@ +#pragma once + +#include "cuda_error.hpp" + +/** +@file cuda_device.hpp +@brief CUDA device utilities include file +*/ + +namespace tf { + +/** +@brief queries the number of available devices +*/ +inline size_t cuda_get_num_devices() { + int N = 0; + TF_CHECK_CUDA(cudaGetDeviceCount(&N), "failed to get device count"); + return static_cast(N); +} + +/** +@brief gets the current device associated with the caller thread +*/ +inline int cuda_get_device() { + int id; + TF_CHECK_CUDA(cudaGetDevice(&id), "failed to get current device id"); + return id; +} + +/** +@brief switches to a given device context +*/ +inline void cuda_set_device(int id) { + TF_CHECK_CUDA(cudaSetDevice(id), "failed to switch to device ", id); +} + +/** +@brief obtains the device property +*/ +inline void cuda_get_device_property(int i, cudaDeviceProp& p) { + TF_CHECK_CUDA( + cudaGetDeviceProperties(&p, i), "failed to get property of device ", i + ); +} + +/** +@brief obtains the device property +*/ +inline cudaDeviceProp cuda_get_device_property(int i) { + cudaDeviceProp p; + TF_CHECK_CUDA( + cudaGetDeviceProperties(&p, i), "failed to get property of device ", i + ); + return p; +} + +/** +@brief dumps the device property +*/ +inline void cuda_dump_device_property(std::ostream& os, const cudaDeviceProp& p) { + + os << "Major revision number: " << p.major << '\n' + << "Minor revision number: " << p.minor << '\n' + << "Name: " << p.name << '\n' + << "Total global memory: " << p.totalGlobalMem << '\n' + << "Total shared memory per block: " << p.sharedMemPerBlock << '\n' + << "Total registers per block: " << p.regsPerBlock << '\n' + << "Warp size: " << p.warpSize << '\n' + << "Maximum memory pitch: " << p.memPitch << '\n' + << "Maximum threads per block: " << p.maxThreadsPerBlock << '\n'; + + os << "Maximum dimension of block: "; + for (int i = 0; i < 3; ++i) { + if(i) os << 'x'; + os << p.maxThreadsDim[i]; + } + os << '\n'; + + os << "Maximum dimension of grid: "; + for (int i = 0; i < 3; ++i) { + if(i) os << 'x'; + os << p.maxGridSize[i];; + } + os << '\n'; + os << "Total constant memory: " << p.totalConstMem << '\n' + << "Texture alignment: " << p.textureAlignment << '\n' + << "Number of multiprocessors: " << p.multiProcessorCount << '\n' + << "GPU sharing Host Memory: " << p.integrated << '\n' + << "Host page-locked mem mapping: " << p.canMapHostMemory << '\n' + << "Alignment for Surfaces: " << p.surfaceAlignment << '\n' + << "Device has ECC support: " << p.ECCEnabled << '\n' + << "Unified Addressing (UVA): " << p.unifiedAddressing << '\n'; +} + +/** +@brief queries the maximum threads per block on a device +*/ +inline size_t cuda_get_device_max_threads_per_block(int d) { + int threads = 0; + TF_CHECK_CUDA( + cudaDeviceGetAttribute(&threads, cudaDevAttrMaxThreadsPerBlock, d), + "failed to query the maximum threads per block on device ", d + ) + return threads; +} + +/** +@brief queries the maximum x-dimension per block on a device +*/ +inline size_t cuda_get_device_max_x_dim_per_block(int d) { + int dim = 0; + TF_CHECK_CUDA( + cudaDeviceGetAttribute(&dim, cudaDevAttrMaxBlockDimX, d), + "failed to query the maximum x-dimension per block on device ", d + ) + return dim; +} + +/** +@brief queries the maximum y-dimension per block on a device +*/ +inline size_t cuda_get_device_max_y_dim_per_block(int d) { + int dim = 0; + TF_CHECK_CUDA( + cudaDeviceGetAttribute(&dim, cudaDevAttrMaxBlockDimY, d), + "failed to query the maximum y-dimension per block on device ", d + ) + return dim; +} + +/** +@brief queries the maximum z-dimension per block on a device +*/ +inline size_t cuda_get_device_max_z_dim_per_block(int d) { + int dim = 0; + TF_CHECK_CUDA( + cudaDeviceGetAttribute(&dim, cudaDevAttrMaxBlockDimZ, d), + "failed to query the maximum z-dimension per block on device ", d + ) + return dim; +} + +/** +@brief queries the maximum x-dimension per grid on a device +*/ +inline size_t cuda_get_device_max_x_dim_per_grid(int d) { + int dim = 0; + TF_CHECK_CUDA( + cudaDeviceGetAttribute(&dim, cudaDevAttrMaxGridDimX, d), + "failed to query the maximum x-dimension per grid on device ", d + ) + return dim; +} + +/** +@brief queries the maximum y-dimension per grid on a device +*/ +inline size_t cuda_get_device_max_y_dim_per_grid(int d) { + int dim = 0; + TF_CHECK_CUDA( + cudaDeviceGetAttribute(&dim, cudaDevAttrMaxGridDimY, d), + "failed to query the maximum y-dimension per grid on device ", d + ) + return dim; +} + +/** +@brief queries the maximum z-dimension per grid on a device +*/ +inline size_t cuda_get_device_max_z_dim_per_grid(int d) { + int dim = 0; + TF_CHECK_CUDA( + cudaDeviceGetAttribute(&dim, cudaDevAttrMaxGridDimZ, d), + "failed to query the maximum z-dimension per grid on device ", d + ) + return dim; +} + +/** +@brief queries the maximum shared memory size in bytes per block on a device +*/ +inline size_t cuda_get_device_max_shm_per_block(int d) { + int num = 0; + TF_CHECK_CUDA( + cudaDeviceGetAttribute(&num, cudaDevAttrMaxSharedMemoryPerBlock, d), + "failed to query the maximum shared memory per block on device ", d + ) + return num; +} + +/** +@brief queries the warp size on a device +*/ +inline size_t cuda_get_device_warp_size(int d) { + int num = 0; + TF_CHECK_CUDA( + cudaDeviceGetAttribute(&num, cudaDevAttrWarpSize, d), + "failed to query the warp size per block on device ", d + ) + return num; +} + +/** +@brief queries the major number of compute capability of a device +*/ +inline int cuda_get_device_compute_capability_major(int d) { + int num = 0; + TF_CHECK_CUDA( + cudaDeviceGetAttribute(&num, cudaDevAttrComputeCapabilityMajor, d), + "failed to query the major number of compute capability of device ", d + ) + return num; +} + +/** +@brief queries the minor number of compute capability of a device +*/ +inline int cuda_get_device_compute_capability_minor(int d) { + int num = 0; + TF_CHECK_CUDA( + cudaDeviceGetAttribute(&num, cudaDevAttrComputeCapabilityMinor, d), + "failed to query the minor number of compute capability of device ", d + ) + return num; +} + +/** +@brief queries if the device supports unified addressing +*/ +inline bool cuda_get_device_unified_addressing(int d) { + int num = 0; + TF_CHECK_CUDA( + cudaDeviceGetAttribute(&num, cudaDevAttrUnifiedAddressing, d), + "failed to query unified addressing status on device ", d + ) + return num; +} + +// ---------------------------------------------------------------------------- +// CUDA Version +// ---------------------------------------------------------------------------- + +/** +@brief queries the latest CUDA version (1000 * major + 10 * minor) supported by the driver +*/ +inline int cuda_get_driver_version() { + int num = 0; + TF_CHECK_CUDA( + cudaDriverGetVersion(&num), + "failed to query the latest cuda version supported by the driver" + ); + return num; +} + +/** +@brief queries the CUDA Runtime version (1000 * major + 10 * minor) +*/ +inline int cuda_get_runtime_version() { + int num = 0; + TF_CHECK_CUDA( + cudaRuntimeGetVersion(&num), "failed to query cuda runtime version" + ); + return num; +} + +// ---------------------------------------------------------------------------- +// cudaScopedDevice +// ---------------------------------------------------------------------------- + +/** @class cudaScopedDevice + +@brief class to create an RAII-styled context switch + +Sample usage: + +@code{.cpp} +{ + tf::cudaScopedDevice device(1); // switch to the device context 1 + + // create a stream under device context 1 + cudaStream_t stream; + cudaStreamCreate(&stream); + +} // leaving the scope and goes back to the previous device context +@endcode + +%cudaScopedDevice is neither movable nor copyable. +*/ +class cudaScopedDevice { + + public: + + /** + @brief constructs a RAII-styled device switcher + + @param device device context to scope in the guard + */ + explicit cudaScopedDevice(int device); + + /** + @brief destructs the guard and switches back to the previous device context + */ + ~cudaScopedDevice(); + + private: + + cudaScopedDevice() = delete; + cudaScopedDevice(const cudaScopedDevice&) = delete; + cudaScopedDevice(cudaScopedDevice&&) = delete; + + int _p; +}; + +// Constructor +inline cudaScopedDevice::cudaScopedDevice(int dev) { + TF_CHECK_CUDA(cudaGetDevice(&_p), "failed to get current device scope"); + if(_p == dev) { + _p = -1; + } + else { + TF_CHECK_CUDA(cudaSetDevice(dev), "failed to scope on device ", dev); + } +} + +// Destructor +inline cudaScopedDevice::~cudaScopedDevice() { + if(_p != -1) { + cudaSetDevice(_p); + //TF_CHECK_CUDA(cudaSetDevice(_p), "failed to scope back to device ", _p); + } +} + +} // end of namespace cuda --------------------------------------------------- + + + + + diff --git a/projects/hipfile/third-party/taskflow/cuda/cuda_error.hpp b/projects/hipfile/third-party/taskflow/cuda/cuda_error.hpp new file mode 100644 index 000000000000..0e56e5ac7d1e --- /dev/null +++ b/projects/hipfile/third-party/taskflow/cuda/cuda_error.hpp @@ -0,0 +1,34 @@ +#pragma once + +#include +#include +#include +#include + +#include "../utility/stream.hpp" + +#define TF_CUDA_EXPAND( x ) x +#define TF_CUDA_REMOVE_FIRST_HELPER(N, ...) __VA_ARGS__ +#define TF_CUDA_REMOVE_FIRST(...) TF_CUDA_EXPAND(TF_CUDA_REMOVE_FIRST_HELPER(__VA_ARGS__)) +#define TF_CUDA_GET_FIRST_HELPER(N, ...) N +#define TF_CUDA_GET_FIRST(...) TF_CUDA_EXPAND(TF_CUDA_GET_FIRST_HELPER(__VA_ARGS__)) + +#define TF_CHECK_CUDA(...) \ +if(TF_CUDA_GET_FIRST(__VA_ARGS__) != cudaSuccess) { \ + std::ostringstream oss; \ + auto __ev__ = TF_CUDA_GET_FIRST(__VA_ARGS__); \ + oss << "[" << __FILE__ << ":" << __LINE__ << "] " \ + << (cudaGetErrorString(__ev__)) << " (" \ + << (cudaGetErrorName(__ev__)) << ") - "; \ + tf::ostreamize(oss, TF_CUDA_REMOVE_FIRST(__VA_ARGS__)); \ + throw std::runtime_error(oss.str()); \ +} + +#if __CUDACC_VER_MAJOR__ >= 13 +#define TF_CUDA_POST13(X) X +#define TF_CUDA_PRE13(X) +#else +#define TF_CUDA_PRE13(X) X +#define TF_CUDA_POST13(X) +#endif + diff --git a/projects/hipfile/third-party/taskflow/cuda/cuda_execution_policy.hpp b/projects/hipfile/third-party/taskflow/cuda/cuda_execution_policy.hpp new file mode 100644 index 000000000000..c33eaa1d5d94 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/cuda/cuda_execution_policy.hpp @@ -0,0 +1,136 @@ +#pragma once + +#include "cuda_error.hpp" + +/** +@file cuda_execution_policy.hpp +@brief CUDA execution policy include file +*/ + +namespace tf { + +/** +@class cudaExecutionPolicy + +@brief class to define execution policy for CUDA standard algorithms + +@tparam NT number of threads per block +@tparam VT number of work units per thread + +Execution policy configures the kernel execution parameters in CUDA algorithms. +The first template argument, @c NT, the number of threads per block should +always be a power-of-two number. +The second template argument, @c VT, the number of work units per thread +is recommended to be an odd number to avoid bank conflict. + +Details can be referred to @ref CUDASTDExecutionPolicy. +*/ +template +class cudaExecutionPolicy { + + static_assert(is_pow2(NT), "max # threads per block must be a power of two"); + + public: + + /** @brief static constant for getting the number of threads per block */ + const static unsigned nt = NT; + + /** @brief static constant for getting the number of work units per thread */ + const static unsigned vt = VT; + + /** @brief static constant for getting the number of elements to process per block */ + const static unsigned nv = NT*VT; + + /** + @brief constructs an execution policy object + */ + cudaExecutionPolicy() = default; + + /** + @brief queries the number of blocks to accommodate N elements + */ + static unsigned num_blocks(unsigned N) { return (N + nv - 1) / nv; } + + // -------------------------------------------------------------------------- + // Buffer Sizes for Standard Algorithms + // -------------------------------------------------------------------------- + + /** + @brief queries the buffer size in bytes needed to call reduce kernels + + @tparam T value type + + @param count number of elements to reduce + + The function is used to allocate a buffer for calling tf::cuda_reduce, + tf::cuda_uninitialized_reduce, tf::cuda_transform_reduce, and + tf::cuda_uninitialized_transform_reduce. + */ + template + static unsigned reduce_bufsz(unsigned count); + + /** + @brief queries the buffer size in bytes needed to call tf::cuda_min_element + + @tparam T value type + + @param count number of elements to search + + The function is used to decide the buffer size in bytes for calling + tf::cuda_min_element. + */ + template + static unsigned min_element_bufsz(unsigned count); + + /** + @brief queries the buffer size in bytes needed to call tf::cuda_max_element + + @tparam T value type + + @param count number of elements to search + + The function is used to decide the buffer size in bytes for calling + tf::cuda_max_element. + */ + template + static unsigned max_element_bufsz(unsigned count); + + /** + @brief queries the buffer size in bytes needed to call scan kernels + + @tparam T value type + + @param count number of elements to scan + + The function is used to allocate a buffer for calling + tf::cuda_inclusive_scan, tf::cuda_exclusive_scan, + tf::cuda_transform_inclusive_scan, and tf::cuda_transform_exclusive_scan. + */ + template + static unsigned scan_bufsz(unsigned count); + + /** + @brief queries the buffer size in bytes needed for CUDA merge algorithms + + @param a_count number of elements in the first vector to merge + @param b_count number of elements in the second vector to merge + + The buffer size of merge algorithm does not depend on the data type. + The buffer is purely used only for storing temporary indices + (of type @c unsigned) required during the merge process. + + The function is used to allocate a buffer for calling + tf::cuda_merge and tf::cuda_merge_by_key. + */ + inline static unsigned merge_bufsz(unsigned a_count, unsigned b_count); +}; + +/** +@brief default execution policy + */ +using cudaDefaultExecutionPolicy = cudaExecutionPolicy<512, 7>; + +} // end of namespace tf ----------------------------------------------------- + + + diff --git a/projects/hipfile/third-party/taskflow/cuda/cuda_graph.hpp b/projects/hipfile/third-party/taskflow/cuda/cuda_graph.hpp new file mode 100644 index 000000000000..285acdd1e82b --- /dev/null +++ b/projects/hipfile/third-party/taskflow/cuda/cuda_graph.hpp @@ -0,0 +1,1126 @@ +#pragma once + +#include + +#include "cuda_memory.hpp" +#include "cuda_stream.hpp" +#include "cuda_meta.hpp" + +#include "../utility/traits.hpp" + +namespace tf { + +// ---------------------------------------------------------------------------- +// cudaGraph_t routines +// ---------------------------------------------------------------------------- + +/** +@brief gets the memcpy node parameter of a copy task +*/ +template , void>* = nullptr +> +cudaMemcpy3DParms cuda_get_copy_parms(T* tgt, const T* src, size_t num) { + + using U = std::decay_t; + + cudaMemcpy3DParms p; + + p.srcArray = nullptr; + p.srcPos = ::make_cudaPos(0, 0, 0); + p.srcPtr = ::make_cudaPitchedPtr(const_cast(src), num*sizeof(U), num, 1); + p.dstArray = nullptr; + p.dstPos = ::make_cudaPos(0, 0, 0); + p.dstPtr = ::make_cudaPitchedPtr(tgt, num*sizeof(U), num, 1); + p.extent = ::make_cudaExtent(num*sizeof(U), 1, 1); + p.kind = cudaMemcpyDefault; + + return p; +} + +/** +@brief gets the memcpy node parameter of a memcpy task (untyped) +*/ +inline cudaMemcpy3DParms cuda_get_memcpy_parms( + void* tgt, const void* src, size_t bytes +) { + + // Parameters in cudaPitchedPtr + // d - Pointer to allocated memory + // p - Pitch of allocated memory in bytes + // xsz - Logical width of allocation in elements + // ysz - Logical height of allocation in elements + cudaMemcpy3DParms p; + p.srcArray = nullptr; + p.srcPos = ::make_cudaPos(0, 0, 0); + p.srcPtr = ::make_cudaPitchedPtr(const_cast(src), bytes, bytes, 1); + p.dstArray = nullptr; + p.dstPos = ::make_cudaPos(0, 0, 0); + p.dstPtr = ::make_cudaPitchedPtr(tgt, bytes, bytes, 1); + p.extent = ::make_cudaExtent(bytes, 1, 1); + p.kind = cudaMemcpyDefault; + + return p; +} + +/** +@brief gets the memset node parameter of a memcpy task (untyped) +*/ +inline cudaMemsetParams cuda_get_memset_parms(void* dst, int ch, size_t count) { + + cudaMemsetParams p; + p.dst = dst; + p.value = ch; + p.pitch = 0; + //p.elementSize = (count & 1) == 0 ? ((count & 3) == 0 ? 4 : 2) : 1; + //p.width = (count & 1) == 0 ? ((count & 3) == 0 ? count >> 2 : count >> 1) : count; + p.elementSize = 1; // either 1, 2, or 4 + p.width = count; + p.height = 1; + + return p; +} + +/** +@brief gets the memset node parameter of a fill task (typed) +*/ +template && (sizeof(T)==1 || sizeof(T)==2 || sizeof(T)==4), void>* = nullptr +> +cudaMemsetParams cuda_get_fill_parms(T* dst, T value, size_t count) { + + cudaMemsetParams p; + p.dst = dst; + + // perform bit-wise copy + p.value = 0; // crucial + static_assert(sizeof(T) <= sizeof(p.value), "internal error"); + std::memcpy(&p.value, &value, sizeof(T)); + + p.pitch = 0; + p.elementSize = sizeof(T); // either 1, 2, or 4 + p.width = count; + p.height = 1; + + return p; +} + +/** +@brief gets the memset node parameter of a zero task (typed) +*/ +template && (sizeof(T)==1 || sizeof(T)==2 || sizeof(T)==4), void>* = nullptr +> +cudaMemsetParams cuda_get_zero_parms(T* dst, size_t count) { + + cudaMemsetParams p; + p.dst = dst; + p.value = 0; + p.pitch = 0; + p.elementSize = sizeof(T); // either 1, 2, or 4 + p.width = count; + p.height = 1; + + return p; +} + +/** +@brief queries the number of root nodes in a native CUDA graph +*/ +inline size_t cuda_graph_get_num_root_nodes(cudaGraph_t graph) { + size_t num_nodes; + TF_CHECK_CUDA( + cudaGraphGetRootNodes(graph, nullptr, &num_nodes), + "failed to get native graph root nodes" + ); + return num_nodes; +} + +/** +@brief queries the number of nodes in a native CUDA graph +*/ +inline size_t cuda_graph_get_num_nodes(cudaGraph_t graph) { + size_t num_nodes; + TF_CHECK_CUDA( + cudaGraphGetNodes(graph, nullptr, &num_nodes), + "failed to get native graph nodes" + ); + return num_nodes; +} + +/** +@brief Handles compatibility with CUDA <= 12.x and CUDA == 13.x + */ +inline size_t cuda_graph_get_num_edges(cudaGraph_t graph, cudaGraphNode_t* from, cudaGraphNode_t* to) { + size_t num_edges; + TF_CHECK_CUDA( + TF_CUDA_PRE13(cudaGraphGetEdges(graph, from, to, &num_edges)) + TF_CUDA_POST13(cudaGraphGetEdges(graph, from, to, nullptr, &num_edges)), + "failed to get native graph edges" + ); + return num_edges; +} + +/** +@brief Handles compatibility with CUDA <= 12.x and CUDA 13 +* @param node +* @param dependencies +* @return + */ +inline size_t cuda_graph_node_get_dependencies(cudaGraphNode_t node, cudaGraphNode_t* dependencies) { + size_t num_predecessors; + TF_CHECK_CUDA( + TF_CUDA_PRE13(cudaGraphNodeGetDependencies(node, dependencies, &num_predecessors)) + TF_CUDA_POST13(cudaGraphNodeGetDependencies(node, dependencies, nullptr, &num_predecessors)), + "Failed to get number of dependencies"); + return num_predecessors; +} + +/** +@brief Handles compatibility with CUDA <= 12.x and CUDA 13 +@param node +@param dependent_nodes +@return + */ +inline size_t cuda_graph_node_get_dependent_nodes(cudaGraphNode_t node, cudaGraphNode_t *dependent_nodes) { + size_t num_successors; + TF_CHECK_CUDA( + TF_CUDA_PRE13(cudaGraphNodeGetDependentNodes(node, dependent_nodes, &num_successors)) + TF_CUDA_POST13(cudaGraphNodeGetDependentNodes(node, dependent_nodes, nullptr, &num_successors)), + "Failed to get CUDA dependent nodes"); + return num_successors; +} + +/** +@brief Handles compatibility with CUDA <= 12.x and CUDA 13 +@param graph +@param from +@param to +@param numDependencies + */ +inline void cuda_graph_add_dependencies(cudaGraph_t graph, const cudaGraphNode_t *from, const cudaGraphNode_t *to, size_t numDependencies) { + TF_CHECK_CUDA( + TF_CUDA_PRE13(cudaGraphAddDependencies(graph, from, to, numDependencies)) + TF_CUDA_POST13(cudaGraphAddDependencies(graph, from, to, nullptr, numDependencies)), + "Failed to add CUDA graph node dependencies" + ); +} + +/** +@brief queries the number of edges in a native CUDA graph +*/ +inline size_t cuda_graph_get_num_edges(cudaGraph_t graph) { + return cuda_graph_get_num_edges(graph, nullptr, nullptr); +} + + + +/** +@brief acquires the nodes in a native CUDA graph +*/ +inline std::vector cuda_graph_get_nodes(cudaGraph_t graph) { + size_t num_nodes = cuda_graph_get_num_nodes(graph); + std::vector nodes(num_nodes); + TF_CHECK_CUDA( + cudaGraphGetNodes(graph, nodes.data(), &num_nodes), + "failed to get native graph nodes" + ); + return nodes; +} + +/** +@brief acquires the root nodes in a native CUDA graph +*/ +inline std::vector cuda_graph_get_root_nodes(cudaGraph_t graph) { + size_t num_nodes = cuda_graph_get_num_root_nodes(graph); + std::vector nodes(num_nodes); + TF_CHECK_CUDA( + cudaGraphGetRootNodes(graph, nodes.data(), &num_nodes), + "failed to get native graph nodes" + ); + return nodes; +} + +/** +@brief acquires the edges in a native CUDA graph +*/ +inline std::vector> +cuda_graph_get_edges(cudaGraph_t graph) { + size_t num_edges = cuda_graph_get_num_edges(graph); + std::vector froms(num_edges), tos(num_edges); + num_edges = cuda_graph_get_num_edges(graph, froms.data(), tos.data()); + std::vector> edges(num_edges); + for(size_t i=0; i + friend class cudaGraphBase; + + template + friend class cudaGraphExecBase; + + friend class cudaFlow; + friend class cudaFlowCapturer; + friend class cudaFlowCapturerBase; + + friend std::ostream& operator << (std::ostream&, const cudaTask&); + + public: + + /** + @brief constructs an empty cudaTask + */ + cudaTask() = default; + + /** + @brief copy-constructs a cudaTask + */ + cudaTask(const cudaTask&) = default; + + /** + @brief copy-assigns a cudaTask + */ + cudaTask& operator = (const cudaTask&) = default; + + /** + @brief adds precedence links from this to other tasks + + @tparam Ts parameter pack + + @param tasks one or multiple tasks + + @return @c *this + */ + template + cudaTask& precede(Ts&&... tasks); + + /** + @brief adds precedence links from other tasks to this + + @tparam Ts parameter pack + + @param tasks one or multiple tasks + + @return @c *this + */ + template + cudaTask& succeed(Ts&&... tasks); + + /** + @brief queries the number of successors + */ + size_t num_successors() const; + + /** + @brief queries the number of dependents + */ + size_t num_predecessors() const; + + /** + @brief queries the type of this task + */ + auto type() const; + + /** + @brief dumps the task through an output stream + + @param os an output stream target + */ + void dump(std::ostream& os) const; + + private: + + cudaTask(cudaGraph_t, cudaGraphNode_t); + + cudaGraph_t _native_graph {nullptr}; + cudaGraphNode_t _native_node {nullptr}; +}; + +// Constructor +inline cudaTask::cudaTask(cudaGraph_t native_graph, cudaGraphNode_t native_node) : + _native_graph {native_graph}, _native_node {native_node} { +} + +// Function: precede +template +cudaTask& cudaTask::precede(Ts&&... tasks) { + ( + cuda_graph_add_dependencies( + _native_graph, &_native_node, &(tasks._native_node), 1 + ), ... + ); + return *this; +} + +// Function: succeed +template +cudaTask& cudaTask::succeed(Ts&&... tasks) { + (tasks.precede(*this), ...); + return *this; +} + +// Function: num_predecessors +inline size_t cudaTask::num_predecessors() const { + return cuda_graph_node_get_dependencies(_native_node, nullptr); +} + +// Function: num_successors +inline size_t cudaTask::num_successors() const { + return cuda_graph_node_get_dependent_nodes(_native_node, nullptr); +} + +// Function: type +inline auto cudaTask::type() const { + cudaGraphNodeType type; + cudaGraphNodeGetType(_native_node, &type); + return type; +} + +// Function: dump +inline void cudaTask::dump(std::ostream& os) const { + os << "cudaTask [type=" << to_string(type()) << ']'; +} + +/** +@brief overload of ostream inserter operator for cudaTask +*/ +inline std::ostream& operator << (std::ostream& os, const cudaTask& ct) { + ct.dump(os); + return os; +} + +// ---------------------------------------------------------------------------- +// cudaGraph +// ---------------------------------------------------------------------------- + +/** + @class cudaGraphCreator + + @brief class to create functors that construct CUDA graphs + + This class define functors to new CUDA graphs using `cudaGraphCreate`. + +*/ +class cudaGraphCreator { + + public: + + /** + * @brief creates a new CUDA graph + * + * Calls `cudaGraphCreate` to generate a CUDA native graph and returns it. + * If the graph creation fails, an error is reported. + * + * @return A newly created `cudaGraph_t` instance. + * @throws If CUDA graph creation fails, an error is logged. + */ + cudaGraph_t operator () () const { + cudaGraph_t g; + TF_CHECK_CUDA(cudaGraphCreate(&g, 0), "failed to create a CUDA native graph"); + return g; + } + + /** + @brief return the given CUDA graph + */ + cudaGraph_t operator () (cudaGraph_t graph) const { + return graph; + } + +}; + +/** + @class cudaGraphDeleter + + @brief class to create a functor that deletes a CUDA graph + + This structure provides an overloaded function call operator to safely + destroy a CUDA graph using `cudaGraphDestroy`. + +*/ +class cudaGraphDeleter { + + public: + + /** + * @brief deletes a CUDA graph + * + * Calls `cudaGraphDestroy` to release the CUDA graph resource if it is valid. + * + * @param g the CUDA graph to be destroyed + */ + void operator () (cudaGraph_t g) const { + cudaGraphDestroy(g); + } +}; + + +/** +@class cudaGraphBase + +@brief class to create a CUDA graph with uunique ownership + +@tparam Creator functor to create the stream (used in constructor) +@tparam Deleter functor to delete the stream (used in destructor) + +This class wraps a `cudaGraph_t` handle with std::unique_ptr to ensure proper +resource management and automatic cleanup. +*/ +template +class cudaGraphBase : public std::unique_ptr, cudaGraphDeleter> { + + static_assert(std::is_pointer_v, "cudaGraph_t is not a pointer type"); + + public: + + /** + @brief base std::unique_ptr type + */ + using base_type = std::unique_ptr, Deleter>; + + /** + @brief constructs a `cudaGraph` object by passing the given arguments to the executable CUDA graph creator + + Constructs a `cudaGraph` object by passing the given arguments to the executable CUDA graph creator + + @param args arguments to pass to the executable CUDA graph creator + */ + template + explicit cudaGraphBase(ArgsT&& ... args) : base_type( + Creator{}(std::forward(args)...), Deleter() + ) { + } + + /** + @brief constructs a `cudaGraph` from the given rhs using move semantics + */ + cudaGraphBase(cudaGraphBase&&) = default; + + /** + @brief assign the rhs to `*this` using move semantics + */ + cudaGraphBase& operator = (cudaGraphBase&&) = default; + + /** + @brief queries the number of nodes in a native CUDA graph + */ + size_t num_nodes() const; + + /** + @brief queries the number of edges in a native CUDA graph + */ + size_t num_edges() const; + + /** + @brief queries if the graph is empty + */ + bool empty() const; + + /** + @brief dumps the CUDA graph to a DOT format through the given output stream + + @param os target output stream + */ + void dump(std::ostream& os); + + // ------------------------------------------------------------------------ + // Graph building routines + // ------------------------------------------------------------------------ + + /** + @brief creates a no-operation task + + @return a tf::cudaTask handle + + An empty node performs no operation during execution, + but can be used for transitive ordering. + For example, a phased execution graph with 2 groups of @c n nodes + with a barrier between them can be represented using an empty node + and @c 2*n dependency edges, + rather than no empty node and @c n^2 dependency edges. + */ + cudaTask noop(); + + /** + @brief creates a host task that runs a callable on the host + + @tparam C callable type + + @param callable a callable object with neither arguments nor return + (i.e., constructible from @c std::function) + @param user_data a pointer to the user data + + @return a tf::cudaTask handle + + A host task can only execute CPU-specific functions and cannot do any CUDA calls + (e.g., @c cudaMalloc). + */ + template + cudaTask host(C&& callable, void* user_data); + + /** + @brief creates a kernel task + + @tparam F kernel function type + @tparam ArgsT kernel function parameters type + + @param g configured grid + @param b configured block + @param s configured shared memory size in bytes + @param f kernel function + @param args arguments to forward to the kernel function by copy + + @return a tf::cudaTask handle + */ + template + cudaTask kernel(dim3 g, dim3 b, size_t s, F f, ArgsT... args); + + /** + @brief creates a memset task that fills untyped data with a byte value + + @param dst pointer to the destination device memory area + @param v value to set for each byte of specified memory + @param count size in bytes to set + + @return a tf::cudaTask handle + + A memset task fills the first @c count bytes of device memory area + pointed by @c dst with the byte value @c v. + */ + cudaTask memset(void* dst, int v, size_t count); + + /** + @brief creates a memcpy task that copies untyped data in bytes + + @param tgt pointer to the target memory block + @param src pointer to the source memory block + @param bytes bytes to copy + + @return a tf::cudaTask handle + + A memcpy task transfers @c bytes of data from a source location + to a target location. Direction can be arbitrary among CPUs and GPUs. + */ + cudaTask memcpy(void* tgt, const void* src, size_t bytes); + + /** + @brief creates a memset task that sets a typed memory block to zero + + @tparam T element type (size of @c T must be either 1, 2, or 4) + @param dst pointer to the destination device memory area + @param count number of elements + + @return a tf::cudaTask handle + + A zero task zeroes the first @c count elements of type @c T + in a device memory area pointed by @c dst. + */ + template && (sizeof(T)==1 || sizeof(T)==2 || sizeof(T)==4), void>* = nullptr + > + cudaTask zero(T* dst, size_t count); + + /** + @brief creates a memset task that fills a typed memory block with a value + + @tparam T element type (size of @c T must be either 1, 2, or 4) + + @param dst pointer to the destination device memory area + @param value value to fill for each element of type @c T + @param count number of elements + + @return a tf::cudaTask handle + + A fill task fills the first @c count elements of type @c T with @c value + in a device memory area pointed by @c dst. + The value to fill is interpreted in type @c T rather than byte. + */ + template && (sizeof(T)==1 || sizeof(T)==2 || sizeof(T)==4), void>* = nullptr + > + cudaTask fill(T* dst, T value, size_t count); + + /** + @brief creates a memcopy task that copies typed data + + @tparam T element type (non-void) + + @param tgt pointer to the target memory block + @param src pointer to the source memory block + @param num number of elements to copy + + @return a tf::cudaTask handle + + A copy task transfers num*sizeof(T) bytes of data from a source location + to a target location. Direction can be arbitrary among CPUs and GPUs. + */ + template , void>* = nullptr + > + cudaTask copy(T* tgt, const T* src, size_t num); + + // ------------------------------------------------------------------------ + // generic algorithms + // ------------------------------------------------------------------------ + + /** + @brief runs a callable with only a single kernel thread + + @tparam C callable type + + @param c callable to run by a single kernel thread + + @return a tf::cudaTask handle + */ + template + cudaTask single_task(C c); + + /** + @brief applies a callable to each dereferenced element of the data array + + @tparam I iterator type + @tparam C callable type + @tparam E execution poligy (default tf::cudaDefaultExecutionPolicy) + + @param first iterator to the beginning (inclusive) + @param last iterator to the end (exclusive) + @param callable a callable object to apply to the dereferenced iterator + + @return a tf::cudaTask handle + + This method is equivalent to the parallel execution of the following loop on a GPU: + + @code{.cpp} + for(auto itr = first; itr != last; itr++) { + callable(*itr); + } + @endcode + */ + template + cudaTask for_each(I first, I last, C callable); + + /** + @brief applies a callable to each index in the range with the step size + + @tparam I index type + @tparam C callable type + @tparam E execution poligy (default tf::cudaDefaultExecutionPolicy) + + @param first beginning index + @param last last index + @param step step size + @param callable the callable to apply to each element in the data array + + @return a tf::cudaTask handle + + This method is equivalent to the parallel execution of the following loop on a GPU: + + @code{.cpp} + // step is positive [first, last) + for(auto i=first; ilast; i+=step) { + callable(i); + } + @endcode + */ + template + cudaTask for_each_index(I first, I last, I step, C callable); + + /** + @brief applies a callable to a source range and stores the result in a target range + + @tparam I input iterator type + @tparam O output iterator type + @tparam C unary operator type + @tparam E execution poligy (default tf::cudaDefaultExecutionPolicy) + + @param first iterator to the beginning of the input range + @param last iterator to the end of the input range + @param output iterator to the beginning of the output range + @param op the operator to apply to transform each element in the range + + @return a tf::cudaTask handle + + This method is equivalent to the parallel execution of the following loop on a GPU: + + @code{.cpp} + while (first != last) { + *output++ = callable(*first++); + } + @endcode + */ + template + cudaTask transform(I first, I last, O output, C op); + + /** + @brief creates a task to perform parallel transforms over two ranges of items + + @tparam I1 first input iterator type + @tparam I2 second input iterator type + @tparam O output iterator type + @tparam C unary operator type + @tparam E execution poligy (default tf::cudaDefaultExecutionPolicy) + + @param first1 iterator to the beginning of the input range + @param last1 iterator to the end of the input range + @param first2 iterato + @param output iterator to the beginning of the output range + @param op binary operator to apply to transform each pair of items in the + two input ranges + + @return cudaTask handle + + This method is equivalent to the parallel execution of the following loop on a GPU: + + @code{.cpp} + while (first1 != last1) { + *output++ = op(*first1++, *first2++); + } + @endcode + */ + template + cudaTask transform(I1 first1, I1 last1, I2 first2, O output, C op); + + private: + + cudaGraphBase(const cudaGraphBase&) = delete; + cudaGraphBase& operator = (const cudaGraphBase&) = delete; +}; + +// query the number of nodes +template +size_t cudaGraphBase::num_nodes() const { + size_t n; + TF_CHECK_CUDA( + cudaGraphGetNodes(this->get(), nullptr, &n), + "failed to get native graph nodes" + ); + return n; +} + +// query the emptiness +template +bool cudaGraphBase::empty() const { + return num_nodes() == 0; +} + +// query the number of edges +template +size_t cudaGraphBase::num_edges() const { + return cuda_graph_get_num_edges(this->get()); +} + +//// dump the graph +//inline void cudaGraph::dump(std::ostream& os) { +// +// // acquire the native handle +// auto g = this->get(); +// +// os << "digraph cudaGraph {\n"; +// +// std::stack> stack; +// stack.push(std::make_tuple(g, nullptr, 1)); +// +// int pl = 0; +// +// while(stack.empty() == false) { +// +// auto [graph, parent, l] = stack.top(); +// stack.pop(); +// +// for(int i=0; i " << 'p' << to << ";\n"; +// } +// +// for(auto& node : nodes) { +// auto type = cuda_get_graph_node_type(node); +// if(type == cudaGraphNodeTypeGraph) { +// +// cudaGraph_t child_graph; +// TF_CHECK_CUDA(cudaGraphChildGraphNodeGetGraph(node, &child_graph), ""); +// stack.push(std::make_tuple(child_graph, node, l+1)); +// +// os << 'p' << node << "[" +// << "shape=folder, style=filled, fontcolor=white, fillcolor=purple, " +// << "label=\"cudaGraph-L" << l+1 +// << "\"];\n"; +// } +// else { +// os << 'p' << node << "[label=\"" +// << to_string(type) +// << "\"];\n"; +// } +// } +// +// // precede to parent +// if(parent != nullptr) { +// std::unordered_set successors; +// for(const auto& p : edges) { +// successors.insert(p.first); +// } +// for(auto node : nodes) { +// if(successors.find(node) == successors.end()) { +// os << 'p' << node << " -> " << 'p' << parent << ";\n"; +// } +// } +// } +// +// // set the previous level +// pl = l; +// } +// +// for(int i=0; i<=pl; i++) { +// os << "}\n"; +// } +//} + +// dump the graph +template +void cudaGraphBase::dump(std::ostream& os) { + + // Generate a unique temporary filename in the system's temp directory using filesystem + auto temp_path = std::filesystem::temp_directory_path() / "graph_"; + std::random_device rd; + std::uniform_int_distribution dist(100000, 999999); // Generates a random number + temp_path += std::to_string(dist(rd)) + ".dot"; + + // Call the original function with the temporary file + TF_CHECK_CUDA(cudaGraphDebugDotPrint(this->get(), temp_path.string().c_str(), 0), ""); + + // Read the file and write to the output stream + std::ifstream file(temp_path); + if (file) { + os << file.rdbuf(); // Copy file contents to the stream + file.close(); + std::filesystem::remove(temp_path); // Clean up the temporary file + } else { + TF_THROW("failed to open ", temp_path, " for dumping the CUDA graph"); + } +} + +// Function: noop +template +cudaTask cudaGraphBase::noop() { + + cudaGraphNode_t node; + + TF_CHECK_CUDA( + cudaGraphAddEmptyNode(&node, this->get(), nullptr, 0), + "failed to create a no-operation (empty) node" + ); + + return cudaTask(this->get(), node); +} + +// Function: host +template +template +cudaTask cudaGraphBase::host(C&& callable, void* user_data) { + + cudaGraphNode_t node; + cudaHostNodeParams p {callable, user_data}; + + TF_CHECK_CUDA( + cudaGraphAddHostNode(&node, this->get(), nullptr, 0, &p), + "failed to create a host node" + ); + + return cudaTask(this->get(), node); +} + +// Function: kernel +template +template +cudaTask cudaGraphBase::kernel( + dim3 g, dim3 b, size_t s, F f, ArgsT... args +) { + + cudaGraphNode_t node; + cudaKernelNodeParams p; + + void* arguments[sizeof...(ArgsT)] = { (void*)(&args)... }; + + p.func = (void*)f; + p.gridDim = g; + p.blockDim = b; + p.sharedMemBytes = s; + p.kernelParams = arguments; + p.extra = nullptr; + + TF_CHECK_CUDA( + cudaGraphAddKernelNode(&node, this->get(), nullptr, 0, &p), + "failed to create a kernel task" + ); + + return cudaTask(this->get(), node); +} + +// Function: zero +template +template && (sizeof(T)==1 || sizeof(T)==2 || sizeof(T)==4), void>* +> +cudaTask cudaGraphBase::zero(T* dst, size_t count) { + + cudaGraphNode_t node; + auto p = cuda_get_zero_parms(dst, count); + + TF_CHECK_CUDA( + cudaGraphAddMemsetNode(&node, this->get(), nullptr, 0, &p), + "failed to create a memset (zero) task" + ); + + return cudaTask(this->get(), node); +} + +// Function: fill +template +template && (sizeof(T)==1 || sizeof(T)==2 || sizeof(T)==4), void>* +> +cudaTask cudaGraphBase::fill(T* dst, T value, size_t count) { + + cudaGraphNode_t node; + auto p = cuda_get_fill_parms(dst, value, count); + TF_CHECK_CUDA( + cudaGraphAddMemsetNode(&node, this->get(), nullptr, 0, &p), + "failed to create a memset (fill) task" + ); + + return cudaTask(this->get(), node); +} + +// Function: copy +template +template < + typename T, + std::enable_if_t, void>* +> +cudaTask cudaGraphBase::copy(T* tgt, const T* src, size_t num) { + + cudaGraphNode_t node; + auto p = cuda_get_copy_parms(tgt, src, num); + + TF_CHECK_CUDA( + cudaGraphAddMemcpyNode(&node, this->get(), nullptr, 0, &p), + "failed to create a memcpy (copy) task" + ); + + return cudaTask(this->get(), node); +} + +// Function: memset +template +cudaTask cudaGraphBase::memset(void* dst, int ch, size_t count) { + + cudaGraphNode_t node; + auto p = cuda_get_memset_parms(dst, ch, count); + + TF_CHECK_CUDA( + cudaGraphAddMemsetNode(&node, this->get(), nullptr, 0, &p), + "failed to create a memset task" + ); + + return cudaTask(this->get(), node); +} + +// Function: memcpy +template +cudaTask cudaGraphBase::memcpy(void* tgt, const void* src, size_t bytes) { + + cudaGraphNode_t node; + auto p = cuda_get_memcpy_parms(tgt, src, bytes); + + TF_CHECK_CUDA( + cudaGraphAddMemcpyNode(&node, this->get(), nullptr, 0, &p), + "failed to create a memcpy task" + ); + + return cudaTask(this->get(), node); +} + + + + + +} // end of namespace tf ----------------------------------------------------- + + + + diff --git a/projects/hipfile/third-party/taskflow/cuda/cuda_graph_exec.hpp b/projects/hipfile/third-party/taskflow/cuda/cuda_graph_exec.hpp new file mode 100644 index 000000000000..912c9f6c6f0a --- /dev/null +++ b/projects/hipfile/third-party/taskflow/cuda/cuda_graph_exec.hpp @@ -0,0 +1,384 @@ +#pragma once + +#include "cuda_graph.hpp" + + +namespace tf { + +// ---------------------------------------------------------------------------- +// cudaGraphExec +// ---------------------------------------------------------------------------- + +/** +@class cudaGraphExecCreator +@brief class to create functors for constructing executable CUDA graphs + +This class provides an overloaded function call operator to create a +new executable CUDA graph using `cudaGraphCreate`. +*/ +class cudaGraphExecCreator { + + public: + + /** + @brief returns a null executable CUDA graph + */ + cudaGraphExec_t operator () () const { + return nullptr; + } + + /** + @brief returns the given executable graph + */ + cudaGraphExec_t operator () (cudaGraphExec_t exec) const { + return exec; + } + + /** + @brief returns a newly instantiated executable graph from the given CUDA graph + */ + cudaGraphExec_t operator () (cudaGraph_t graph) const { + cudaGraphExec_t exec; + TF_CHECK_CUDA( + cudaGraphInstantiate(&exec, graph, nullptr, nullptr, 0), + "failed to create an executable graph" + ); + return exec; + } + + /** + @brief returns a newly instantiated executable graph from the given CUDA graph + */ + template + cudaGraphExec_t operator () (const cudaGraphBase& graph) const { + return this->operator()(graph.get()); + } +}; + +/** +@class cudaGraphExecDeleter +@brief class to create a functor for deleting an executable CUDA graph + +This class provides an overloaded function call operator to safely +destroy a CUDA graph using `cudaGraphDestroy`. +*/ +class cudaGraphExecDeleter { + + public: + + /** + @brief deletes an executable CUDA graph + + Calls `cudaGraphDestroy` to release the CUDA graph resource if it is valid. + + @param executable the executable CUDA graph to be destroyed + */ + void operator () (cudaGraphExec_t executable) const { + cudaGraphExecDestroy(executable); + } +}; + +/** +@class cudaGraphExecBase + +@brief class to create an executable CUDA graph with unique ownership + +@tparam Creator functor to create the stream (used in constructor) +@tparam Deleter functor to delete the stream (used in destructor) + +This class wraps a `cudaGraphExec_t` handle with `std::unique_ptr` to ensure proper +resource management and automatic cleanup. +*/ +template +class cudaGraphExecBase : public std::unique_ptr, Deleter> { + + static_assert(std::is_pointer_v, "cudaGraphExec_t is not a pointer type"); + + public: + + /** + @brief base std::unique_ptr type + */ + using base_type = std::unique_ptr, Deleter>; + + /** + @brief constructs a `cudaGraphExec` object by passing the given arguments to the executable CUDA graph creator + + Constructs a `cudaGraphExec` object by passing the given arguments to the executable CUDA graph creator + + @param args arguments to pass to the executable CUDA graph creator + */ + template + explicit cudaGraphExecBase(ArgsT&& ... args) : base_type( + Creator{}(std::forward(args)...), Deleter() + ) {} + + /** + @brief constructs a `cudaGraphExec` from the given rhs using move semantics + */ + cudaGraphExecBase(cudaGraphExecBase&&) = default; + + /** + @brief assign the rhs to `*this` using move semantics + */ + cudaGraphExecBase& operator = (cudaGraphExecBase&&) = default; + + // ---------------------------------------------------------------------------------------------- + // Update Methods + // ---------------------------------------------------------------------------------------------- + + /** + @brief updates parameters of a host task + + This method updates the parameter of the given host task (similar to tf::cudaFlow::host). + */ + template + void host(cudaTask task, C&& callable, void* user_data); + + /** + @brief updates parameters of a kernel task + + The method is similar to tf::cudaFlow::kernel but operates on a task + of type tf::cudaTaskType::KERNEL. + The kernel function name must NOT change. + */ + template + void kernel( + cudaTask task, dim3 g, dim3 b, size_t shm, F f, ArgsT... args + ); + + /** + @brief updates parameters of a memset task + + The method is similar to tf::cudaFlow::memset but operates on a task + of type tf::cudaTaskType::MEMSET. + The source/destination memory may have different address values but + must be allocated from the same contexts as the original + source/destination memory. + */ + void memset(cudaTask task, void* dst, int ch, size_t count); + + /** + @brief updates parameters of a memcpy task + + The method is similar to tf::cudaFlow::memcpy but operates on a task + of type tf::cudaTaskType::MEMCPY. + The source/destination memory may have different address values but + must be allocated from the same contexts as the original + source/destination memory. + */ + void memcpy(cudaTask task, void* tgt, const void* src, size_t bytes); + + /** + @brief updates parameters of a memset task to a zero task + + The method is similar to tf::cudaFlow::zero but operates on + a task of type tf::cudaTaskType::MEMSET. + + The source/destination memory may have different address values but + must be allocated from the same contexts as the original + source/destination memory. + */ + template && (sizeof(T)==1 || sizeof(T)==2 || sizeof(T)==4), void>* = nullptr + > + void zero(cudaTask task, T* dst, size_t count); + + /** + @brief updates parameters of a memset task to a fill task + + The method is similar to tf::cudaFlow::fill but operates on a task + of type tf::cudaTaskType::MEMSET. + + The source/destination memory may have different address values but + must be allocated from the same contexts as the original + source/destination memory. + */ + template && (sizeof(T)==1 || sizeof(T)==2 || sizeof(T)==4), void>* = nullptr + > + void fill(cudaTask task, T* dst, T value, size_t count); + + /** + @brief updates parameters of a memcpy task to a copy task + + The method is similar to tf::cudaFlow::copy but operates on a task + of type tf::cudaTaskType::MEMCPY. + The source/destination memory may have different address values but + must be allocated from the same contexts as the original + source/destination memory. + */ + template , void>* = nullptr + > + void copy(cudaTask task, T* tgt, const T* src, size_t num); + + //--------------------------------------------------------------------------- + // Algorithm Primitives + //--------------------------------------------------------------------------- + + /** + @brief updates a single-threaded kernel task + + This method is similar to cudaFlow::single_task but operates + on an existing task. + */ + template + void single_task(cudaTask task, C c); + + /** + @brief updates parameters of a `for_each` kernel task created from the CUDA graph of `*this` + */ + template + void for_each(cudaTask task, I first, I last, C callable); + + /** + @brief updates parameters of a `for_each_index` kernel task created from the CUDA graph of `*this` + */ + template + void for_each_index(cudaTask task, I first, I last, I step, C callable); + + /** + @brief updates parameters of a `transform` kernel task created from the CUDA graph of `*this` + */ + template + void transform(cudaTask task, I first, I last, O output, C c); + + /** + @brief updates parameters of a `transform` kernel task created from the CUDA graph of `*this` + */ + template + void transform(cudaTask task, I1 first1, I1 last1, I2 first2, O output, C c); + + + private: + + cudaGraphExecBase(const cudaGraphExecBase&) = delete; + + cudaGraphExecBase& operator = (const cudaGraphExecBase&) = delete; +}; + +// ------------------------------------------------------------------------------------------------ +// update methods +// ------------------------------------------------------------------------------------------------ + +// Function: host +template +template +void cudaGraphExecBase::host(cudaTask task, C&& func, void* user_data) { + cudaHostNodeParams p {func, user_data}; + TF_CHECK_CUDA( + cudaGraphExecHostNodeSetParams(this->get(), task._native_node, &p), + "failed to update kernel parameters on ", task + ); +} + +// Function: update kernel parameters +template +template +void cudaGraphExecBase::kernel( + cudaTask task, dim3 g, dim3 b, size_t s, F f, ArgsT... args +) { + cudaKernelNodeParams p; + + void* arguments[sizeof...(ArgsT)] = { (void*)(&args)... }; + p.func = (void*)f; + p.gridDim = g; + p.blockDim = b; + p.sharedMemBytes = s; + p.kernelParams = arguments; + p.extra = nullptr; + + TF_CHECK_CUDA( + cudaGraphExecKernelNodeSetParams(this->get(), task._native_node, &p), + "failed to update kernel parameters on ", task + ); +} + +// Function: update copy parameters +template +template , void>*> +void cudaGraphExecBase::copy(cudaTask task, T* tgt, const T* src, size_t num) { + auto p = cuda_get_copy_parms(tgt, src, num); + TF_CHECK_CUDA( + cudaGraphExecMemcpyNodeSetParams(this->get(), task._native_node, &p), + "failed to update memcpy parameters on ", task + ); +} + +// Function: update memcpy parameters +template +void cudaGraphExecBase::memcpy( + cudaTask task, void* tgt, const void* src, size_t bytes +) { + auto p = cuda_get_memcpy_parms(tgt, src, bytes); + + TF_CHECK_CUDA( + cudaGraphExecMemcpyNodeSetParams(this->get(), task._native_node, &p), + "failed to update memcpy parameters on ", task + ); +} + +// Procedure: memset +template +void cudaGraphExecBase::memset(cudaTask task, void* dst, int ch, size_t count) { + auto p = cuda_get_memset_parms(dst, ch, count); + TF_CHECK_CUDA( + cudaGraphExecMemsetNodeSetParams(this->get(), task._native_node, &p), + "failed to update memset parameters on ", task + ); +} + +// Procedure: fill +template +template && (sizeof(T)==1 || sizeof(T)==2 || sizeof(T)==4), void>* +> +void cudaGraphExecBase::fill(cudaTask task, T* dst, T value, size_t count) { + auto p = cuda_get_fill_parms(dst, value, count); + TF_CHECK_CUDA( + cudaGraphExecMemsetNodeSetParams(this->get(), task._native_node, &p), + "failed to update memset parameters on ", task + ); +} + +// Procedure: zero +template +template && (sizeof(T)==1 || sizeof(T)==2 || sizeof(T)==4), void>* +> +void cudaGraphExecBase::zero(cudaTask task, T* dst, size_t count) { + auto p = cuda_get_zero_parms(dst, count); + TF_CHECK_CUDA( + cudaGraphExecMemsetNodeSetParams(this->get(), task._native_node, &p), + "failed to update memset parameters on ", task + ); +} + +//------------------------------------------------------------------------------------------------- +// forward declaration +//------------------------------------------------------------------------------------------------- + +/** +@private +*/ +template +cudaStreamBase& cudaStreamBase::run(cudaGraphExec_t exec) { + TF_CHECK_CUDA( + cudaGraphLaunch(exec, this->get()), "failed to launch a CUDA executable graph" + ); + return *this; +} + +/** +@private +*/ +template +template +cudaStreamBase& cudaStreamBase::run(const cudaGraphExecBase& exec) { + return run(exec.get()); +} + + + +} // end of namespace tf ------------------------------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/cuda/cuda_memory.hpp b/projects/hipfile/third-party/taskflow/cuda/cuda_memory.hpp new file mode 100644 index 000000000000..76aa10167d96 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/cuda/cuda_memory.hpp @@ -0,0 +1,839 @@ +#pragma once + +#include "cuda_device.hpp" + +/** +@file cuda_memory.hpp +@brief CUDA memory utilities include file +*/ + +namespace tf { + +// ---------------------------------------------------------------------------- +// memory +// ---------------------------------------------------------------------------- + +/** +@brief queries the free memory (expensive call) +*/ +inline size_t cuda_get_free_mem(int d) { + cudaScopedDevice ctx(d); + size_t free, total; + TF_CHECK_CUDA( + cudaMemGetInfo(&free, &total), "failed to get mem info on device ", d + ); + return free; +} + +/** +@brief queries the total available memory (expensive call) +*/ +inline size_t cuda_get_total_mem(int d) { + cudaScopedDevice ctx(d); + size_t free, total; + TF_CHECK_CUDA( + cudaMemGetInfo(&free, &total), "failed to get mem info on device ", d + ); + return total; +} + +/** +@brief allocates memory on the given device for holding @c N elements of type @c T + +The function calls @c cudaMalloc to allocate N*sizeof(T) bytes of memory +on the given device @c d and returns a pointer to the starting address of +the device memory. +*/ +template +T* cuda_malloc_device(size_t N, int d) { + cudaScopedDevice ctx(d); + T* ptr {nullptr}; + TF_CHECK_CUDA( + cudaMalloc(&ptr, N*sizeof(T)), + "failed to allocate memory (", N*sizeof(T), "bytes) on device ", d + ) + return ptr; +} + +/** +@brief allocates memory on the current device associated with the caller + +The function calls malloc_device from the current device associated +with the caller. +*/ +template +T* cuda_malloc_device(size_t N) { + T* ptr {nullptr}; + TF_CHECK_CUDA( + cudaMalloc(&ptr, N*sizeof(T)), + "failed to allocate memory (", N*sizeof(T), "bytes)" + ) + return ptr; +} + +/** +@brief allocates shared memory for holding @c N elements of type @c T + +The function calls @c cudaMallocManaged to allocate N*sizeof(T) bytes +of memory and returns a pointer to the starting address of the shared memory. +*/ +template +T* cuda_malloc_shared(size_t N) { + T* ptr {nullptr}; + TF_CHECK_CUDA( + cudaMallocManaged(&ptr, N*sizeof(T)), + "failed to allocate shared memory (", N*sizeof(T), "bytes)" + ) + return ptr; +} + +/** +@brief frees memory on the GPU device + +@tparam T pointer type +@param ptr device pointer to memory to free +@param d device context identifier + +This methods call @c cudaFree to free the memory space pointed to by @c ptr +using the given device context. +*/ +template +void cuda_free(T* ptr, int d) { + cudaScopedDevice ctx(d); + TF_CHECK_CUDA(cudaFree(ptr), "failed to free memory ", ptr, " on GPU ", d); +} + +/** +@brief frees memory on the GPU device + +@tparam T pointer type +@param ptr device pointer to memory to free + +This methods call @c cudaFree to free the memory space pointed to by @c ptr +using the current device context of the caller. +*/ +template +void cuda_free(T* ptr) { + TF_CHECK_CUDA(cudaFree(ptr), "failed to free memory ", ptr); +} + +/** +@brief copies data between host and device asynchronously through a stream + +@param stream stream identifier +@param dst destination memory address +@param src source memory address +@param count size in bytes to copy + +The method calls @c cudaMemcpyAsync with the given @c stream +using @c cudaMemcpyDefault to infer the memory space of the source and +the destination pointers. The memory areas may not overlap. +*/ +inline void cuda_memcpy_async( + cudaStream_t stream, void* dst, const void* src, size_t count +) { + TF_CHECK_CUDA( + cudaMemcpyAsync(dst, src, count, cudaMemcpyDefault, stream), + "failed to perform cudaMemcpyAsync" + ); +} + +/** +@brief initializes or sets GPU memory to the given value byte by byte + +@param stream stream identifier +@param devPtr pointer to GPU memory +@param value value to set for each byte of the specified memory +@param count size in bytes to set + +The method calls @c cudaMemsetAsync with the given @c stream +to fill the first @c count bytes of the memory area pointed to by @c devPtr +with the constant byte value @c value. +*/ +inline void cuda_memset_async( + cudaStream_t stream, void* devPtr, int value, size_t count +){ + TF_CHECK_CUDA( + cudaMemsetAsync(devPtr, value, count, stream), + "failed to perform cudaMemsetAsync" + ); +} + +// ---------------------------------------------------------------------------- +// Shared Memory +// ---------------------------------------------------------------------------- +// +// Because dynamically sized shared memory arrays are declared "extern", +// we can't templatize them directly. To get around this, we declare a +// simple wrapper struct that will declare the extern array with a different +// name depending on the type. This avoids compiler errors about duplicate +// definitions. +// +// To use dynamically allocated shared memory in a templatized __global__ or +// __device__ function, just replace code like this: +// +// template +// __global__ void +// foo( T* g_idata, T* g_odata) +// { +// // Shared mem size is determined by the host app at run time +// extern __shared__ T sdata[]; +// ... +// doStuff(sdata); +// ... +// } +// +// With this: +// +// template +// __global__ void +// foo( T* g_idata, T* g_odata) +// { +// // Shared mem size is determined by the host app at run time +// cudaSharedMemory smem; +// T* sdata = smem.get(); +// ... +// doStuff(sdata); +// ... +// } +// ---------------------------------------------------------------------------- + +// This is the un-specialized struct. Note that we prevent instantiation of this +// struct by putting an undefined symbol in the function body so it won't compile. +/** +@private +*/ +template +struct cudaSharedMemory +{ + // Ensure that we won't compile any un-specialized types + __device__ T *get() + { + extern __device__ void error(void); + error(); + return NULL; + } +}; + +// Following are the specializations for the following types. +// int, uint, char, uchar, short, ushort, long, ulong, bool, float, and double +// One could also specialize it for user-defined types. + +/** +@private +*/ +template <> +struct cudaSharedMemory +{ + __device__ int *get() + { + extern __shared__ int s_int[]; + return s_int; + } +}; + +/** +@private +*/ +template <> +struct cudaSharedMemory +{ + __device__ unsigned int *get() + { + extern __shared__ unsigned int s_uint[]; + return s_uint; + } +}; + +/** +@private +*/ +template <> +struct cudaSharedMemory +{ + __device__ char *get() + { + extern __shared__ char s_char[]; + return s_char; + } +}; + +/** +@private +*/ +template <> +struct cudaSharedMemory +{ + __device__ unsigned char *get() + { + extern __shared__ unsigned char s_uchar[]; + return s_uchar; + } +}; + +/** +@private +*/ +template <> +struct cudaSharedMemory +{ + __device__ short *get() + { + extern __shared__ short s_short[]; + return s_short; + } +}; + +/** +@private +*/ +template <> +struct cudaSharedMemory +{ + __device__ unsigned short *get() + { + extern __shared__ unsigned short s_ushort[]; + return s_ushort; + } +}; + +/** +@private +*/ +template <> +struct cudaSharedMemory +{ + __device__ long *get() + { + extern __shared__ long s_long[]; + return s_long; + } +}; + +/** +@private +*/ +template <> +struct cudaSharedMemory +{ + __device__ unsigned long *get() + { + extern __shared__ unsigned long s_ulong[]; + return s_ulong; + } +}; + +//template <> +//struct cudaSharedMemory +//{ +// __device__ size_t *get() +// { +// extern __shared__ size_t s_sizet[]; +// return s_sizet; +// } +//}; + +/** +@private +*/ +template <> +struct cudaSharedMemory +{ + __device__ bool *get() + { + extern __shared__ bool s_bool[]; + return s_bool; + } +}; + +/** +@private +*/ +template <> +struct cudaSharedMemory +{ + __device__ float *get() + { + extern __shared__ float s_float[]; + return s_float; + } +}; + +/** +@private +*/ +template <> +struct cudaSharedMemory +{ + __device__ double *get() + { + extern __shared__ double s_double[]; + return s_double; + } +}; + + + +// ---------------------------------------------------------------------------- +// cudaDeviceAllocator +// ---------------------------------------------------------------------------- + +/** +@private +*/ +template +class cudaDeviceAllocator { + + public: + + /** + @brief element type + */ + using value_type = T; + + /** + @brief element pointer type + */ + using pointer = T*; + + /** + @brief element reference type + */ + using reference = T&; + + /** + @brief const element pointer type + */ + using const_pointer = const T*; + + /** + @brief constant element reference type + */ + using const_reference = const T&; + + /** + @brief size type + */ + using size_type = std::size_t; + + /** + @brief pointer difference type + */ + using difference_type = std::ptrdiff_t; + + /** + @brief its member type @c U is the equivalent allocator type to allocate elements of type U + */ + template + struct rebind { + /** + @brief allocator of a different data type + */ + using other = cudaDeviceAllocator; + }; + + /** + @brief Constructs a device allocator object. + */ + cudaDeviceAllocator() noexcept {} + + /** + @brief Constructs a device allocator object from another device allocator object. + */ + cudaDeviceAllocator( const cudaDeviceAllocator& ) noexcept {} + + /** + @brief Constructs a device allocator object from another device allocator + object with a different element type. + */ + template + cudaDeviceAllocator( const cudaDeviceAllocator& ) noexcept {} + + /** + @brief Destructs the device allocator object. + */ + ~cudaDeviceAllocator() noexcept {} + + /** + @brief Returns the address of x. + + This effectively means returning &x. + + @param x reference to an object + @return a pointer to the object + */ + pointer address( reference x ) { return &x; } + + /** + @brief Returns the address of x. + + This effectively means returning &x. + + @param x reference to an object + @return a pointer to the object + */ + const_pointer address( const_reference x ) const { return &x; } + + /** + @brief allocates block of storage. + + Attempts to allocate a block of storage with a size large enough to contain + @c n elements of member type, @c value_type, and returns a pointer + to the first element. + + The storage is aligned appropriately for object of type @c value_type, + but they are not constructed. + + The block of storage is allocated using cudaMalloc and throws std::bad_alloc + if it cannot allocate the total amount of storage requested. + + @param n number of elements (each of size sizeof(value_type)) to be allocated + @return a pointer to the initial element in the block of storage. + */ + pointer allocate( size_type n, const void* = 0 ) + { + void* ptr = NULL; + TF_CHECK_CUDA( + cudaMalloc( &ptr, n*sizeof(T) ), + "failed to allocate ", n, " elements (", n*sizeof(T), "bytes)" + ) + return static_cast(ptr); + } + + /** + @brief Releases a block of storage previously allocated with member allocate and not yet released + + The elements in the array are not destroyed by a call to this member function. + + @param ptr pointer to a block of storage previously allocated with allocate + */ + void deallocate( pointer ptr, size_type ) + { + if(ptr){ + cudaFree(ptr); + } + } + + /** + @brief returns the maximum number of elements that could potentially + be allocated by this allocator + + A call to member allocate with the value returned by this function + can still fail to allocate the requested storage. + + @return the number of elements that might be allocated as maximum + by a call to member allocate + */ + size_type max_size() const noexcept { return size_type {-1}; } + + /** + @brief ignored to avoid de-referencing device pointer from the host + */ + void construct( pointer, const_reference) { } + + /** + @brief ignored to avoid de-referencing device pointer from the host + */ + void destroy( pointer) { } + + /** + @brief compares two allocator of different types using @c == + + Device allocators of different types are always equal to each other + because the storage allocated by the allocator @c a1 can be deallocated + through @c a2. + */ + template + bool operator == (const cudaDeviceAllocator&) const noexcept { + return true; + } + + /** + @brief compares two allocator of different types using @c != + + Device allocators of different types are always equal to each other + because the storage allocated by the allocator @c a1 can be deallocated + through @c a2. + */ + template + bool operator != (const cudaDeviceAllocator&) const noexcept { + return false; + } + +}; + +// ---------------------------------------------------------------------------- +// cudaUSMAllocator +// ---------------------------------------------------------------------------- + +/** +@private +*/ +template +class cudaUSMAllocator { + + public: + + /** + @brief element type + */ + using value_type = T; + + /** + @brief element pointer type + */ + using pointer = T*; + + /** + @brief element reference type + */ + using reference = T&; + + /** + @brief const element pointer type + */ + using const_pointer = const T*; + + /** + @brief constant element reference type + */ + using const_reference = const T&; + + /** + @brief size type + */ + using size_type = std::size_t; + + /** + @brief pointer difference type + */ + using difference_type = std::ptrdiff_t; + + /** + @brief its member type @c U is the equivalent allocator type to allocate elements of type U + */ + template + struct rebind { + /** + @brief allocator of a different data type + */ + using other = cudaUSMAllocator; + }; + + /** + @brief Constructs a device allocator object. + */ + cudaUSMAllocator() noexcept {} + + /** + @brief Constructs a device allocator object from another device allocator object. + */ + cudaUSMAllocator( const cudaUSMAllocator& ) noexcept {} + + /** + @brief Constructs a device allocator object from another device allocator + object with a different element type. + */ + template + cudaUSMAllocator( const cudaUSMAllocator& ) noexcept {} + + /** + @brief Destructs the device allocator object. + */ + ~cudaUSMAllocator() noexcept {} + + /** + @brief Returns the address of x. + + This effectively means returning &x. + + @param x reference to an object + @return a pointer to the object + */ + pointer address( reference x ) { return &x; } + + /** + @brief Returns the address of x. + + This effectively means returning &x. + + @param x reference to an object + @return a pointer to the object + */ + const_pointer address( const_reference x ) const { return &x; } + + /** + @brief allocates block of storage. + + Attempts to allocate a block of storage with a size large enough to contain + @c n elements of member type, @c value_type, and returns a pointer + to the first element. + + The storage is aligned appropriately for object of type @c value_type, + but they are not constructed. + + The block of storage is allocated using cudaMalloc and throws std::bad_alloc + if it cannot allocate the total amount of storage requested. + + @param n number of elements (each of size sizeof(value_type)) to be allocated + @return a pointer to the initial element in the block of storage. + */ + pointer allocate( size_type n, const void* = 0 ) + { + void* ptr {nullptr}; + TF_CHECK_CUDA( + cudaMallocManaged( &ptr, n*sizeof(T) ), + "failed to allocate ", n, " elements (", n*sizeof(T), "bytes)" + ) + return static_cast(ptr); + } + + /** + @brief Releases a block of storage previously allocated with member allocate and not yet released + + The elements in the array are not destroyed by a call to this member function. + + @param ptr pointer to a block of storage previously allocated with allocate + */ + void deallocate( pointer ptr, size_type ) + { + if(ptr){ + cudaFree(ptr); + } + } + + /** + @brief returns the maximum number of elements that could potentially + be allocated by this allocator + + A call to member allocate with the value returned by this function + can still fail to allocate the requested storage. + + @return the number of elements that might be allocated as maximum + by a call to member allocate + */ + size_type max_size() const noexcept { return size_type {-1}; } + + /** + @brief Constructs an element object on the location pointed by ptr. + @param ptr pointer to a location with enough storage soace to contain + an element of type @c value_type + + @param val value to initialize the constructed element to + */ + void construct( pointer ptr, const_reference val ) { + new ((void*)ptr) value_type(val); + } + + /** + @brief destroys in-place the object pointed by @c ptr + + Notice that this does not deallocate the storage for the element but calls + its destructor. + + @param ptr pointer to the object to be destroye + */ + void destroy( pointer ptr ) { + ptr->~value_type(); + } + + /** + @brief compares two allocator of different types using @c == + + USM allocators of different types are always equal to each other + because the storage allocated by the allocator @c a1 can be deallocated + through @c a2. + */ + template + bool operator == (const cudaUSMAllocator&) const noexcept { + return true; + } + + /** + @brief compares two allocator of different types using @c != + + USM allocators of different types are always equal to each other + because the storage allocated by the allocator @c a1 can be deallocated + through @c a2. + */ + template + bool operator != (const cudaUSMAllocator&) const noexcept { + return false; + } + +}; + +// ---------------------------------------------------------------------------- +// GPU vector object +// ---------------------------------------------------------------------------- + +//template +//using cudaDeviceVector = std::vector, cudaDeviceAllocator>>; + +//template +//using cudaUSMVector = std::vector>; + +/** +@private +*/ +template +class cudaDeviceVector { + + public: + + cudaDeviceVector() = default; + + cudaDeviceVector(size_t N) : _N {N} { + if(N) { + TF_CHECK_CUDA( + cudaMalloc(&_data, N*sizeof(T)), + "failed to allocate device memory (", N*sizeof(T), " bytes)" + ); + } + } + + cudaDeviceVector(cudaDeviceVector&& rhs) : + _data{rhs._data}, _N {rhs._N} { + rhs._data = nullptr; + rhs._N = 0; + } + + ~cudaDeviceVector() { + if(_data) { + cudaFree(_data); + } + } + + cudaDeviceVector& operator = (cudaDeviceVector&& rhs) { + if(_data) { + cudaFree(_data); + } + _data = rhs._data; + _N = rhs._N; + rhs._data = nullptr; + rhs._N = 0; + return *this; + } + + size_t size() const { return _N; } + + T* data() { return _data; } + const T* data() const { return _data; } + + cudaDeviceVector(const cudaDeviceVector&) = delete; + cudaDeviceVector& operator = (const cudaDeviceVector&) = delete; + + private: + + T* _data {nullptr}; + size_t _N {0}; +}; + + +} // end of namespace tf ----------------------------------------------------- + + + + + + diff --git a/projects/hipfile/third-party/taskflow/cuda/cuda_meta.hpp b/projects/hipfile/third-party/taskflow/cuda/cuda_meta.hpp new file mode 100644 index 000000000000..b08eb29b08fe --- /dev/null +++ b/projects/hipfile/third-party/taskflow/cuda/cuda_meta.hpp @@ -0,0 +1,452 @@ +#pragma once + +#include "cuda_execution_policy.hpp" + +namespace tf { + +// default warp size +inline constexpr unsigned CUDA_WARP_SIZE = 32; + +// empty type +struct cudaEmpty { }; + +// ---------------------------------------------------------------------------- +// iterator unrolling +// ---------------------------------------------------------------------------- + +// Template unrolled looping construct. +template +struct cudaIterate { + template + __device__ static void eval(F f) { + f(i); + cudaIterate::eval(f); + } +}; + +template +struct cudaIterate { + template + __device__ static void eval(F) { } +}; + +template +__device__ void cuda_iterate(F f) { + cudaIterate::eval(f); +} + +template +__device__ void cuda_iterate(F f) { + cuda_iterate<0, count>(f); +} + +template +__device__ T reduce(const T(&x)[count]) { + T y; + cuda_iterate([&](auto i) { y = i ? x[i] + y : x[i]; }); + return y; +} + +template +__device__ void fill(T(&x)[count], T val) { + cuda_iterate([&](auto i) { x[i] = val; }); +} + +// Invoke unconditionally. +template +__device__ void cuda_strided_iterate(F f, unsigned tid) { + cuda_iterate([=](auto i) { f(i, nt * i + tid); }); +} + +// Check range. +template +__device__ void cuda_strided_iterate(F f, unsigned tid, unsigned count) { + // Unroll the first vt0 elements of each thread. + if(vt0 > 1 && count >= nt * vt0) { + cuda_strided_iterate(f, tid); // No checking + } else { + cuda_iterate([=](auto i) { + auto j = nt * i + tid; + if(j < count) f(i, j); + }); + } + + // TODO: seems dummy when vt0 == vt + cuda_iterate([=](auto i) { + auto j = nt * i + tid; + if(j < count) f(i, j); + }); +} + +template +__device__ void cuda_thread_iterate(F f, unsigned tid) { + cuda_iterate([=](auto i) { f(i, vt * tid + i); }); +} + +// ---------------------------------------------------------------------------- +// cudaRange +// ---------------------------------------------------------------------------- + +// cudaRange +struct cudaRange { + unsigned begin, end; + __device__ unsigned size() const { return end - begin; } + __device__ unsigned count() const { return size(); } + __device__ bool valid() const { return end > begin; } +}; + +inline __device__ cudaRange cuda_get_tile(unsigned b, unsigned nv, unsigned count) { + return cudaRange { nv * b, min(count, nv * (b + 1)) }; +} + + +// ---------------------------------------------------------------------------- +// cudaArray +// ---------------------------------------------------------------------------- + +template +struct cudaArray { + T data[size]; + + __device__ T operator[](unsigned i) const { return data[i]; } + __device__ T& operator[](unsigned i) { return data[i]; } + + cudaArray() = default; + cudaArray(const cudaArray&) = default; + cudaArray& operator=(const cudaArray&) = default; + + // Fill the array with x. + __device__ cudaArray(T x) { + cuda_iterate([&](unsigned i) { data[i] = x; }); + } +}; + +template +struct cudaArray { + __device__ T operator[](unsigned) const { return T(); } + __device__ T& operator[](unsigned) { return *(T*)nullptr; } +}; + +template +struct cudaKVArray { + cudaArray keys; + cudaArray vals; +}; + +// ---------------------------------------------------------------------------- +// thread reg <-> global mem +// ---------------------------------------------------------------------------- + +template +__device__ auto cuda_mem_to_reg_strided(I mem, unsigned tid, unsigned count) { + using T = typename std::iterator_traits::value_type; + cudaArray x; + cuda_strided_iterate( + [&](auto i, auto j) { x[i] = mem[j]; }, tid, count + ); + return x; +} + +template +__device__ void cuda_reg_to_mem_strided( + cudaArray x, unsigned tid, unsigned count, it_t mem) { + + cuda_strided_iterate( + [=](auto i, auto j) { mem[j] = x[i]; }, tid, count + ); +} + +template +__device__ auto cuda_transform_mem_to_reg_strided( + I mem, unsigned tid, unsigned count, O op +) { + using T = std::invoke_result_t::value_type>; + cudaArray x; + cuda_strided_iterate( + [&](auto i, auto j) { x[i] = op(mem[j]); }, tid, count + ); + return x; +} + +// ---------------------------------------------------------------------------- +// thread reg <-> shared +// ---------------------------------------------------------------------------- + +template +__device__ void cuda_reg_to_shared_thread( + cudaArray x, unsigned tid, T (&shared)[shared_size], bool sync = true +) { + + static_assert(shared_size >= nt * vt, + "reg_to_shared_thread must have at least nt * vt storage"); + + cuda_thread_iterate([&](auto i, auto j) { shared[j] = x[i]; }, tid); + + if(sync) __syncthreads(); +} + +template +__device__ auto cuda_shared_to_reg_thread( + const T (&shared)[shared_size], unsigned tid, bool sync = true +) { + + static_assert(shared_size >= nt * vt, + "reg_to_shared_thread must have at least nt * vt storage"); + + cudaArray x; + cuda_thread_iterate([&](auto i, auto j) { + x[i] = shared[j]; + }, tid); + + if(sync) __syncthreads(); + + return x; +} + +template +__device__ void cuda_reg_to_shared_strided( + cudaArray x, unsigned tid, T (&shared)[shared_size], bool sync = true +) { + + static_assert(shared_size >= nt * vt, + "reg_to_shared_strided must have at least nt * vt storage"); + + cuda_strided_iterate( + [&](auto i, auto j) { shared[j] = x[i]; }, tid + ); + + if(sync) __syncthreads(); +} + +template +__device__ auto cuda_shared_to_reg_strided( + const T (&shared)[shared_size], unsigned tid, bool sync = true +) { + + static_assert(shared_size >= nt * vt, + "shared_to_reg_strided must have at least nt * vt storage"); + + cudaArray x; + cuda_strided_iterate([&](auto i, auto j) { x[i] = shared[j]; }, tid); + if(sync) __syncthreads(); + + return x; +} + +template< + unsigned nt, unsigned vt, unsigned vt0 = vt, typename T, typename it_t, + unsigned shared_size +> +__device__ auto cuda_reg_to_mem_thread( + cudaArray x, unsigned tid, + unsigned count, it_t mem, T (&shared)[shared_size] +) { + cuda_reg_to_shared_thread(x, tid, shared); + auto y = cuda_shared_to_reg_strided(shared, tid); + cuda_reg_to_mem_strided(y, tid, count, mem); +} + +template< + unsigned nt, unsigned vt, unsigned vt0 = vt, typename T, typename it_t, + unsigned shared_size +> +__device__ auto cuda_mem_to_reg_thread( + it_t mem, unsigned tid, unsigned count, T (&shared)[shared_size] +) { + + auto x = cuda_mem_to_reg_strided(mem, tid, count); + cuda_reg_to_shared_strided(x, tid, shared); + auto y = cuda_shared_to_reg_thread(shared, tid); + return y; +} + +template +__device__ auto cuda_shared_gather( + const T(&data)[S], cudaArray indices, bool sync = true +) { + + static_assert(S >= nt * vt, + "shared_gather must have at least nt * vt storage"); + + cudaArray x; + cuda_iterate([&](auto i) { x[i] = data[indices[i]]; }); + + if(sync) __syncthreads(); + + return x; +} + + + +// ---------------------------------------------------------------------------- +// reg<->reg +// ---------------------------------------------------------------------------- + +template +__device__ auto cuda_reg_thread_to_strided( + cudaArray x, unsigned tid, T (&shared)[S] +) { + cuda_reg_to_shared_thread(x, tid, shared); + return cuda_shared_to_reg_strided(shared, tid); +} + +template +__device__ auto cuda_reg_strided_to_thread( + cudaArray x, unsigned tid, T (&shared)[S] +) { + cuda_reg_to_shared_strided(x, tid, shared); + return cuda_shared_to_reg_thread(shared, tid); +} + +// ---------------------------------------------------------------------------- +// cudaLoadStoreIterator +// ---------------------------------------------------------------------------- + +template +struct cudaLoadStoreIterator : std::iterator_traits { + + L load; + S store; + I base; + + cudaLoadStoreIterator(L load_, S store_, I base_) : + load(load_), store(store_), base(base_) { } + + struct assign_t { + L load; + S store; + I index; + + __device__ assign_t& operator=(T rhs) { + static_assert(!std::is_same::value, + "load_iterator is being stored to."); + store(rhs, index); + return *this; + } + __device__ operator T() const { + static_assert(!std::is_same::value, + "store_iterator is being loaded from."); + return load(index); + } + }; + + __device__ assign_t operator[](I index) const { + return assign_t { load, store, base + index }; + } + + __device__ assign_t operator*() const { + return assign_t { load, store, base }; + } + + __device__ cudaLoadStoreIterator operator+(I offset) const { + cudaLoadStoreIterator cp = *this; + cp += offset; + return cp; + } + + __device__ cudaLoadStoreIterator& operator+=(I offset) { + base += offset; + return *this; + } + + __device__ cudaLoadStoreIterator operator-(I offset) const { + cudaLoadStoreIterator cp = *this; + cp -= offset; + return cp; + } + + __device__ cudaLoadStoreIterator& operator-=(I offset) { + base -= offset; + return *this; + } +}; + +//template +//struct trivial_load_functor { +// template +// __device__ T operator()(I index) const { +// return T(); +// } +//}; + +//template +//struct trivial_store_functor { +// template +// __device__ void operator()(T v, I index) const { } +//}; + +template +auto cuda_make_load_store_iterator(L load, S store, I base = 0) { + return cudaLoadStoreIterator(load, store, base); +} + +template +auto cuda_make_load_iterator(L load, I base = 0) { + return cuda_make_load_store_iterator(load, cudaEmpty(), base); +} + +template +auto cuda_make_store_iterator(S store, I base = 0) { + return cuda_make_load_store_iterator(cudaEmpty(), store, base); +} + +// ---------------------------------------------------------------------------- +// swap +// ---------------------------------------------------------------------------- + +template +__device__ void cuda_swap(T& a, T& b) { + auto c = a; + a = b; + b = c; +} + +// ---------------------------------------------------------------------------- +// launch kernel +// ---------------------------------------------------------------------------- + +template +__global__ void cuda_kernel(F f, args_t... args) { + f(threadIdx.x, blockIdx.x, args...); +} + +// ---------------------------------------------------------------------------- +// operators +// ---------------------------------------------------------------------------- + +template +struct cuda_plus{ + __device__ T operator()(T a, T b) const { return a + b; } +}; + + template +struct cuda_minus{ + __device__ T operator()(T a, T b) const { return a - b; } +}; + +template +struct cuda_multiplies{ + __device__ T operator()(T a, T b) const { return a * b; } +}; + +template +struct cuda_maximum{ + __device__ T operator()(T a, T b) const { return a > b ? a : b; } +}; + +template +struct cuda_minimum{ + __device__ T operator()(T a, T b) const { return a < b ? a : b; } +}; + +template +struct cuda_less{ + __device__ T operator()(T a, T b) const { return a < b; } +}; + +template +struct cuda_greater{ + __device__ T operator()(T a, T b) const { return a > b; } +}; + +} // end of namespace tf ----------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/cuda/cuda_optimizer.hpp b/projects/hipfile/third-party/taskflow/cuda/cuda_optimizer.hpp new file mode 100644 index 000000000000..60efed1a7941 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/cuda/cuda_optimizer.hpp @@ -0,0 +1,404 @@ +#pragma once + +#include "cuda_graph.hpp" + +/** +@file cuda_optimizer.hpp +@brief %cudaFlow capturing algorithms include file +*/ + +namespace tf { + +// ---------------------------------------------------------------------------- +// cudaFlowOptimizerBase +// ---------------------------------------------------------------------------- + +/** +@private + +@brief class to provide helper common methods for optimization algorithms +*/ +class cudaFlowOptimizerBase { + + protected: + + std::vector _toposort(cudaFlowGraph&); + std::vector> _levelize(cudaFlowGraph&); +}; + +// Function: _toposort +inline std::vector cudaFlowOptimizerBase::_toposort(cudaFlowGraph& graph) { + + std::vector res; + std::queue bfs; + + res.reserve(graph._nodes.size()); + + // insert the first level of nodes into the queue + for(auto& u : graph._nodes) { + + auto hu = std::get_if(&u->_handle); + hu->level = u->_dependents.size(); + + if(hu->level == 0) { + bfs.push(u.get()); + } + } + + // levelize the graph using bfs + while(!bfs.empty()) { + + auto u = bfs.front(); + bfs.pop(); + + res.push_back(u); + + for(auto v : u->_successors) { + auto hv = std::get_if(&v->_handle); + if(--hv->level == 0) { + bfs.push(v); + } + } + } + + return res; +} + +// Function: _levelize +inline std::vector> +cudaFlowOptimizerBase::_levelize(cudaFlowGraph& graph) { + + std::queue bfs; + + size_t max_level = 0; + + // insert the first level of nodes into the queue + for(auto& u : graph._nodes) { + + auto hu = std::get_if(&u->_handle); + hu->level = u->_dependents.size(); + + if(hu->level == 0) { + bfs.push(u.get()); + } + } + + // levelize the graph using bfs + while(!bfs.empty()) { + + auto u = bfs.front(); + bfs.pop(); + + auto hu = std::get_if(&u->_handle); + + for(auto v : u->_successors) { + auto hv = std::get_if(&v->_handle); + if(--hv->level == 0) { + hv->level = hu->level + 1; + if(hv->level > max_level) { + max_level = hv->level; + } + bfs.push(v); + } + } + } + + // set level_graph and each node's idx + std::vector> level_graph(max_level+1); + for(auto& u : graph._nodes) { + auto hu = std::get_if(&u->_handle); + hu->lid = level_graph[hu->level].size(); + level_graph[hu->level].emplace_back(u.get()); + + //for(auto s : u->_successors) { + // assert(hu.level < std::get_if(&s->_handle)->level); + //} + } + + return level_graph; +} + +// ---------------------------------------------------------------------------- +// class definition: cudaFlowSequentialOptimizer +// ---------------------------------------------------------------------------- + +/** +@class cudaFlowSequentialOptimizer + +@brief class to capture a CUDA graph using a sequential stream + +A sequential capturing algorithm finds a topological order of +the described graph and captures dependent GPU tasks using a single stream. +All GPU tasks run sequentially without breaking inter dependencies. +*/ +class cudaFlowSequentialOptimizer : public cudaFlowOptimizerBase { + + friend class cudaFlowCapturer; + + public: + + /** + @brief constructs a sequential optimizer + */ + cudaFlowSequentialOptimizer() = default; + + private: + + cudaGraph_t _optimize(cudaFlowGraph& graph); +}; + +inline cudaGraph_t cudaFlowSequentialOptimizer::_optimize(cudaFlowGraph& graph) { + + // acquire per-thread stream and turn it into capture mode + // we must use ThreadLocal mode to avoid clashing with CUDA global states + + cudaStream stream; + + stream.begin_capture(cudaStreamCaptureModeThreadLocal); + + auto ordered = _toposort(graph); + for(auto node : ordered) { + std::get_if(&node->_handle)->work(stream); + } + + return stream.end_capture(); +} + +// ---------------------------------------------------------------------------- +// class definition: cudaFlowLinearOptimizer +// ---------------------------------------------------------------------------- + +/** +@class cudaFlowLinearOptimizer + +@brief class to capture a linear CUDA graph using a sequential stream + +A linear capturing algorithm is a special case of tf::cudaFlowSequentialOptimizer +and assumes the input task graph to be a single linear chain of tasks +(i.e., a straight line). +This assumption allows faster optimization during the capturing process. +If the input task graph is not a linear chain, the behavior is undefined. +*/ +class cudaFlowLinearOptimizer : public cudaFlowOptimizerBase { + + friend class cudaFlowCapturer; + + public: + + /** + @brief constructs a linear optimizer + */ + cudaFlowLinearOptimizer() = default; + + private: + + cudaGraph_t _optimize(cudaFlowGraph& graph); +}; + +inline cudaGraph_t cudaFlowLinearOptimizer::_optimize(cudaFlowGraph& graph) { + + // acquire per-thread stream and turn it into capture mode + // we must use ThreadLocal mode to avoid clashing with CUDA global states + cudaStream stream; + + stream.begin_capture(cudaStreamCaptureModeThreadLocal); + + // find the source node + cudaFlowNode* src {nullptr}; + for(auto& u : graph._nodes) { + if(u->_dependents.size() == 0) { + src = u.get(); + while(src) { + std::get_if(&src->_handle)->work(stream); + src = src->_successors.empty() ? nullptr : src->_successors[0]; + } + break; + } + // ideally, there should be only one source + } + + return stream.end_capture(); +} + +// ---------------------------------------------------------------------------- +// class definition: cudaFlowRoundRobinOptimizer +// ---------------------------------------------------------------------------- + +/** +@class cudaFlowRoundRobinOptimizer + +@brief class to capture a CUDA graph using a round-robin algorithm + +A round-robin capturing algorithm levelizes the user-described graph +and assign streams to nodes in a round-robin order level by level. +The algorithm is based on the following paper published in Euro-Par 2021: + + Dian-Lun Lin and Tsung-Wei Huang, "Efficient GPU Computation using %Task Graph Parallelism," European Conference on Parallel and Distributed Computing (Euro-Par), 2021 + +The round-robin optimization algorithm is best suited for large %cudaFlow graphs +that compose hundreds of or thousands of GPU operations +(e.g., kernels and memory copies) with many of them being able to run in parallel. +You can configure the number of streams to the optimizer to adjust the +maximum kernel currency in the captured CUDA graph. +*/ +class cudaFlowRoundRobinOptimizer : public cudaFlowOptimizerBase { + + friend class cudaFlowCapturer; + + public: + + /** + @brief constructs a round-robin optimizer with 4 streams by default + */ + cudaFlowRoundRobinOptimizer() = default; + + /** + @brief constructs a round-robin optimizer with the given number of streams + */ + explicit cudaFlowRoundRobinOptimizer(size_t num_streams); + + /** + @brief queries the number of streams used by the optimizer + */ + size_t num_streams() const; + + /** + @brief sets the number of streams used by the optimizer + */ + void num_streams(size_t n); + + private: + + size_t _num_streams {4}; + + cudaGraph_t _optimize(cudaFlowGraph& graph); + + void _reset(std::vector>& graph); + +}; + +// Constructor +inline cudaFlowRoundRobinOptimizer::cudaFlowRoundRobinOptimizer(size_t num_streams) : + _num_streams {num_streams} { + + if(num_streams == 0) { + TF_THROW("number of streams must be at least one"); + } +} + +// Function: num_streams +inline size_t cudaFlowRoundRobinOptimizer::num_streams() const { + return _num_streams; +} + +// Procedure: num_streams +inline void cudaFlowRoundRobinOptimizer::num_streams(size_t n) { + if(n == 0) { + TF_THROW("number of streams must be at least one"); + } + _num_streams = n; +} + +inline void cudaFlowRoundRobinOptimizer::_reset( + std::vector>& graph +) { + //level == global id + //idx == stream id we want to skip + size_t id{0}; + for(auto& each_level: graph) { + for(auto& node: each_level) { + auto hn = std::get_if(&node->_handle); + hn->level = id++; + hn->idx = _num_streams; + hn->event = nullptr; + } + } +} + +// Function: _optimize +inline cudaGraph_t cudaFlowRoundRobinOptimizer::_optimize(cudaFlowGraph& graph) { + + // levelize the graph + auto levelized = _levelize(graph); + + // initialize the data structure + _reset(levelized); + + // begin to capture + std::vector streams(_num_streams); + + streams[0].begin_capture(cudaStreamCaptureModeThreadLocal); + + // reserve space for scoped events + std::vector events; + events.reserve((_num_streams >> 1) + levelized.size()); + + // fork + cudaEvent_t fork_event = events.emplace_back(); + streams[0].record(fork_event); + + for(size_t i = 1; i < streams.size(); ++i) { + streams[i].wait(fork_event); + } + + // assign streams to levelized nodes in a round-robin manner + for(auto& each_level: levelized) { + for(auto& node: each_level) { + auto hn = std::get_if(&node->_handle); + size_t sid = hn->lid % _num_streams; + + //wait events + cudaFlowNode* wait_node{nullptr}; + for(auto& pn: node->_dependents) { + auto phn = std::get_if(&pn->_handle); + size_t psid = phn->lid % _num_streams; + + //level == global id + //idx == stream id we want to skip + if(psid == hn->idx) { + if(wait_node == nullptr || + std::get_if(&wait_node->_handle)->level < phn->level) { + wait_node = pn; + } + } + else if(psid != sid) { + streams[sid].wait(phn->event); + } + } + + if(wait_node != nullptr) { + assert(std::get_if(&wait_node->_handle)->event); + streams[sid].wait(std::get_if(&wait_node->_handle)->event); + } + + //capture + hn->work(streams[sid]); + + //create/record stream + for(auto& sn: node->_successors) { + auto shn = std::get_if(&sn->_handle); + size_t ssid = shn->lid % _num_streams; + if(ssid != sid) { + if(!hn->event) { + hn->event = events.emplace_back(); + streams[sid].record(hn->event); + } + //idx == stream id we want to skip + shn->idx = sid; + } + } + } + } + + // join + for(size_t i=1; i<_num_streams; ++i) { + cudaEvent_t join_event = events.emplace_back(); + streams[i].record(join_event); + streams[0].wait(join_event); + } + + return streams[0].end_capture(); +} + + +} // end of namespace tf ----------------------------------------------------- + diff --git a/projects/hipfile/third-party/taskflow/cuda/cuda_stream.hpp b/projects/hipfile/third-party/taskflow/cuda/cuda_stream.hpp new file mode 100644 index 000000000000..cbcb7fd45dd3 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/cuda/cuda_stream.hpp @@ -0,0 +1,345 @@ +#pragma once + +#include "cuda_error.hpp" + +/** +@file cuda_stream.hpp +@brief CUDA stream utilities include file +*/ + +namespace tf { + + +// ---------------------------------------------------------------------------- +// cudaEventBase +// ---------------------------------------------------------------------------- + +/** +@class cudaEventCreator + +@brief class to create functors that construct CUDA events +*/ +class cudaEventCreator { + + public: + + /** + @brief creates a new `cudaEvent_t` object using `cudaEventCreate` + */ + cudaEvent_t operator () () const { + cudaEvent_t event; + TF_CHECK_CUDA(cudaEventCreate(&event), "failed to create a CUDA event"); + return event; + } + + /** + @brief creates a new `cudaEvent_t` object using `cudaEventCreate` with the given `flag` + */ + cudaEvent_t operator () (unsigned int flag) const { + cudaEvent_t event; + TF_CHECK_CUDA( + cudaEventCreateWithFlags(&event, flag), + "failed to create a CUDA event with flag=", flag + ); + return event; + } + + /** + @brief returns the given `cudaEvent_t` object + */ + cudaEvent_t operator () (cudaEvent_t event) const { + return event; + } +}; + +/** +@class cudaEventDeleter + +@brief class to create a functor that deletes a CUDA event +*/ +class cudaEventDeleter { + public: + /** + @brief deletes the given `cudaEvent_t` object using `cudaEventDestroy` + */ + void operator () (cudaEvent_t event) const { + cudaEventDestroy(event); + } +}; + +/** +@class cudaEventBase + +@brief class to create a CUDA event with unique ownership + +@tparam Creator functor to create the stream (used in constructor) +@tparam Deleter functor to delete the stream (used in destructor) + +The `cudaEventBase` class encapsulates a `cudaEvent_t` using `std::unique_ptr`, ensuring that +CUDA events are properly created and destroyed with a unique ownership. +*/ +template +class cudaEventBase : public std::unique_ptr, Deleter> { + + static_assert(std::is_pointer_v, "cudaEvent_t is not a pointer type"); + + public: + + /** + @brief base type for the underlying unique pointer + + This alias provides a shorthand for the underlying `std::unique_ptr` type that manages + CUDA event resources with an associated deleter. + */ + using base_type = std::unique_ptr, Deleter>; + + /** + @brief constructs a `cudaEvent` object by passing the given arguments to the event creator + + Constructs a `cudaEvent` object by passing the given arguments to the event creator + + @param args arguments to pass to the event creator + */ + template + explicit cudaEventBase(ArgsT&& ... args) : base_type( + Creator{}(std::forward(args)...), Deleter() + ) { + } + + /** + @brief constructs a `cudaEvent` from the given rhs using move semantics + */ + cudaEventBase(cudaEventBase&&) = default; + + /** + @brief assign the rhs to `*this` using move semantics + */ + cudaEventBase& operator = (cudaEventBase&&) = default; + + private: + + cudaEventBase(const cudaEventBase&) = delete; + cudaEventBase& operator = (const cudaEventBase&) = delete; +}; + +/** +@brief default smart pointer type to manage a `cudaEvent_t` object with unique ownership +*/ +using cudaEvent = cudaEventBase; + +// ---------------------------------------------------------------------------- +// cudaStream +// ---------------------------------------------------------------------------- + +/** +@class cudaStreamCreator + +@brief class to create functors that construct CUDA streams +*/ +class cudaStreamCreator { + + public: + + /** + @brief constructs a new `cudaStream_t` object using `cudaStreamCreate` + */ + cudaStream_t operator () () const { + cudaStream_t stream; + TF_CHECK_CUDA(cudaStreamCreate(&stream), "failed to create a CUDA stream"); + return stream; + } + + /** + @brief returns the given `cudaStream_t` object + */ + cudaStream_t operator () (cudaStream_t stream) const { + return stream; + } +}; + +/** +@class cudaStreamDeleter + +@brief class to create a functor that deletes a CUDA stream +*/ +class cudaStreamDeleter { + + public: + + /** + @brief deletes the given `cudaStream_t` object + */ + void operator () (cudaStream_t stream) const { + cudaStreamDestroy(stream); + } +}; + +/** +@class cudaStreamBase + +@brief class to create a CUDA stream with unique ownership + +@tparam Creator functor to create the stream (used in constructor) +@tparam Deleter functor to delete the stream (used in destructor) + +The `cudaStream` class encapsulates a `cudaStream_t` using `std::unique_ptr`, ensuring that +CUDA events are properly created and destroyed with a unique ownership. +*/ +template +class cudaStreamBase : public std::unique_ptr, Deleter> { + + static_assert(std::is_pointer_v, "cudaStream_t is not a pointer type"); + + public: + + /** + @brief base type for the underlying unique pointer + + This alias provides a shorthand for the underlying `std::unique_ptr` type that manages + CUDA stream resources with an associated deleter. + */ + using base_type = std::unique_ptr, Deleter>; + + /** + @brief constructs a `cudaStream` object by passing the given arguments to the stream creator + + Constructs a `cudaStream` object by passing the given arguments to the stream creator + + @param args arguments to pass to the stream creator + */ + template + explicit cudaStreamBase(ArgsT&& ... args) : base_type( + Creator{}(std::forward(args)...), Deleter() + ) { + } + + /** + @brief constructs a `cudaStream` from the given rhs using move semantics + */ + cudaStreamBase(cudaStreamBase&&) = default; + + /** + @brief assign the rhs to `*this` using move semantics + */ + cudaStreamBase& operator = (cudaStreamBase&&) = default; + + /** + @brief synchronizes the associated stream + + Equivalently calling @c cudaStreamSynchronize to block + until this stream has completed all operations. + */ + cudaStreamBase& synchronize() { + TF_CHECK_CUDA( + cudaStreamSynchronize(this->get()), "failed to synchronize a CUDA stream" + ); + return *this; + } + + /** + @brief begins graph capturing on the stream + + When a stream is in capture mode, all operations pushed into the stream + will not be executed, but will instead be captured into a graph, + which will be returned via cudaStream::end_capture. + + A thread's mode can be one of the following: + + @c cudaStreamCaptureModeGlobal: This is the default mode. + If the local thread has an ongoing capture sequence that was not initiated + with @c cudaStreamCaptureModeRelaxed at @c cuStreamBeginCapture, + or if any other thread has a concurrent capture sequence initiated with + @c cudaStreamCaptureModeGlobal, this thread is prohibited from potentially + unsafe API calls. + + + @c cudaStreamCaptureModeThreadLocal: If the local thread has an ongoing capture + sequence not initiated with @c cudaStreamCaptureModeRelaxed, + it is prohibited from potentially unsafe API calls. + Concurrent capture sequences in other threads are ignored. + + + @c cudaStreamCaptureModeRelaxed: The local thread is not prohibited + from potentially unsafe API calls. Note that the thread is still prohibited + from API calls which necessarily conflict with stream capture, for example, + attempting @c cudaEventQuery on an event that was last recorded + inside a capture sequence. + */ + void begin_capture(cudaStreamCaptureMode m = cudaStreamCaptureModeGlobal) const { + TF_CHECK_CUDA( + cudaStreamBeginCapture(this->get(), m), + "failed to begin capture on stream ", this->get(), " with thread mode ", m + ); + } + + /** + @brief ends graph capturing on the stream + + Equivalently calling @c cudaStreamEndCapture to + end capture on stream and returning the captured graph. + Capture must have been initiated on stream via a call to cudaStream::begin_capture. + If capture was invalidated, due to a violation of the rules of stream capture, + then a NULL graph will be returned. + */ + cudaGraph_t end_capture() const { + cudaGraph_t native_g; + TF_CHECK_CUDA( + cudaStreamEndCapture(this->get(), &native_g), + "failed to end capture on stream ", this->get() + ); + return native_g; + } + + /** + @brief records an event on the stream + + Equivalently calling @c cudaEventRecord to record an event on this stream, + both of which must be on the same CUDA context. + */ + void record(cudaEvent_t event) const { + TF_CHECK_CUDA( + cudaEventRecord(event, this->get()), + "failed to record event ", event, " on stream ", this->get() + ); + } + + /** + @brief waits on an event + + Equivalently calling @c cudaStreamWaitEvent to make all future work + submitted to stream wait for all work captured in event. + */ + void wait(cudaEvent_t event) const { + TF_CHECK_CUDA( + cudaStreamWaitEvent(this->get(), event, 0), + "failed to wait for event ", event, " on stream ", this->get() + ); + } + + /** + @brief runs the given executable CUDA graph + + @param exec the given `cudaGraphExec` + */ + template + cudaStreamBase& run(const cudaGraphExecBase& exec); + + /** + @brief runs the given executable CUDA graph + + @param exec the given `cudaGraphExec_t` + */ + cudaStreamBase& run(cudaGraphExec_t exec); + + private: + + cudaStreamBase(const cudaStreamBase&) = delete; + cudaStreamBase& operator = (const cudaStreamBase&) = delete; +}; + +/** +@brief default smart pointer type to manage a `cudaStream_t` object with unique ownership +*/ +using cudaStream = cudaStreamBase; + +} // end of namespace tf ----------------------------------------------------- + + + diff --git a/projects/hipfile/third-party/taskflow/cuda/cudaflow.hpp b/projects/hipfile/third-party/taskflow/cuda/cudaflow.hpp new file mode 100644 index 000000000000..770de9c15a67 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/cuda/cudaflow.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include "../taskflow.hpp" +#include "cuda_graph.hpp" +#include "cuda_graph_exec.hpp" +#include "algorithm/single_task.hpp" + +/** +@file taskflow/cuda/cudaflow.hpp +@brief cudaFlow include file +*/ + +namespace tf { + +/** +@brief default smart pointer type to manage a `cudaGraph_t` object with unique ownership +*/ +using cudaGraph = cudaGraphBase; + +/** +@brief default smart pointer type to manage a `cudaGraphExec_t` object with unique ownership +*/ +using cudaGraphExec = cudaGraphExecBase; + +} // end of namespace tf ----------------------------------------------------- + + diff --git a/projects/hipfile/third-party/taskflow/observer/chrome.hpp b/projects/hipfile/third-party/taskflow/observer/chrome.hpp new file mode 100644 index 000000000000..5a1af00ce6fe --- /dev/null +++ b/projects/hipfile/third-party/taskflow/observer/chrome.hpp @@ -0,0 +1,223 @@ +#pragma once + +#include "interface.hpp" + +/** +@file chrome.hpp +@brief ChromeObserver include file +*/ + +namespace tf { + +// ---------------------------------------------------------------------------- +// ChromeObserver definition +// ---------------------------------------------------------------------------- + +/** +@class: ChromeObserver + +@brief class to create an observer based on Chrome tracing format + +A tf::ChromeObserver inherits tf::ObserverInterface and defines methods to dump +the observed thread activities into a format that can be visualized through +@ChromeTracing. + +@code{.cpp} +tf::Taskflow taskflow; +tf::Executor executor; + +// insert tasks into taskflow +// ... + +// create a custom observer +std::shared_ptr observer = executor.make_observer(); + +// run the taskflow +executor.run(taskflow).wait(); + +// dump the thread activities to a chrome-tracing format. +observer->dump(std::cout); +@endcode +*/ +class ChromeObserver : public ObserverInterface { + + friend class Executor; + + // data structure to record each task execution + struct Segment { + + std::string name; + + observer_stamp_t beg; + observer_stamp_t end; + + Segment( + const std::string& n, + observer_stamp_t b, + observer_stamp_t e + ); + }; + + // data structure to store the entire execution timeline + struct Timeline { + observer_stamp_t origin; + std::vector> segments; + std::vector> stacks; + }; + + public: + + /** + @brief dumps the timelines into a @ChromeTracing format through + an output stream + */ + void dump(std::ostream& ostream) const; + + /** + @brief dumps the timelines into a @ChromeTracing format + */ + inline std::string dump() const; + + /** + @brief clears the timeline data + */ + inline void clear(); + + /** + @brief queries the number of tasks observed + */ + inline size_t num_tasks() const; + + private: + + inline void set_up(size_t num_workers) override final; + inline void on_entry(WorkerView w, TaskView task_view) override final; + inline void on_exit(WorkerView w, TaskView task_view) override final; + + Timeline _timeline; +}; + +// constructor +inline ChromeObserver::Segment::Segment( + const std::string& n, observer_stamp_t b, observer_stamp_t e +) : + name {n}, beg {b}, end {e} { +} + +// Procedure: set_up +inline void ChromeObserver::set_up(size_t num_workers) { + _timeline.segments.resize(num_workers); + _timeline.stacks.resize(num_workers); + + for(size_t w=0; w 0) { + break; + } + } + + os << '['; + + for(size_t w=first; w<_timeline.segments.size(); w++) { + + if(w != first && _timeline.segments[w].size() > 0) { + os << ','; + } + + for(size_t i=0; i<_timeline.segments[w].size(); i++) { + + os << '{'<< "\"cat\":\"ChromeObserver\","; + + // name field + os << "\"name\":\""; + if(_timeline.segments[w][i].name.empty()) { + os << w << '_' << i; + } + else { + os << _timeline.segments[w][i].name; + } + os << "\","; + + // segment field + os << "\"ph\":\"X\"," + << "\"pid\":1," + << "\"tid\":" << w << ',' + << "\"ts\":" << duration_cast( + _timeline.segments[w][i].beg - _timeline.origin + ).count() << ',' + << "\"dur\":" << duration_cast( + _timeline.segments[w][i].end - _timeline.segments[w][i].beg + ).count(); + + if(i != _timeline.segments[w].size() - 1) { + os << "},"; + } + else { + os << '}'; + } + } + } + os << "]\n"; +} + +// Function: dump +inline std::string ChromeObserver::dump() const { + std::ostringstream oss; + dump(oss); + return oss.str(); +} + +// Function: num_tasks +inline size_t ChromeObserver::num_tasks() const { + return std::accumulate( + _timeline.segments.begin(), _timeline.segments.end(), size_t{0}, + [](size_t sum, const auto& exe){ + return sum + exe.size(); + } + ); +} + +} // end of namespace tf ----------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/observer/interface.hpp b/projects/hipfile/third-party/taskflow/observer/interface.hpp new file mode 100644 index 000000000000..3a6cab08638e --- /dev/null +++ b/projects/hipfile/third-party/taskflow/observer/interface.hpp @@ -0,0 +1,111 @@ +#pragma once + +#include "../core/task.hpp" +#include "../core/worker.hpp" + +/** +@file interface.hpp +@brief ObserverInterface include file +*/ + +namespace tf { + +// ---------------------------------------------------------------------------- +// common observer types +// ---------------------------------------------------------------------------- + +/** +@brief default time point type of observers +*/ +using observer_stamp_t = std::chrono::time_point; + +// ---------------------------------------------------------------------------- +// ObserverInterface +// ---------------------------------------------------------------------------- + +/** +@class: ObserverInterface + +@brief class to derive an executor observer + +The tf::ObserverInterface class allows users to define custom methods to monitor +the behaviors of an executor. This is particularly useful when you want to +inspect the performance of an executor and visualize when each thread +participates in the execution of a task. +To prevent users from direct access to the internal threads and tasks, +tf::ObserverInterface provides immutable wrappers, +tf::WorkerView and tf::TaskView, over workers and tasks. + +Please refer to tf::WorkerView and tf::TaskView for details. + +Example usage: + +@code{.cpp} + +struct MyObserver : public tf::ObserverInterface { + + MyObserver(const std::string& name) { + std::cout << "constructing observer " << name << '\n'; + } + + void set_up(size_t num_workers) override final { + std::cout << "setting up observer with " << num_workers << " workers\n"; + } + + void on_entry(WorkerView w, tf::TaskView tv) override final { + std::ostringstream oss; + oss << "worker " << w.id() << " ready to run " << tv.name() << '\n'; + std::cout << oss.str(); + } + + void on_exit(WorkerView w, tf::TaskView tv) override final { + std::ostringstream oss; + oss << "worker " << w.id() << " finished running " << tv.name() << '\n'; + std::cout << oss.str(); + } +}; + +tf::Taskflow taskflow; +tf::Executor executor; + +// insert tasks into taskflow +// ... + +// create a custom observer +std::shared_ptr observer = executor.make_observer("MyObserver"); + +// run the taskflow +executor.run(taskflow).wait(); +@endcode +*/ +class ObserverInterface { + + public: + + /** + @brief virtual destructor + */ + virtual ~ObserverInterface() = default; + + /** + @brief constructor-like method to call when the executor observer is fully created + @param num_workers the number of the worker threads in the executor + */ + virtual void set_up(size_t num_workers) = 0; + + /** + @brief method to call before a worker thread executes a closure + @param wv an immutable view of this worker thread + @param task_view a constant wrapper object to the task + */ + virtual void on_entry(WorkerView wv, TaskView task_view) = 0; + + /** + @brief method to call after a worker thread executed a closure + @param wv an immutable view of this worker thread + @param task_view a constant wrapper object to the task + */ + virtual void on_exit(WorkerView wv, TaskView task_view) = 0; +}; + +} // end of namespace tf ----------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/observer/tfprof.hpp b/projects/hipfile/third-party/taskflow/observer/tfprof.hpp new file mode 100644 index 000000000000..8890f4b06872 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/observer/tfprof.hpp @@ -0,0 +1,979 @@ +#pragma once + +#include "interface.hpp" +#include "../utility/os.hpp" + +/** +@file observer/tfprof_observer.hpp +@brief TFProfObserver include file +*/ + +namespace tf { + +// ---------------------------------------------------------------------------- +// timeline data structure +// ---------------------------------------------------------------------------- + +/** +@private +*/ +struct Segment { + + std::string name; + TaskType type; + + observer_stamp_t beg; + observer_stamp_t end; + + Segment() = default; + + Segment( + const std::string& n, TaskType t, observer_stamp_t b, observer_stamp_t e + ) : name {n}, type {t}, beg {b}, end {e} { + } + + auto span() const { + return end-beg; + } +}; + +/** +@private +*/ +struct Timeline { + + size_t uid; + + observer_stamp_t origin; + std::vector>> segments; + + Timeline() = default; + + Timeline(const Timeline& rhs) = delete; + Timeline(Timeline&& rhs) = default; + + Timeline& operator = (const Timeline& rhs) = delete; + Timeline& operator = (Timeline&& rhs) = default; +}; + +// ---------------------------------------------------------------------------- +// TFProfObserver definition +// ---------------------------------------------------------------------------- + +/** +@class TFProfObserver + +@brief class to create an observer based on the built-in taskflow profiler format + +A tf::TFProfObserver inherits tf::ObserverInterface and defines methods to dump +the observed thread activities into a format that can be visualized through +@TFProf. + +@code{.cpp} +tf::Taskflow taskflow; +tf::Executor executor; + +// insert tasks into taskflow +// ... + +// create a custom observer +std::shared_ptr observer = executor.make_observer(); + +// run the taskflow +executor.run(taskflow).wait(); + +// dump the thread activities to Taskflow Profiler format. +observer->dump(std::cout); +@endcode + +*/ +class TFProfObserver : public ObserverInterface { + + friend class Executor; + friend class TFProfManager; + + public: + + + struct TaskSummary { + size_t count {0}; + size_t total_span {0}; + size_t min_span {0}; // valid when count > 0 + size_t max_span {0}; // valid when count > 0 + + double avg_span() const { + return count ? static_cast(total_span) / count : 0.0; + } + + // update with a new observation of duration t (microseconds) + void update(size_t t) { + total_span += t; + min_span = (count == 0) ? t : (std::min)(min_span, t); + max_span = (count == 0) ? t : (std::max)(max_span, t); + count += 1; + } + }; + + /** + @private + @brief per-worker aggregated statistics, collapsed across all nesting levels + + busy_us = sum of all task durations on this worker (across all levels) + idle_us = wall_us - busy_us (wall_us = global max(end) - global min(beg)) + util_pct = busy_us / wall_us * 100 + */ + struct WorkerSummary { + size_t id {0}; + size_t count {0}; + size_t busy_us {0}; + size_t idle_us {0}; + size_t min_span {0}; // valid when count > 0 + size_t max_span {0}; // valid when count > 0 + + double avg_span() const { + return count ? static_cast(busy_us) / count : 0.0; + } + + double util_pct(size_t wall_us) const { + return wall_us ? static_cast(busy_us) / wall_us * 100.0 : 0.0; + } + + void update(size_t t) { + busy_us += t; + min_span = (count == 0) ? t : (std::min)(min_span, t); + max_span = (count == 0) ? t : (std::max)(max_span, t); + count += 1; + } + }; + + /** + @private + @brief top-level summary container built by summary() and consumed by dump_* + + worker_histogram[b] = distinct workers active during bin b (0..num_workers). + task_histogram[b] = total concurrent tasks during bin b (can exceed num_workers). + num_bins is computed at runtime from the 80-char line width and scaled tick label width. + Each bin covers (wall_us / num_bins) microseconds of wall time. + */ + struct Summary { + std::array tsum; + std::vector wsum; + std::vector worker_histogram; // sized at runtime to num_bins + std::vector task_histogram; // sized at runtime to num_bins + + size_t wall_us {0}; + size_t num_all_workers {0}; + + // avg_utilization = sum of (busy_us / wall_us) across ALL workers + // (including idle ones) divided by num_all_workers. + // this answers "how well did I use my thread pool overall?" — a worker + // that ran no tasks contributes 0% to the sum, correctly dragging the + // average down when the pool was underutilized. + // result is in [0, 100]%. + double avg_utilization() const { + if(wall_us == 0 || num_all_workers == 0) return 0.0; + double sum = 0.0; + for(const auto& w : wsum) { + sum += static_cast(w.busy_us) / static_cast(wall_us) * 100.0; + } + return sum / static_cast(num_all_workers); + } + + void dump_overview (std::ostream&, size_t uid, size_t num_tasks) const; + void dump_tsum (std::ostream&) const; + void dump_wsum (std::ostream&) const; + void dump (std::ostream&, size_t uid, size_t num_tasks) const; + }; + + public: + + /** + @brief dumps the timelines into a @TFProf format through + an output stream + */ + + /** + @brief dumps the timelines into a JSON string + */ + + /** + @brief dumps this executor's data as a self-contained .tfp executor block. + Writes its own string table followed by all worker-level segment data. + Called by TFProfManager::dump after the file header is written. + */ + void dump(std::ostream& ostream) const; + + /** + @brief shows the summary report through an output stream + */ + void summary(std::ostream& ostream) const; + + /** + @brief returns the summary report in a string + */ + std::string summary() const; + + /** + @brief clears the timeline data + */ + void clear(); + + /** + @brief queries the number of tasks observed + */ + size_t num_tasks() const; + + /** + @brief queries the number of observed workers + */ + size_t num_workers() const; + + private: + + Timeline _timeline; + + std::vector> _stacks; + + inline void set_up(size_t num_workers) override final; + inline void on_entry(WorkerView, TaskView) override final; + inline void on_exit(WorkerView, TaskView) override final; + + // ----------------------------------------------------------------------- + // .tfp binary format helpers + // Kept private so they have direct access to _timeline without + // any friend declarations or public/protected exposure. + // ----------------------------------------------------------------------- + + // Portable little-endian write helpers + static void _tfp_write_u8 (std::ostream& os, uint8_t v) { + os.put(static_cast(v)); + } + static void _tfp_write_u16(std::ostream& os, uint16_t v) { + uint8_t buf[2] = { uint8_t(v), uint8_t(v >> 8) }; + os.write(reinterpret_cast(buf), 2); + } + static void _tfp_write_u32(std::ostream& os, uint32_t v) { + uint8_t buf[4] = { + uint8_t(v), uint8_t(v >> 8), + uint8_t(v >> 16), uint8_t(v >> 24) + }; + os.write(reinterpret_cast(buf), 4); + } + static void _tfp_write_u64(std::ostream& os, uint64_t v) { + uint8_t buf[8] = { + uint8_t(v), uint8_t(v >> 8), + uint8_t(v >> 16), uint8_t(v >> 24), + uint8_t(v >> 32), uint8_t(v >> 40), + uint8_t(v >> 48), uint8_t(v >> 56) + }; + os.write(reinterpret_cast(buf), 8); + } + + // Variable-length unsigned integer (varint) — LEB128 encoding. + // Values 0-127 encode in 1 byte. Each byte uses 7 bits of data; + // the MSB is a continuation flag (1 = more bytes follow). + // Typical delta timestamps (1-1000 µs) encode in 1-2 bytes vs 8 bytes fixed. + static void _tfp_write_varint(std::ostream& os, uint64_t v) { + do { + uint8_t byte = v & 0x7F; + v >>= 7; + if(v) byte |= 0x80; // set continuation bit + os.put(static_cast(byte)); + } while(v); + } + + // TaskType -> uint8 (must match JS TYPE_NAMES array order: + // 0=static 1=subflow 2=condition 3=module 4=async) + static uint8_t _tfp_type_byte(TaskType t) { + switch(t) { + case TaskType::STATIC: return 0; + case TaskType::SUBFLOW: return 1; + case TaskType::CONDITION: return 2; + case TaskType::MODULE: return 3; + case TaskType::ASYNC: return 4; + default: return 0; + } + } + + // Deduplicating string table builder (used by dump) + struct _StringTable { + std::vector data; + std::unordered_map idx; + + // Returns {byte_offset, clamped_length}. + // Empty name => {0, 0} meaning anonymous; viewer generates "W{w}_{i}". + std::pair intern(const std::string& name) { + if(name.empty()) return { uint32_t{0}, uint8_t{0} }; + const uint8_t len = static_cast( + name.size() > 31 ? 31 : name.size() + ); + auto it = idx.find(name); + if(it != idx.end()) return { it->second, len }; + const uint32_t off = static_cast(data.size()); + data.insert(data.end(), name.begin(), name.begin() + len); + idx.emplace(name, off); + return { off, len }; + } + }; +}; + + +// ---------------------------------------------------------------------------- +// TFProfObserver::Summary output methods +// ---------------------------------------------------------------------------- + +// helper: emit a horizontal rule of width w +static inline void _tf_rule(std::ostream& os, size_t w, char c = '-') { + for(size_t i = 0; i < w; ++i) os << c; + os << '\n'; +} + +// Helper: _tf_time_scale +// Given a duration in microseconds, returns a human-readable scaled value +// and sets unit to the appropriate suffix string. +// Thresholds: +// < 10,000 us -> us (microseconds) +// < 10,000 ms -> ms (milliseconds) +// < 10,000 s -> s (seconds) +// otherwise -> min (minutes) +static inline double _tf_time_scale(size_t us, const char*& unit) { + if(us < 10000ULL) { + unit = "us"; + return static_cast(us); + } + if(us < 10000ULL * 1000ULL) { + unit = "ms"; + return static_cast(us) / 1e3; + } + if(us < 10000ULL * 1000ULL * 1000ULL) { + unit = "s"; + return static_cast(us) / 1e6; + } + unit = "min"; + return static_cast(us) / 60e6; +} + +// Procedure: dump_overview +// Emits the single-line header with wall time, worker count, task count, +// and average worker utilization — the single number that captures how +// well the thread pool was used (mean of busy/wall across active workers). +inline void TFProfObserver::Summary::dump_overview( + std::ostream& os, size_t uid, size_t num_tasks +) const { + const char* wall_unit; + double wall_val = _tf_time_scale(wall_us, wall_unit); + + os << std::string(80, '=') << '\n'; + os << std::fixed << std::setprecision(2); + os << " Observer " << uid + << " | Wall: " << wall_val << " " << wall_unit + << " | Workers: " << num_all_workers + << " | Tasks: " << num_tasks + << " | Avg Utilization: "<< avg_utilization() << "%\n"; + os << std::string(80, '=') << '\n'; +} + +// Procedure: dump_tsum +// Emits the aggregate task statistics table broken down by task type. +// Only task types that were actually observed are shown. +inline void TFProfObserver::Summary::dump_tsum(std::ostream& os) const { + + // check if there is anything to show + bool any = false; + for(const auto& t : tsum) { if(t.count) { any = true; break; } } + if(!any) return; + + // column widths - compute from data so numbers never overflow their columns + size_t type_w = 9; + size_t count_w = 5; + size_t tot_w = 10; + size_t avg_w = 10; + size_t min_w = 9; + size_t max_w = 9; + + for(const auto& t : tsum) { + if(!t.count) continue; + count_w = (std::max)(count_w, std::to_string(t.count).size()); + tot_w = (std::max)(tot_w, std::to_string(t.total_span).size()); + min_w = (std::max)(min_w, std::to_string(t.min_span).size()); + max_w = (std::max)(max_w, std::to_string(t.max_span).size()); + } + + os << "\n[Aggregate Task Statistics]\n"; + _tf_rule(os, 2 + type_w + count_w + tot_w + avg_w + min_w + max_w + 12); + + os << std::setw(type_w) << "Type" + << std::setw(count_w+2) << "Count" + << std::setw(tot_w+2) << "Total(us)" + << std::setw(avg_w+2) << "Avg(us)" + << std::setw(min_w+2) << "Min(us)" + << std::setw(max_w+2) << "Max(us)" + << '\n'; + _tf_rule(os, 2 + type_w + count_w + tot_w + avg_w + min_w + max_w + 12); + + for(size_t i = 0; i < TASK_TYPES.size(); ++i) { + const auto& t = tsum[i]; + if(!t.count) continue; + os << std::setw(type_w) << to_string(TASK_TYPES[i]) + << std::setw(count_w+2) << t.count + << std::setw(tot_w+2) << t.total_span + << std::setw(avg_w+2) << std::fixed << std::setprecision(2) << t.avg_span() + << std::setw(min_w+2) << t.min_span + << std::setw(max_w+2) << t.max_span + << '\n'; + } +} + +// Procedure: dump_wsum +// Emits the per-worker utilization table. Each row shows a worker's task +// count, total busy time, idle time, avg/min/max task duration, and +// utilization percentage. Workers that ran no tasks are skipped. +// A totals row at the bottom summarises across all active workers. +inline void TFProfObserver::Summary::dump_wsum(std::ostream& os) const { + + if(wsum.empty()) return; + + // column widths + size_t w_w = 6; + size_t count_w = 5; + size_t busy_w = 9; + size_t idle_w = 9; + size_t avg_w = 9; + size_t min_w = 8; + size_t max_w = 8; + size_t util_w = 6; + + for(const auto& w : wsum) { + w_w = (std::max)(w_w, std::to_string(w.id).size()); + count_w = (std::max)(count_w, std::to_string(w.count).size()); + busy_w = (std::max)(busy_w, std::to_string(w.busy_us).size()); + idle_w = (std::max)(idle_w, std::to_string(w.idle_us).size()); + min_w = (std::max)(min_w, std::to_string(w.min_span).size()); + max_w = (std::max)(max_w, std::to_string(w.max_span).size()); + } + + size_t row_w = w_w + count_w + busy_w + idle_w + avg_w + min_w + max_w + util_w + 16; + + os << "\n[Worker Utilization]\n"; + _tf_rule(os, row_w); + + os << std::setw(w_w+2) << "Worker" + << std::setw(count_w+2) << "Tasks" + << std::setw(busy_w+2) << "Busy(us)" + << std::setw(idle_w+2) << "Idle(us)" + << std::setw(avg_w+2) << "Avg(us)" + << std::setw(min_w+2) << "Min(us)" + << std::setw(max_w+2) << "Max(us)" + << std::setw(util_w+2) << "Util%" + << '\n'; + _tf_rule(os, row_w); + + for(const auto& w : wsum) { + os << std::setw(w_w+2) << w.id + << std::setw(count_w+2) << w.count + << std::setw(busy_w+2) << w.busy_us + << std::setw(idle_w+2) << w.idle_us + << std::setw(avg_w+2) << std::fixed << std::setprecision(2) << w.avg_span() + << std::setw(min_w+2) << w.min_span + << std::setw(max_w+2) << w.max_span + << std::setw(util_w+2) << std::fixed << std::setprecision(1) + << w.util_pct(wall_us) << "%" + << '\n'; + } + + // totals row: Util% shows avg across ALL num_all_workers (including idle), + // consistent with the header's "Avg Utilization" figure + _tf_rule(os, row_w); + size_t total_count = 0; + size_t total_busy = 0; + size_t total_idle = 0; + for(const auto& w : wsum) { + total_count += w.count; + total_busy += w.busy_us; + total_idle += w.idle_us; + } + os << std::setw(w_w+2) << "Total" + << std::setw(count_w+2) << total_count + << std::setw(busy_w+2) << total_busy + << std::setw(idle_w+2) << total_idle + << std::setw(avg_w+2) << "" + << std::setw(min_w+2) << "" + << std::setw(max_w+2) << "" + << std::setw(util_w+2) << std::fixed << std::setprecision(1) + << avg_utilization() << "% (avg)" + << '\n'; +} + +// Procedure: dump +// Emits overview header, task type stats, and worker utilization table. +inline void TFProfObserver::Summary::dump( + std::ostream& os, size_t uid, size_t num_tasks +) const { + dump_overview (os, uid, num_tasks); + dump_tsum (os); + dump_wsum (os); + os << '\n'; +} + +// Procedure: set_up +inline void TFProfObserver::set_up(size_t num_workers) { + _timeline.uid = unique_id(); + _timeline.origin = observer_stamp_t::clock::now(); + _timeline.segments.resize(num_workers); + _stacks.resize(num_workers); +} + +// Procedure: on_entry +inline void TFProfObserver::on_entry(WorkerView wv, TaskView) { + _stacks[wv.id()].push(observer_stamp_t::clock::now()); +} + +// Procedure: on_exit +inline void TFProfObserver::on_exit(WorkerView wv, TaskView tv) { + + size_t w = wv.id(); + + assert(!_stacks[w].empty()); + + if(_stacks[w].size() > _timeline.segments[w].size()) { + _timeline.segments[w].resize(_stacks[w].size()); + } + + auto beg = _stacks[w].top(); + _stacks[w].pop(); + + _timeline.segments[w][_stacks[w].size()].emplace_back( + tv.name(), tv.type(), beg, observer_stamp_t::clock::now() + ); +} + +// Function: clear +inline void TFProfObserver::clear() { + for(size_t w=0; w<_timeline.segments.size(); ++w) { + for(size_t l=0; l<_timeline.segments[w].size(); ++l) { + _timeline.segments[w][l].clear(); + } + while(!_stacks[w].empty()) { + _stacks[w].pop(); + } + } +} + +// Procedure: dump +// Writes this executor's data as a self-contained block: +// +// FILE FORMAT (all integers little-endian): +// Executor header (24 bytes): +// uid : u64 — unique executor id +// origin_us : u64 — wall-clock origin (µs since epoch) +// str_table_len: u32 — byte length of the string table below +// num_wl : u32 — number of non-empty (worker, level) blocks +// String table : str_table_len bytes of flat UTF-8 task names +// Per non-empty (worker, level): +// Worker-level header (12 bytes): worker_id(u32) level(u32) num_segs(u32) +// Per segment (variable length, delta+varint encoded): +// delta_beg : varint — beg_us minus previous beg_us (0 for first) +// duration : varint — (end_us - beg_us), always non-negative +// name_off : u32 — byte offset into this executor's string table +// type|nlen : u8 — upper 3 bits = task type, lower 5 = name length +// +// Each executor block is fully self-contained with its own string table, +// so TFProfManager::dump can simply write the file header and call dump() +// on each observer with no shared state. +inline void TFProfObserver::dump(std::ostream& os) const { + using namespace std::chrono; + using SegRef = std::pair; + + // ---- pass 1: build this executor's string table ---- + _StringTable strtab; + const size_t nw = _timeline.segments.size(); + std::vector>> segRefs(nw); + for(size_t w = 0; w < nw; ++w) { + segRefs[w].resize(_timeline.segments[w].size()); + for(size_t l = 0; l < _timeline.segments[w].size(); ++l) { + const auto& segs = _timeline.segments[w][l]; + segRefs[w][l].resize(segs.size()); + for(size_t s = 0; s < segs.size(); ++s) { + segRefs[w][l][s] = strtab.intern(segs[s].name); + } + } + } + + // ---- count non-empty worker-level blocks ---- + uint32_t numWL = 0; + for(size_t w = 0; w < nw; ++w) + for(size_t l = 0; l < _timeline.segments[w].size(); ++l) + if(!_timeline.segments[w][l].empty()) ++numWL; + + // ---- executor header (24 bytes) ---- + const int64_t origin_us = duration_cast( + _timeline.origin.time_since_epoch() + ).count(); + _tfp_write_u64(os, static_cast(_timeline.uid)); + _tfp_write_u64(os, static_cast(origin_us)); + _tfp_write_u32(os, static_cast(strtab.data.size())); // str_table_len + _tfp_write_u32(os, numWL); + + // ---- string table ---- + if(!strtab.data.empty()) { + os.write(strtab.data.data(), static_cast(strtab.data.size())); + } + + // ---- worker-level blocks ---- + for(size_t w = 0; w < nw; ++w) { + for(size_t l = 0; l < _timeline.segments[w].size(); ++l) { + const auto& segs = _timeline.segments[w][l]; + if(segs.empty()) continue; + + _tfp_write_u32(os, static_cast(w)); + _tfp_write_u32(os, static_cast(l)); + _tfp_write_u32(os, static_cast(segs.size())); + + int64_t prev_beg = 0; + for(size_t s = 0; s < segs.size(); ++s) { + const auto& seg = segs[s]; + const auto& ref = segRefs[w][l][s]; + const int64_t beg_us = duration_cast(seg.beg - _timeline.origin).count(); + const int64_t end_us = duration_cast(seg.end - _timeline.origin).count(); + _tfp_write_varint(os, static_cast(beg_us - prev_beg)); // delta_beg + _tfp_write_varint(os, static_cast(end_us - beg_us)); // duration + _tfp_write_u32(os, ref.first); // name_off + _tfp_write_u8 (os, static_cast( + (_tfp_type_byte(seg.type) << 5) | (ref.second & 0x1F) + )); // type|name_len + prev_beg = beg_us; + } + } + } +} + +// Procedure: summary +inline void TFProfObserver::summary(std::ostream& os) const { + + using namespace std::chrono; + + Summary summary; + summary.num_all_workers = _timeline.segments.size(); + + // --- pass 1: scan all segments to compute time range, task stats, + // per-worker busy time, and global task type breakdown --- + // + // per-worker busy time is computed via a merge-intervals line sweep across + // ALL nesting levels to correctly handle recursive tasking. a worker that + // spawns subflow children has overlapping segments across levels — simply + // summing durations would double-count the overlapping wall-clock time. + // the sweep merges [beg, end] intervals per worker into non-overlapping + // spans and sums only those, giving true wall-clock busy time in [0, wall_us]. + std::optional view_beg, view_end; + + // per-worker accumulators indexed by worker id; sized to num_workers + std::vector wmap(_timeline.segments.size()); + for(size_t w = 0; w < wmap.size(); ++w) { + wmap[w].id = w; + } + + // collect flat interval lists per worker (across all levels) for line sweep + // intervals stored as (beg_us, end_us) relative to timeline origin + // (we use the raw time points for sorting, convert to us after merging) + using Interval = std::pair; + std::vector> worker_intervals(_timeline.segments.size()); + + for(size_t w = 0; w < _timeline.segments.size(); ++w) { + for(size_t l = 0; l < _timeline.segments[w].size(); ++l) { + for(const auto& s : _timeline.segments[w][l]) { + + // update global time bounds (all levels) + view_beg = view_beg ? (std::min)(*view_beg, s.beg) : s.beg; + view_end = view_end ? (std::max)(*view_end, s.end) : s.end; + + size_t t = duration_cast(s.end - s.beg).count(); + + // global per-type stats count all levels + summary.tsum[static_cast(s.type)].update(t); + + // update count/min/max on WorkerSummary (these are task-level stats, + // not time stats, so all levels count) + wmap[w].count += 1; + wmap[w].min_span = (wmap[w].count == 1) ? t : (std::min)(wmap[w].min_span, t); + wmap[w].max_span = (wmap[w].count == 1) ? t : (std::max)(wmap[w].max_span, t); + + // collect interval for line sweep + worker_intervals[w].emplace_back(s.beg, s.end); + } + } + } + + // nothing to show + if(!view_beg || !view_end) { + os << "==Observer " << _timeline.uid << ": no tasks recorded\n"; + return; + } + + summary.wall_us = duration_cast(*view_end - *view_beg).count(); + if(summary.wall_us == 0) summary.wall_us = 1; + + // line sweep per worker: sort intervals by start time, merge overlaps, + // sum merged span durations to get true wall-clock busy time + for(size_t w = 0; w < worker_intervals.size(); ++w) { + auto& ivs = worker_intervals[w]; + if(ivs.empty()) continue; + + // sort by beg + std::sort(ivs.begin(), ivs.end(), + [](const Interval& a, const Interval& b){ return a.first < b.first; }); + + size_t busy_us = 0; + auto cur_beg = ivs[0].first; + auto cur_end = ivs[0].second; + + for(size_t i = 1; i < ivs.size(); ++i) { + if(ivs[i].first <= cur_end) { + // overlapping or adjacent: extend current merged interval + cur_end = (std::max)(cur_end, ivs[i].second); + } + else { + // gap: emit completed merged interval and start a new one + busy_us += duration_cast(cur_end - cur_beg).count(); + cur_beg = ivs[i].first; + cur_end = ivs[i].second; + } + } + busy_us += duration_cast(cur_end - cur_beg).count(); // emit last + + wmap[w].busy_us = busy_us; + wmap[w].idle_us = (summary.wall_us > busy_us) ? (summary.wall_us - busy_us) : 0; + summary.wsum.push_back(wmap[w]); + } + + // --- pass 2: build concurrency histograms --- + // + // num_bins is derived from the available 80-char line width and the width + // of the widest scaled tick label, so the chart fills the terminal naturally. + // + // worker_histogram[b] = distinct workers with at least one task active in bin b. + // task_histogram[b] = total concurrent tasks in bin b. + static constexpr size_t Y_PREFIX = 6; + static constexpr size_t LINE_WIDTH = 80; + + // compute num_bins: scale wall_us, format the last tick, derive col_w and num_bins + { + // start with a provisional num_bins to get the unit; iterate once to converge. + // in practice the unit is stable after the first estimate. + size_t probe_bins = 20; + size_t probe_bin_us = (summary.wall_us + probe_bins - 1) / probe_bins; + if(probe_bin_us == 0) probe_bin_us = 1; + + const char* bin_unit; + _tf_time_scale(probe_bin_us, bin_unit); + double divisor = 1.0; + if (std::string(bin_unit) == "ms") divisor = 1e3; + else if(std::string(bin_unit) == "s") divisor = 1e6; + else if(std::string(bin_unit) == "min") divisor = 60e6; + + // format the last tick at probe_bins-1 to get its rendered width + std::string last_tick; + if(divisor == 1.0) { + last_tick = std::to_string((probe_bins - 1) * probe_bin_us); + } else { + std::ostringstream oss; + oss << std::fixed << std::setprecision(1) + << static_cast((probe_bins - 1) * probe_bin_us) / divisor; + last_tick = oss.str(); + } + + // col_w = label width + 1 space padding, minimum 3 + size_t col_w = (std::max)(last_tick.size() + 1, size_t(3)); + size_t num_bins = (LINE_WIDTH - Y_PREFIX) / col_w; + if(num_bins < 2) num_bins = 2; // need at least 2 bins to be meaningful + + summary.worker_histogram.assign(num_bins, 0); + summary.task_histogram.assign(num_bins, 0); + } + + size_t num_bins = summary.worker_histogram.size(); + size_t bin_us = (summary.wall_us + num_bins - 1) / num_bins; + if(bin_us == 0) bin_us = 1; + + size_t nw = _timeline.segments.size(); + + // active[b][w] = true if worker w had any task overlapping bin b + std::vector> active(num_bins, std::vector(nw, false)); + + for(size_t w = 0; w < nw; ++w) { + for(size_t l = 0; l < _timeline.segments[w].size(); ++l) { + for(const auto& s : _timeline.segments[w][l]) { + + size_t s_beg = duration_cast(s.beg - *view_beg).count(); + size_t s_end = duration_cast(s.end - *view_beg).count(); + + size_t first_bin = s_beg / bin_us; + size_t last_bin = (s_end == 0) ? 0 : (s_end - 1) / bin_us; + first_bin = (std::min)(first_bin, num_bins - 1); + last_bin = (std::min)(last_bin, num_bins - 1); + + for(size_t b = first_bin; b <= last_bin; ++b) { + active[b][w] = true; + summary.task_histogram[b] += 1; + } + } + } + } + + for(size_t b = 0; b < num_bins; ++b) { + for(size_t w = 0; w < nw; ++w) { + summary.worker_histogram[b] += active[b][w] ? 1 : 0; + } + } + + summary.dump(os, _timeline.uid, num_tasks()); +} + +// Procedure: summary +inline std::string TFProfObserver::summary() const { + std::ostringstream oss; + summary(oss); + return oss.str(); +} + +// Function: num_tasks +inline size_t TFProfObserver::num_tasks() const { + size_t s = 0; + for(size_t w=0; w<_timeline.segments.size(); ++w) { + for(size_t l=0; l<_timeline.segments[w].size(); ++l) { + s += _timeline.segments[w][l].size(); + } + } + return s; +} + +// Function: num_workers +inline size_t TFProfObserver::num_workers() const { + size_t w = 0; + for(size_t i=0; i<_timeline.segments.size(); ++i) { + w += (!_timeline.segments[i].empty()); + } + return w; +} + + +// ---------------------------------------------------------------------------- +// TFProfManager +// ---------------------------------------------------------------------------- + +/** +@private +*/ +class TFProfManager { + + friend class Executor; + + public: + + ~TFProfManager(); + + TFProfManager(const TFProfManager&) = delete; + TFProfManager& operator=(const TFProfManager&) = delete; + + static TFProfManager& get(); + + + private: + + const std::string _fpath; + + std::mutex _mutex; + std::vector> _observers; + + TFProfManager(); + + void _manage(std::shared_ptr observer); + + // Writes a complete .tfp file covering all observers to 'os'. + // Builds a single global string table across all observers (maximises + // deduplication), then writes the file header, string table, and each + // executor block by calling into TFProfObserver's private helpers. + void dump(std::ostream& os) const; +}; + +// Constructor +inline TFProfManager::TFProfManager() : + _fpath {get_env(TF_ENABLE_PROFILER)} { +} + +// Procedure: _manage +inline void TFProfManager::_manage(std::shared_ptr observer) { + std::lock_guard lock(_mutex); + _observers.push_back(std::move(observer)); +} + +// Procedure: dump (JSON, used for streaming / manual calls) + + +// Procedure: dump +// Writes a complete .tfp file to os. +// File header (12 bytes): magic(4) version(u16) flags(u16) num_exec(u32) +// Followed by each executor's self-contained block (see TFProfObserver::dump). +inline void TFProfManager::dump(std::ostream& os) const { + // File header — 12 bytes + os.write("TFPX", 4); + TFProfObserver::_tfp_write_u16(os, 1); // version + TFProfObserver::_tfp_write_u16(os, 0); // flags + TFProfObserver::_tfp_write_u32(os, static_cast(_observers.size())); // num_exec + + // Each observer writes its own self-contained executor block + for(size_t i = 0; i < _observers.size(); ++i) { + _observers[i]->dump(os); + } +} + +// Destructor +// If a file path was given, write the binary .tfp file. +// Otherwise print a text summary to stderr. +inline TFProfManager::~TFProfManager() { + std::ofstream ofs(_fpath, std::ios::binary); + if(ofs) { + dump(ofs); + } + else { + std::ostringstream oss; + for(size_t i=0; i<_observers.size(); ++i) { + _observers[i]->summary(oss); + } + fprintf(stderr, "%s", oss.str().c_str()); + } +} + +// Function: get +inline TFProfManager& TFProfManager::get() { + static TFProfManager mgr; + return mgr; +} + +// ---------------------------------------------------------------------------- +// Identifier for Each Built-in Observer +// ---------------------------------------------------------------------------- + +/** @enum ObserverType + +@brief enumeration of all observer types + +*/ +enum class ObserverType : int { + TFPROF = 0, + CHROME, + UNDEFINED +}; + +/** +@brief convert an observer type to a human-readable string +*/ +inline const char* to_string(ObserverType type) { + switch(type) { + case ObserverType::TFPROF: return "tfprof"; + case ObserverType::CHROME: return "chrome"; + default: return "undefined"; + } +} + + +} // end of namespace tf ----------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/taskflow.hpp b/projects/hipfile/third-party/taskflow/taskflow.hpp new file mode 100644 index 000000000000..ded7269bdd44 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/taskflow.hpp @@ -0,0 +1,95 @@ +#pragma once + +#include "core/executor.hpp" +#include "core/runtime.hpp" +#include "core/async.hpp" +#include "core/task_group.hpp" +#include "algorithm/algorithm.hpp" + +/** +@dir taskflow +@brief root taskflow include dir +*/ + +/** +@dir taskflow/core +@brief taskflow core include dir +*/ + +/** +@dir taskflow/algorithm +@brief taskflow algorithms include dir +*/ + +/** +@dir taskflow/cuda +@brief taskflow CUDA include dir +*/ + +/** +@file taskflow/taskflow.hpp +@brief main taskflow include file +*/ + + + +/** +@def TF_VERSION + +@brief version of the %Taskflow (currently 4.1.0) + +The version system is made of a major version number, a minor version number, +and a patch number: + + TF_VERSION % 100 is the patch level + + TF_VERSION / 100 % 1000 is the minor version + + TF_VERSION / 100000 is the major version +*/ +#define TF_VERSION 400100 + +/** +@def TF_MAJOR_VERSION + +@brief major version of %Taskflow, which is equal to `TF_VERSION/100000` +*/ +#define TF_MAJOR_VERSION TF_VERSION/100000 + +/** +@def TF_MINOR_VERSION + +@brief minor version of %Taskflow, which is equal to `TF_VERSION / 100 % 1000` +*/ +#define TF_MINOR_VERSION TF_VERSION/100%1000 + +/** +@def TF_PATCH_VERSION + +@brief patch version of %Taskflow, which is equal to `TF_VERSION % 100` +*/ +#define TF_PATCH_VERSION TF_VERSION%100 + + + +/** +@brief taskflow namespace +*/ +namespace tf { + +/** +@private +*/ +namespace detail { } + + +/** +@brief queries the version information in a string format @c major.minor.patch + +Release notes are available here: https://taskflow.github.io/taskflow/Releases.html +*/ +constexpr const char* version() { + return "4.1.0"; +} + + +} // end of namespace tf ----------------------------------------------------- + + diff --git a/projects/hipfile/third-party/taskflow/utility/iterator.hpp b/projects/hipfile/third-party/taskflow/utility/iterator.hpp new file mode 100644 index 000000000000..1b62c5244bec --- /dev/null +++ b/projects/hipfile/third-party/taskflow/utility/iterator.hpp @@ -0,0 +1,1035 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace tf { + +/** +@brief checks if the given index range is invalid + +@tparam T integral type of the indices and step + +@param beg starting index of the range +@param end ending index of the range +@param step step size to traverse the range + +@return returns @c true if the range is invalid; @c false otherwise. + +A range is considered invalid under the following conditions: + + The step is zero and the begin and end values are not equal. + + A positive range (begin < end) with a non-positive step. + + A negative range (begin > end) with a non-negative step. +*/ +template +constexpr bool is_index_range_invalid(T beg, T end, T step) { + return ((step == T{0} && beg != end) || + (beg < end && step <= T{0}) || // positive range + (beg > end && step >= T{0})); // negative range +} + +/** +@brief calculates the number of iterations in the given index range + +@tparam T integral type of the indices and step + +@param beg starting index of the range +@param end ending index of the range +@param step step size to traverse the range + +@return returns the number of required iterations to traverse the range + +The distance of a range represents the number of required iterations to traverse the range +from the beginning index to the ending index (exclusive) with the given step size. + +Example 1: +@code{.cpp} +// Range: 0 to 10 with step size 2 +size_t dist = distance(0, 10, 2); // Returns 5, the sequence is [0, 2, 4, 6, 8] +@endcode + +Example 2: +@code{.cpp} +// Range: 10 to 0 with step size -2 +size_t dist = distance(10, 0, -2); // Returns 5, the sequence is [10, 8, 6, 4, 2] +@endcode + +Example 3: +@code{.cpp} +// Range: 5 to 20 with step size 5 +size_t dist = distance(5, 20, 5); // Returns 3, the sequence is [5, 10, 15] +@endcode + +@attention +An invalid index range will return 0. +*/ +template +constexpr size_t distance(T beg, T end, T step) { + if constexpr (std::is_unsigned_v) { + return end > beg + ? static_cast((end - beg + step - T{1}) / step) + : size_t{0}; + } else { + return static_cast( + std::max(T{0}, (end - beg + step + (step > T{0} ? T{-1} : T{1})) / step) + ); + } +} + +// ============================================================================ +// IndexRanges +// +// A single class template representing an N-dimensional index range, the +// Cartesian product of N independent 1D ranges (begin, end, step). Each +// dimension is stored as a std::tuple, so dim(d) returns a tuple +// that can be read or mutated directly, including via structured bindings: +// +// auto& [beg, end, step] = ranges.dim(0); +// +// Members that only make sense for a single dimension (begin(), end(), +// step_size(), reset(), unravel()) are gated with `requires (N == 1)`. +// Members that only make sense for more than one dimension (ceil(), floor(), +// upper_slice(), lower_slice()) are +// gated with `requires (N > 1)`. This keeps +// everything in one class body — instead of a primary template plus a +// partial specialization — which Doxygen can parse and cross-reference +// cleanly. +// +// tf::IndexRange is an alias for tf::IndexRanges, the common 1D +// case. +// +// Iteration order for N > 1 is row-major (the last dimension is innermost / +// fastest), matching the natural loop nesting: +// +// for i in dim[0]: // outermost +// for j in dim[1]: +// ... +// for k in dim[N-1]: // innermost +// +// Flat index 0 corresponds to (beg[0], beg[1], ..., beg[N-1]). +// ============================================================================ + +/** +@class IndexRanges + +@brief class to create an N-dimensional index range of integral indices + +@tparam T the integral type of the indices +@tparam N the number of dimensions (defaults to 1) + +This class represents the Cartesian product of @c N independent 1D index +ranges, each defined by a starting index, ending index, and step size. Each +dimension is stored as a std::tuple of (begin, end, step), +accessible and mutable through @c dim(d). + +For @c N == 1, the class behaves like a plain 1D range: @c tf::IndexRange +(an alias for tf::IndexRanges) exposes convenience accessors +@c begin(), @c end(), @c step_size(), @c reset(), and @c unravel() directly, +without going through @c dim(0). + +@code{.cpp} +tf::IndexRange range(0, 10, 2); +for(auto i=range.begin(); i range; +range.reset(0, 10, 2); +for(auto i=range.begin(); i 1, iteration order is row-major: the last dimension varies +fastest, matching the natural nesting of C-style for-loops. + +@code{.cpp} +// 3D range: i in [0,4), j in [0,6), k in [0,8), all step 1 +tf::IndexRanges r( + tf::IndexRange(0, 4, 1), + tf::IndexRange(0, 6, 1), + tf::IndexRange(0, 8, 1) +); +printf("%zu\n", r.size()); // 4*6*8 = 192 +@endcode + +@note +If any dimension has zero size (e.g. an empty range such as @c [0,0)), the +active iteration space stops at that dimension. @c size() returns the product +of all outer dimensions before the first zero, and @c upper_slice / @c lower_slice +copy the zero-size dimension and all inner +dimensions as full extent into each +returned sub-box. This matches the behaviour of sequential nested loops: + +@code{.cpp} +// Zero in the middle dimension: outer i-loop still runs, j/k body never executes. +tf::IndexRanges r( + tf::IndexRange(0, 100, 1), // i: 100 iters (active) + tf::IndexRange(0, 0, 1), // j: 0 iters (stops accumulation here) + tf::IndexRange(0, 100, 1) // k: 100 iters (inactive — never reached) +); +r.size(); // 100 (only the i-dimension contributes) +@endcode +*/ +template +class IndexRanges { + +public: + + /** + @brief alias for the index type + */ + using index_type = T; + + /** + @brief the number of dimensions + */ + static constexpr size_t rank = N; + + // -------------------------------------------------------------------------- + // Construction + // -------------------------------------------------------------------------- + + /** + @brief constructs an index range without initialization + + The per-dimension ranges are left in an indeterminate state. + Use this when the bounds will be set later via @c reset() (N==1) or + @c dim(d) (any N). + */ + IndexRanges() = default; + + /** + @brief constructs a 1D index range (only available when @c N == 1) + + @param beg starting index of the range + @param end ending index of the range (exclusive) + @param step_size step size between consecutive indices in the range + + @code{.cpp} + tf::IndexRange range(0, 10, 2); // elements: 0, 2, 4, 6, 8 + @endcode + */ + explicit IndexRanges(T beg, T end, T step_size) requires (N == 1) + : _dims{ std::tuple{beg, end, step_size} } {} + + /** + @brief constructs an N-D index range from N 1D ranges + + @param ranges exactly N 1D ranges (each a tf::IndexRanges, + i.e. a @c tf::IndexRange), one per dimension in order + from outermost (dim 0) to innermost (dim N-1) + + Each 1D range defines the @c begin, @c end, and @c step_size for its + dimension. Dimensions are independent — any combination of positive, + negative, or zero step sizes is supported, as long as each 1D range is + individually valid. + + @code{.cpp} + // 3D: mixed step sizes + tf::IndexRanges r3( + tf::IndexRange(0, 4, 1), // dim 0: 4 elements + tf::IndexRange(0, 10, 2), // dim 1: 5 elements (0,2,4,6,8) + tf::IndexRange(0, 6, 1) // dim 2: 6 elements + ); + r3.size(); // 120 + @endcode + */ + template + requires (sizeof...(Ranges) == N) && + (std::same_as, IndexRanges> && ...) + explicit IndexRanges(Ranges&&... ranges) + : _dims{ ranges.dim(0)... } {} + + /** + @brief constructs an index range from an array of (begin, end, step) tuples + + @param dims @c std::array of exactly N (begin, end, step) tuples + + Useful when the per-dimension bounds are constructed programmatically. + + @code{.cpp} + std::array, 2> dims = { + std::tuple{0, 4, 1}, + std::tuple{0, 5, 1} + }; + tf::IndexRanges r(dims); + r.size(); // 20 + @endcode + */ + explicit IndexRanges(const std::array, N>& dims) : _dims{dims} {} + + // -------------------------------------------------------------------------- + // Dimension access + // -------------------------------------------------------------------------- + + /** + @brief returns the (begin, end, step) tuple for dimension @c d (read-only) + + @param d zero-based dimension index in @c [0, N) + + @return a const reference to the std::tuple for + dimension @c d, in (begin, end, step) order + + @code{.cpp} + tf::IndexRanges r( + tf::IndexRange(0, 4, 1), + tf::IndexRange(0, 10, 2), + tf::IndexRange(0, 6, 1) + ); + + auto [beg, end, step] = r.dim(1); // beg=0, end=10, step=2 + @endcode + */ + const std::tuple& dim(size_t d) const { return _dims[d]; } + + /** + @brief returns the (begin, end, step) tuple for dimension @c d (mutable) + + @param d zero-based dimension index in @c [0, N) + + @return a mutable reference to the std::tuple for + dimension @c d + + Use this to update the bounds of an individual dimension at runtime, for + example inside an upstream init task when using stateful ranges: + + @code{.cpp} + tf::IndexRanges range; + + auto init = taskflow.emplace([&]() { + range.dim(0) = {0, rows, 1}; // set row range at runtime + range.dim(1) = {0, cols, 1}; // set col range at runtime + }); + + auto loop = taskflow.for_each_by_index(std::ref(range), callable); + init.precede(loop); + @endcode + + Native structured bindings can bind directly to the tuple's elements: + + @code{.cpp} + // mutate a single field through a structured binding + auto& [beg, end, step] = range.dim(0); + beg = 0; end = rows; step = 1; + @endcode + */ + std::tuple& dim(size_t d) { return _dims[d]; } + + // -------------------------------------------------------------------------- + // 1D convenience accessors (only available when N == 1) + // -------------------------------------------------------------------------- + + /** + @brief queries the starting index of the range (only available when @c N == 1) + + @return the starting index of the range + + @code{.cpp} + tf::IndexRange range(0, 10, 2); + auto b = range.begin(); // b == 0 + @endcode + */ + T begin() const requires (N == 1) { return std::get<0>(_dims[0]); } + + /** + @brief queries the ending index of the range (only available when @c N == 1) + + @return the ending index (exclusive) of the range + + @code{.cpp} + tf::IndexRange range(0, 10, 2); + auto e = range.end(); // e == 10 + @endcode + */ + T end() const requires (N == 1) { return std::get<1>(_dims[0]); } + + /** + @brief queries the step size of the range (only available when @c N == 1) + + @return the step size between consecutive indices in the range + + @code{.cpp} + tf::IndexRange range(0, 10, 2); + auto s = range.step_size(); // s == 2 + @endcode + */ + T step_size() const requires (N == 1) { return std::get<2>(_dims[0]); } + + /** + @brief updates the range with a new starting index, ending index, and step + size (only available when @c N == 1) + + @param beg new starting index of the range + @param end new ending index of the range (exclusive) + @param step_size new step size between consecutive indices in the range + + @return a reference to @c *this + + Use this to rebind a stateful range to new bounds at runtime, for example + inside an upstream init task: + + @code{.cpp} + tf::IndexRange range; + + auto init = taskflow.emplace([&]() { + range.reset(0, 10, 2); // elements: 0, 2, 4, 6, 8 + }); + + auto loop = taskflow.for_each_by_index(std::ref(range), callable); + init.precede(loop); + @endcode + */ + IndexRanges& reset(T beg, T end, T step_size) requires (N == 1); + + /** + @brief updates the starting index of the range (only available when @c N == 1) + + @param new_begin the new starting index of the range + + @return a reference to @c *this + + @code{.cpp} + tf::IndexRange range(0, 10, 2); // elements: 0, 2, 4, 6, 8 + range.begin(4); // elements become: 4, 6, 8 + @endcode + */ + IndexRanges& begin(T new_begin) requires (N == 1); + + /** + @brief updates the ending index of the range (only available when @c N == 1) + + @param new_end the new ending index (exclusive) of the range + + @return a reference to @c *this + + @code{.cpp} + tf::IndexRange range(0, 10, 2); // elements: 0, 2, 4, 6, 8 + range.end(6); // elements become: 0, 2, 4 + @endcode + */ + IndexRanges& end(T new_end) requires (N == 1); + + /** + @brief updates the step size of the range (only available when @c N == 1) + + @param new_step_size the new step size between consecutive indices + + @return a reference to @c *this + + @code{.cpp} + tf::IndexRange range(0, 10, 1); // elements: 0, 1, 2, ..., 9 + range.step_size(3); // elements become: 0, 3, 6, 9 + @endcode + */ + IndexRanges& step_size(T new_step_size) requires (N == 1); + + /** + @brief maps a contiguous index partition back to the corresponding subrange + (only available when @c N == 1) + + @param part_beg beginning index of the partition (inclusive) + @param part_end ending index of the partition (exclusive) + @return a new range covering the elements at positions + [@c part_beg, @c part_end) in the original range + + Each element of the range can be addressed by a zero-based position index + from @c 0 to @c size()-1. This function unravels a contiguous slice of those + position indices back into the original iteration space, returning the + sub-range whose elements correspond exactly to positions + [@c part_beg, @c part_end). + + @code{.cpp} + tf::IndexRange range(0, 10, 2); // elements: 0, 2, 4, 6, 8 + auto sub = range.unravel(1, 4); // elements at positions [1,4): 2, 4, 6 + // sub.begin() == 2, sub.end() == 8, sub.step_size() == 2 + @endcode + + @attention + Users must ensure [@c part_beg, @c part_end) is a valid partition of + [0, @c size()), i.e., @c part_end <= size(). + */ + IndexRanges unravel(size_t part_beg, size_t part_end) const requires (N == 1); + + // -------------------------------------------------------------------------- + // Size queries (available for any N) + // -------------------------------------------------------------------------- + + /** + @brief returns the number of iterations along dimension @c d + + @param d zero-based dimension index in @c [0, N) + + @return the element count of dimension @c d + + This is a per-dimension count, independent of other dimensions and of the + zero-size stopping rule that @c size() applies. + + @code{.cpp} + tf::IndexRanges r( + tf::IndexRange(0, 4, 1), + tf::IndexRange(0, 10, 2), + tf::IndexRange(0, 6, 1) + ); + + r.size(1); // 5, since the range [0,10) with step 2 has 5 elements + @endcode + */ + size_t size(size_t d) const { return std::apply(distance, _dims[d]); } + + /** + @brief returns the number of active flat iterations + + For @c N == 1 this is simply the element count of the range. For @c N > 1 + it returns the product of dimension sizes from the outermost dimension + inward, stopping before the first zero-size dimension. This matches the + behaviour of sequential nested loops: a zero-size dimension prevents its + own iterations and those of all deeper dimensions, but outer dimensions + are unaffected. + + @code{.cpp} + // All non-zero: 4 * 6 * 8 = 192 + tf::IndexRanges r1( + tf::IndexRange(0, 4, 1), + tf::IndexRange(0, 6, 1), + tf::IndexRange(0, 8, 1) + ); + r1.size(); // 192 + + // Zero in middle (d=1): outer dim only -> 4 + tf::IndexRanges r2( + tf::IndexRange(0, 4, 1), + tf::IndexRange(0, 0, 1), // stops accumulation here + tf::IndexRange(0, 8, 1) + ); + r2.size(); // 4 + @endcode + */ + size_t size() const; + + // -------------------------------------------------------------------------- + // N-dimensional algorithms (only available when N > 1) + // -------------------------------------------------------------------------- + + /** + @brief returns the smallest hyperplane-aligned chunk size that is >= chunk_size, + capped at size() when chunk_size exceeds the total active range size + (only available when @c N > 1) + + @param chunk_size hint for the desired number of flat elements + + @return the smallest natural per-step volume of the active dimensions that is + >= @c chunk_size, or @c size() if @c chunk_size > @c size(), or 0 if + the outermost dimension is zero-size + + Analogous to @c std::ceil but operating on the discrete set of hyperplane + boundary sizes of the active dimensions (those before the first zero-size + dimension). Only active suffix products are considered — inactive inner + dimensions do not contribute boundaries. + + @code{.cpp} + // 6D range: 3 x 4 x 8 x 0 x 2 x 3 + // Active dims: d=0(3), d=1(4), d=2(8) -> size()=96 + // Active boundaries: 1, 8, 32, 96 (inactive dims d=3,4,5 contribute nothing) + tf::IndexRanges r( + tf::IndexRange(0, 3, 1), + tf::IndexRange(0, 4, 1), + tf::IndexRange(0, 8, 1), + tf::IndexRange(0, 0, 1), + tf::IndexRange(0, 2, 1), + tf::IndexRange(0, 3, 1) + ); + r.ceil(1); // 1 — already a boundary + r.ceil(5); // 8 — rounds up to next active boundary + r.ceil(33); // 96 — rounds up to full active size + r.ceil(200); // 96 — capped at size() + @endcode + */ + size_t ceil(size_t chunk_size) const requires (N > 1); + + /** + @brief returns the largest hyperplane-aligned chunk size that is <= chunk_size + (only available when @c N > 1) + + @param chunk_size hint for the desired number of flat elements + + @return the largest natural per-step volume of the active dimensions that is + <= @c chunk_size, or 0 if the outermost dimension is zero-size + + Analogous to @c std::floor; see @c ceil for the boundary semantics. + + @code{.cpp} + // 6D range: 3 x 4 x 8 x 0 x 2 x 3 + // Active dims: d=0(3), d=1(4), d=2(8) -> size()=96 + // Active boundaries: 1, 8, 32, 96 (inactive dims d=3,4,5 contribute nothing) + tf::IndexRanges r( + tf::IndexRange(0, 3, 1), + tf::IndexRange(0, 4, 1), + tf::IndexRange(0, 8, 1), + tf::IndexRange(0, 0, 1), + tf::IndexRange(0, 2, 1), + tf::IndexRange(0, 3, 1) + ); + r.floor(5); // 1 — rounds down to previous active boundary + r.floor(33); // 32 — rounds down + r.floor(200); // 96 — capped at size() + @endcode + */ + size_t floor(size_t chunk_size) const requires (N > 1); + + /** + @brief returns the smallest perfectly-aligned hyperbox reachable from flat_beg, + rounding up to the next hyperplane boundary when chunk_size is not + aligned (only available when @c N > 1) + + @param flat_beg starting flat index (row-major) into the active ND space + @param chunk_size hint for the desired number of elements + + @return the sub-box; its @c size() gives the number of elements consumed + + Analogous to @c std::ceil: if @c chunk_size already aligns to a hyperplane + boundary the returned box is exact; otherwise it rounds up to the next clean + orthogonal boundary, so @c size() >= @c chunk_size. + + Only the active dimensions — those before the first zero-size dimension — are + partitioned. Dimensions from the first zero onward are copied into the + returned box as full extent and do not affect the flat index space. + + The trailing-zeros rule applies within the active dimensions: a dimension can + only expand if all active dimensions inner to it start at coordinate 0. + When this fires the function returns the best geometry-constrained box + reachable from @c flat_beg. + + Used by dynamic partitioners: the atomic cursor advances by the returned + box's @c size() and any overshoot is self-correcting. + */ + IndexRanges upper_slice(size_t flat_beg, size_t chunk_size) const requires (N > 1); + + /** + @brief returns the largest perfectly-aligned hyperbox reachable from flat_beg + whose size does not exceed chunk_size, rounding down to the previous + hyperplane boundary when chunk_size is not aligned (only available + when @c N > 1) + + @param flat_beg starting flat index (row-major) into the active ND space + @param chunk_size hint for the desired number of elements + + @return the sub-box; its @c size() gives the number of elements consumed + + Analogous to @c std::floor: if @c chunk_size already aligns to a hyperplane + boundary the returned box is exact (identical to upper_slice); otherwise + it rounds down to the largest box that does not exceed @c chunk_size, so + @c size() <= @c chunk_size. + + Used by static partitioners: each worker's pre-assigned flat quota is never + exceeded, so workers do not double-process elements at quota boundaries. + */ + IndexRanges lower_slice(size_t flat_beg, size_t chunk_size) const requires (N > 1); + + /** + @brief returns a box whose @c size() is 0 (only available when @c N > 1) + + @return a degenerate box with the same dimensions as @c *this, except + dimension 0 is collapsed to a single point (begin == end) + + Collapsing dimension 0 alone is sufficient: @c size() stops accumulating + at the first zero-size dimension, so the returned box reports 0 regardless + of the other dimensions, which are copied through verbatim. + + Used by @c upper_slice and @c lower_slice for the @c chunk_size @c == @c 0 + case, since the returned box's @c size() doubles as the "consumed" count. + */ + IndexRanges empty_box() const requires (N > 1); + +private: + + std::array, N> _dims; +}; + +// ============================================================================ +// Out-of-class definitions — 1D convenience accessors and size queries +// ============================================================================ + +template +IndexRanges& +IndexRanges::reset(T beg, T end, T step_size) requires (N == 1) { + _dims[0] = {beg, end, step_size}; + return *this; +} + +template +IndexRanges& +IndexRanges::begin(T new_begin) requires (N == 1) { + std::get<0>(_dims[0]) = new_begin; + return *this; +} + +template +IndexRanges& +IndexRanges::end(T new_end) requires (N == 1) { + std::get<1>(_dims[0]) = new_end; + return *this; +} + +template +IndexRanges& +IndexRanges::step_size(T new_step_size) requires (N == 1) { + std::get<2>(_dims[0]) = new_step_size; + return *this; +} + +template +IndexRanges +IndexRanges::unravel(size_t part_beg, size_t part_end) const requires (N == 1) { + auto [beg, end, step] = _dims[0]; + return IndexRanges( + static_cast(part_beg) * step + beg, + static_cast(part_end) * step + beg, + step + ); +} + +template +size_t IndexRanges::size() const { + if constexpr (N == 1) { + return size(0); + } else { + // Compile-time-unrolled recursion: D is a template parameter, so each + // depth is a distinct instantiation and the recursion fully unrolls + // (no runtime loop). `self` is passed explicitly because C++20 lambdas + // cannot otherwise name themselves for recursion. + auto compute = [this](auto& self, size_t total) -> size_t { + if constexpr (D == N) { + return total; + } else { + size_t s = this->size(D); + if (s == 0) return D == 0 ? 0 : total; // outermost zero -> 0, inner zero -> outer product + return self.template operator()(self, total * s); + } + }; + return compute.template operator()<0>(compute, 1); + } +} + +// ============================================================================ +// Out-of-class definitions — N-dimensional algorithms (N > 1) +// ============================================================================ + +template +IndexRanges +IndexRanges::empty_box() const requires (N > 1) { + std::array, N> box_dims = _dims; + box_dims[0] = std::tuple{ + std::get<0>(_dims[0]), std::get<0>(_dims[0]), std::get<2>(_dims[0]) + }; + return IndexRanges(box_dims); +} + +template +size_t IndexRanges::ceil(size_t chunk_size) const requires (N > 1) { + // Pass 1: cache all dim sizes (one call each) and find d_active. + size_t dim_sizes[N]; + size_t d_active = N; + for (size_t d = 0; d < N; ++d) { + dim_sizes[d] = size(d); + if (dim_sizes[d] == 0) { d_active = d; break; } + } + if (d_active == 0) return 0; + + // Pass 2: walk suffix products backward within [0, d_active) only. + // Return the first suffix product >= chunk_size, capped at size(). + size_t inner_volume = 1; + if (inner_volume >= chunk_size) return inner_volume; + for (size_t d = d_active; d-- > 0; ) { + inner_volume *= dim_sizes[d]; + if (inner_volume >= chunk_size) return inner_volume; + } + return inner_volume; // == size() of active dims, capped +} + +template +size_t IndexRanges::floor(size_t chunk_size) const requires (N > 1) { + // Pass 1: cache all dim sizes (one call each) and find d_active. + size_t dim_sizes[N]; + size_t d_active = N; + for (size_t d = 0; d < N; ++d) { + dim_sizes[d] = size(d); + if (dim_sizes[d] == 0) { d_active = d; break; } + } + if (d_active == 0) return 0; + + // Pass 2: walk suffix products backward within [0, d_active) only. + // Return the largest suffix product <= chunk_size. + size_t inner_volume = 1; + size_t last_fit = 1; + for (size_t d = d_active; d-- > 0; ) { + size_t next = inner_volume * dim_sizes[d]; + if (next > chunk_size) break; + inner_volume = next; + last_fit = inner_volume; + } + return last_fit; +} + +template +IndexRanges +IndexRanges::upper_slice(size_t flat_beg, size_t chunk_size) const requires (N > 1) { + + if (chunk_size == 0) { + return empty_box(); + } + + // Pass 1 (forward): cache dim sizes and find d_active — the first zero-size + // dimension. Only [0, d_active) participates in the flat iteration space; + // dims [d_active, N) are copied as full extent into the returned box. + size_t dim_sizes[N]; + size_t d_active = N; + for (size_t d = 0; d < N; ++d) { + dim_sizes[d] = size(d); + if (dim_sizes[d] == 0) { d_active = d; break; } + } + + if (d_active == 0) return *this; + + // Pass 2a (backward): decode flat_beg into per-dimension coords. + size_t coords[N] = {}; + size_t temp = flat_beg; + for (size_t d = d_active; d-- > 0; ) { + coords[d] = temp % dim_sizes[d]; + temp /= dim_sizes[d]; + } + + // Pass 2b (backward, fused with grow_dim search): find grow_dim. + // Ceil variant: commit grow_dim BEFORE the budget check (round up). + size_t grow_dim = d_active - 1; + size_t inner_volume = 1; + size_t active_inner_vol = 1; + + for (size_t d = d_active; d-- > 0; ) { + // trailing-zeros rule: inner coord non-zero -> can't expand further out + if (d + 1 < d_active && coords[d + 1] != 0) break; + + // ceil: commit BEFORE budget check + grow_dim = d; + active_inner_vol = inner_volume; + + if (inner_volume >= chunk_size) break; + + inner_volume *= dim_sizes[d]; + } + + // Steps along grow_dim: ceil division, at least 1 for forward progress. + size_t steps_left = dim_sizes[grow_dim] - coords[grow_dim]; + size_t steps_needed = (std::max)(size_t{1}, chunk_size / active_inner_vol); + size_t steps_to_take = (std::min)(steps_left, steps_needed); + + // Pass 3: construct the sub-box in three clean segments: + // [0, grow_dim) — locked dims (one element each) + // [grow_dim] — the grow dimension + // [grow_dim+1, N) — full inner/inactive extent + std::array, N> box_dims; + + for (size_t d = 0; d < grow_dim; ++d) { + const T beg = std::get<0>(_dims[d]); + const T step = std::get<2>(_dims[d]); + box_dims[d] = std::tuple{ + beg + static_cast(coords[d]) * step, + beg + static_cast(coords[d] + 1) * step, + step + }; + } + + { + const T beg = std::get<0>(_dims[grow_dim]); + const T step = std::get<2>(_dims[grow_dim]); + box_dims[grow_dim] = std::tuple{ + beg + static_cast(coords[grow_dim]) * step, + beg + static_cast(coords[grow_dim] + steps_to_take) * step, + step + }; + } + + for (size_t d = grow_dim + 1; d < N; ++d) { + box_dims[d] = _dims[d]; // full inner or inactive extent + } + + return IndexRanges(box_dims); +} + +template +IndexRanges +IndexRanges::lower_slice(size_t flat_beg, size_t chunk_size) const requires (N > 1) { + + if (chunk_size == 0) { + return empty_box(); + } + + // Pass 1 (forward): cache dim sizes and find d_active. + size_t dim_sizes[N]; + size_t d_active = N; + for (size_t d = 0; d < N; ++d) { + dim_sizes[d] = size(d); + if (dim_sizes[d] == 0) { d_active = d; break; } + } + + if (d_active == 0) return *this; + + // Pass 2a (backward): decode flat_beg into per-dimension coords. + size_t coords[N] = {}; + size_t temp = flat_beg; + for (size_t d = d_active; d-- > 0; ) { + coords[d] = temp % dim_sizes[d]; + temp /= dim_sizes[d]; + } + + // Pass 2b (backward, fused with grow_dim search): find grow_dim. + // Floor variant: commit grow_dim AFTER the budget check (round down). + size_t grow_dim = d_active - 1; + size_t inner_volume = 1; + size_t active_inner_vol = 1; + + for (size_t d = d_active; d-- > 0; ) { + // trailing-zeros rule + if (d + 1 < d_active && coords[d + 1] != 0) break; + + // floor: budget check BEFORE committing + size_t next_vol = inner_volume * dim_sizes[d]; + if (next_vol > chunk_size && inner_volume > 1) break; + + grow_dim = d; + active_inner_vol = inner_volume; + inner_volume = next_vol; + } + + // Steps along grow_dim: floor division, at least 1 for forward progress. + size_t steps_left = dim_sizes[grow_dim] - coords[grow_dim]; + size_t steps_needed = (std::max)(size_t{1}, chunk_size / active_inner_vol); + size_t steps_to_take = (std::min)(steps_left, steps_needed); + + // Pass 3: construct the sub-box in three clean segments. + std::array, N> box_dims; + + for (size_t d = 0; d < grow_dim; ++d) { + const T beg = std::get<0>(_dims[d]); + const T step = std::get<2>(_dims[d]); + box_dims[d] = std::tuple{ + beg + static_cast(coords[d]) * step, + beg + static_cast(coords[d] + 1) * step, + step + }; + } + + { + const T beg = std::get<0>(_dims[grow_dim]); + const T step = std::get<2>(_dims[grow_dim]); + box_dims[grow_dim] = std::tuple{ + beg + static_cast(coords[grow_dim]) * step, + beg + static_cast(coords[grow_dim] + steps_to_take) * step, + step + }; + } + + for (size_t d = grow_dim + 1; d < N; ++d) { + box_dims[d] = _dims[d]; // full inner or inactive extent + } + + return IndexRanges(box_dims); +} + +// ---------------------------------------------------------------------------- +// IndexRange — alias for the common 1D case +// ---------------------------------------------------------------------------- + +/** +@brief alias for the common 1D case of tf::IndexRanges + +@tparam T the integral type of the indices + +tf::IndexRange is equivalent to tf::IndexRanges. +Class template argument deduction works through the alias, so the +three-argument constructor can be used without an explicit template argument: + +@code{.cpp} +tf::IndexRange r(0, 10, 2); // deduced as tf::IndexRanges +tf::IndexRange s(0, 100, 5); // same type, written explicitly +@endcode +*/ +template +using IndexRange = IndexRanges; + +// ========================================== +// traits +// ========================================== + +/** +@brief base type trait to detect if a type is a tf::IndexRanges +@tparam T The type to inspect. +*/ +template +constexpr bool is_index_ranges_v = false; + +/** +@brief specialization of the detector for tf::IndexRanges + +Matches an IndexRanges of ANY dimensionality (1D, 2D, 3D, etc.). + +@tparam T the underlying coordinate type (e.g., size_t, int) +@tparam N the number of dimensions +*/ +template +constexpr bool is_index_ranges_v> = true; + +/** +@brief concept to check if a type is a tf::IndexRanges, regardless of dimensionality +@tparam R the range type to evaluate + +This concept strips cv-qualifiers and references (using std::unwrap_ref_decay_t) +before evaluating, allowing const and reference (including std::reference_wrapper) +types to satisfy the constraint. +*/ +template +concept IndexRangesLike = is_index_ranges_v>>; + +/** +@brief concept to check if a type is a tf::IndexRanges (i.e., tf::IndexRange) +@tparam R the range type to evaluate + +@code{.cpp} +static_assert( tf::IndexRanges1DLike>); // true +static_assert( tf::IndexRanges1DLike>); // true +static_assert(!tf::IndexRanges1DLike>); // false +@endcode +*/ +template +concept IndexRanges1DLike = IndexRangesLike && + (std::decay_t>::rank == 1); + +/** +@brief concept to check if a type is a tf::IndexRanges with rank > 1 +@tparam R the range type to evaluate + +@code{.cpp} +static_assert(!tf::IndexRangesMDLike>); // false, rank == 1 +static_assert(!tf::IndexRangesMDLike>); // false, rank == 1 +static_assert( tf::IndexRangesMDLike>); // true +static_assert( tf::IndexRangesMDLike>); // true +@endcode +*/ +template +concept IndexRangesMDLike = IndexRangesLike && + (std::decay_t>::rank > 1); + +} // end of namespace tf ----------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/utility/lazy_string.hpp b/projects/hipfile/third-party/taskflow/utility/lazy_string.hpp new file mode 100644 index 000000000000..dce2340f68dc --- /dev/null +++ b/projects/hipfile/third-party/taskflow/utility/lazy_string.hpp @@ -0,0 +1,73 @@ +#pragma once + +#include +#include + +namespace tf { + +class LazyString { + + public: + + LazyString() = default; + + LazyString(const std::string& str) : + _str(str.empty() ? nullptr : std::make_unique(str)) { + } + + LazyString(std::string&& str) : + _str(str.empty() ? nullptr : std::make_unique(std::move(str))) { + } + + LazyString(const char* str) : + _str((!str || str[0] == '\0') ? nullptr : std::make_unique(str)) { + } + + // Modify the operator to return a const reference + operator const std::string& () const noexcept { + static const std::string empty_string; + return _str ? *_str : empty_string; + } + + LazyString& operator = (const std::string& str) { + if(_str == nullptr) { + _str = std::make_unique(str); + } + else { + *_str = str; + } + return *this; + } + + LazyString& operator = (std::string&& str) { + if(_str == nullptr) { + _str = std::make_unique(std::move(str)); + } + else { + *_str = std::move(str); + } + return *this; + } + + bool empty() const noexcept { + return !_str || _str->empty(); + } + + size_t size() const noexcept { + return _str ? _str->size() : 0; + } + + friend std::ostream& operator<<(std::ostream& os, const LazyString& ls) { + os << (ls._str ? *ls._str : ""); + return os; + } + + private: + + std::unique_ptr _str; + +}; + + + +} // end of namespace tf ------------------------------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/utility/macros.hpp b/projects/hipfile/third-party/taskflow/utility/macros.hpp new file mode 100644 index 000000000000..f9baabecb826 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/utility/macros.hpp @@ -0,0 +1,41 @@ +#pragma once + +// ============================================================================ +// C++ Versions +// ============================================================================ +#define TF_CPP98 199711L +#define TF_CPP11 201103L +#define TF_CPP14 201402L +#define TF_CPP17 201703L +#define TF_CPP20 202002L + +// ============================================================================ +// inline and no-inline +// ============================================================================ + +#if defined(_MSC_VER) + #define TF_FORCE_INLINE __forceinline +#elif defined(__GNUC__) && __GNUC__ > 3 + #define TF_FORCE_INLINE __attribute__((__always_inline__)) inline +#else + #define TF_FORCE_INLINE inline +#endif + +#if defined(_MSC_VER) + #define TF_NO_INLINE __declspec(noinline) +#elif defined(__GNUC__) && __GNUC__ > 3 + #define TF_NO_INLINE __attribute__((__noinline__)) +#else + #define TF_NO_INLINE +#endif + +// ---------------------------------------------------------------------------- + + + + + + + + + diff --git a/projects/hipfile/third-party/taskflow/utility/math.hpp b/projects/hipfile/third-party/taskflow/utility/math.hpp new file mode 100644 index 000000000000..7d8b97eb5c0d --- /dev/null +++ b/projects/hipfile/third-party/taskflow/utility/math.hpp @@ -0,0 +1,505 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace tf { + +/** +@brief rounds the given 64-bit unsigned integer to the nearest power of 2 + +@tparam T 64-bit unsigned integral type +@param x the number to round up +@return the smallest power of 2 that is greater than or equal to @c x + +This overload participates in overload resolution only when @c T is an +8-byte unsigned integral type. It repeatedly fills in the lower bits of +x - 1 until all bits below the highest set bit are 1, then adds 1 +to obtain the next power of 2. + +@code{.cpp} +tf::next_pow2(uint64_t{17}); // returns 32 +tf::next_pow2(uint64_t{32}); // returns 32 +@endcode +*/ +template +requires (std::is_unsigned_v> && sizeof(T) == 8) +constexpr T next_pow2(T x) { + if(x == 0) return 1; + x--; + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + x |= x >> 8; + x |= x >> 16; + x |= x >> 32; + x++; + return x; +} + +/** +@brief rounds the given 32-bit unsigned integer to the nearest power of 2 + +@tparam T 32-bit unsigned integral type +@param y the number to round up +@return the smallest power of 2 that is greater than or equal to @c y + +This overload participates in overload resolution only when @c T is a +4-byte unsigned integral type. It uses the same bit-filling technique as +the 64-bit overload, but only propagates bits up to the 32-bit width. + +@code{.cpp} +tf::next_pow2(uint32_t{17}); // returns 32 +tf::next_pow2(uint32_t{32}); // returns 32 +@endcode +*/ +template +requires (std::is_unsigned_v> && sizeof(T) == 4) +constexpr T next_pow2(T y) { + if(y == 0) return 1; + y--; + y |= y >> 1; + y |= y >> 2; + y |= y >> 4; + y |= y >> 8; + y |= y >> 16; + y++; + return y; +} + +/** +@brief checks if the given number is a power of 2 + +@tparam T integral type of the input +@param x The integer to check. +@return `true` if `x` is a power of 2, otherwise `false`. + +This function determines if the given integer is a power of 2 by testing +that exactly one bit is set, i.e., x & (x - 1) == 0, while also +excluding zero. + +@code{.cpp} +tf::is_pow2(8); // true +tf::is_pow2(10); // false +@endcode + +@note This function is constexpr and can be evaluated at compile time. +*/ +template +constexpr bool is_pow2(const T& x) { + return x && (!(x&(x-1))); +} + +/** +@brief returns the floor of `log2(N)` at compile time + +@tparam N the input value +@return the largest integer `k` such that `2^k <= N` + +This function recursively halves @c N until it is smaller than 2, +counting the number of halving steps performed, which equals the floor +of the base-2 logarithm of @c N. + +@code{.cpp} +tf::static_floor_log2<16>(); // returns 4 +tf::static_floor_log2<17>(); // returns 4 +@endcode +*/ +template +constexpr size_t static_floor_log2() { + return (N < 2) ? 0 : 1 + static_floor_log2(); + //auto log = 0; + //while (N >>= 1) { + // ++log; + //} + //return log; +} + + +/** +@brief finds the median of three numbers pointed to by iterators using the given comparator + +@tparam RandItr The type of the random-access iterator. +@tparam C The type of the comparator. +@param l Iterator to the first element. +@param m Iterator to the second element. +@param r Iterator to the third element. +@param cmp The comparator used to compare the dereferenced iterator values. +@return The iterator pointing to the median value among the three elements. + +This function determines the median value of the elements pointed to by +three random-access iterators using the provided comparator. + +@code{.cpp} +std::vector v = {5, 1, 3}; +auto it = tf::median_of_three(v.begin(), v.begin()+1, v.begin()+2, std::less{}); +// *it == 3 +@endcode +*/ +template +RandItr median_of_three(RandItr l, RandItr m, RandItr r, C cmp) { + return cmp(*l, *m) ? (cmp(*m, *r) ? m : (cmp(*l, *r) ? r : l )) + : (cmp(*r, *m) ? m : (cmp(*r, *l) ? r : l )); +} + +/** +@brief finds the pseudo median of a range of items using a spread of nine numbers + +@tparam RandItr The type of the random-access iterator. +@tparam C The type of the comparator. +@param beg Iterator to the beginning of the range. +@param end Iterator to the end of the range. +@param cmp The comparator used to compare the dereferenced iterator values. +@return The iterator pointing to the pseudo median of the range. + +This function computes an approximate median of a range of items by sampling +nine values spread across the range and finding their median. It uses a +combination of the `median_of_three` function to determine the pseudo median. + +@code{.cpp} +std::vector v = {9, 4, 1, 7, 3, 8, 2, 6, 5}; +auto it = tf::pseudo_median_of_nine(v.begin(), v.end(), std::less{}); +@endcode + +@note The pseudo median is an approximation of the true median and may not + be the exact middle value of the range. +*/ +template +RandItr pseudo_median_of_nine(RandItr beg, RandItr end, C cmp) { + size_t N = std::distance(beg, end); + size_t offset = N >> 3; + return median_of_three( + median_of_three(beg, beg+offset, beg+(offset*2), cmp), + median_of_three(beg+(offset*3), beg+(offset*4), beg+(offset*5), cmp), + median_of_three(beg+(offset*6), beg+(offset*7), end-1, cmp), + cmp + ); +} + +/** +@brief sorts two elements of dereferenced iterators using the given comparison function + +@tparam Iter The type of the iterator. +@tparam Compare The type of the comparator. +@param a Iterator to the first element. +@param b Iterator to the second element. +@param comp The comparator used to compare the dereferenced iterator values. + +This function compares two elements pointed to by iterators and swaps them +if they are out of order according to the provided comparator. + +@code{.cpp} +std::vector v = {3, 1}; +tf::sort2(v.begin(), v.begin()+1, std::less{}); +// v == {1, 3} +@endcode +*/ +template +void sort2(Iter a, Iter b, Compare comp) { + if (comp(*b, *a)) std::iter_swap(a, b); +} + +/** +@brief Sorts three elements of dereferenced iterators using the given comparison function. + +@tparam Iter The type of the iterator. +@tparam Compare The type of the comparator. +@param a Iterator to the first element. +@param b Iterator to the second element. +@param c Iterator to the third element. +@param comp The comparator used to compare the dereferenced iterator values. + +This function sorts three elements pointed to by iterators in ascending order +according to the provided comparator. The sorting is performed using a sequence +of calls to the `sort2` function to ensure the correct order of elements. + +@code{.cpp} +std::vector v = {3, 1, 2}; +tf::sort3(v.begin(), v.begin()+1, v.begin()+2, std::less{}); +// v == {1, 2, 3} +@endcode +*/ +template +void sort3(Iter a, Iter b, Iter c, Compare comp) { + sort2(a, b, comp); + sort2(b, c, comp); + sort2(a, b, comp); +} + +/** +@brief generates a program-wide unique ID of the given type in a thread-safe manner + +@tparam T integral type of the ID to generate +@return A unique ID of type `T`. + +This function provides a globally unique identifier of the specified integral type. +It uses a static `std::atomic` counter to ensure thread safety and increments the +counter in a relaxed memory ordering for efficiency. + +@code{.cpp} +size_t id1 = tf::unique_id(); +size_t id2 = tf::unique_id(); +// id1 != id2 +@endcode + +@note The uniqueness of the ID is guaranteed only within the program's lifetime. +@note The function does not throw exceptions. +*/ +template +T unique_id() { + static std::atomic counter{0}; + return counter.fetch_add(1, std::memory_order_relaxed); +} + +/** +@brief updates an atomic variable with the maximum value + +@tparam T The type of the atomic variable. Must be trivially copyable and comparable. +@param v The atomic variable to update. +@param max_v The value to compare with the current value of `v`. + +This function atomically updates the provided atomic variable `v` to hold +the maximum of its current value and `max_v`. The update is performed using +a relaxed memory ordering for efficiency in non-synchronizing contexts. + +@code{.cpp} +std::atomic v{5}; +tf::atomic_max(v, 10); +// v.load() == 10 +@endcode + +@note If multiple threads call this function concurrently, the value of `v` + will be the maximum value seen across all threads. +*/ +template +inline void atomic_max(std::atomic& v, const T& max_v) noexcept { + T prev = v.load(std::memory_order_relaxed); + while(prev < max_v && + !v.compare_exchange_weak(prev, max_v, std::memory_order_relaxed, + std::memory_order_relaxed)) { + } +} + +/** +@brief updates an atomic variable with the minimum value + +@tparam T The type of the atomic variable. Must be trivially copyable and comparable. +@param v The atomic variable to update. +@param min_v The value to compare with the current value of `v`. + +This function atomically updates the provided atomic variable `v` to hold +the minimum of its current value and `min_v`. The update is performed using +a relaxed memory ordering for efficiency in non-synchronizing contexts. + +@code{.cpp} +std::atomic v{5}; +tf::atomic_min(v, 2); +// v.load() == 2 +@endcode + +@note If multiple threads call this function concurrently, the value of `v` + will be the minimum value seen across all threads. +*/ +template +inline void atomic_min(std::atomic& v, const T& min_v) noexcept { + T prev = v.load(std::memory_order_relaxed); + while(prev > min_v && + !v.compare_exchange_weak(prev, min_v, std::memory_order_relaxed, + std::memory_order_relaxed)) { + } +} + +/** +@brief generates a random seed based on the current system clock + +@tparam T The type of the returned seed. Must be an integral type. +@return A seed value based on the system clock. + +This function returns a seed value derived from the number of clock ticks +since the epoch as measured by the system clock. The seed can be used +to initialize random number generators. + +@code{.cpp} +auto s = tf::seed(); +tf::Xorshift rng(s); +@endcode +*/ +template +inline T seed() noexcept { + return std::chrono::system_clock::now().time_since_epoch().count(); +} + +// ------------------------------------------------------------------------------------------------ +// coprime +// ------------------------------------------------------------------------------------------------ + +/** +@brief computes a coprime of a given number + +@param N input number for which a coprime is to be found. +@return the largest number < @c N that is coprime to N + +This function finds the largest number less than N that is coprime (i.e., has a greatest common divisor of 1) with @c N. +If @c N is less than 3, it returns 1 as a default coprime. + +@code{.cpp} +tf::coprime(10); // returns 9 +@endcode +*/ +constexpr size_t coprime(size_t N) { + if(N < 3) { + return 1; + } + for (size_t x = N; --x > 0;) { + if (std::gcd(x, N) == 1) { + return x; + } + } + return 1; +} + +/** +@brief generates a compile-time array of coprimes for numbers from 0 to N-1 + +@tparam N the size of the array to generate (should be greater than 0). +@return a constexpr array of size @c N where each index holds a coprime of its value. + +This function constructs a constexpr array where each element at index `i` contains a coprime of `i` +(the largest number less than `i` that is coprime to it). + +@code{.cpp} +constexpr auto lut = tf::make_coprime_lut<8>(); +// lut[5] holds a coprime of 5 +@endcode +*/ +template +constexpr std::array make_coprime_lut() { + static_assert(N>0, "N must be greater than 0"); + std::array coprimes{}; + for (size_t n = 0; n < N; ++n) { + coprimes[n] = coprime(n); + } + return coprimes; +} + +//template +//constexpr T lemire_range(T x, T range) { +// return (uint32_t)(((uint64_t)x * (uint64_t)range) >> 32); +//} + + +/** +@class Xorshift + +@brief class to create a fast xorshift-based pseudo-random number generator + +@tparam T unsigned integral type used as the internal state (supported uint32_t and uint64_t) + +This class implements a lightweight xorshift pseudo-random number generator +suitable for performance-critical paths such as schedulers, work-stealing victim selection, +and randomized backoff. +The implementation is branchless on the hot path and has a very small state +footprint (one machine word). All operations are integer-only. + +@note +The internal state must be seeded with a non-zero value. +This class is not thread-safe. Each thread should maintain its own instance. +*/ +template +class Xorshift { + static_assert(std::is_unsigned::value, "Xorshift requires an unsigned integral type."); + + public: + + /** + @brief constructs an uninitialized xor-shift generator + + The internal state is not initialized. The user must call `seed()` + with a non-zero value before generating numbers. + + @code{.cpp} + tf::Xorshift rng; + rng.seed(12345); + @endcode + */ + Xorshift() = default; + + /** + @brief constructs a xor-shift generator with the given seed + + @param value the new seed value to use + + The seed value must be non-zero. + + @code{.cpp} + tf::Xorshift rng(12345); + auto r = rng(); + @endcode + */ + Xorshift(T value) : _state(value) {} + + /** + @brief seeds the generator with a new value + + @param value the new seed value + + The seed value must be non-zero. A zero seed results in a degenerated + generator that always returns zero. + + @code{.cpp} + tf::Xorshift rng; + rng.seed(12345); + @endcode + */ + void seed(T value) { + _state = value; + } + + /** + @brief generates the next pseudo-random value + + @return a pseudo-random value of type `T` + + For 32-bit state, this function implements the Xorshift32 algorithm. + For 64-bit state, this function implements the Xorshift64 algorithm with + a multiplicative output transformation to improve distribution. + + @code{.cpp} + tf::Xorshift rng(12345); + uint64_t r1 = rng(); + uint64_t r2 = rng(); + @endcode + + @warning + Calling this function before seeding the generator with a non-zero value + results in undefined behavior. + */ + T operator()() { + if constexpr (sizeof(T) == 8) { + // Xorshift64 constants + _state ^= _state << 13; + _state ^= _state >> 7; + _state ^= _state << 17; + return _state * 0x2545F4914F6CDD1DULL; + } + else if constexpr (sizeof(T) == 4) { + // Xorshift32 constants + _state ^= _state << 13; + _state ^= _state >> 17; + _state ^= _state << 5; + return _state; + } + else { + static_assert(sizeof(T) == 0, "Unsupported bit-width for Xorshift. Use uint32_t or uint64_t."); + } + } + + private: + + T _state; // must be initialized with non-zero +}; + +} // end of namespace tf ----------------------------------------------------- diff --git a/projects/hipfile/third-party/taskflow/utility/mpmc.hpp b/projects/hipfile/third-party/taskflow/utility/mpmc.hpp new file mode 100644 index 000000000000..75c775fcf8a4 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/utility/mpmc.hpp @@ -0,0 +1,508 @@ +#pragma once + +#include +#include +#include + +#include "os.hpp" + +namespace tf { + +/** + * A 'lockless' bounded multi-producer, multi-consumer queue + * + * Has the caveat that the queue can *appear* empty even if there are + * returned items within it as a single thread can block progression + * of the queue. + */ +template +class MPMC { + + constexpr static uint64_t BufferSize = 1ull << LogSize; + constexpr static uint64_t BufferMask = (BufferSize - 1); + + static_assert((BufferSize >= 2) && ((BufferSize & (BufferSize - 1)) == 0)); + +public: + + /** + * Constructs a bounded multi-producer, multi-consumer queue + * + * Note: Due to the algorithm used, buffer_size must be a power + * of two and must be greater than or equal to two. + * + * @param buffer_size Number of spaces available in the queue. + */ + explicit MPMC() { + for (size_t i = 0; i < _buffer.size(); i++) { + _buffer[i].sequence.store(i, std::memory_order_relaxed); + } + _enqueue_pos.store(0, std::memory_order_relaxed); + _dequeue_pos.store(0, std::memory_order_relaxed); + } + + + /** + * Enqueues an item into the queue + * + * @param data Argument to place into the array + * @return false if the queue was full (and enqueing failed), + * true otherwise + */ + bool try_enqueue(T data) { + Cell *cell; + auto pos = _enqueue_pos.load(std::memory_order_relaxed); + for (; ;) { + cell = &_buffer[pos & BufferMask]; + auto seq = cell->sequence.load(std::memory_order_acquire); + if (seq == pos) { + if (_enqueue_pos.compare_exchange_weak(pos, pos + 1, + std::memory_order_relaxed)) { + break; + } + } else if (seq < pos) { + return false; + } else { + pos = _enqueue_pos.load(std::memory_order_relaxed); + } + } + + cell->data = data; + cell->sequence.store(pos + 1, std::memory_order_release); + + return true; + } + + void enqueue(T data) { + + Cell *cell; + auto pos = _enqueue_pos.load(std::memory_order_relaxed); + + for (; ;) { + cell = &_buffer[pos & BufferMask]; + auto seq = cell->sequence.load(std::memory_order_acquire); + if (seq == pos) { + if (_enqueue_pos.compare_exchange_weak(pos, pos + 1, + std::memory_order_relaxed)) { + break; + } + } + else { + pos = _enqueue_pos.load(std::memory_order_relaxed); + } + } + + cell->data = data; + cell->sequence.store(pos + 1, std::memory_order_release); + } + + /** + * Dequeues an item from the queue + * + * @param[out] data Reference to place item into + * @return false if the queue was empty (and dequeuing failed), + * true if successful + */ + std::optional try_dequeue() { + Cell *cell; + auto pos = _dequeue_pos.load(std::memory_order_relaxed); + for (; ;) { + cell = &_buffer[pos & BufferMask]; + auto seq = cell->sequence.load(std::memory_order_acquire); + if (seq == pos + 1) { + if (_dequeue_pos.compare_exchange_weak(pos, pos + 1, + std::memory_order_relaxed)) { + break; + } + } else if (seq < (pos + 1)) { + return std::nullopt; + } else { + pos = _dequeue_pos.load(std::memory_order_relaxed); + } + } + + T data = cell->data; + cell->sequence.store(pos + BufferMask + 1, std::memory_order_release); + + return data; + } + + bool empty() const { + auto beg = _dequeue_pos.load(std::memory_order_relaxed); + auto end = _enqueue_pos.load(std::memory_order_relaxed); + return beg >= end; + } + + size_t capacity() const { + return BufferSize; + } + +private: + + struct Cell { + T data; + std::atomic sequence; + }; + + //static const size_t cacheline_size = 64; + + alignas(TF_CACHELINE_SIZE) std::array _buffer; + alignas(TF_CACHELINE_SIZE) std::atomic _enqueue_pos; + alignas(TF_CACHELINE_SIZE) std::atomic _dequeue_pos; +}; + +// ------------------------------------------------------------------------------------------------ +// specialization for pointer type +// ------------------------------------------------------------------------------------------------ + +template +class MPMC { + + constexpr static uint64_t BufferSize = 1ull << LogSize; + constexpr static uint64_t BufferMask = (BufferSize - 1); + + static_assert((BufferSize >= 2) && ((BufferSize & (BufferSize - 1)) == 0)); + +public: + + /** + * Constructs a bounded multi-producer, multi-consumer queue + * + * Note: Due to the algorithm used, buffer_size must be a power + * of two and must be greater than or equal to two. + * + * @param buffer_size Number of spaces available in the queue. + */ + explicit MPMC() { + for (size_t i = 0; i < _buffer.size(); i++) { + _buffer[i].sequence.store(i, std::memory_order_relaxed); + } + _enqueue_pos.store(0, std::memory_order_relaxed); + _dequeue_pos.store(0, std::memory_order_relaxed); + } + + + /** + * Enqueues an item into the queue + * + * @param data Argument to place into the array + * @return false if the queue was full (and enqueing failed), + * true otherwise + */ + bool try_enqueue(T* data) { + Cell *cell; + auto pos = _enqueue_pos.load(std::memory_order_relaxed); + for (; ;) { + cell = &_buffer[pos & BufferMask]; + auto seq = cell->sequence.load(std::memory_order_acquire); + if (seq == pos) { + if (_enqueue_pos.compare_exchange_weak(pos, pos + 1, + std::memory_order_relaxed)) { + break; + } + } else if (seq < pos) { + return false; + } else { + pos = _enqueue_pos.load(std::memory_order_relaxed); + } + } + + cell->data = data; + cell->sequence.store(pos + 1, std::memory_order_release); + + return true; + } + + void enqueue(T* data) { + + Cell *cell; + auto pos = _enqueue_pos.load(std::memory_order_relaxed); + + for (; ;) { + cell = &_buffer[pos & BufferMask]; + auto seq = cell->sequence.load(std::memory_order_acquire); + if (seq == pos) { + if (_enqueue_pos.compare_exchange_weak(pos, pos + 1, + std::memory_order_relaxed)) { + break; + } + } + else { + pos = _enqueue_pos.load(std::memory_order_relaxed); + } + } + + cell->data = data; + cell->sequence.store(pos + 1, std::memory_order_release); + } + + /** + * Dequeues an item from the queue + * + * @param[out] data Reference to place item into + * @return false if the queue was empty (and dequeuing failed), + * true if successful + */ + T* try_dequeue() { + Cell *cell; + auto pos = _dequeue_pos.load(std::memory_order_relaxed); + for (; ;) { + cell = &_buffer[pos & BufferMask]; + auto seq = cell->sequence.load(std::memory_order_acquire); + if (seq == pos + 1) { + if (_dequeue_pos.compare_exchange_weak(pos, pos + 1, + std::memory_order_relaxed)) { + break; + } + } else if (seq < (pos + 1)) { + return nullptr; + } else { + pos = _dequeue_pos.load(std::memory_order_relaxed); + } + } + + auto data = cell->data; + cell->sequence.store(pos + BufferMask + 1, std::memory_order_release); + + return data; + } + + bool empty() const { + auto beg = _dequeue_pos.load(std::memory_order_relaxed); + auto end = _enqueue_pos.load(std::memory_order_relaxed); + return beg >= end; + } + + size_t capacity() const { + return BufferSize; + } + +private: + + struct Cell { + T* data; + std::atomic sequence; + }; + + //static const size_t cacheline_size = 64; + + alignas(TF_CACHELINE_SIZE) std::array _buffer; + alignas(TF_CACHELINE_SIZE) std::atomic _enqueue_pos; + alignas(TF_CACHELINE_SIZE) std::atomic _dequeue_pos; +}; + +/** + * RunQueue is a fixed-size, partially non-blocking deque or Work items. + * Operations on front of the queue must be done by a single thread (owner), + * operations on back of the queue can be done by multiple threads concurrently. + * + * Algorithm outline: + * All remote threads operating on the queue back are serialized by a mutex. + * This ensures that at most two threads access state: owner and one remote + * thread (Size aside). The algorithm ensures that the occupied region of the + * underlying array is logically continuous (can wraparound, but no stray + * occupied elements). Owner operates on one end of this region, remote thread + * operates on the other end. Synchronization between these threads + * (potential consumption of the last element and take up of the last empty + * element) happens by means of state variable in each element. States are: + * empty, busy (in process of insertion of removal) and ready. Threads claim + * elements (empty->busy and ready->busy transitions) by means of a CAS + * operation. The finishing transition (busy->empty and busy->ready) are done + * with plain store as the element is exclusively owned by the current thread. + * + * Note: we could permit only pointers as elements, then we would not need + * separate state variable as null/non-null pointer value would serve as state, + * but that would require malloc/free per operation for large, complex values + * (and this is designed to store std::function<()>). +template +class RunQueue { + public: + RunQueue() : front_(0), back_(0) { + // require power-of-two for fast masking + eigen_plain_assert((kSize & (kSize - 1)) == 0); + eigen_plain_assert(kSize > 2); // why would you do this? + eigen_plain_assert(kSize <= (64 << 10)); // leave enough space for counter + for (unsigned i = 0; i < kSize; i++) array_[i].state.store(kEmpty, std::memory_order_relaxed); + } + + ~RunQueue() { eigen_plain_assert(Size() == 0); } + + // PushFront inserts w at the beginning of the queue. + // If queue is full returns w, otherwise returns default-constructed Work. + Work PushFront(Work w) { + unsigned front = front_.load(std::memory_order_relaxed); + Elem* e = &array_[front & kMask]; + uint8_t s = e->state.load(std::memory_order_relaxed); + if (s != kEmpty || !e->state.compare_exchange_strong(s, kBusy, std::memory_order_acquire)) return w; + front_.store(front + 1 + (kSize << 1), std::memory_order_relaxed); + e->w = std::move(w); + e->state.store(kReady, std::memory_order_release); + return Work(); + } + + // PopFront removes and returns the first element in the queue. + // If the queue was empty returns default-constructed Work. + Work PopFront() { + unsigned front = front_.load(std::memory_order_relaxed); + Elem* e = &array_[(front - 1) & kMask]; + uint8_t s = e->state.load(std::memory_order_relaxed); + if (s != kReady || !e->state.compare_exchange_strong(s, kBusy, std::memory_order_acquire)) return Work(); + Work w = std::move(e->w); + e->state.store(kEmpty, std::memory_order_release); + front = ((front - 1) & kMask2) | (front & ~kMask2); + front_.store(front, std::memory_order_relaxed); + return w; + } + + // PushBack adds w at the end of the queue. + // If queue is full returns w, otherwise returns default-constructed Work. + Work PushBack(Work w) { + EIGEN_MUTEX_LOCK lock(mutex_); + unsigned back = back_.load(std::memory_order_relaxed); + Elem* e = &array_[(back - 1) & kMask]; + uint8_t s = e->state.load(std::memory_order_relaxed); + if (s != kEmpty || !e->state.compare_exchange_strong(s, kBusy, std::memory_order_acquire)) return w; + back = ((back - 1) & kMask2) | (back & ~kMask2); + back_.store(back, std::memory_order_relaxed); + e->w = std::move(w); + e->state.store(kReady, std::memory_order_release); + return Work(); + } + + // PopBack removes and returns the last elements in the queue. + Work PopBack() { + if (Empty()) return Work(); + EIGEN_MUTEX_LOCK lock(mutex_); + unsigned back = back_.load(std::memory_order_relaxed); + Elem* e = &array_[back & kMask]; + uint8_t s = e->state.load(std::memory_order_relaxed); + if (s != kReady || !e->state.compare_exchange_strong(s, kBusy, std::memory_order_acquire)) return Work(); + Work w = std::move(e->w); + e->state.store(kEmpty, std::memory_order_release); + back_.store(back + 1 + (kSize << 1), std::memory_order_relaxed); + return w; + } + + // PopBackHalf removes and returns half last elements in the queue. + // Returns number of elements removed. + unsigned PopBackHalf(std::vector* result) { + if (Empty()) return 0; + EIGEN_MUTEX_LOCK lock(mutex_); + unsigned back = back_.load(std::memory_order_relaxed); + unsigned size = Size(); + unsigned mid = back; + if (size > 1) mid = back + (size - 1) / 2; + unsigned n = 0; + unsigned start = 0; + for (; static_cast(mid - back) >= 0; mid--) { + Elem* e = &array_[mid & kMask]; + uint8_t s = e->state.load(std::memory_order_relaxed); + if (n == 0) { + if (s != kReady || !e->state.compare_exchange_strong(s, kBusy, std::memory_order_acquire)) continue; + start = mid; + } else { + // Note: no need to store temporal kBusy, we exclusively own these + // elements. + eigen_plain_assert(s == kReady); + } + result->push_back(std::move(e->w)); + e->state.store(kEmpty, std::memory_order_release); + n++; + } + if (n != 0) back_.store(start + 1 + (kSize << 1), std::memory_order_relaxed); + return n; + } + + // Size returns current queue size. + // Can be called by any thread at any time. + unsigned Size() const { return SizeOrNotEmpty(); } + + // Empty tests whether container is empty. + // Can be called by any thread at any time. + bool Empty() const { return SizeOrNotEmpty() == 0; } + + // Delete all the elements from the queue. + void Flush() { + while (!Empty()) { + PopFront(); + } + } + + private: + static const unsigned kMask = kSize - 1; + static const unsigned kMask2 = (kSize << 1) - 1; + + enum State { + kEmpty, + kBusy, + kReady, + }; + + struct Elem { + std::atomic state; + Work w; + }; + + // Low log(kSize) + 1 bits in front_ and back_ contain rolling index of + // front/back, respectively. The remaining bits contain modification counters + // that are incremented on Push operations. This allows us to (1) distinguish + // between empty and full conditions (if we would use log(kSize) bits for + // position, these conditions would be indistinguishable); (2) obtain + // consistent snapshot of front_/back_ for Size operation using the + // modification counters. + EIGEN_ALIGN_TO_AVOID_FALSE_SHARING std::atomic front_; + EIGEN_ALIGN_TO_AVOID_FALSE_SHARING std::atomic back_; + EIGEN_MUTEX mutex_; // guards `PushBack` and `PopBack` (accesses `back_`) + + EIGEN_ALIGN_TO_AVOID_FALSE_SHARING Elem array_[kSize]; + + // SizeOrNotEmpty returns current queue size; if NeedSizeEstimate is false, + // only whether the size is 0 is guaranteed to be correct. + // Can be called by any thread at any time. + template + unsigned SizeOrNotEmpty() const { + // Emptiness plays critical role in thread pool blocking. So we go to great + // effort to not produce false positives (claim non-empty queue as empty). + unsigned front = front_.load(std::memory_order_acquire); + for (;;) { + // Capture a consistent snapshot of front/tail. + unsigned back = back_.load(std::memory_order_acquire); + unsigned front1 = front_.load(std::memory_order_relaxed); + if (front != front1) { + front = front1; + std::atomic_thread_fence(std::memory_order_acquire); + continue; + } + if (NeedSizeEstimate) { + return CalculateSize(front, back); + } else { + // This value will be 0 if the queue is empty, and undefined otherwise. + unsigned maybe_zero = ((front ^ back) & kMask2); + // Queue size estimate must agree with maybe zero check on the queue + // empty/non-empty state. + eigen_assert((CalculateSize(front, back) == 0) == (maybe_zero == 0)); + return maybe_zero; + } + } + } + + EIGEN_ALWAYS_INLINE unsigned CalculateSize(unsigned front, unsigned back) const { + int size = (front & kMask2) - (back & kMask2); + // Fix overflow. + if (EIGEN_PREDICT_FALSE(size < 0)) size += 2 * kSize; + // Order of modification in push/pop is crafted to make the queue look + // larger than it is during concurrent modifications. E.g. push can + // increment size before the corresponding pop has decremented it. + // So the computed size can be up to kSize + 1, fix it. + if (EIGEN_PREDICT_FALSE(size > static_cast(kSize))) size = kSize; + return static_cast(size); + } + + RunQueue(const RunQueue&) = delete; + void operator=(const RunQueue&) = delete; +}; +*/ + + +} // end of namespace tf ----------------------------------------------------- + diff --git a/projects/hipfile/third-party/taskflow/utility/object_pool.hpp b/projects/hipfile/third-party/taskflow/utility/object_pool.hpp new file mode 100644 index 000000000000..83aeb3aebbde --- /dev/null +++ b/projects/hipfile/third-party/taskflow/utility/object_pool.hpp @@ -0,0 +1,684 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include "os.hpp" + +/** +@file object_pool.hpp +@brief object pool include file +*/ + +namespace tf { + +// ---------------------------------------------------------------------------- +// TaggedHead128 +// ---------------------------------------------------------------------------- + +/** +@brief tagged free-list head using a 128-bit (pointer, version) pair + +TaggedHead128 stores the free-list head pointer and an ABA version counter +as two independent 64-bit words, yielding a 16-byte representation. +@c std::atomic requires a 128-bit compare-and-swap: +CMPXCHG16B on x86-64 or CASP on ARMv8.1+. On platforms that lack a native +128-bit CAS (e.g. RISC-V, baseline ARMv8.0), the atomic falls back to a +lock-based implementation — still correct, but no longer lock-free. + +The full 64-bit version counter makes ABA wrap-around effectively impossible +under any realistic workload. + +@code{.cpp} +// explicitly select the 128-bit head (this is also the default) +tf::ObjectPool pool; +@endcode + +@see TaggedHead64 for an 8-byte alternative that is lock-free on all 64-bit + platforms at the cost of a narrower version counter. +*/ +struct TaggedHead128 { + + /** + @brief block address representation + */ + using pointer_type = uintptr_t; + + /** + @brief ABA version counter representation + */ + using tag_type = uintptr_t; + + /** + @brief block address (reinterpret-cast to/from @c ObjectBlock*) + */ + pointer_type ptr {0}; + + /** + @brief ABA version counter; incremented on every push and pop + */ + tag_type tag {0}; + + /** + @brief constructs a null, zero-tagged head + */ + TaggedHead128() = default; + + /** + @brief constructs a head with an explicit block address and version counter + + @param p block address as @c uintptr_t + @param t ABA version counter + + @code{.cpp} + tf::TaggedHead128 head(reinterpret_cast(block), 0); + @endcode + */ + TaggedHead128(pointer_type p, tag_type t) noexcept : ptr{p}, tag{t} {} + + /** + @brief returns the stored block address + + @code{.cpp} + tf::TaggedHead128 head(reinterpret_cast(block), 7); + auto p = head.get_ptr(); // p == reinterpret_cast(block) + @endcode + */ + pointer_type get_ptr() const noexcept { return ptr; } + + /** + @brief returns the ABA version counter + + @code{.cpp} + tf::TaggedHead128 head(reinterpret_cast(block), 7); + auto t = head.get_tag(); // t == 7 + @endcode + */ + tag_type get_tag() const noexcept { return tag; } +}; + +// ---------------------------------------------------------------------------- +// TaggedHead64 +// ---------------------------------------------------------------------------- + +/** +@brief tagged free-list head packed into a single 64-bit word + +@tparam PtrBits number of low bits reserved for the block address; the + remaining @c (64 - PtrBits) bits hold the ABA version counter. + Must be at most 48 so that at least 16 tag bits are available. + Defaults to @c TF_POINTER_BITS (48 on x86-64, AArch64, and + RISC-V SV48). Override for non-standard VA layouts: + @li @c 39 for RISC-V SV39, giving 25 tag bits + @li Values above 48 (e.g. LA57's 57) are rejected by a + @c static_assert — use TaggedHead128 in those cases + +TaggedHead64 fits both the free-list head pointer and an ABA version counter +into a single @c uintptr_t: the low @c PtrBits bits store the block address +and the remaining high bits store the version counter. Because the entire +state is one 64-bit word, @c std::atomic> is always lock-free +on every 64-bit platform — including those without a native 128-bit CAS +instruction (RISC-V, baseline ARMv8.0). + +The default @c PtrBits of 48 assumes user-space virtual addresses occupy at +most 48 bits, which holds for: +@li x86-64 with 4-level paging — bits 48–63 are zero in user space +@li AArch64 with 48-bit VA space — TTBR0 range +@li RISC-V SV39 / SV48 — provides even more headroom (25 / 16 free bits) + +The resulting @c (64 - PtrBits) -bit version counter wraps at +@c 2^(64-PtrBits). With the default of 48, the counter is 16 bits and wraps +at 65 536. ABA would require a competing thread to complete exactly that many +push/pop cycles on the same shard between another thread's load and its CAS — +negligible probability in any task-parallel workload. + +@code{.cpp} +// default PtrBits (48) — correct for x86-64, AArch64, RISC-V SV48 +tf::ObjectPool> pool; + +// RISC-V SV39: only 39 address bits, 25 bits available for the tag +tf::ObjectPool> pool_sv39; +@endcode + +@see TaggedHead128 for a variant with a full 64-bit version counter that may + use a lock-based fallback on platforms without 128-bit CAS. +*/ +template +struct TaggedHead64 { + static_assert(64 - PtrBits >= 16, + "TaggedHead64 requires at least 16 tag bits for ABA safety " + "(PtrBits must be <= 48); use TaggedHead128 instead"); + + /** + @brief block address representation + */ + using pointer_type = uintptr_t; + + /** + @brief ABA version counter representation + */ + using tag_type = uint16_t; + + /** + @brief bits reserved for the block address + */ + static constexpr int PTR_BITS = PtrBits; + + /** + @brief bits reserved for the version counter + */ + static constexpr int TAG_BITS = 64 - PtrBits; + + /** + @brief mask isolating the address field + */ + static constexpr pointer_type PTR_MASK = (pointer_type{1} << PTR_BITS) - 1; + + /** + @brief packed word: high @c TAG_BITS bits hold the version tag, low + @c PTR_BITS bits hold the address + */ + uintptr_t bits {0}; + + /** + @brief constructs a null, zero-tagged head + */ + TaggedHead64() = default; + + /** + @brief constructs a head with an explicit block address and version counter + + @param p block address as @c uintptr_t; only the low @c PtrBits bits are stored + @param t ABA version counter; only the low @c TAG_BITS bits are stored + + @code{.cpp} + tf::TaggedHead64<> head(reinterpret_cast(block), 0); + @endcode + */ + TaggedHead64(pointer_type p, tag_type t) noexcept + : bits{ (p & PTR_MASK) | (static_cast(t) << PTR_BITS) } {} + + /** + @brief returns the block address + + @code{.cpp} + tf::TaggedHead64<> head(reinterpret_cast(block), 7); + auto p = head.get_ptr(); // p == reinterpret_cast(block) + @endcode + */ + pointer_type get_ptr() const noexcept { return bits & PTR_MASK; } + + /** + @brief returns the 16-bit ABA version counter + + @code{.cpp} + tf::TaggedHead64<> head(reinterpret_cast(block), 7); + auto t = head.get_tag(); // t == 7 + @endcode + */ + tag_type get_tag() const noexcept { return static_cast(bits >> PTR_BITS); } +}; + +// ---------------------------------------------------------------------------- +// ObjectBlock +// ---------------------------------------------------------------------------- + +/** +@private + +@brief internal storage block wrapping an object with pool metadata + +@tparam T object type stored in this block + +ObjectBlock wraps an object of type @c T together with the shard index +(@c pool_id) and an intrusive free-list link (@c next_free) in a +standard-layout struct. Using @c std::byte storage instead of a @c T +member directly ensures that @c offsetof(ObjectBlock, storage) is +well-defined regardless of @c T's layout, which allows safe recovery of +the block pointer from a bare @c T pointer in @c from_object. + +@c next_free is @c std::atomic rather than a plain pointer. +C++20 [atomics.types.generic.general] guarantees that @c std::atomic is +standard-layout, so ObjectBlock remains standard-layout and @c offsetof +stays well-defined. See the @c next_free field declaration below for the +race scenario that motivates the atomic type. +*/ +template +struct ObjectBlock { + + /** + @brief index of the shard that owns this block + */ + uint16_t pool_id; + + // Intrusive free-list link. Must be atomic to avoid a formal data race + // between push_free writing next_free and a concurrent pop_free reading it. + // + // The race arises from a *stale pointer* held by a thread that loses a CAS. + // Consider this interleaving (free list: [b -> null]): + // + // Thread A (pop_free): loads _free_head = {b, 5} <-- cur.ptr = b + // Thread B (pop_free): also loads {b, 5}, wins CAS first, + // pops b, returns it to the caller + // Thread C (push_free): caller recycles b; + // C writes b->next_free <-- non-atomic WRITE + // Thread A (pop_free): reads cur.ptr->next_free <-- non-atomic READ + // (cur.ptr is the stale b!) + // + // Thread A's CAS will ultimately fail (the version tag changed), so there + // is no algorithmic corruption — but the concurrent non-atomic read + write + // on the same memory location is a formal data race and undefined behavior + // per the C++ memory model. Making next_free atomic eliminates the race by + // definition: two concurrent atomic accesses are never a data race. + std::atomic next_free {nullptr}; + + /** + @brief raw storage for one @c T + */ + alignas(T) std::byte storage[sizeof(T)]; + + /** + @brief returns a pointer to the stored object + */ + T* object() noexcept { + return std::launder(reinterpret_cast(storage)); + } + + /** + @brief returns a const pointer to the stored object + */ + const T* object() const noexcept { + return std::launder(reinterpret_cast(storage)); + } + + /** + @brief recovers the enclosing ObjectBlock from a bare object pointer + + Uses @c offsetof(ObjectBlock, storage), which is well-defined because + ObjectBlock is standard-layout (guaranteed by C++20 for @c std::atomic + member types). + */ + static ObjectBlock* from_object(T* obj) noexcept { + return reinterpret_cast( + reinterpret_cast(obj) - offsetof(ObjectBlock, storage) + ); + } +}; + +// ---------------------------------------------------------------------------- +// ObjectPool +// ---------------------------------------------------------------------------- + +/** +@brief sharded fixed-size object allocator with a lock-free hot path + +@tparam T object type to allocate +@tparam H tagged-pointer policy that controls the free-stack head + representation. The type must provide: + @li @c pointer_type — typedef for the address representation + @li @c tag_type — typedef for the ABA counter representation + @li @c get_ptr() — returns the stored address as @c pointer_type + @li @c get_tag() — returns the ABA counter as @c tag_type + @li @c H(pointer_type, tag_type) — two-argument constructor + Built-in choices: + @li tf::TaggedHead128 (default) — 16-byte head, 64-bit counter; + lock-free on x86-64 and ARMv8.1+, mutex-based elsewhere. + @li tf::TaggedHead64<> — 8-byte head, 16-bit counter; + lock-free on all 64-bit platforms; requires pointers to + fit in 48 bits. +@tparam LogSize log2 of the number of shards (default @c 5, giving 32 shards); + must be in [1, 15] to fit the shard index in a @c uint16_t + +%ObjectPool is a high-performance allocator for a single fixed-size type @c T, +designed for concurrent task-parallel workloads where objects are frequently +created and destroyed across many threads. + +@dotfile images/object_pool_flow.dot + +Internally, allocations are distributed across @c 2^LogSize independent +shards. Each shard maintains two independent components (separated by cache +lines to prevent false sharing): + +Hot Path (99% of operations): A lock-free Treiber stack of recycled blocks. +When tf::ObjectPool::animate is called, it tries to pop a recycled block from +this stack with a single atomic CAS. On success, the block is reused with +no mutex acquisition. Blocks returned by tf::ObjectPool::recycle are pushed +back onto this stack without acquiring any mutex. + +Cold Path (1% of operations): A @c std::pmr::synchronized_pool_resource +as backing storage for fresh block allocations. This mutex-protected pool is +only touched when the shard's hot-path stack is empty. When accessed, it +allocates a whole chunk (configured to hold up to 1024 blocks via +@c max_blocks_per_chunk = 1024), amortizing the synchronization cost: one +mutex acquisition yields ~1024 blocks for the hot path. + +The tagged-pointer policy @c H attaches a version counter to each free-stack +head to prevent the ABA problem. This counter increments on every push and pop, +making ABA wrap-around effectively impossible under realistic workloads. +Shards are aligned to the cache line size to eliminate false sharing between +concurrent threads accessing different shards' hot-path stacks. + +@code{.cpp} +// default: TaggedHead128, 32 shards +tf::ObjectPool pool; + +// lock-free on all 64-bit platforms (default PtrBits=48), 32 shards +tf::ObjectPool> pool64; + +// construct a MyTask in the pool, forwarding constructor arguments +MyTask* t = pool.animate(arg1, arg2); + +// ... use t ... + +// destruct and return storage to the pool for reuse +pool.recycle(t); +@endcode + +@note +All pointers returned by tf::ObjectPool::animate must be passed to +tf::ObjectPool::recycle before the allocator is destroyed. Destroying +the allocator with live objects is undefined behavior. + +@par Two-Level Freelist Design + +The combination of lock-free and mutex-protected freelists is deliberate: +recycled blocks remain on the lock-free stack indefinitely, avoiding mutex +costs on every allocation. The backing pool's internal freelist is rarely +used directly because blocks do not call @c deallocate() in the normal +hot path — they stay on the lock-free stack for immediate reuse. This design +trades chunk-level memory reuse efficiency for atomic-fast allocation on the +hot path, which is the right trade-off for task-parallel workloads where the +hot path is hit millions of times. + +@par Chunk Amortization + +When the hot-path stack is empty, a single @c std::pmr::synchronized_pool_resource::allocate +call acquires a mutex and either reuses a chunk or allocates a new one from +the system allocator. With @c max_blocks_per_chunk = 1024, one mutex acquisition +amortizes to ~1024 subsequent lock-free pops, yielding negligible mutex overhead +(roughly 0.001 mutex cost per allocation). + +*/ +template +class ObjectPool { + + static_assert(LogSize >= 1 && LogSize <= 15, + "LogSize must be in [1, 15]"); + + using Block = ObjectBlock; + + static constexpr size_t NumPools = 1u << LogSize; + + struct alignas(TF_CACHELINE_SIZE) Shard { + + // Hot path: lock-free Treiber stack of recycled blocks. + // _free_head sits on its own cache line (via the Shard alignas) so that + // hot-path CAS does not invalidate the line holding _backing's mutex. + std::atomic _free_head {H{}}; + + // Cold path: backing allocator for fresh block memory. + // alignas pushes _backing to the next cache line, separating it from the + // hot _free_head above and preventing hot/cold false sharing. + alignas(TF_CACHELINE_SIZE) std::pmr::synchronized_pool_resource _backing { + std::pmr::pool_options { + .max_blocks_per_chunk = 1024, + .largest_required_pool_block = sizeof(Block) + } + }; + + void push_free(Block* b) noexcept { + H cur = _free_head.load(std::memory_order_relaxed); + H next; + do { + // relaxed: the release CAS below synchronises-with pop_free's acquire, + // making this store visible to any thread that subsequently observes b + // at the head of the list. + b->next_free.store( + reinterpret_cast(cur.get_ptr()), std::memory_order_relaxed); + next = H( + reinterpret_cast(b), + static_cast(cur.get_tag() + 1) + ); + } while (!_free_head.compare_exchange_weak( + cur, next, + std::memory_order_release, // publish next_free write to pop_free + std::memory_order_relaxed + )); + } + + Block* pop_free() noexcept { + H cur = _free_head.load(std::memory_order_acquire); + while (cur.get_ptr()) { + auto* p = reinterpret_cast(cur.get_ptr()); + // relaxed on next_free: the acquire on _free_head (either the load + // above or the acquire failure ordering below) synchronises-with the + // release CAS in push_free, so the next_free store that preceded it + // is already visible to this thread. + Block* nx = p->next_free.load(std::memory_order_relaxed); + H next( + reinterpret_cast(nx), + static_cast(cur.get_tag() + 1) + ); + if (_free_head.compare_exchange_weak( + cur, next, + std::memory_order_acquire, // success: synchronise with push_free + std::memory_order_acquire // failure: fresh cur must also synchronise + )) { // before the next next_free read + return p; + } + } + return nullptr; + } + + Block* allocate_from_backing() { + return static_cast( + _backing.allocate(sizeof(Block), alignof(Block)) + ); + } + + void deallocate_to_backing(Block* b) { + _backing.deallocate(b, sizeof(Block), alignof(Block)); + } + }; + + std::array _shards; + + // Returns the next shard index for this thread. The thread_local counter + // is seeded once from the calling thread's ID hash, spreading different + // threads across different starting shards with zero shared state. + // Subsequent calls are a bare local increment — no atomic, no cache-line + // traffic. + // + // If thread_local is broken (e.g. MSVC DLL with improper TLS), the counter + // degrades to a single shared value and causes contention, but shard + // selection remains correct. + static size_t _next_shard() noexcept { + thread_local size_t counter = + std::hash{}(std::this_thread::get_id()); + return counter++ & (NumPools - 1); + } + + public: + + /** + @brief constructs the allocator with @c 2^LogSize empty shards + + Each shard is default-constructed with an empty free stack and an + uninitialized backing pool. No memory is allocated from the OS until + the first call to tf::ObjectPool::animate. + */ + ObjectPool() = default; + + /** + @brief disabled copy constructor + + ObjectPool owns its shards and backing memory; copying is not meaningful. + Declare the allocator as a global or long-lived member and share it by + reference or pointer. + */ + ObjectPool(const ObjectPool&) = delete; + + /** + @brief disabled copy assignment operator + */ + ObjectPool& operator=(const ObjectPool&) = delete; + + /** + @brief destroys the allocator and releases all backing memory to upstream + + The destructor of each shard's @c std::pmr::synchronized_pool_resource + returns all allocated chunks to the system allocator, including memory + backing blocks that are currently on the free stack. No per-block destructor + is called; callers are responsible for recycling all live objects before + destroying the allocator. + */ + ~ObjectPool() = default; + + /** + @brief constructs an object of type @c T in the pool and returns a pointer + + @tparam Args constructor argument types + @param args arguments forwarded to the constructor of @c T + + @return pointer to the newly constructed @c T; never null + + On the hot path, animate pops a previously recycled block from the shard's + lock-free free stack and constructs @c T in it via @c std::construct_at, + with no mutex acquisition. On a cache miss (empty free stack), a fresh block + is carved from the shard's backing @c std::pmr::synchronized_pool_resource, + which amortizes system allocation cost over chunks of up to 1024 blocks. + + Allocations are distributed across shards via a per-thread round-robin + counter seeded from the thread ID hash, balancing load with zero shared + state after initialization. + + @code{.cpp} + tf::ObjectPool pool; + + // default-construct + MyTask* t1 = pool.animate(); + + // construct with arguments + MyTask* t2 = pool.animate(42, "hello"); + + pool.recycle(t1); + pool.recycle(t2); + @endcode + + @note + The returned pointer must eventually be passed to tf::ObjectPool::recycle. + Discarding it without recycling leaks both the object's resources and the + underlying block. + */ + template + [[nodiscard]] T* animate(Args&&... args) { + auto sid = _next_shard(); + auto& shard = _shards[sid]; + + Block* block = shard.pop_free(); // hot path: lock-free + if (!block) block = shard.allocate_from_backing(); // cold path: mutex, amortized + + block->pool_id = static_cast(sid); + return std::construct_at(block->object(), std::forward(args)...); + } + + /** + @brief destructs the object and returns its storage to the pool + + @param obj pointer to a @c T previously returned by tf::ObjectPool::animate, + or @c nullptr (no-op) + + recycle calls the destructor of @c *obj via @c std::destroy_at, then pushes + the underlying block onto its shard's lock-free free stack without acquiring + any mutex. The block becomes immediately available for the next call to + tf::ObjectPool::animate on any thread. + + The correct shard is identified via the @c pool_id stored in the block + header, so recycle may be called from any thread regardless of which thread + called animate. + + @code{.cpp} + tf::ObjectPool pool; + + MyTask* t = pool.animate(arg1, arg2); + + // ... use t ... + + pool.recycle(t); // destructor called here; memory returned to pool + t = nullptr; // pointer is now dangling; do not dereference + @endcode + + @note + Passing a pointer not obtained from this allocator is undefined behavior. + After recycle returns, @c obj is a dangling pointer and must not be + dereferenced. + */ + void recycle(T* obj) { + if (!obj) return; + auto* block = Block::from_object(obj); + std::destroy_at(block->object()); + _shards[block->pool_id].push_free(block); // hot path: lock-free + } + + /** + @brief returns all recycled blocks and backing memory to the system allocator + + release calls @c std::pmr::synchronized_pool_resource::release on each + shard's backing pool, returning all chunks to the upstream system allocator + in one shot, then atomically resets each shard's free stack to null. This + is an O(1) operation per shard — no per-block work is performed because the + backing pool owns memory at the chunk level and frees entire chunks + regardless of how many individual blocks were returned to it. + + After this call the allocator is in the same state as after construction: + empty free stacks, no memory held from the OS. + + This method is optional and is not required before destruction. It is useful + for reclaiming pool memory between distinct workload phases without + destroying the allocator itself. + + @code{.cpp} + tf::ObjectPool pool; + + // --- phase 1 --- + for (auto& task : phase1_tasks) { + MyTask* t = pool.animate(task); + // ... run t ... + pool.recycle(t); + } + + pool.release(); // return OS memory before phase 2 begins + + // --- phase 2 --- + for (auto& task : phase2_tasks) { + MyTask* t = pool.animate(task); + // ... run t ... + pool.recycle(t); + } + @endcode + + @note + All live objects must be recycled before calling release. Calling release + while objects are still alive is undefined behavior because the backing + memory they reside in is freed. + */ + void release() { + for (auto& shard : _shards) { + // Release all backing chunks to upstream first — this covers both blocks + // on the free stack and any that were never recycled, since the backing + // pool owns memory at the chunk level, not per block. + shard._backing.release(); + // Reset the free stack to null in O(1). Pointers it held are now + // dangling (their backing chunks were just freed), so they must be + // cleared before the allocator is used again. + shard._free_head.store(H{}, std::memory_order_relaxed); + } + } +}; + +} // namespace tf diff --git a/projects/hipfile/third-party/taskflow/utility/os.hpp b/projects/hipfile/third-party/taskflow/utility/os.hpp new file mode 100644 index 000000000000..472c79c347c1 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/utility/os.hpp @@ -0,0 +1,338 @@ +#pragma once + +#include +#include +#include +#include +#include + +#define TF_OS_LINUX 0 +#define TF_OS_DRAGONFLY 0 +#define TF_OS_FREEBSD 0 +#define TF_OS_NETBSD 0 +#define TF_OS_OPENBSD 0 +#define TF_OS_DARWIN 0 +#define TF_OS_WINDOWS 0 +#define TF_OS_CNK 0 +#define TF_OS_HURD 0 +#define TF_OS_SOLARIS 0 +#define TF_OS_UNIX 0 + +#ifdef _WIN32 +#undef TF_OS_WINDOWS +#define TF_OS_WINDOWS 1 +#endif + +#ifdef __CYGWIN__ +#undef TF_OS_WINDOWS +#define TF_OS_WINDOWS 1 +#endif + +#if (defined __APPLE__ && defined __MACH__) +#undef TF_OS_DARWIN +#define TF_OS_DARWIN 1 +#endif + +// in some ppc64 linux installations, only the second condition is met +#if (defined __linux) +#undef TF_OS_LINUX +#define TF_OS_LINUX 1 +#elif (defined __linux__) +#undef TF_OS_LINUX +#define TF_OS_LINUX 1 +#else +#endif + +#if (defined __DragonFly__) +#undef TF_OS_DRAGONFLY +#define TF_OS_DRAGONFLY 1 +#endif + +#if (defined __FreeBSD__) +#undef TF_OS_FREEBSD +#define TF_OS_FREEBSD 1 +#endif + +#if (defined __NetBSD__) +#undef TF_OS_NETBSD +#define TF_OS_NETBSD 1 +#endif + +#if (defined __OpenBSD__) +#undef TF_OS_OPENBSD +#define TF_OS_OPENBSD 1 +#endif + +#if (defined __bgq__) +#undef TF_OS_CNK +#define TF_OS_CNK 1 +#endif + +#if (defined __GNU__) +#undef TF_OS_HURD +#define TF_OS_HURD 1 +#endif + +#if (defined __sun) +#undef TF_OS_SOLARIS +#define TF_OS_SOLARIS 1 +#endif + +#if (1 != \ + TF_OS_LINUX + TF_OS_DRAGONFLY + TF_OS_FREEBSD + TF_OS_NETBSD + \ + TF_OS_OPENBSD + TF_OS_DARWIN + TF_OS_WINDOWS + TF_OS_HURD + \ + TF_OS_SOLARIS) +#define TF_OS_UNKNOWN 1 +#endif + +#if TF_OS_LINUX || TF_OS_DRAGONFLY || TF_OS_FREEBSD || TF_OS_NETBSD || \ + TF_OS_OPENBSD || TF_OS_DARWIN || TF_OS_HURD || TF_OS_SOLARIS +#undef TF_OS_UNIX +#define TF_OS_UNIX 1 +#endif + +// ------------------------------------------------------------------------------------------------ +// Number of bits used by the OS for user-space virtual addresses +// Used as the default PtrBits for TaggedHead64 to leave the remaining +// high bits free for the ABA version counter. +// ------------------------------------------------------------------------------------------------ +// +// No standard C++ mechanism exposes the VA width; we derive it from +// architecture-specific predefined macros. Override by defining +// TF_POINTER_BITS before including this header if your environment +// differs (e.g. x86-64 with LA57 5-level paging uses 57 bits). + +#if defined(TF_POINTER_BITS) + // user-defined override — accepted as-is + +#elif defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) + // 4-level paging: 48 bits. If your kernel enables LA57 (5-level paging, + // 57-bit VA), compile with -DTF_POINTER_BITS=57. Note that only 7 bits + // remain for the ABA tag in that case; TaggedHead128 is a better choice. + #define TF_POINTER_BITS 48 + +#elif defined(__aarch64__) || defined(_M_ARM64) + #define TF_POINTER_BITS 48 // ARMv8 48-bit VA (TTBR0 range) + +#elif defined(__riscv) && __riscv_xlen == 64 + #define TF_POINTER_BITS 48 // SV48 worst case; SV39 gives 25 free bits + +#else + #define TF_POINTER_BITS (sizeof(void*) * CHAR_BIT) // 32-bit or unknown +#endif + + +// ------------------------------------------------------------------------------------------------ +// Cache line size detection. +// +// Underestimating causes false sharing (hurts performance). +// Overestimating wastes memory. +// 64B is correct for the vast majority of modern server/desktop CPUs. +// ------------------------------------------------------------------------------------------------ + +#if defined(__i386__) || defined(__x86_64__) || \ + defined(_M_IX86) || defined(_M_AMD64) + // All modern x86/x86_64 CPUs (Intel, AMD) since Pentium 4. + #define TF_CACHELINE_SIZE 64 + +#elif defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) + // 64-bit ARM: Apple Silicon (M1/M2/M3), AWS Graviton, Qualcomm Oryon, etc. + #define TF_CACHELINE_SIZE 64 + +#elif defined(__arm__) || defined(_M_ARM) + // 32-bit ARM — cache line size depends on the ARM architecture revision. + #if defined(__ARM_ARCH_5T__) || \ + defined(__ARM_ARCH_5TE__) || \ + defined(__ARM_ARCH_6__) + #define TF_CACHELINE_SIZE 32 + #else + // ARMv7-A (Cortex-A5/A7/A8/A9/A15) and later 32-bit ARM. + #define TF_CACHELINE_SIZE 64 + #endif + +#elif defined(__powerpc64__) || defined(__ppc64__) + // IBM POWER7/8/9/10 and PowerPC64 (e.g., original Xbox 360-era G5). + #define TF_CACHELINE_SIZE 128 + +#elif defined(__powerpc__) || defined(__ppc__) + // Older 32-bit PowerPC (G3/G4 era embedded systems). + #define TF_CACHELINE_SIZE 32 + +#elif defined(__s390x__) || defined(__zarch__) + // IBM Z (z13 and later). Unusually large at 256B. + #define TF_CACHELINE_SIZE 256 + +#elif defined(__riscv) + // RISC-V: SiFive U74, Alibaba T-Head, etc. + #define TF_CACHELINE_SIZE 64 + +#elif defined(__mips__) || defined(__mips64) + // MIPS32/MIPS64 (embedded and networking SoCs). + #define TF_CACHELINE_SIZE 64 + +#elif defined(__sparc__) || defined(__sparc64__) + // Oracle/Fujitsu SPARC. + #define TF_CACHELINE_SIZE 64 + +#elif defined(__loongarch64) + // LoongArch (Loongson 3A5000 and later). + #define TF_CACHELINE_SIZE 64 + +#elif defined(__alpha__) + // DEC/Compaq Alpha. + #define TF_CACHELINE_SIZE 64 + +#endif + +#ifndef TF_CACHELINE_SIZE + // Conservative fallback. If we land here, the architecture is unknown. + // 64B is correct for virtually all modern CPUs. Overestimating wastes + // a small amount of space; underestimating causes false sharing. + #define TF_CACHELINE_SIZE 64 +#endif + +namespace tf { + +/** +@class CachelineAligned + +@brief class to ensure cacheline-aligned storage for an object. + +@tparam T The type of the stored object. + +This utility class aligns the stored object `data` to twice the size of a cacheline. +The alignment improves performance by optimizing data access in cache-sensitive scenarios. + +@code{.cpp} +// create two integers on two separate cachelines to avoid false sharing +tf::CachelineAligned counter1; +tf::CachelineAligned counter2; + +// two threads access the two counters without false sharing +std::thread t1([&]{ counter1.get() = 1; }); +std::thread t2([&]{ counter2.get() = 2; }); +t1.join(); +t2.join(); +@endcode +*/ +template +class CachelineAligned { + public: + /** + @brief The stored object, aligned to twice the cacheline size. + */ + alignas (TF_CACHELINE_SIZE) T data; + + /** + @brief accesses the underlying object + + @return a reference to the underlying object. + + Returns a mutable reference to the stored object so it can be read + or written directly without copying. + + @code{.cpp} + tf::CachelineAligned counter; + counter.get() = 1; + @endcode + */ + T& get() { return data; } + + /** + @brief accesses the underlying object as a constant reference + + @return a constant reference to the underlying object. + + Returns a read-only reference to the stored object, for use on + const-qualified instances of `CachelineAligned`. + + @code{.cpp} + const tf::CachelineAligned counter; + int v = counter.get(); + @endcode + */ + const T& get() const { return data; } +}; + +/** +@brief retrieves the value of an environment variable + +@param str The name of the environment variable to retrieve. +@return The value of the environment variable as a string, or an empty string if not found. + +This function fetches the value of an environment variable by name. +If the variable is not found, it returns an empty string. + +@code{.cpp} +std::string path = tf::get_env("PATH"); +@endcode + +@note The implementation differs between Windows and POSIX platforms: + - On Windows, it uses `_dupenv_s` to fetch the value. + - On POSIX, it uses `std::getenv`. +*/ +inline std::string get_env(const std::string& str) { +#ifdef _MSC_VER + char *ptr = nullptr; + size_t len = 0; + + if(_dupenv_s(&ptr, &len, str.c_str()) == 0 && ptr != nullptr) { + std::string res(ptr, len); + std::free(ptr); + return res; + } + return ""; + +#else + auto ptr = std::getenv(str.c_str()); + return ptr ? ptr : ""; +#endif +} + +/** +@brief checks whether an environment variable is defined + +@param str The name of the environment variable to check. +@return `true` if the environment variable exists, `false` otherwise. + +This function determines if a specific environment variable exists in the current environment. + +@code{.cpp} +if(tf::has_env("TF_NUM_THREADS")) { + // ... +} +@endcode + +@note The implementation differs between Windows and POSIX platforms: + - On Windows, it uses `_dupenv_s` to check for the variable's presence. + - On POSIX, it uses `std::getenv` to check for the variable's presence. +*/ +inline bool has_env(const std::string& str) { +#ifdef _MSC_VER + char *ptr = nullptr; + size_t len = 0; + + if(_dupenv_s(&ptr, &len, str.c_str()) == 0 && ptr != nullptr) { + std::string res(ptr, len); + std::free(ptr); + return true; + } + return false; + +#else + auto ptr = std::getenv(str.c_str()); + return ptr ? true : false; +#endif +} + + +} // end of namespace tf ----------------------------------------------------- + + + + + + + + + diff --git a/projects/hipfile/third-party/taskflow/utility/serializer.hpp b/projects/hipfile/third-party/taskflow/utility/serializer.hpp new file mode 100644 index 000000000000..2659905e93d7 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/utility/serializer.hpp @@ -0,0 +1,771 @@ +// This file is part of Taskflow project +// Copyright (c) 2018-2023 Dr. Tsung-Wei Huang +// MIT License - see LICENSE file for details + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include // C++20 + +namespace tf { + +// ---------------------------------------------------------------------------- +// Type detection traits (partial specializations — unchanged as per spec) +// ---------------------------------------------------------------------------- + +template +struct is_std_basic_string : std::false_type {}; +template +struct is_std_basic_string> : std::true_type {}; +template +inline constexpr bool is_std_basic_string_v = is_std_basic_string::value; + +template +struct is_std_vector : std::false_type {}; +template +struct is_std_vector> : std::true_type {}; +template +inline constexpr bool is_std_vector_v = is_std_vector::value; + +template +struct is_std_deque : std::false_type {}; +template +struct is_std_deque> : std::true_type {}; +template +inline constexpr bool is_std_deque_v = is_std_deque::value; + +template +struct is_std_map : std::false_type {}; +template +struct is_std_map> : std::true_type {}; +template +inline constexpr bool is_std_map_v = is_std_map::value; + +template +struct is_std_unordered_map : std::false_type {}; +template +struct is_std_unordered_map> : std::true_type {}; +template +inline constexpr bool is_std_unordered_map_v = is_std_unordered_map::value; + +template +struct is_std_set : std::false_type {}; +template +struct is_std_set> : std::true_type {}; +template +inline constexpr bool is_std_set_v = is_std_set::value; + +template +struct is_std_unordered_set : std::false_type {}; +template +struct is_std_unordered_set> : std::true_type {}; +template +inline constexpr bool is_std_unordered_set_v = is_std_unordered_set::value; + +template +struct is_std_forward_list : std::false_type {}; +template +struct is_std_forward_list> : std::true_type {}; +template +inline constexpr bool is_std_forward_list_v = is_std_forward_list::value; + +template +struct is_std_variant : std::false_type {}; +template +struct is_std_variant> : std::true_type {}; +template +inline constexpr bool is_std_variant_v = is_std_variant::value; + +template +struct is_std_optional : std::false_type {}; +template +struct is_std_optional> : std::true_type {}; +template +inline constexpr bool is_std_optional_v = is_std_optional::value; + +template +struct is_std_tuple : std::false_type {}; +template +struct is_std_tuple> : std::true_type {}; +template +inline constexpr bool is_std_tuple_v = is_std_tuple::value; + +template +struct is_std_array : std::false_type {}; +template +struct is_std_array> : std::true_type {}; +template +inline constexpr bool is_std_array_v = is_std_array::value; + +template +struct is_std_duration : std::false_type {}; +template +struct is_std_duration> : std::true_type {}; +template +inline constexpr bool is_std_duration_v = is_std_duration::value; + +template +struct is_std_time_point : std::false_type {}; +template +struct is_std_time_point> : std::true_type {}; +template +inline constexpr bool is_std_time_point_v = is_std_time_point::value; + +template +inline constexpr bool is_std_enum_v = std::is_enum_v; + +// is_default_serializable_v: compound trait — unchanged as per spec +template +inline constexpr bool is_default_serializable_v = + std::is_arithmetic_v || + is_std_basic_string_v || + is_std_vector_v || + is_std_deque_v || + is_std_map_v || + is_std_unordered_map_v || + is_std_set_v || + is_std_unordered_set_v || + is_std_forward_list_v || + is_std_variant_v || + is_std_optional_v || + is_std_tuple_v || + is_std_array_v || + is_std_duration_v || + is_std_time_point_v || + is_std_enum_v; + +// ---------------------------------------------------------------------------- +// C++20 concepts derived from the _v traits above +// ---------------------------------------------------------------------------- + +/** + * @brief concept for std::basic_string specializations + */ +template +concept StdBasicString = is_std_basic_string_v>; + +/** + * @brief concept for std::vector specializations + */ +template +concept StdVector = is_std_vector_v>; + +/** + * @brief concept for std::deque specializations + */ +template +concept StdDeque = is_std_deque_v>; + +/** + * @brief concept for std::map specializations + */ +template +concept StdMap = is_std_map_v>; + +/** + * @brief concept for std::unordered_map specializations + */ +template +concept StdUnorderedMap = is_std_unordered_map_v>; + +/** + * @brief concept for std::set specializations + */ +template +concept StdSet = is_std_set_v>; + +/** + * @brief concept for std::unordered_set specializations + */ +template +concept StdUnorderedSet = is_std_unordered_set_v>; + +/** + * @brief concept for std::forward_list specializations + */ +template +concept StdForwardList = is_std_forward_list_v>; + +/** + * @brief concept for std::variant specializations + */ +template +concept StdVariant = is_std_variant_v>; + +/** + * @brief concept for std::optional specializations + */ +template +concept StdOptional = is_std_optional_v>; + +/** + * @brief concept for std::tuple specializations + */ +template +concept StdTuple = is_std_tuple_v>; + +/** + * @brief concept for std::array specializations + */ +template +concept StdArray = is_std_array_v>; + +/** + * @brief concept for std::chrono::duration specializations + */ +template +concept StdDuration = is_std_duration_v>; + +/** + * @brief concept for std::chrono::time_point specializations + */ +template +concept StdTimePoint = is_std_time_point_v>; + +/** + * @brief concept for enum types + */ +template +concept StdEnum = is_std_enum_v>; + +// ---------------------------------------------------------------------------- +// Helper: index sequence iteration over a tuple +// ---------------------------------------------------------------------------- + +namespace detail { + +template +void for_each_in_tuple(F&& f, Tuple&& t, std::index_sequence) { + (f(std::get(std::forward(t))), ...); +} + +template +void for_each_in_tuple(F&& f, Tuple&& t) { + for_each_in_tuple( + std::forward(f), + std::forward(t), + std::make_index_sequence>>{} + ); +} + +} // namespace detail + +// ---------------------------------------------------------------------------- +// Serializer +// ---------------------------------------------------------------------------- + +/** + * @brief serializer class + * + * @tparam Device output device type (must support write()) + * @tparam SizeType integer type used to encode sizes + */ +template +class Serializer { + +public: + + /** + * @brief constructs the serializer with a reference to the output device + */ + explicit Serializer(Device& device) : _device(device) {} + + /** + * @brief serializes one or more values to the device + * + * @tparam Ts value types to serialize + * @param args values to serialize + * @return total number of bytes written + */ + template + SizeType operator()(Ts&&... args) { + return (_save(std::forward(args)) + ... + SizeType{0}); + } + +private: + + Device& _device; + + // ------------------------------------------------------------------ + // Dispatch overloads — ordered from most- to least-specific so that + // the requires clauses form a clean, non-overlapping partition. + // The negated !is_default_serializable_v guard stays as a + // requires clause per project conventions. + // ------------------------------------------------------------------ + + // (1) Custom / user-defined serializable: T has a .save(ar) method + // and is NOT one of the standard types we handle ourselves. + template + requires (!is_default_serializable_v>) + SizeType _save(T&& t) { + SizeType sz = 0; + auto ar = [&](auto&&... args) { + sz += (*this)(std::forward(args)...); + }; + t.save(ar); + return sz; + } + + // (2) Arithmetic (int, float, double, bool, …) + template + requires std::is_arithmetic_v> + SizeType _save(T&& t) { + _device.write(reinterpret_cast(&t), sizeof(std::decay_t)); + return sizeof(std::decay_t); + } + + // (3) std::basic_string + template + SizeType _save(T&& t) { + SizeType sz = _save(static_cast(t.size())); + _device.write(t.data(), t.size() * sizeof(typename std::decay_t::value_type)); + sz += static_cast(t.size() * sizeof(typename std::decay_t::value_type)); + return sz; + } + + // (4) std::vector — bulk write for arithmetic value_type, element-wise otherwise + template + SizeType _save(T&& t) { + using U = std::decay_t; + SizeType sz = _save(static_cast(t.size())); + if constexpr (std::is_arithmetic_v) { + _device.write( + reinterpret_cast(t.data()), + t.size() * sizeof(typename U::value_type) + ); + sz += static_cast(t.size() * sizeof(typename U::value_type)); + } else { + for (auto&& item : t) { + sz += _save(item); + } + } + return sz; + } + + // (5) std::deque + template + SizeType _save(T&& t) { + SizeType sz = _save(static_cast(t.size())); + for (auto&& item : t) { + sz += _save(item); + } + return sz; + } + + // (6) std::map + template + SizeType _save(T&& t) { + SizeType sz = _save(static_cast(t.size())); + for (auto&& [k, v] : t) { + sz += _save(k); + sz += _save(v); + } + return sz; + } + + // (7) std::unordered_map + template + SizeType _save(T&& t) { + SizeType sz = _save(static_cast(t.size())); + for (auto&& [k, v] : t) { + sz += _save(k); + sz += _save(v); + } + return sz; + } + + // (8) std::set + template + SizeType _save(T&& t) { + SizeType sz = _save(static_cast(t.size())); + for (auto&& item : t) { + sz += _save(item); + } + return sz; + } + + // (9) std::unordered_set + template + SizeType _save(T&& t) { + SizeType sz = _save(static_cast(t.size())); + for (auto&& item : t) { + sz += _save(item); + } + return sz; + } + + // (10) std::forward_list — must count elements first (no size()) + template + SizeType _save(T&& t) { + SizeType count = static_cast(std::distance(t.begin(), t.end())); + SizeType sz = _save(count); + for (auto&& item : t) { + sz += _save(item); + } + return sz; + } + + // (11) std::variant — save index, then the active alternative + template + SizeType _save(T&& t) { + SizeType sz = _save(static_cast(t.index())); + sz += std::visit([&](auto&& v) { return _save(v); }, std::forward(t)); + return sz; + } + + // (12) std::optional — save has_value flag, then the value if present + template + SizeType _save(T&& t) { + SizeType sz = _save(static_cast(t.has_value())); + if (t.has_value()) { + sz += _save(*t); + } + return sz; + } + + // (13) std::tuple — fold over elements + template + SizeType _save(T&& t) { + SizeType sz{0}; + detail::for_each_in_tuple([&](auto&& v) { sz += _save(v); }, std::forward(t)); + return sz; + } + + // (14) std::array — bulk write for arithmetic, element-wise otherwise + template + SizeType _save(T&& t) { + using U = std::decay_t; + SizeType sz{0}; + if constexpr (std::is_arithmetic_v) { + _device.write( + reinterpret_cast(t.data()), + t.size() * sizeof(typename U::value_type) + ); + sz = static_cast(t.size() * sizeof(typename U::value_type)); + } else { + for (auto&& item : t) { + sz += _save(item); + } + } + return sz; + } + + // (15) std::chrono::duration — serialize the underlying rep + template + SizeType _save(T&& t) { + return _save(t.count()); + } + + // (16) std::chrono::time_point — serialize as duration since epoch + template + SizeType _save(T&& t) { + return _save(t.time_since_epoch()); + } + + // (17) enum — serialize as underlying integer type + template + SizeType _save(T&& t) { + return _save(static_cast>>(t)); + } +}; + +// ---------------------------------------------------------------------------- +// Deserializer +// ---------------------------------------------------------------------------- + +/** + * @brief deserializer class + * + * @tparam Device input device type (must support read()) + * @tparam SizeType integer type used to decode sizes (must match Serializer) + */ +template +class Deserializer { + +public: + + /** + * @brief constructs the deserializer with a reference to the input device + */ + explicit Deserializer(Device& device) : _device(device) {} + + /** + * @brief deserializes one or more values from the device + * + * @tparam Ts value types to deserialize + * @param args values to populate + * @return total number of bytes read + */ + template + SizeType operator()(Ts&&... args) { + return (_load(std::forward(args)) + ... + SizeType{0}); + } + +private: + + Device& _device; + + // ------------------------------------------------------------------ + // Dispatch overloads — mirror of Serializer::_save + // ------------------------------------------------------------------ + + // (1) Custom / user-defined deserializable + template + requires (!is_default_serializable_v>) + SizeType _load(T&& t) { + SizeType sz = 0; + auto ar = [&](auto&&... args) { + sz += (*this)(std::forward(args)...); + }; + t.load(ar); + return sz; + } + + // (2) Arithmetic + template + requires std::is_arithmetic_v> + SizeType _load(T&& t) { + _device.read(reinterpret_cast(&t), sizeof(std::decay_t)); + return sizeof(std::decay_t); + } + + // (3) std::basic_string + template + SizeType _load(T&& t) { + SizeType n{0}; + SizeType sz = _load(n); + t.resize(static_cast(n)); + _device.read( + reinterpret_cast(t.data()), + static_cast(n) * sizeof(typename std::decay_t::value_type) + ); + sz += static_cast(n * sizeof(typename std::decay_t::value_type)); + return sz; + } + + // (4) std::vector + template + SizeType _load(T&& t) { + using U = std::decay_t; + SizeType n{0}; + SizeType sz = _load(n); + t.resize(static_cast(n)); + if constexpr (std::is_arithmetic_v) { + _device.read( + reinterpret_cast(t.data()), + static_cast(n) * sizeof(typename U::value_type) + ); + sz += static_cast(n * sizeof(typename U::value_type)); + } else { + for (auto& item : t) { + sz += _load(item); + } + } + return sz; + } + + // (5) std::deque + template + SizeType _load(T&& t) { + SizeType n{0}; + SizeType sz = _load(n); + t.resize(static_cast(n)); + for (auto& item : t) { + sz += _load(item); + } + return sz; + } + + // (6) std::map + template + SizeType _load(T&& t) { + using U = std::decay_t; + SizeType n{0}; + SizeType sz = _load(n); + t.clear(); + for (SizeType i = 0; i < n; ++i) { + typename U::key_type k{}; + typename U::mapped_type v{}; + sz += _load(k); + sz += _load(v); + t.emplace(std::move(k), std::move(v)); + } + return sz; + } + + // (7) std::unordered_map + template + SizeType _load(T&& t) { + using U = std::decay_t; + SizeType n{0}; + SizeType sz = _load(n); + t.clear(); + t.reserve(static_cast(n)); + for (SizeType i = 0; i < n; ++i) { + typename U::key_type k{}; + typename U::mapped_type v{}; + sz += _load(k); + sz += _load(v); + t.emplace(std::move(k), std::move(v)); + } + return sz; + } + + // (8) std::set + template + SizeType _load(T&& t) { + using U = std::decay_t; + SizeType n{0}; + SizeType sz = _load(n); + t.clear(); + for (SizeType i = 0; i < n; ++i) { + typename U::value_type v{}; + sz += _load(v); + t.emplace(std::move(v)); + } + return sz; + } + + // (9) std::unordered_set + template + SizeType _load(T&& t) { + using U = std::decay_t; + SizeType n{0}; + SizeType sz = _load(n); + t.clear(); + t.reserve(static_cast(n)); + for (SizeType i = 0; i < n; ++i) { + typename U::value_type v{}; + sz += _load(v); + t.emplace(std::move(v)); + } + return sz; + } + + // (10) std::forward_list + template + SizeType _load(T&& t) { + using U = std::decay_t; + SizeType n{0}; + SizeType sz = _load(n); + t.clear(); + // Build in reverse then reverse — O(n) without needing push_back + for (SizeType i = 0; i < n; ++i) { + typename U::value_type v{}; + sz += _load(v); + t.push_front(std::move(v)); + } + t.reverse(); + return sz; + } + + // (11) std::variant — load the index, then construct the right alternative + template + SizeType _load(T&& t) { + SizeType idx{0}; + SizeType sz = _load(idx); + sz += _load_variant_at(t, static_cast(idx), + std::make_index_sequence< + std::variant_size_v>>{}); + return sz; + } + + template + SizeType _load_variant_at(V& v, std::size_t idx, std::index_sequence) { + SizeType sz{0}; + bool matched = false; + ([&]() { + if (!matched && idx == Is) { + matched = true; + std::variant_alternative_t> alt{}; + sz = _load(alt); + v = std::move(alt); + } + }(), ...); + return sz; + } + + // (12) std::optional + template + SizeType _load(T&& t) { + bool has{false}; + SizeType sz = _load(has); + if (has) { + typename std::decay_t::value_type v{}; + sz += _load(v); + t = std::move(v); + } else { + t = std::nullopt; + } + return sz; + } + + // (13) std::tuple + template + SizeType _load(T&& t) { + SizeType sz{0}; + detail::for_each_in_tuple([&](auto& v) { sz += _load(v); }, std::forward(t)); + return sz; + } + + // (14) std::array + template + SizeType _load(T&& t) { + using U = std::decay_t; + SizeType sz{0}; + if constexpr (std::is_arithmetic_v) { + _device.read( + reinterpret_cast(t.data()), + t.size() * sizeof(typename U::value_type) + ); + sz = static_cast(t.size() * sizeof(typename U::value_type)); + } else { + for (auto& item : t) { + sz += _load(item); + } + } + return sz; + } + + // (15) std::chrono::duration + template + SizeType _load(T&& t) { + typename std::decay_t::rep rep{}; + SizeType sz = _load(rep); + t = std::decay_t{rep}; + return sz; + } + + // (16) std::chrono::time_point + template + SizeType _load(T&& t) { + typename std::decay_t::duration dur{}; + SizeType sz = _load(dur); + t = std::decay_t{dur}; + return sz; + } + + // (17) enum + template + SizeType _load(T&& t) { + std::underlying_type_t> val{}; + SizeType sz = _load(val); + t = static_cast>(val); + return sz; + } +}; + +} // namespace tf diff --git a/projects/hipfile/third-party/taskflow/utility/small_vector.hpp b/projects/hipfile/third-party/taskflow/utility/small_vector.hpp new file mode 100644 index 000000000000..3a001b1cf95e --- /dev/null +++ b/projects/hipfile/third-party/taskflow/utility/small_vector.hpp @@ -0,0 +1,1056 @@ +// small vector modified from llvm + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + + +/** +@file small_vector.hpp +@brief small vector include file +*/ + +namespace tf { namespace detail { + +/** +@private +@brief NextCapacity - Returns the next power of two (in 64-bits) + that is strictly greater than A. Returns zero on overflow. + this function assumes A to be positive +*/ +inline uint64_t NextCapacity(uint64_t A) { + A |= (A >> 1); + A |= (A >> 2); + A |= (A >> 4); + A |= (A >> 8); + A |= (A >> 16); + A |= (A >> 32); + return A + 1; +} + +}} // end of namespace tf::detail -------------------------------------------- + + +namespace tf { + +/** +@private +*/ +template +struct IsPod : std::integral_constant::value && + std::is_trivial::value> {}; + +/** +@private +*/ +class SmallVectorBase { +protected: + void *BeginX, *EndX, *CapacityX; + +protected: + SmallVectorBase(void *FirstEl, size_t Size) + : BeginX(FirstEl), EndX(FirstEl), CapacityX((char*)FirstEl+Size) {} + + /// This is an implementation of the grow() method which only works + /// on POD-like data types and is out of line to reduce code duplication. + void grow_pod(void *FirstEl, size_t MinSizeInBytes, size_t TSize){ + size_t CurSizeBytes = size_in_bytes(); + size_t NewCapacityInBytes = 2 * capacity_in_bytes() + TSize; // Always grow. + if (NewCapacityInBytes < MinSizeInBytes) { + NewCapacityInBytes = MinSizeInBytes; + } + + void *NewElts; + if (BeginX == FirstEl) { + NewElts = std::malloc(NewCapacityInBytes); + + // Copy the elements over. No need to run dtors on PODs. + memcpy(NewElts, this->BeginX, CurSizeBytes); + } else { + // If this wasn't grown from the inline copy, grow the allocated space. + NewElts = realloc(this->BeginX, NewCapacityInBytes); + } + //assert(NewElts && "Out of memory"); + + this->EndX = (char*)NewElts+CurSizeBytes; + this->BeginX = NewElts; + this->CapacityX = (char*)this->BeginX + NewCapacityInBytes; + } + +public: + /// This returns size()*sizeof(T). + /** + @private + */ + size_t size_in_bytes() const { + return size_t((char*)EndX - (char*)BeginX); + } + + /// capacity_in_bytes - This returns capacity()*sizeof(T). + /** + @private + */ + size_t capacity_in_bytes() const { + return size_t((char*)CapacityX - (char*)BeginX); + } + + bool empty() const { return BeginX == EndX; } +}; + +/** +@private +*/ +template struct SmallVectorStorage; + +/** +@private +*/ +template +class SmallVectorTemplateCommon : public SmallVectorBase { + + private: + template friend struct SmallVectorStorage; + + //template + //struct AlignedUnionType { + // alignas(X) std::byte buff[std::max(sizeof(std::byte), sizeof(X))]; + //}; + + template + struct AlignedUnionType { + static constexpr std::size_t max_size = (sizeof(std::byte) > sizeof(X)) ? sizeof(std::byte) : sizeof(X); + alignas(X) std::byte buff[max_size]; + }; + + // Allocate raw space for N elements of type T. If T has a ctor or dtor, we + // don't want it to be automatically run, so we need to represent the space as + // something else. Use an array of char of sufficient alignment. + + // deprecated in c++23 + //typedef typename std::aligned_union<1, T>::type U; + typedef AlignedUnionType U; + + U FirstEl; + // Space after 'FirstEl' is clobbered, do not add any instance vars after it. + + protected: + SmallVectorTemplateCommon(size_t Size) : SmallVectorBase(&FirstEl, Size) {} + + void grow_pod(size_t MinSizeInBytes, size_t TSize) { + SmallVectorBase::grow_pod(&FirstEl, MinSizeInBytes, TSize); + } + + /// Return true if this is a smallvector which has not had dynamic + /// memory allocated for it. + bool isSmall() const { + return BeginX == static_cast(&FirstEl); + } + + /// Put this vector in a state of being small. + void resetToSmall() { + BeginX = EndX = CapacityX = &FirstEl; + } + + void setEnd(T *P) { this->EndX = P; } + + public: + typedef size_t size_type; + typedef ptrdiff_t difference_type; + typedef T value_type; + typedef T *iterator; + typedef const T *const_iterator; + + typedef std::reverse_iterator const_reverse_iterator; + typedef std::reverse_iterator reverse_iterator; + + typedef T &reference; + typedef const T &const_reference; + typedef T *pointer; + typedef const T *const_pointer; + + // forward iterator creation methods. + inline iterator begin() { return (iterator)this->BeginX; } + inline const_iterator begin() const { return (const_iterator)this->BeginX; } + inline iterator end() { return (iterator)this->EndX; } + inline const_iterator end() const { return (const_iterator)this->EndX; } + + protected: + + iterator capacity_ptr() { return (iterator)this->CapacityX; } + const_iterator capacity_ptr() const { return (const_iterator)this->CapacityX;} + + public: + + // reverse iterator creation methods. + reverse_iterator rbegin() { return reverse_iterator(end()); } + const_reverse_iterator rbegin() const{ return const_reverse_iterator(end()); } + reverse_iterator rend() { return reverse_iterator(begin()); } + const_reverse_iterator rend() const { return const_reverse_iterator(begin());} + + inline size_type size() const { return end()-begin(); } + inline size_type max_size() const { return size_type(-1) / sizeof(T); } + + /// Return the total number of elements in the currently allocated buffer. + size_t capacity() const { return capacity_ptr() - begin(); } + + /// Return a pointer to the vector's buffer, even if empty(). + pointer data() { return pointer(begin()); } + /// Return a pointer to the vector's buffer, even if empty(). + const_pointer data() const { return const_pointer(begin()); } + + inline reference operator[](size_type idx) { + //assert(idx < size()); + return begin()[idx]; + } + + inline const_reference operator[](size_type idx) const { + //assert(idx < size()); + return begin()[idx]; + } + + reference front() { + //assert(!empty()); + return begin()[0]; + } + + const_reference front() const { + //assert(!empty()); + return begin()[0]; + } + + reference back() { + //assert(!empty()); + return end()[-1]; + } + + const_reference back() const { + //assert(!empty()); + return end()[-1]; + } +}; + +/** +@private +*/ +template +class SmallVectorTemplateBase : public SmallVectorTemplateCommon { + +protected: + SmallVectorTemplateBase(size_t Size) : SmallVectorTemplateCommon(Size) {} + + static void destroy_range(T *S, T *E) { + while (S != E) { + --E; + E->~T(); + } + } + + /// Move the range [I, E) into the uninitialized memory starting with "Dest", + /// constructing elements as needed. + template + static void uninitialized_move(It1 I, It1 E, It2 Dest) { + std::uninitialized_copy(std::make_move_iterator(I), + std::make_move_iterator(E), Dest); + } + + /// Copy the range [I, E) onto the uninitialized memory starting with "Dest", + /// constructing elements as needed. + template + static void uninitialized_copy(It1 I, It1 E, It2 Dest) { + std::uninitialized_copy(I, E, Dest); + } + + /// Grow the allocated memory (without initializing new elements), doubling + /// the size of the allocated memory. Guarantees space for at least one more + /// element, or MinSize more elements if specified. + void grow(size_t MinSize = 0); + +public: + void push_back(const T &Elt) { + if ((this->EndX >= this->CapacityX)) [[unlikely]] + this->grow(); + ::new ((void*) this->end()) T(Elt); + this->setEnd(this->end()+1); + } + + void push_back(T &&Elt) { + if ((this->EndX >= this->CapacityX)) [[unlikely]] + this->grow(); + ::new ((void*) this->end()) T(::std::move(Elt)); + this->setEnd(this->end()+1); + } + + void pop_back() { + this->setEnd(this->end()-1); + this->end()->~T(); + } +}; + +/** +@private +*/ +template +void SmallVectorTemplateBase::grow(size_t MinSize) { + size_t CurCapacity = this->capacity(); + size_t CurSize = this->size(); + // Always grow, even from zero. + size_t NewCapacity = size_t(tf::detail::NextCapacity(CurCapacity+2)); + if (NewCapacity < MinSize) + NewCapacity = MinSize; + T *NewElts = static_cast(std::malloc(NewCapacity*sizeof(T))); + + // Move the elements over. + this->uninitialized_move(this->begin(), this->end(), NewElts); + + // Destroy the original elements. + destroy_range(this->begin(), this->end()); + + // If this wasn't grown from the inline copy, deallocate the old space. + if (!this->isSmall()) + std::free(this->begin()); + + this->setEnd(NewElts+CurSize); + this->BeginX = NewElts; + this->CapacityX = this->begin()+NewCapacity; +} + +/** +@private +*/ +template +class SmallVectorTemplateBase : public SmallVectorTemplateCommon { +protected: + SmallVectorTemplateBase(size_t Size) : SmallVectorTemplateCommon(Size) {} + + // No need to do a destroy loop for POD's. + static void destroy_range(T *, T *) {} + + /// Move the range [I, E) onto the uninitialized memory + /// starting with "Dest", constructing elements into it as needed. + template + static void uninitialized_move(It1 I, It1 E, It2 Dest) { + // Just do a copy. + uninitialized_copy(I, E, Dest); + } + + /// Copy the range [I, E) onto the uninitialized memory + /// starting with "Dest", constructing elements into it as needed. + template + static void uninitialized_copy(It1 I, It1 E, It2 Dest) { + // Arbitrary iterator types; just use the basic implementation. + std::uninitialized_copy(I, E, Dest); + } + + /// Copy the range [I, E) onto the uninitialized memory + /// starting with "Dest", constructing elements into it as needed. + template + static void uninitialized_copy( + T1 *I, T1 *E, T2 *Dest, + typename std::enable_if::type, + T2>::value>::type * = nullptr) { + // Use memcpy for PODs iterated by pointers (which includes SmallVector + // iterators): std::uninitialized_copy optimizes to memmove, but we can + // use memcpy here. Note that I and E are iterators and thus might be + // invalid for memcpy if they are equal. + if (I != E) + memcpy(Dest, I, (E - I) * sizeof(T)); + } + + /// Double the size of the allocated memory, guaranteeing space for at + /// least one more element or MinSize if specified. + void grow(size_t MinSize = 0) { + this->grow_pod(MinSize*sizeof(T), sizeof(T)); + } +public: + void push_back(const T &Elt) { + if ((this->EndX >= this->CapacityX)) [[unlikely]] + this->grow(); + memcpy(this->end(), &Elt, sizeof(T)); + this->setEnd(this->end()+1); + } + + void pop_back() { + this->setEnd(this->end()-1); + } +}; + +/** +@private +*/ +template +class SmallVectorImpl : public SmallVectorTemplateBase::value> { + typedef SmallVectorTemplateBase::value> SuperClass; + + SmallVectorImpl(const SmallVectorImpl&) = delete; + +public: + typedef typename SuperClass::iterator iterator; + typedef typename SuperClass::const_iterator const_iterator; + typedef typename SuperClass::size_type size_type; + +protected: + // Default ctor - Initialize to empty. + explicit SmallVectorImpl(unsigned N) + : SmallVectorTemplateBase::value>(N*sizeof(T)) { + } + +public: + ~SmallVectorImpl() { + // Destroy the constructed elements in the vector. + this->destroy_range(this->begin(), this->end()); + + // If this wasn't grown from the inline copy, deallocate the old space. + if (!this->isSmall()) + std::free(this->begin()); + } + + + void clear() { + this->destroy_range(this->begin(), this->end()); + this->EndX = this->BeginX; + } + + void resize(size_type N) { + if (N < this->size()) { + this->destroy_range(this->begin()+N, this->end()); + this->setEnd(this->begin()+N); + } else if (N > this->size()) { + if (this->capacity() < N) + this->grow(N); + for (auto I = this->end(), E = this->begin() + N; I != E; ++I) + new (&*I) T(); + this->setEnd(this->begin()+N); + } + } + + void resize(size_type N, const T &NV) { + if (N < this->size()) { + this->destroy_range(this->begin()+N, this->end()); + this->setEnd(this->begin()+N); + } else if (N > this->size()) { + if (this->capacity() < N) + this->grow(N); + std::uninitialized_fill(this->end(), this->begin()+N, NV); + this->setEnd(this->begin()+N); + } + } + + void reserve(size_type N) { + if (this->capacity() < N) + this->grow(N); + } + + T pop_back_val() { + T Result = ::std::move(this->back()); + this->pop_back(); + return Result; + } + + void swap(SmallVectorImpl &RHS); + + /// Add the specified range to the end of the SmallVector. + template + void append(in_iter in_start, in_iter in_end) { + size_type NumInputs = std::distance(in_start, in_end); + // Grow allocated space if needed. + if (NumInputs > size_type(this->capacity_ptr()-this->end())) + this->grow(this->size()+NumInputs); + + // Copy the new elements over. + this->uninitialized_copy(in_start, in_end, this->end()); + this->setEnd(this->end() + NumInputs); + } + + /// Add the specified range to the end of the SmallVector. + void append(size_type NumInputs, const T &Elt) { + // Grow allocated space if needed. + if (NumInputs > size_type(this->capacity_ptr()-this->end())) + this->grow(this->size()+NumInputs); + + // Copy the new elements over. + std::uninitialized_fill_n(this->end(), NumInputs, Elt); + this->setEnd(this->end() + NumInputs); + } + + void append(std::initializer_list IL) { + append(IL.begin(), IL.end()); + } + + void assign(size_type NumElts, const T &Elt) { + clear(); + if (this->capacity() < NumElts) + this->grow(NumElts); + this->setEnd(this->begin()+NumElts); + std::uninitialized_fill(this->begin(), this->end(), Elt); + } + + void assign(std::initializer_list IL) { + clear(); + append(IL); + } + + iterator erase(const_iterator CI) { + // Just cast away constness because this is a non-const member function. + iterator I = const_cast(CI); + + //assert(I >= this->begin() && "Iterator to erase is out of bounds."); + //assert(I < this->end() && "Erasing at past-the-end iterator."); + + iterator N = I; + // Shift all elts down one. + std::move(I+1, this->end(), I); + // Drop the last elt. + this->pop_back(); + return(N); + } + + iterator erase(const_iterator CS, const_iterator CE) { + // Just cast away constness because this is a non-const member function. + iterator S = const_cast(CS); + iterator E = const_cast(CE); + + //assert(S >= this->begin() && "Range to erase is out of bounds."); + //assert(S <= E && "Trying to erase invalid range."); + //assert(E <= this->end() && "Trying to erase past the end."); + + iterator N = S; + // Shift all elts down. + iterator I = std::move(E, this->end(), S); + // Drop the last elts. + this->destroy_range(I, this->end()); + this->setEnd(I); + return(N); + } + + iterator insert(iterator I, T &&Elt) { + if (I == this->end()) { // Important special case for empty vector. + this->push_back(::std::move(Elt)); + return this->end()-1; + } + + //assert(I >= this->begin() && "Insertion iterator is out of bounds."); + //assert(I <= this->end() && "Inserting past the end of the vector."); + + if (this->EndX >= this->CapacityX) { + size_t EltNo = I-this->begin(); + this->grow(); + I = this->begin()+EltNo; + } + + ::new ((void*) this->end()) T(::std::move(this->back())); + // Push everything else over. + std::move_backward(I, this->end()-1, this->end()); + this->setEnd(this->end()+1); + + // If we just moved the element we're inserting, be sure to update + // the reference. + T *EltPtr = &Elt; + if (I <= EltPtr && EltPtr < this->EndX) + ++EltPtr; + + *I = ::std::move(*EltPtr); + return I; + } + + iterator insert(iterator I, const T &Elt) { + if (I == this->end()) { // Important special case for empty vector. + this->push_back(Elt); + return this->end()-1; + } + + //assert(I >= this->begin() && "Insertion iterator is out of bounds."); + //assert(I <= this->end() && "Inserting past the end of the vector."); + + if (this->EndX >= this->CapacityX) { + size_t EltNo = I-this->begin(); + this->grow(); + I = this->begin()+EltNo; + } + ::new ((void*) this->end()) T(std::move(this->back())); + // Push everything else over. + std::move_backward(I, this->end()-1, this->end()); + this->setEnd(this->end()+1); + + // If we just moved the element we're inserting, be sure to update + // the reference. + const T *EltPtr = &Elt; + if (I <= EltPtr && EltPtr < this->EndX) + ++EltPtr; + + *I = *EltPtr; + return I; + } + + iterator insert(iterator I, size_type NumToInsert, const T &Elt) { + // Convert iterator to elt# to avoid invalidating iterator when we reserve() + size_t InsertElt = I - this->begin(); + + if (I == this->end()) { // Important special case for empty vector. + append(NumToInsert, Elt); + return this->begin()+InsertElt; + } + + //assert(I >= this->begin() && "Insertion iterator is out of bounds."); + //assert(I <= this->end() && "Inserting past the end of the vector."); + + // Ensure there is enough space. + reserve(this->size() + NumToInsert); + + // Uninvalidate the iterator. + I = this->begin()+InsertElt; + + // If there are more elements between the insertion point and the end of the + // range than there are being inserted, we can use a simple approach to + // insertion. Since we already reserved space, we know that this won't + // reallocate the vector. + if (size_t(this->end()-I) >= NumToInsert) { + T *OldEnd = this->end(); + append(std::move_iterator(this->end() - NumToInsert), + std::move_iterator(this->end())); + + // Copy the existing elements that get replaced. + std::move_backward(I, OldEnd-NumToInsert, OldEnd); + + std::fill_n(I, NumToInsert, Elt); + return I; + } + + // Otherwise, we're inserting more elements than exist already, and we're + // not inserting at the end. + + // Move over the elements that we're about to overwrite. + T *OldEnd = this->end(); + this->setEnd(this->end() + NumToInsert); + size_t NumOverwritten = OldEnd-I; + this->uninitialized_move(I, OldEnd, this->end()-NumOverwritten); + + // Replace the overwritten part. + std::fill_n(I, NumOverwritten, Elt); + + // Insert the non-overwritten middle part. + std::uninitialized_fill_n(OldEnd, NumToInsert-NumOverwritten, Elt); + return I; + } + + template + iterator insert(iterator I, ItTy From, ItTy To) { + // Convert iterator to elt# to avoid invalidating iterator when we reserve() + size_t InsertElt = I - this->begin(); + + if (I == this->end()) { // Important special case for empty vector. + append(From, To); + return this->begin()+InsertElt; + } + + //assert(I >= this->begin() && "Insertion iterator is out of bounds."); + //assert(I <= this->end() && "Inserting past the end of the vector."); + + size_t NumToInsert = std::distance(From, To); + + // Ensure there is enough space. + reserve(this->size() + NumToInsert); + + // Uninvalidate the iterator. + I = this->begin()+InsertElt; + + // If there are more elements between the insertion point and the end of the + // range than there are being inserted, we can use a simple approach to + // insertion. Since we already reserved space, we know that this won't + // reallocate the vector. + if (size_t(this->end()-I) >= NumToInsert) { + T *OldEnd = this->end(); + append(std::move_iterator(this->end() - NumToInsert), + std::move_iterator(this->end())); + + // Copy the existing elements that get replaced. + std::move_backward(I, OldEnd-NumToInsert, OldEnd); + + std::copy(From, To, I); + return I; + } + + // Otherwise, we're inserting more elements than exist already, and we're + // not inserting at the end. + + // Move over the elements that we're about to overwrite. + T *OldEnd = this->end(); + this->setEnd(this->end() + NumToInsert); + size_t NumOverwritten = OldEnd-I; + this->uninitialized_move(I, OldEnd, this->end()-NumOverwritten); + + // Replace the overwritten part. + for (T *J = I; NumOverwritten > 0; --NumOverwritten) { + *J = *From; + ++J; ++From; + } + + // Insert the non-overwritten middle part. + this->uninitialized_copy(From, To, OldEnd); + return I; + } + + void insert(iterator I, std::initializer_list IL) { + insert(I, IL.begin(), IL.end()); + } + + template void emplace_back(ArgTypes &&... Args) { + if ((this->EndX >= this->CapacityX)) [[unlikely]] + this->grow(); + ::new ((void *)this->end()) T(std::forward(Args)...); + this->setEnd(this->end() + 1); + } + + SmallVectorImpl &operator=(const SmallVectorImpl &RHS); + + SmallVectorImpl &operator=(SmallVectorImpl &&RHS); + + bool operator==(const SmallVectorImpl &RHS) const { + if (this->size() != RHS.size()) return false; + return std::equal(this->begin(), this->end(), RHS.begin()); + } + bool operator!=(const SmallVectorImpl &RHS) const { + return !(*this == RHS); + } + + bool operator<(const SmallVectorImpl &RHS) const { + return std::lexicographical_compare(this->begin(), this->end(), + RHS.begin(), RHS.end()); + } + + /// Set the array size to \p N, which the current array must have enough + /// capacity for. + /// + /// This does not construct or destroy any elements in the vector. + /// + /// Clients can use this in conjunction with capacity() to write past the end + /// of the buffer when they know that more elements are available, and only + /// update the size later. This avoids the cost of value initializing elements + /// which will only be overwritten. + void set_size(size_type N) { + //assert(N <= this->capacity()); + this->setEnd(this->begin() + N); + } +}; + + +template +void SmallVectorImpl::swap(SmallVectorImpl &RHS) { + if (this == &RHS) return; + + // We can only avoid copying elements if neither vector is small. + if (!this->isSmall() && !RHS.isSmall()) { + std::swap(this->BeginX, RHS.BeginX); + std::swap(this->EndX, RHS.EndX); + std::swap(this->CapacityX, RHS.CapacityX); + return; + } + if (RHS.size() > this->capacity()) + this->grow(RHS.size()); + if (this->size() > RHS.capacity()) + RHS.grow(this->size()); + + // Swap the shared elements. + size_t NumShared = this->size(); + if (NumShared > RHS.size()) NumShared = RHS.size(); + for (size_type i = 0; i != NumShared; ++i) + std::swap((*this)[i], RHS[i]); + + // Copy over the extra elts. + if (this->size() > RHS.size()) { + size_t EltDiff = this->size() - RHS.size(); + this->uninitialized_copy(this->begin()+NumShared, this->end(), RHS.end()); + RHS.setEnd(RHS.end()+EltDiff); + this->destroy_range(this->begin()+NumShared, this->end()); + this->setEnd(this->begin()+NumShared); + } else if (RHS.size() > this->size()) { + size_t EltDiff = RHS.size() - this->size(); + this->uninitialized_copy(RHS.begin()+NumShared, RHS.end(), this->end()); + this->setEnd(this->end() + EltDiff); + this->destroy_range(RHS.begin()+NumShared, RHS.end()); + RHS.setEnd(RHS.begin()+NumShared); + } +} + +template +SmallVectorImpl &SmallVectorImpl:: + operator=(const SmallVectorImpl &RHS) { + // Avoid self-assignment. + if (this == &RHS) return *this; + + // If we already have sufficient space, assign the common elements, then + // destroy any excess. + size_t RHSSize = RHS.size(); + size_t CurSize = this->size(); + if (CurSize >= RHSSize) { + // Assign common elements. + iterator NewEnd; + if (RHSSize) + NewEnd = std::copy(RHS.begin(), RHS.begin()+RHSSize, this->begin()); + else + NewEnd = this->begin(); + + // Destroy excess elements. + this->destroy_range(NewEnd, this->end()); + + // Trim. + this->setEnd(NewEnd); + return *this; + } + + // If we have to grow to have enough elements, destroy the current elements. + // This allows us to avoid copying them during the grow. + // FIXME: don't do this if they're efficiently moveable. + if (this->capacity() < RHSSize) { + // Destroy current elements. + this->destroy_range(this->begin(), this->end()); + this->setEnd(this->begin()); + CurSize = 0; + this->grow(RHSSize); + } else if (CurSize) { + // Otherwise, use assignment for the already-constructed elements. + std::copy(RHS.begin(), RHS.begin()+CurSize, this->begin()); + } + + // Copy construct the new elements in place. + this->uninitialized_copy(RHS.begin()+CurSize, RHS.end(), + this->begin()+CurSize); + + // Set end. + this->setEnd(this->begin()+RHSSize); + return *this; +} + +template +SmallVectorImpl &SmallVectorImpl::operator=(SmallVectorImpl &&RHS) { + // Avoid self-assignment. + if (this == &RHS) return *this; + + // If the RHS isn't small, clear this vector and then steal its buffer. + if (!RHS.isSmall()) { + this->destroy_range(this->begin(), this->end()); + if (!this->isSmall()) std::free(this->begin()); + this->BeginX = RHS.BeginX; + this->EndX = RHS.EndX; + this->CapacityX = RHS.CapacityX; + RHS.resetToSmall(); + return *this; + } + + // If we already have sufficient space, assign the common elements, then + // destroy any excess. + size_t RHSSize = RHS.size(); + size_t CurSize = this->size(); + if (CurSize >= RHSSize) { + // Assign common elements. + iterator NewEnd = this->begin(); + if (RHSSize) + NewEnd = std::move(RHS.begin(), RHS.end(), NewEnd); + + // Destroy excess elements and trim the bounds. + this->destroy_range(NewEnd, this->end()); + this->setEnd(NewEnd); + + // Clear the RHS. + RHS.clear(); + + return *this; + } + + // If we have to grow to have enough elements, destroy the current elements. + // This allows us to avoid copying them during the grow. + // FIXME: this may not actually make any sense if we can efficiently move + // elements. + if (this->capacity() < RHSSize) { + // Destroy current elements. + this->destroy_range(this->begin(), this->end()); + this->setEnd(this->begin()); + CurSize = 0; + this->grow(RHSSize); + } else if (CurSize) { + // Otherwise, use assignment for the already-constructed elements. + std::move(RHS.begin(), RHS.begin()+CurSize, this->begin()); + } + + // Move-construct the new elements in place. + this->uninitialized_move(RHS.begin()+CurSize, RHS.end(), + this->begin()+CurSize); + + // Set end. + this->setEnd(this->begin()+RHSSize); + + RHS.clear(); + return *this; +} + +/** +@private +*/ +template +struct SmallVectorStorage { + /** + @private + */ + typename SmallVectorTemplateCommon::U InlineElts[N - 1]; +}; + +/** +@private +*/ +template struct SmallVectorStorage {}; + +/** +@private +*/ +template struct SmallVectorStorage {}; + +/** +@brief class to define a vector optimized for small array + +@tparam T data type +@tparam N threshold of the number of elements in the initial storage + +The class defines a C++ STL-styled vector (a variable-sized array) +optimized for the case when the array is small. +It contains some number of elements in-place, +which allows it to avoid heap allocation when the actual number of +elements is below that threshold. This allows normal @em small cases to be +fast without losing generality for large inputs. +All the methods in [std::vector](https://en.cppreference.com/w/cpp/container/vector) +can apply to this class. + +The class is stripped from the LLVM codebase. +*/ +template +class SmallVector : public SmallVectorImpl { + /// Inline space for elements which aren't stored in the base class. + SmallVectorStorage Storage; + +public: + + /** + @brief constructs an empty vector + */ + SmallVector() : SmallVectorImpl(N) { + } + + /** + @brief constructs a vector with @c Size copies of elements with value @c value + */ + explicit SmallVector(size_t Size, const T &Value = T()) + : SmallVectorImpl(N) { + this->assign(Size, Value); + } + + /** + @brief constructs a vector with the contents of the range + [S, E) + */ + template + SmallVector(ItTy S, ItTy E) : SmallVectorImpl(N) { + this->append(S, E); + } + + //template + //explicit SmallVector(const tf::iterator_range &R) + // : SmallVectorImpl(N) { + // this->append(R.begin(), R.end()); + //} + + /** + @brief constructs a vector with the contents of the initializer list @c IL + */ + SmallVector(std::initializer_list IL) : SmallVectorImpl(N) { + this->assign(IL); + } + + /** + @brief constructs the vector with the copy of the contents of @c RHS + */ + SmallVector(const SmallVector &RHS) : SmallVectorImpl(N) { + if (!RHS.empty()) + SmallVectorImpl::operator=(RHS); + } + + /** + @brief constructs the vector with the contents of @c RHS using move semantics + */ + SmallVector(SmallVector &&RHS) : SmallVectorImpl(N) { + if (!RHS.empty()) + SmallVectorImpl::operator=(::std::move(RHS)); + } + + /** + @brief replaces the contents with a copy of the contents of @c RHS + */ + const SmallVector &operator=(const SmallVector &RHS) { + SmallVectorImpl::operator=(RHS); + return *this; + } + + /** + @brief replaces the contents with the contents of @c RHS using move semantics + */ + const SmallVector &operator=(SmallVector &&RHS) { + SmallVectorImpl::operator=(::std::move(RHS)); + return *this; + } + + /** + @brief constructs a vector with the contents of @c RHS using move semantics + */ + SmallVector(SmallVectorImpl &&RHS) : SmallVectorImpl(N) { + if (!RHS.empty()) + SmallVectorImpl::operator=(::std::move(RHS)); + } + + /** + @brief replaces the contents with the contents of @c RHS using move semantics + */ + const SmallVector &operator=(SmallVectorImpl &&RHS) { + SmallVectorImpl::operator=(::std::move(RHS)); + return *this; + } + + /** + @brief replaces the contents with the copy of the contents of an initializer list @c IL + */ + const SmallVector &operator=(std::initializer_list IL) { + this->assign(IL); + return *this; + } +}; + +/** +@private +*/ +template +inline size_t capacity_in_bytes(const SmallVector &X) { + return X.capacity_in_bytes(); +} + +} // end tf namespace --------------------------------------------------------- + +namespace std { + /// Implement std::swap in terms of SmallVector swap. + template + inline void + swap(tf::SmallVectorImpl &LHS, tf::SmallVectorImpl &RHS) { + LHS.swap(RHS); + } + + /// Implement std::swap in terms of SmallVector swap. + template + inline void + swap(tf::SmallVector &LHS, tf::SmallVector &RHS) { + LHS.swap(RHS); + } +} // end of namespace std ---------------------------------------------------- + + diff --git a/projects/hipfile/third-party/taskflow/utility/spsc.hpp b/projects/hipfile/third-party/taskflow/utility/spsc.hpp new file mode 100644 index 000000000000..bdb2caa13f38 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/utility/spsc.hpp @@ -0,0 +1,242 @@ +#pragma once + +#include +#include +#include +#include +#include +#include "os.hpp" + +/** +@file spsc.hpp +@brief lock-free single-producer / single-consumer ring buffer + +Designed and implemented by Matthew Busel. +*/ + +namespace tf { + +// ---------------------------------------------------------------------------- +// SPSCRing +// ---------------------------------------------------------------------------- + +/** +@class SPSCRing + +@tparam T element type; must be noexcept move-constructible and + noexcept move-assignable +@tparam LogSize base-2 logarithm of the internal buffer size; + capacity = 2^LogSize - 1; LogSize must be in [1, 30] + +@brief lock-free single-producer / single-consumer ring buffer + +%SPSCRing provides a fixed-capacity, zero-heap-allocation queue safe +for exactly one producer thread and one consumer thread. Unlike +@ref UnboundedWSQ and @ref BoundedWSQ (which support work-stealing +by multiple thief threads), %SPSCRing is specialised for the strict +1-producer / 1-consumer case and pays no cost for multi-consumer +coordination. + +@par Guarantees +- Lock-free: no mutexes, condition variables, or spin-locks. +- Zero allocation: the buffer is stored inline (std::array); the + total object size is 2^LogSize * sizeof(T) + 128 bytes. +- Cache-friendly: producer index and consumer index live on + separate cache lines (alignas(64)) to eliminate false sharing. +- noexcept: @ref push and @ref pop are unconditionally noexcept + provided T satisfies the noexcept move requirements. +- Non-blocking: @ref push returns @c false when full; @ref pop + returns an empty @ref value_type when empty. + +@par Capacity +The usable capacity is 2^LogSize - 1. One slot is reserved to +distinguish the full state from the empty state without a separate counter. + +@par Return-type specialization +Like @ref UnboundedWSQ, the return type of @ref pop is specialized on T: +- For pointer types, @ref pop returns @c T (using @c nullptr as the empty sentinel). +- For non-pointer types, @ref pop returns @c std::optional (using + @c std::nullopt as the empty sentinel). + +@par Typical usage + +@code{.cpp} +// shared between producer and consumer threads +tf::SPSCRing ring; // capacity = 2^7 - 1 = 127 + +// --- producer thread --- +ring.push(42); // returns false if full + +// --- consumer thread --- +if (auto v = ring.pop()) { + process(*v); // std::nullopt / nullptr when empty +} +@endcode + +@note Create one %SPSCRing per producer-consumer pair. + %SPSCRing is not safe for multiple producers or multiple consumers. +*/ +template +class SPSCRing { + + static_assert(LogSize >= 1, + "tf::SPSCRing: LogSize must be at least 1"); + static_assert(LogSize <= 30, + "tf::SPSCRing: LogSize must be at most 30"); + static_assert(std::is_nothrow_move_constructible_v, + "tf::SPSCRing: T must be noexcept move-constructible"); + static_assert(std::is_nothrow_move_assignable_v, + "tf::SPSCRing: T must be noexcept move-assignable"); + + static constexpr size_t SIZE = size_t{1} << LogSize; + static constexpr size_t MASK = SIZE - 1u; + + public: + + /** + @brief the return type of @ref pop + + For pointer element types @c T, `value_type` is @c T itself and uses + @c nullptr to indicate an empty result. For non-pointer types, it is + @c std::optional, where @c std::nullopt denotes the absence of a value. + + @code{.cpp} + static_assert(std::is_same_v::value_type, std::optional>); + static_assert(std::is_same_v::value_type, int*>); + @endcode + */ + using value_type = std::conditional_t, T, std::optional>; + + /** + @brief returns the maximum number of elements the ring can hold simultaneously + + The capacity is 2^LogSize - 1; one slot is reserved for + full/empty disambiguation. + */ + [[nodiscard]] static constexpr size_t capacity() noexcept { + return SIZE - 1u; + } + + /** + @brief returns the empty sentinel value appropriate for @ref value_type + + For pointer types this is @c nullptr; for non-pointer types it is + @c std::nullopt. + */ + static constexpr value_type empty_value() noexcept { + if constexpr (std::is_pointer_v) { + return T{nullptr}; + } else { + return std::optional{std::nullopt}; + } + } + + // -------------------------------------------------------------------------- + // Producer API - call from a single producer thread only + // -------------------------------------------------------------------------- + + /** + @brief tries to enqueue one element + + Moves @p item into the ring. Returns immediately without blocking. + + @param item element to enqueue (moved from on success) + + @return @c true on success; @c false if the ring is full (@p item is + left in a valid but unspecified state) + + @note Call from the producer thread only. + */ + [[nodiscard]] bool push(T&& item) noexcept { + const size_t tail = _tail.load(std::memory_order_relaxed); + const size_t next = (tail + 1u) & MASK; + if (next == _head.load(std::memory_order_acquire)) { + return false; // full + } + _buf[tail] = std::move(item); + _tail.store(next, std::memory_order_release); + return true; + } + + /** + @brief tries to enqueue a copy of one element + + Convenience overload for copy-constructible types. + + @param item element to copy-enqueue + @return @c true on success; @c false if full + */ + [[nodiscard]] bool push(const T& item) noexcept(std::is_nothrow_copy_constructible_v) { + T copy{item}; + return push(std::move(copy)); + } + + // -------------------------------------------------------------------------- + // Consumer API - call from a single consumer thread only + // -------------------------------------------------------------------------- + + /** + @brief tries to dequeue one element + + Returns immediately without blocking. + + @return @ref value_type containing the dequeued element on success, + or the empty sentinel (@c std::nullopt / @c nullptr) if empty + + @note Call from the consumer thread only. + */ + [[nodiscard]] value_type pop() noexcept { + const size_t head = _head.load(std::memory_order_relaxed); + if (head == _tail.load(std::memory_order_acquire)) { + return empty_value(); // empty + } + T item{std::move(_buf[head])}; + _head.store((head + 1u) & MASK, std::memory_order_release); + if constexpr (std::is_pointer_v) { + return item; + } else { + return std::optional{std::move(item)}; + } + } + + // -------------------------------------------------------------------------- + // Diagnostic queries (approximate - non-atomic snapshot) + // -------------------------------------------------------------------------- + + /** + @brief returns an approximate element count + + Not guaranteed to be exact under concurrent use. Suitable for + monitoring and metrics only. + */ + [[nodiscard]] size_t size_approx() const noexcept { + const size_t t = _tail.load(std::memory_order_acquire); + const size_t h = _head.load(std::memory_order_acquire); + return (t - h + SIZE) & MASK; + } + + /** + @brief returns @c true if the ring appears empty (approximate) + */ + [[nodiscard]] bool empty_approx() const noexcept { + return _tail.load(std::memory_order_acquire) == + _head.load(std::memory_order_acquire); + } + + /** + @brief returns @c true if the ring appears full (approximate) + */ + [[nodiscard]] bool full_approx() const noexcept { + const size_t t = _tail.load(std::memory_order_acquire); + const size_t h = _head.load(std::memory_order_acquire); + return ((t + 1u) & MASK) == h; + } + + private: + + alignas(TF_CACHELINE_SIZE) std::atomic _tail{0}; + alignas(TF_CACHELINE_SIZE) std::atomic _head{0}; + alignas(TF_CACHELINE_SIZE) std::array _buf{}; +}; + +} // namespace tf diff --git a/projects/hipfile/third-party/taskflow/utility/stream.hpp b/projects/hipfile/third-party/taskflow/utility/stream.hpp new file mode 100644 index 000000000000..34a86ff56996 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/utility/stream.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include +#include +#include + +namespace tf { + +// Procedure: ostreamize +template +void ostreamize(std::ostream& os, T&& token) { + os << std::forward(token); +} + +// Procedure: ostreamize +template +void ostreamize(std::ostream& os, T&& token, Rest&&... rest) { + os << std::forward(token); + ostreamize(os, std::forward(rest)...); +} + +// Function: stringify +template +std::string stringify(ArgsT&&... args) { + std::ostringstream oss; + ostreamize(oss, std::forward(args)...); + return oss.str(); +} + + +} // end of namespace tf ----------------------------------------------------- + diff --git a/projects/hipfile/third-party/taskflow/utility/traits.hpp b/projects/hipfile/third-party/taskflow/utility/traits.hpp new file mode 100644 index 000000000000..da01148b59f4 --- /dev/null +++ b/projects/hipfile/third-party/taskflow/utility/traits.hpp @@ -0,0 +1,316 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "os.hpp" + +namespace tf { + +//----------------------------------------------------------------------------- +// Traits +//----------------------------------------------------------------------------- + +//// Struct: dependent_false +//template +//struct dependent_false { +// static constexpr bool value = false; +//}; +// +//template +//constexpr auto dependent_false_v = dependent_false::value; + +template inline constexpr bool dependent_false_v = false; + +// ---------------------------------------------------------------------------- +// is_pod +//----------------------------------------------------------------------------- +template +struct is_pod { + static const bool value = std::is_trivial_v && + std::is_standard_layout_v; +}; + +template +constexpr bool is_pod_v = is_pod::value; + +//----------------------------------------------------------------------------- +// NoInit +//----------------------------------------------------------------------------- + +template +struct NoInit { + + //static_assert(is_pod_v, "NoInit only supports POD type"); + + // constructor without initialization + NoInit () noexcept {} + + // implicit conversion T -> NoInit + constexpr NoInit (T value) noexcept : v{value} {} + + // implicit conversion NoInit -> T + constexpr operator T () const noexcept { return v; } + + T v; +}; + +//----------------------------------------------------------------------------- +// Move-On-Copy +//----------------------------------------------------------------------------- + +// Struct: MoveOnCopyWrapper +template +struct MoC { + + MoC(T&& rhs) : object(std::move(rhs)) {} + MoC(const MoC& other) : object(std::move(other.object)) {} + + T& get() { return object; } + + mutable T object; +}; + +template +auto make_moc(T&& m) { + return MoC(std::forward(m)); +} + +//----------------------------------------------------------------------------- +// Visitors. +//----------------------------------------------------------------------------- + +//// Overloadded. +//template +//struct Visitors : Ts... { +// using Ts::operator()... ; +//}; +// +//template +//Visitors(Ts...) -> Visitors; + +// ---------------------------------------------------------------------------- +// std::variant +// ---------------------------------------------------------------------------- +template +struct get_index; + +template +struct get_index_impl {}; + +template +struct get_index_impl : std::integral_constant{}; + +template +struct get_index_impl : get_index_impl{}; + +template +struct get_index> : get_index_impl<0, T, Ts...>{}; + +template +constexpr auto get_index_v = get_index::value; + +// ---------------------------------------------------------------------------- +// visit a tuple with a functor at runtime +// ---------------------------------------------------------------------------- + +template +void visit_tuple(Func&& func, Tuple& tup, size_t idx) { + [&](std::index_sequence) { + ([&]{ if(Is == idx) { std::invoke(func, std::get(tup)); } }(), ...); + }(std::make_index_sequence>{}); +} + +// ---------------------------------------------------------------------------- +// unroll loop +// ---------------------------------------------------------------------------- + +template +constexpr void unroll(F&& f) { + [&](std::index_sequence) { + (f(beg + Is * step), ...); + }(std::make_index_sequence<(end - beg + step - 1) / step>{}); +} + +// stops at the first true; returns true if any f(i) was true (||-fold) +template +constexpr auto unroll_until(F&& f) { + return [&](std::index_sequence) { + return (f(beg + Is * step) || ...); + }(std::make_index_sequence<(end - beg + step - 1) / step>{}); +} + +// stops at the first false; returns true if every f(i) was true (&&-fold) +template +constexpr auto unroll_while(F&& f) { + return [&](std::index_sequence) { + return (f(beg + Is * step) && ...); + }(std::make_index_sequence<(end - beg + step - 1) / step>{}); +} + +// ---------------------------------------------------------------------------- +// make types of variant unique +// ---------------------------------------------------------------------------- + +template +struct filter_duplicates { using type = T; }; + +template