diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 6bfea599ee1f..4ba016d0da6c 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1045,6 +1045,8 @@ add_library( src/strings/merge/merge.cu src/strings/padding.cu src/strings/positions.cu + src/strings/regex/gkexec.cpp + src/strings/regex/glushkov_regcomp.cpp src/strings/regex/regcomp.cpp src/strings/regex/regexec.cpp src/strings/regex/regex_program.cpp diff --git a/cpp/src/strings/char_types/char_flags.h b/cpp/src/strings/char_types/char_flags.h index 4cfc0dc03eb6..093fc7c892c0 100644 --- a/cpp/src/strings/char_types/char_flags.h +++ b/cpp/src/strings/char_types/char_flags.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2023, NVIDIA CORPORATION. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once @@ -23,7 +23,8 @@ // 0 - decimal // -uint8_t const g_character_codepoint_flags[] = { +// clang-format off +uint8_t const g_character_codepoint_flags[] = { // NOLINT 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 7, 7, @@ -3475,3 +3476,4 @@ uint8_t const g_character_codepoint_flags[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; +// clang-format on diff --git a/cpp/src/strings/contains.cu b/cpp/src/strings/contains.cu index fd249d6ff3d0..b16199c077d5 100644 --- a/cpp/src/strings/contains.cu +++ b/cpp/src/strings/contains.cu @@ -35,16 +35,16 @@ struct contains_fn { column_device_view const d_strings; bool const beginning_only; - __device__ bool operator()(size_type const idx, - reprog_device const prog, - int32_t const thread_idx) + template + __device__ bool operator()(size_type const idx, ProgDevice const prog, int32_t const thread_idx) { if (d_strings.is_null(idx)) return false; auto const d_str = d_strings.element(idx); size_type end = beginning_only ? 1 // match only the beginning of the string; : -1; // match anywhere in the string - return prog.find(thread_idx, d_str, d_str.begin(), end).has_value(); + return prog.template find(thread_idx, d_str, d_str.begin(), end) + .has_value(); } }; @@ -62,15 +62,18 @@ std::unique_ptr contains_impl(strings_column_view const& input, mr); if (input.is_empty()) { return results; } - auto d_prog = regex_device_builder::create_prog_device(prog, stream); - auto d_results = results->mutable_view().data(); auto const d_strings = column_device_view::create(input.parent(), stream); - launch_transform_kernel( - contains_fn{*d_strings, beginning_only}, *d_prog, d_results, input.size(), stream); - - results->set_null_count(input.null_count()); + if (regex_device_builder::glushkov_fast_path_supported(prog)) { + auto d_prog = regex_device_builder::create_gkprog_device(prog, stream); + launch_transform_kernel( + contains_fn{*d_strings, beginning_only}, *d_prog, d_results, input.size(), stream); + } else { + auto d_prog = regex_device_builder::create_prog_device(prog, stream); + launch_transform_kernel( + contains_fn{*d_strings, beginning_only}, *d_prog, d_results, input.size(), stream); + } return results; } @@ -133,11 +136,9 @@ std::unique_ptr count_re(strings_column_view const& input, return count(input, target, stream, mr); } - auto d_prog = regex_device_builder::create_prog_device(prog, stream); - auto const d_strings = column_device_view::create(input.parent(), stream); - auto result = count_matches(*d_strings, *d_prog, stream, mr); + auto result = count_matches(*d_strings, prog, stream, mr); if (input.has_nulls()) { result->set_null_mask(cudf::detail::copy_bitmask(input.parent(), stream, mr), input.null_count()); diff --git a/cpp/src/strings/count_matches.cu b/cpp/src/strings/count_matches.cu index 0371b29f8b64..6e5e8dda1779 100644 --- a/cpp/src/strings/count_matches.cu +++ b/cpp/src/strings/count_matches.cu @@ -1,16 +1,20 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #include "strings/count_matches.hpp" +#include "strings/regex/regex_program_impl.h" #include "strings/regex/utilities.cuh" #include #include +#include #include #include +#include + namespace cudf { namespace strings { namespace detail { @@ -23,8 +27,9 @@ template struct count_fn { column_device_view const d_strings; + template __device__ int32_t operator()(size_type const idx, - reprog_device const prog, + ProgDevice const prog, int32_t const thread_idx) { if (d_strings.is_null(idx)) return 0; @@ -34,7 +39,7 @@ struct count_fn { auto itr = d_str.begin(); while (itr.position() <= nchars) { - auto result = prog.find

(thread_idx, d_str, itr); + auto result = prog.template find

(thread_idx, d_str, itr); if (!result) { break; } ++count; // increment the iterator is faster than creating a new one @@ -47,29 +52,64 @@ struct count_fn { } // namespace +template std::unique_ptr count_matches(column_device_view const& d_strings, - reprog_device& d_prog, + ProgDevice& d_prog, + size_type strings_count, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { auto results = make_numeric_column( - data_type{type_to_id()}, d_strings.size(), mask_state::UNALLOCATED, stream, mr); + data_type{type_to_id()}, strings_count, mask_state::UNALLOCATED, stream, mr); - if (d_strings.size() == 0) { return results; } + if (strings_count == 0) { return results; } auto d_results = results->mutable_view().data(); - if (d_prog.is_empty_match_possible()) { + // Glushkov's engine always requires the begin/end positional check; the Thompson + // engine can skip it (cheaper) when an empty match is not possible for this pattern. + if constexpr (std::is_same_v) { launch_transform_kernel( - count_fn{d_strings}, d_prog, d_results, d_strings.size(), stream); + count_fn{d_strings}, d_prog, d_results, strings_count, stream); } else { - launch_transform_kernel( - count_fn{d_strings}, d_prog, d_results, d_strings.size(), stream); + if (d_prog.is_empty_match_possible()) { + launch_transform_kernel( + count_fn{d_strings}, d_prog, d_results, strings_count, stream); + } else { + launch_transform_kernel( + count_fn{d_strings}, d_prog, d_results, strings_count, stream); + } } return results; } +template std::unique_ptr count_matches(column_device_view const&, + reprog_device&, + size_type, + rmm::cuda_stream_view, + rmm::device_async_resource_ref); + +template std::unique_ptr count_matches(column_device_view const&, + gkprog_device&, + size_type, + rmm::cuda_stream_view, + rmm::device_async_resource_ref); + +std::unique_ptr count_matches(column_device_view const& d_strings, + regex_program const& prog, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + auto const strings_count = d_strings.size(); + if (regex_device_builder::glushkov_fast_path_supported(prog)) { + auto d_prog = regex_device_builder::create_gkprog_device(prog, stream); + return count_matches(d_strings, *d_prog, strings_count, stream, mr); + } + auto d_prog = regex_device_builder::create_prog_device(prog, stream); + return count_matches(d_strings, *d_prog, strings_count, stream, mr); +} + } // namespace detail } // namespace strings } // namespace cudf diff --git a/cpp/src/strings/count_matches.hpp b/cpp/src/strings/count_matches.hpp index f869dfe5d208..a948f8e68b7c 100644 --- a/cpp/src/strings/count_matches.hpp +++ b/cpp/src/strings/count_matches.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -15,23 +15,51 @@ namespace cudf { class column_device_view; namespace strings { + +class regex_program; + namespace detail { -class reprog_device; +/** + * @brief Returns a column of regex match counts for each string in the given column. + * + * A null entry will result in a zero count for that output row. + * + * This overload evaluates against an already-built device regex program. Callers that + * also need the device program for other work (e.g. extraction) should build it once + * and pass it here to avoid a redundant device program build. + * + * @tparam The regex prog device instance used for this API + * @param d_strings Device view of the input strings column + * @param d_prog Device regex program to evaluate on each string + * @param strings_count Number of strings (and rows in the output column) + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + * @return Integer column of match counts + */ +template +std::unique_ptr count_matches(column_device_view const& d_strings, + ProgDevice& d_prog, + size_type strings_count, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr); /** * @brief Returns a column of regex match counts for each string in the given column. * * A null entry will result in a zero count for that output row. * + * This overload builds its own device regex program. Prefer the overload above when + * the device program is also needed for other work on the same call site. + * * @param d_strings Device view of the input strings column. - * @param d_prog Regex instance to evaluate on each string. + * @param prog Regex program to evaluate on each string. * @param stream CUDA stream used for device memory operations and kernel launches. * @param mr Device memory resource used to allocate the returned column's device memory. * @return Integer column of match counts */ std::unique_ptr count_matches(column_device_view const& d_strings, - reprog_device& d_prog, + regex_program const& prog, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr); diff --git a/cpp/src/strings/extract/extract_all.cu b/cpp/src/strings/extract/extract_all.cu index 8cded262e80b..bd641badfa59 100644 --- a/cpp/src/strings/extract/extract_all.cu +++ b/cpp/src/strings/extract/extract_all.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -106,9 +106,10 @@ std::unique_ptr extract_all_record(strings_column_view const& input, auto const groups = d_prog->group_counts(); CUDF_EXPECTS(groups > 0, "extract_all requires group indicators in the regex pattern."); - // Get the match counts for each string. + // Get the match counts for each string, reusing the device program built above + // instead of building a second one. // This column will become the output lists child offsets column. - auto counts = count_matches(*d_strings, *d_prog, stream, mr); + auto counts = count_matches(*d_strings, *d_prog, strings_count, stream, mr); auto d_counts = counts->mutable_view().data(); // Compute null output rows diff --git a/cpp/src/strings/regex/common.cuh b/cpp/src/strings/regex/common.cuh new file mode 100644 index 000000000000..39166db24fd5 --- /dev/null +++ b/cpp/src/strings/regex/common.cuh @@ -0,0 +1,90 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include "regcomp.h" + +#include +#include +#include + +#include +#include +#include +#include + +namespace cudf::strings::detail { + +/// Bitmask type: bit i is set when Glushkov position i is active. +using glushkov_state_t = uint64_t; + +/// Maximum number of character-consuming positions (states) in the Glushkov NFA. +/// Patterns with more positions fall back to Thompson NFA automatically. +constexpr int32_t GLUSHKOV_MAX_STATES = sizeof(glushkov_state_t) * 8; + +/// Maximum shift amounts for the Hyperscan-style shift-and optimization. +constexpr int32_t GLUSHKOV_MAX_SHIFTS = 8; + +/// Size of the precomputed ASCII reach table (characters 0–127). +constexpr int32_t GLUSHKOV_ASCII_TABLE_SIZE = 128; + +/** + * @brief Regex class stored on the device and executed by reprog_device. + * + * This class holds the unique data for any regex CCLASS instruction. + */ +struct alignas(16) reclass_device { + int32_t builtins{}; + int32_t count{}; + reclass_range const* literals{}; + + __device__ inline bool is_match(char32_t const ch, uint8_t const* codepoint_flags) const + { + if (thrust::any_of(thrust::seq, literals, literals + count, [ch](auto literal) { + return ((ch >= literal.first) && (ch <= literal.last)); + })) { + return true; + } + + if (!builtins) { return false; } + auto const codept = utf8_to_codepoint(ch); + constexpr uint32_t MAX_CODEPOINT = 0x00'FFFF; + if (codept > MAX_CODEPOINT) { return false; } + auto const fl = codepoint_flags[codept]; + if ((builtins & CCLASS_W) && ((ch == '_') || IS_ALPHANUM(fl))) { return true; } // \w + if ((builtins & CCLASS_S) && IS_SPACE(fl)) { return true; } // \s + if ((builtins & CCLASS_D) && IS_DIGIT(fl)) { return true; } // \d + if ((builtins & NCCLASS_W) && ((ch != '\n') && (ch != '_') && !IS_ALPHANUM(fl))) { // \W + return true; + } + if ((builtins & NCCLASS_S) && !IS_SPACE(fl)) { return true; } // \S + if ((builtins & NCCLASS_D) && ((ch != '\n') && !IS_DIGIT(fl))) { return true; } // \D + + return false; + } +}; + +/** + * @brief Check for supported new-line characters + * + * '\n, \r, \u0085, \u2028, or \u2029' + */ +CUDF_HOST_DEVICE __forceinline__ constexpr bool is_newline(char32_t const ch) +{ + return (ch == '\n' || ch == '\r' || ch == 0x00c285 || ch == 0x00e280a8 || ch == 0x00e280a9); +} + +/** + * @brief Template type used on `find` to specify desired position values in returned match_result + */ +enum class positional : int8_t { + BEGIN_END = 0, /// both begin and end positions are returned + END_ONLY = 1, /// only the end position is returned +}; + +using match_pair = cuda::std::pair; +using match_result = cuda::std::optional; + +} // namespace cudf::strings::detail diff --git a/cpp/src/strings/regex/gkexec.cpp b/cpp/src/strings/regex/gkexec.cpp new file mode 100644 index 000000000000..47ac896f8e17 --- /dev/null +++ b/cpp/src/strings/regex/gkexec.cpp @@ -0,0 +1,180 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "strings/regex/glushkov.cuh" +#include "strings/regex/glushkov_regcomp.hpp" + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +namespace cudf { +namespace strings { +namespace detail { + +/** + * @brief Pack a gkprog into a contiguous device buffer and + * return a device pointer to the embedded gkprog_device struct. + * + * Buffer layout: + * [_positions : num_states × reinst] + * [_shift_masks : GLUSHKOV_MAX_SHIFTS × glushkov_state_t] + * [_shift_amounts: GLUSHKOV_MAX_SHIFTS × uint8_t + padding] + * [_reach_ascii : GLUSHKOV_ASCII_TABLE_SIZE × glushkov_state_t] + * [_exception_successors : num_states × glushkov_state_t + padding] + * [_classes : classes_count × reclass_device + variable-length literals] + */ +std::unique_ptr> gkprog_device::create( + gkprog const& h_gp, rmm::cuda_stream_view stream) +{ + auto const num_states = h_gp.num_states; + auto const classes_cnt = static_cast(h_gp.classes.size()); + + auto const smasks_size = GLUSHKOV_MAX_SHIFTS * sizeof(glushkov_state_t); + auto const reach_ascii_size = GLUSHKOV_ASCII_TABLE_SIZE * sizeof(glushkov_state_t); + auto const cls_size = std::transform_reduce( + h_gp.classes.begin(), + h_gp.classes.end(), + static_cast(classes_cnt) * sizeof(reclass_device), + std::plus{}, + [](auto const& cls) { return cls.literals.size() * sizeof(reclass_range); }); + + // compute size of each section + auto pos_size = num_states * sizeof(reinst); + auto samts_size = GLUSHKOV_MAX_SHIFTS * sizeof(uint8_t); + auto exc_size = num_states * sizeof(glushkov_state_t); + + // make sure each section is aligned for the subsequent section's data type + auto const memsize = + cudf::util::round_up_safe(pos_size, alignof(glushkov_state_t)) + smasks_size + + cudf::util::round_up_safe(samts_size, alignof(glushkov_state_t)) + reach_ascii_size + + cudf::util::round_up_safe(exc_size, alignof(reclass_device)) + + cudf::util::round_up_safe(cls_size, sizeof(char32_t)); + + // allocate memory to store all the prog data in a flat contiguous buffer + auto h_buffer = cudf::detail::make_host_vector(memsize, stream); + auto h_ptr = h_buffer.data(); + auto d_buffer = std::make_unique>(memsize, stream); + auto d_ptr = d_buffer->data(); + + // create our device object; this is managed separately and returned to the caller. + // both allocations are held by unique_ptrs so they are freed if anything below throws; + // ownership is transferred to the returned unique_ptr's deleter only once we cannot throw. + auto d_prog = std::make_unique(); + d_prog->num_states = num_states; + d_prog->shift_count = h_gp.shift_count; + d_prog->first_set = h_gp.first_set; + d_prog->accept_mask = h_gp.accept_mask; + d_prog->exception_mask = h_gp.exception_mask; + d_prog->startchar = h_gp.startchar; + d_prog->_codepoint_flags = get_character_flags_table(stream); + d_prog->_prog_size = memsize; + + // copy the positions array (fixed-sized structs) + std::memcpy(h_ptr, h_gp.pos_insts.data(), pos_size); + d_prog->_positions = reinterpret_cast(d_ptr); + + // advance to next section; align for glushkov_state_t + pos_size = cudf::util::round_up_safe(pos_size, alignof(glushkov_state_t)); + h_ptr += pos_size; + d_ptr += pos_size; + + // copy shift_masks + std::memcpy(h_ptr, h_gp.shift_masks.data(), smasks_size); + d_prog->_shift_masks = reinterpret_cast(d_ptr); + h_ptr += smasks_size; + d_ptr += smasks_size; + + // copy shift_amounts (uint8_t; no alignment padding needed after glushkov_state_t[]) + std::memcpy(h_ptr, h_gp.shift_amounts.data(), samts_size); + d_prog->_shift_amounts = reinterpret_cast(d_ptr); + + // advance to next section; align for glushkov_state_t + samts_size = cudf::util::round_up_safe(samts_size, alignof(glushkov_state_t)); + h_ptr += samts_size; + d_ptr += samts_size; + + // copy reach_ascii table (GLUSHKOV_ASCII_TABLE_SIZE × glushkov_state_t = 1 KB) + std::memcpy(h_ptr, h_gp.reach_ascii.data(), reach_ascii_size); + d_prog->_reach_ascii = reinterpret_cast(d_ptr); + h_ptr += reach_ascii_size; + d_ptr += reach_ascii_size; + + // copy exception_successors + std::memcpy(h_ptr, h_gp.exception_successors.data(), exc_size); + d_prog->_exception_successors = reinterpret_cast(d_ptr); + + // advance to next section; align for reclass_device (alignas(16)) + exc_size = cudf::util::round_up_safe(exc_size, alignof(reclass_device)); + h_ptr += exc_size; + d_ptr += exc_size; + + // copy classes into flat memory: [class1, class2, ...][literals arrays] + auto* classes = reinterpret_cast(h_ptr); + d_prog->_classes = reinterpret_cast(d_ptr); + auto h_end = h_ptr + static_cast(classes_cnt) * sizeof(reclass_device); + auto d_end = d_ptr + static_cast(classes_cnt) * sizeof(reclass_device); + for (int32_t ci = 0; ci < classes_cnt; ++ci) { + auto const& src = h_gp.classes[ci]; + *classes++ = reclass_device{src.builtins, + static_cast(src.literals.size()), + reinterpret_cast(d_end)}; + auto const lit_bytes = src.literals.size() * sizeof(reclass_range); + std::memcpy(h_end, src.literals.data(), lit_bytes); + h_end += lit_bytes; + d_end += lit_bytes; + } + + // copy flat buffer to device + cudf::detail::cuda_memcpy_async(*d_buffer, h_buffer, stream); + + // create a deleter to free both device buffers + auto deleter = [d_buffer = d_buffer.get()](gkprog_device* t) { + t->destroy(); + delete d_buffer; + }; + + stream.synchronize(); // wait for h_buffer to finish copying + + auto result = + std::unique_ptr>(d_prog.get(), deleter); + // ownership now belongs to result's deleter; release from the guarding unique_ptrs + d_prog.release(); + d_buffer.release(); + return result; +} + +void gkprog_device::destroy() { delete this; } + +int32_t gkprog_device::compute_shared_memory_size() const +{ + return static_cast(sizeof(glushkov_shmem_cache)); +} + +std::pair gkprog_device::compute_strided_working_memory(int32_t rows, + int32_t, + std::size_t) const +{ + return std::make_pair(0, rows); +} + +void gkprog_device::set_working_memory(void*, int32_t thread_count, int32_t) +{ + _thread_count = thread_count; +} + +} // namespace detail +} // namespace strings +} // namespace cudf diff --git a/cpp/src/strings/regex/glushkov.cuh b/cpp/src/strings/regex/glushkov.cuh new file mode 100644 index 000000000000..458a8917bf0a --- /dev/null +++ b/cpp/src/strings/regex/glushkov.cuh @@ -0,0 +1,499 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include "common.cuh" + +#include + +#include + +#include +#include +#include + +#include +#include +#include +#include + +/** + * @brief Device-side Glushkov NFA program data structure. + * + * The device program is a compact representation of the Glushkov NFA: + * - State is a 64-bit bitmask (one bit per position, no working memory needed). + * - Transition = shift-and + exception lookup (Hyperscan technique). + * - Character matching reuses the existing reclass_device::is_match path. + */ + +namespace cudf::strings::detail { + +struct gkprog; + +/** + * @brief Block-shared cache of read-only Glushkov program data. + * + * Loaded by thread-0 via gkprog_device::load/store calls. + * Subsequent calls to gkprog_device::find read from shared memory (~20-cycle latency) + * instead of global/L2 (~100 cycles). + * + * Total size: ~1608 bytes — negligible impact on occupancy. + */ +struct glushkov_shmem_cache { + cuda::std::array + reach_ascii; ///< precomputed ASCII reach masks + cuda::std::array + shift_masks; ///< [shift_count] shift-and source masks + cuda::std::array + exception_successors; ///< [num_states] exception successor masks + cuda::std::array shift_amounts; ///< [shift_count] shift amounts +}; + +/** + * @brief GPU-resident Glushkov NFA program. + * + * All array pointers address device-global memory packed into a single flat + * buffer (see gkprog_device::create). + */ +struct gkprog_device { + public: + gkprog_device() = default; + ~gkprog_device() = default; + gkprog_device(gkprog_device const&) = default; + gkprog_device(gkprog_device&&) = default; + gkprog_device& operator=(gkprog_device const&) = default; + gkprog_device& operator=(gkprog_device&&) = default; + + static std::unique_ptr> create( + gkprog const& prog, rmm::cuda_stream_view stream); + + /** + * @brief Called automatically by the unique_ptr returned from create(). + */ + void destroy(); + + [[nodiscard]] __device__ inline bool is_empty() const { return false; } + + [[nodiscard]] CUDF_HOST_DEVICE bool is_empty_match_possible() const { return false; } + + /** + * @brief Stores this instance into the given buffer (shared_memory pointer) + */ + __device__ inline void store(void* buffer) const; + + /** + * @brief Load an instance from a device buffer (e.g. shared memory). + */ + [[nodiscard]] __device__ static inline gkprog_device load(gkprog_device const prog, void* buffer); + + /** + * @brief Returns the size of shared memory required to hold this instance. + * + * This can be called on the CPU for specifying the shared-memory size in the + * kernel launch parameters. + */ + [[nodiscard]] int32_t compute_shared_memory_size() const; + + /** + * @brief Returns amount of working memory needed to hold state for a stride of threads. + */ + [[nodiscard]] std::pair compute_strided_working_memory( + int32_t rows, int32_t min_rows = 0, std::size_t requested_max_size = 0) const; + + /** + * @brief Sets the working memory pointer for this instance. + */ + void set_working_memory(void* buffer, int32_t thread_count, int32_t max_insts = 0); + + /** + * @brief Returns number of threads set by set_working_memory + */ + [[nodiscard]] __device__ inline int32_t thread_count() const { return _thread_count; } + + /** + * @brief Does a find evaluation using the compiled expression on the given string. + * + * @tparam P Desired positional values. Default includes valid begin and end match positions. + * @param thread_idx The index used for mapping the state memory for this string in global memory. + * @param d_str The string to search. + * @param begin Position to begin the search within `d_str`. + * @param end Character position index to end the search within `d_str`. + * Specify -1 to match any virtual positions past the end of the string. + * @return If match found, returns character positions of the matches. + */ + template + [[nodiscard]] __device__ inline match_result find(int32_t const thread_idx, + string_view const d_str, + string_view::const_iterator begin, + cudf::size_type end = -1) const; + + uint32_t num_states{}; ///< Number of Glushkov positions (≤ 64) + uint32_t shift_count{}; ///< Number of shift slots (≤ GLUSHKOV_MAX_SHIFTS_DEV) + glushkov_state_t first_set{}; ///< Initial active positions (before first character) + glushkov_state_t accept_mask{}; ///< Positions whose match completes the pattern + glushkov_state_t exception_mask{}; ///< Positions with non-shift follow transitions + cuda::std::optional startchar{}; ///< First literal + + uint8_t const* _codepoint_flags{}; ///< Character-type lookup table (shared with Thompson) + reclass_device const* _classes{}; ///< Character class data + reinst const* _positions{}; ///< [num_states] per-position descriptors + glushkov_state_t const* _shift_masks{}; ///< [shift_count] shift-and source masks + uint8_t const* _shift_amounts{}; ///< [shift_count] shift amounts + glushkov_state_t const* _reach_ascii{}; ///< precomputed reach bitmasks for ASCII chars + glushkov_state_t const* _exception_successors{}; ///< [num_states] exception successor masks + + std::size_t _prog_size{}; ///< Total buffer size (for potential shmem loading) + int32_t _thread_count{}; ///< Used for strided kernel loops + + [[nodiscard]] __device__ glushkov_state_t priority_kill(glushkov_state_t const state, + glushkov_state_t const accept_mask) const; + [[nodiscard]] __device__ bool position_matches(reinst const& pos, + char32_t const c, + reclass_device const* classes, + uint8_t const* codepoint_flags) const; + [[nodiscard]] __device__ glushkov_state_t compute_follow_impl(glushkov_state_t const state) const; + [[nodiscard]] __device__ glushkov_state_t compute_reach_impl(char32_t const c) const; +}; + +/** + * Matching algorithm + * ------------------ + * One thread processes one input string. The NFA state is a single glushkov_state_t + * (uint64_t) bitmask held in a register – no global working memory required. + * + * Unanchored search: two-phase O(n) per match. + * Phase 1 (forward scan): inject first_set at every character; track the + * earliest injection position (inject_start_pos) since state was last 0. + * When state & accept_mask fires, record the match end; continue extending + * greedily until state dies. + * Phase 2 (BEGIN_END only): rescan from inject_start_pos, trying each valid + * start position with a single-seed NFA until one reaches accept. The + * earliest start that accepts gives the leftmost match. + * END_ONLY (contains_re, matches_re): phase 1 result is returned directly; + * the start position is not needed so phase 2 is skipped. + * + * Anchored (end >= 0): inner-loop from start_pos, already O(n) per string. + * + * + * The shift-and follow computation (Hyperscan-style): + * follow(state) ≈ OR_k( (state & shift_masks[k]) << shift_amounts[k] ) + * | OR over active exception states (exception_successors lookup) + * + */ + +/** + * @brief Emulate Thompson's first-alternative-wins priority by killing lower-priority states. + * + * When multiple alternatives can accept at the same position, the lowest-indexed accepting + * position corresponds to the highest-priority alternative. All bits above the lowest accepting + * bit are cleared so that lower-priority paths are killed while self-loops and continuations of the + * winning path survive. + * + * @param state Current NFA state bitmask. + * @param accept_mask Bitmask of accepting positions. + * @return State with lower-priority alternatives killed. + */ +__device__ __forceinline__ glushkov_state_t +gkprog_device::priority_kill(glushkov_state_t const state, glushkov_state_t const accept_mask) const +{ + auto const lsb_a = static_cast(cuda::std::countr_zero(state & accept_mask)); + return (lsb_a < 63) ? (state & ((glushkov_state_t(1) << (lsb_a + 1)) - 1)) : state; +} + +/** + * @brief Returns true if character @p c is matched by Glushkov position @p pos. + * + * Mirrors the character-dispatch logic in the Thompson NFA regexec loop. + */ +__device__ __forceinline__ bool gkprog_device::position_matches( + reinst const& pos, + char32_t const c, + reclass_device const* classes, + uint8_t const* codepoint_flags) const +{ + switch (pos.type) { + case CHAR: return c == pos.u1.c; + case ANY: { + // pos.ch == 'N' → EXT_NEWLINE: reject all is_newline() characters + // otherwise → default: reject only '\n' (matches Thompson NFA) + return (pos.u1.c == 'N') ? !is_newline(c) : (c != '\n'); + } + case ANYNL: return true; + case CCLASS: return classes[pos.u1.cls_id].is_match(c, codepoint_flags); + case NCCLASS: return !classes[pos.u1.cls_id].is_match(c, codepoint_flags); + default: return false; + } +} + +/** + * @brief Compute the Glushkov follow set for the current state bitmask. + * + * Two-phase: + * 1. Shift masks – fast, covers most forward transitions. + * 2. Exception table – backward and large-span transitions. + */ +__device__ __forceinline__ glushkov_state_t +gkprog_device::compute_follow_impl(glushkov_state_t const state) const +{ + glushkov_state_t follow = 0; + + // Phase 1: shift-and transitions + for (uint32_t k = 0; k < shift_count; ++k) { + follow |= (state & _shift_masks[k]) << _shift_amounts[k]; + } + + // Phase 2: exception transitions (backward / large-span) + glushkov_state_t exc = state & exception_mask; + while (exc) { + auto const p = cuda::std::countr_zero(exc); + exc &= exc - 1; // clear lowest set bit + follow |= _exception_successors[p]; + } + + return follow; +} + +/** + * @brief Compute the reach bitmask for character @p c. + * + * reach(c) = { p | position p matches character c } + * + * Fast path (c < GLUSHKOV_ASCII_TABLE_SIZE): single lookup into the precomputed + * reach_ascii table — O(1). + * Slow path (non-ASCII): O(num_states) loop over position descriptors. + */ +__device__ __forceinline__ glushkov_state_t +gkprog_device::compute_reach_impl(char32_t const c) const +{ + if (c < static_cast(GLUSHKOV_ASCII_TABLE_SIZE)) { return _reach_ascii[c]; } + + // Non-ASCII fallback: must use prog._positions (global memory) + glushkov_state_t reach = 0; + for (uint32_t p = 0; p < num_states; ++p) { + if (position_matches(_positions[p], c, _classes, _codepoint_flags)) { + reach |= glushkov_state_t(1) << p; + } + } + return reach; +} + +/** + * @brief Glushkov NFA find: locate the leftmost (greedy) match in @p d_str. + * + * Semantics match reprog_device::find (Thompson NFA) as closely as possible: + * - Leftmost start position. + * - Greedy (longest match from that start). + * + * For unanchored patterns, uses a two-phase O(n) algorithm: + * Phase 1 injects first_set at every character and extends greedily; when + * state & accept_mask fires, the match end is recorded. Phase 2 (BEGIN_END + * only) rescans from inject_start_pos (the earliest injection position since + * state was last 0) to find the leftmost valid start for that match end. + * + * + * @tparam P Positional mode (BEGIN_END or END_ONLY) + * + * @param d_str Input string. + * @param begin Iterator to the first character to consider. + * @param end Max start-position to try (−1 = try all positions). + * `end=1` means only try starting at 0 + * (used by matches_re / beginning_only) + * @return Match pair or nullopt. + */ +template +__device__ __forceinline__ match_result gkprog_device::find(int32_t const, + string_view const d_str, + string_view::const_iterator begin, + cudf::size_type end) const +{ + auto const size_bytes = static_cast(d_str.size_bytes()); // O(1) + + int32_t const start_pos = begin.position(); + + // --- Anchored (end >= 0): inner-loop approach, O(n) --- + // Outer loop runs at most (end - start_pos) iterations (typically 1 for matches_re). + if (end >= 0) { + auto outer_itr = begin; + for (int32_t start = start_pos;; ++start, ++outer_itr) { + if (start >= end) { break; } + if (outer_itr.byte_offset() >= size_bytes) { break; } + + if (startchar.has_value()) { + while (outer_itr.byte_offset() < size_bytes && *outer_itr != startchar.value()) { + ++outer_itr; + ++start; + } + if (outer_itr.byte_offset() >= size_bytes) { break; } + if (start >= end) { break; } + } else { + char32_t const first_c = *outer_itr; + if ((compute_reach_impl(first_c) & first_set) == 0) { continue; } + } + + match_result cur_match{}; + + glushkov_state_t state = first_set; + auto inner_itr = outer_itr; + for (int32_t pos = start; inner_itr.byte_offset() < size_bytes; ++pos, ++inner_itr) { + char32_t const c = *inner_itr; + if (pos == start) { + state = state & compute_reach_impl(c); + } else { + state = compute_follow_impl(state) & compute_reach_impl(c); + } + if (state == 0) { break; } + if (state & accept_mask) { + if constexpr (P == positional::BEGIN_END) { + cur_match = match_pair{start, pos + 1}; + } else { + cur_match = match_pair{-1, pos + 1}; + } + state = priority_kill(state, accept_mask); + } + } + if (cur_match) { return cur_match; } + } + return {}; + } + + // --- Unanchored: two-phase O(n) search --- + // + // Phase 1 (forward scan): inject first_set at every step; detect match END + // via state & accept_mask (any active seed reaching accept); greedy extension + // until state dies. + // + // inject_start_pos records the earliest position where seeds were injected + // since state was last 0. Phase 2 rescans from there to pinpoint the start. + // + // Phase 2 (BEGIN_END only): rescan from inject_start_pos, trying each valid + // start position with a single-seed NFA (mirrors the anchored inner loop). + // The first start whose NFA reaches accept gives the leftmost match. + // + // Cost: O(n) for phase 1; O(gap + match_length) per match for phase 2. + // Each character is processed at most twice overall. + + glushkov_state_t state = 0; + match_result cur_match{}; + + // Earliest position where seeds were injected since state was last 0. + int32_t inject_start_pos = start_pos; + auto inject_start_itr = begin; + + auto itr = begin; + for (int32_t pos = start_pos; itr.byte_offset() < size_bytes; ++pos, ++itr) { + // Fast skip when no active states and no match found yet. + if (state == 0 && !cur_match) { + if (startchar.has_value()) { + // Tight byte scan for the literal start character. + while (itr.byte_offset() < size_bytes && *itr != startchar.value()) { + ++itr; + ++pos; + } + if (itr.byte_offset() >= size_bytes) { break; } + } + // Record injection start for phase 2 rescan. + inject_start_pos = pos; + inject_start_itr = itr; + } + + char32_t const c = *itr; + auto const reach = compute_reach_impl(c); + + glushkov_state_t state_next = (state != 0) ? compute_follow_impl(state) : glushkov_state_t{0}; + + // Inject fresh starts unless in greedy-extension mode. + if (!cur_match) { state_next |= first_set; } + + state = state_next & reach; + + if (state == 0) { + if (cur_match) { break; } // greedy extension died + continue; + } + + // Detect match: any active state reaching accept. + if (state & accept_mask) { + if constexpr (P == positional::BEGIN_END) { + cur_match = match_pair{inject_start_pos, pos + 1}; // start is provisional + } else { + cur_match = match_pair{-1, pos + 1}; + } + // Greedy: continue extending. Fresh start injection stops via + // the !cur_match gate above. + state = priority_kill(state, accept_mask); + } + } + + if (!cur_match) { return {}; } + + // For END_ONLY (contains_re, matches_re), start position is unused. + if constexpr (P != positional::BEGIN_END) { return cur_match; } + + // --- Phase 2: find the earliest match start (BEGIN_END only) --- + // + // Rescan from inject_start_pos. For each position where first_set + // matches, run a single-seed NFA forward. The first start whose NFA + // reaches accept gives the leftmost match with greedy extension. + + auto rescan_itr = inject_start_itr; + for (int32_t s = inject_start_pos; s < cur_match->second; ++s, ++rescan_itr) { + char32_t const c = *rescan_itr; + auto const reach = compute_reach_impl(c); + glushkov_state_t seed = first_set & reach; + if (seed == 0) { continue; } + + // Check immediate accept (single-position pattern, e.g. `\d`). + int32_t greedy_end = -1; + if (seed & accept_mask) { + greedy_end = s + 1; + seed = priority_kill(seed, accept_mask); + } + + // Advance the single-seed NFA forward. + auto scan_itr = rescan_itr; + ++scan_itr; + for (int32_t p = s + 1; scan_itr.byte_offset() < size_bytes; ++p, ++scan_itr) { + seed = compute_follow_impl(seed) & compute_reach_impl(*scan_itr); + if (seed == 0) { break; } + if (seed & accept_mask) { + greedy_end = p + 1; + seed = priority_kill(seed, accept_mask); + } + } + + if (greedy_end >= 0) { return match_pair{s, greedy_end}; } + } + + // Fallback: should not reach here if phase 1 detected a match correctly. + return cur_match; +} + +__device__ inline void gkprog_device::store(void* buffer) const +{ + auto cache = static_cast(buffer); + for (uint32_t i = 0; i < static_cast(GLUSHKOV_ASCII_TABLE_SIZE); ++i) { + cache->reach_ascii[i] = _reach_ascii[i]; + } + for (uint32_t i = 0; i < shift_count; ++i) { + cache->shift_masks[i] = _shift_masks[i]; + cache->shift_amounts[i] = _shift_amounts[i]; + } + for (uint32_t i = 0; i < num_states; ++i) { + cache->exception_successors[i] = _exception_successors[i]; + } +} + +__device__ gkprog_device gkprog_device::load(gkprog_device const prog, void* buffer) +{ + auto result = gkprog_device(prog); + auto cache = static_cast(buffer); + result._reach_ascii = cache->reach_ascii.data(); + result._shift_amounts = cache->shift_amounts.data(); + result._shift_masks = cache->shift_masks.data(); + result._exception_successors = cache->exception_successors.data(); + return result; +} + +} // namespace cudf::strings::detail diff --git a/cpp/src/strings/regex/glushkov_regcomp.cpp b/cpp/src/strings/regex/glushkov_regcomp.cpp new file mode 100644 index 000000000000..cb740d0cb9af --- /dev/null +++ b/cpp/src/strings/regex/glushkov_regcomp.cpp @@ -0,0 +1,572 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "glushkov_regcomp.hpp" + +#include "strings/char_types/char_flags.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * @brief Glushkov NFA compiler: ε-elimination from a Thompson NFA. + * + * Algorithm overview + * ------------------ + * The Thompson NFA instructions form a directed graph where: + * - Character-consuming nodes (CHAR, ANY, ANYNL, CCLASS, NCCLASS) are the + * "positions" of Glushkov's construction. + * - ε-transition nodes (LBRA, RBRA, OR) are transparent: they are traversed + * during ε-closure without consuming input. + * - END is the accept node. + * + * For each position p we compute: + * follow(p) = ε_closure( p.next_id ) ∩ {char-consuming nodes} + * is_accept(p) = ε_closure( p.next_id ) reaches END + * + * The initial set (first_set) is built from ε_closure(startinst_id). + * + * Shift-and optimization (Hyperscan technique) + * -------------------------------------------- + * Positions are numbered by their instruction index order (low → high). + * Most follow transitions go from a smaller to a slightly larger index, i.e. + * they have a small positive "span" s = q - p. + * We store up to GLUSHKOV_MAX_SHIFTS such spans as shift masks so that the + * GPU can compute most of follow(state) with a handful of shift+OR operations. + * Remaining transitions (backward / large-span) are stored as per-position + * exception successor masks. + */ + +namespace cudf { +namespace strings { +namespace detail { + +namespace { + +/// True for instructions that consume a character (Glushkov positions). +bool is_char_consuming(int32_t const type) +{ + return type == CHAR || type == ANY || type == ANYNL || type == CCLASS || type == NCCLASS; +} + +/** + * @brief Returns true if ASCII character @p c matches the host-side character class @p cls. + * + * Mirrors reclass_device::is_match for ASCII input (codepoint == char value), + * so the same builtin flags and literal ranges produce identical results on host + * and device for characters in [0, 127]. + */ +bool host_reclass_match_ascii(reclass const& cls, uint8_t const c) +{ + for (auto const& lit : cls.literals) { + if (static_cast(c) >= lit.first && static_cast(c) <= lit.last) { + return true; + } + } + if (!cls.builtins) { return false; } + + // ASCII code points map 1:1 into g_character_codepoint_flags, the same table + // used by reclass_device::is_match on the device, so the flags looked up here + // match device-side IS_ALPHANUM/IS_SPACE/IS_DIGIT exactly (e.g. control + // characters 0x1c-0x1f are flagged as whitespace, same as \s on device). + uint8_t const fl = g_character_codepoint_flags[c]; + + if ((cls.builtins & CCLASS_W) && (c == '_' || IS_ALPHANUM(fl))) return true; + if ((cls.builtins & CCLASS_S) && IS_SPACE(fl)) return true; + if ((cls.builtins & CCLASS_D) && IS_DIGIT(fl)) return true; + if ((cls.builtins & NCCLASS_W) && (c != '\n' && c != '_' && !IS_ALPHANUM(fl))) return true; + if ((cls.builtins & NCCLASS_S) && !IS_SPACE(fl)) return true; + if ((cls.builtins & NCCLASS_D) && (c != '\n' && !IS_DIGIT(fl))) return true; + return false; +} + +/// True for zero-width assertion instructions (not handled by Glushkov path). +bool is_assertion(int32_t const type) +{ + return type == BOL || type == EOL || type == BOW || type == NBOW; +} + +// --------------------------------------------------------------------------- +// ε-closure traversal +// --------------------------------------------------------------------------- + +/** + * @brief Iterative (explicit-stack) depth-first ε-closure from @p start. + * + * Traversal order does not matter here: results are OR'd into bitmasks / + * boolean flags by the caller. + * + * @return tuple of: + * - char_positions : IDs of reachable character-consuming instructions. + * - is_accept : true if END is reachable. + * - has_assert : true if an assertion instruction is encountered. + */ +std::tuple, bool, bool> eps_closure(int32_t const start, + reprog const& prog, + std::unordered_set& seen) +{ + std::vector char_positions; + bool is_accept = false; + bool has_assert = false; + + std::vector stack{start}; + while (!stack.empty()) { + int32_t const inst_id = stack.back(); + stack.pop_back(); + if (inst_id < 0 || inst_id >= prog.insts_count()) continue; + if (!seen.insert(inst_id).second) continue; // already visited + + auto const& inst = prog.insts_data()[inst_id]; + switch (inst.type) { + // Character-consuming: record position, do NOT follow further + case CHAR: + case ANY: + case ANYNL: + case CCLASS: + case NCCLASS: char_positions.push_back(inst_id); break; + + // Accept + case END: is_accept = true; break; + + // ε-transitions: follow without consuming + case LBRA: + case RBRA: stack.push_back(inst.u2.next_id); break; + + case OR: + stack.push_back(inst.u2.left_id); + stack.push_back(inst.u1.right_id); + break; + + // Zero-width assertions: Glushkov path does not support these + case BOL: + case EOL: + case BOW: + case NBOW: has_assert = true; break; + + default: break; + } + } + return {std::move(char_positions), is_accept, has_assert}; +} + +/// Wrapper that supplies a fresh `seen` set for a standalone ε-closure. +/// @return tuple of (char_positions, is_accept, has_assert); see eps_closure. +std::tuple, bool, bool> eps_closure_from(int32_t const start, + reprog const& prog) +{ + std::unordered_set seen; + return eps_closure(start, prog, seen); +} + +// --------------------------------------------------------------------------- +// Thompson-priority frontier and conflict detection +// --------------------------------------------------------------------------- + +/// One item in an ordered ε-frontier: either a char-consuming position or END. +struct frontier_item { + enum kind_t : int32_t { CHAR_POS, ACCEPT }; + kind_t kind; + int32_t gpos; ///< Glushkov position index; valid only when kind == CHAR_POS +}; + +/** + * @brief Iterative ε-closure in Thompson priority order (right_id before left_id for OR). + * + * Uses an explicit work-stack instead of recursion so deeply nested patterns + * cannot overflow the host call stack. Records each distinct char-consuming + * position or END in first-visit order. + */ +void ordered_eps_frontier(int32_t const start, + reprog const& prog, + std::vector const& inst_to_pos, + std::unordered_set& seen, + std::vector& items) +{ + std::vector stack{start}; + while (!stack.empty()) { + int32_t const inst_id = stack.back(); + stack.pop_back(); + if (inst_id < 0 || inst_id >= prog.insts_count()) continue; + if (!seen.insert(inst_id).second) continue; + + auto const& inst = prog.insts_data()[inst_id]; + switch (inst.type) { + case CHAR: + case ANY: + case ANYNL: + case CCLASS: + case NCCLASS: items.push_back({frontier_item::CHAR_POS, inst_to_pos[inst_id]}); break; + + case END: items.push_back({frontier_item::ACCEPT, -1}); break; + + case LBRA: + case RBRA: stack.push_back(inst.u2.next_id); break; + + case OR: + // Thompson priority: right_id (first alternative) before left_id (second). + // Push left first so right is popped/visited first. + stack.push_back(inst.u2.left_id); + stack.push_back(inst.u1.right_id); + break; + + default: break; + } + } +} + +/** + * @brief Returns true if Glushkov position @p p matches ASCII character @p c. + */ +bool position_matches_char(gkprog const& gp, uint32_t const p, uint8_t const c) +{ + switch (gp.pos_insts[p].type) { + case CHAR: return gp.pos_insts[p].u1.c == static_cast(c); + case ANY: return (gp.pos_insts[p].u1.c == 'N') ? (c != '\n' && c != '\r') : (c != '\n'); + case ANYNL: return true; + case CCLASS: + case NCCLASS: { + bool const m = host_reclass_match_ascii(gp.classes[gp.pos_insts[p].u1.cls_id], c); + return (gp.pos_insts[p].type == CCLASS) ? m : !m; + } + default: return false; + } +} + +/** + * @brief Returns true if positions @p p and @p q can match a common character. + * + * Iterates all ASCII characters for an exact answer. For non-ASCII input the + * function is conservative: if either side is a wildcard type (ANY/ANYNL/ + * CCLASS/NCCLASS) it may match non-ASCII code points, so overlap is assumed. + */ +bool positions_chars_overlap(gkprog const& gp, uint32_t const p, uint32_t const q) +{ + for (int c = 0; c < GLUSHKOV_ASCII_TABLE_SIZE; ++c) { + if (position_matches_char(gp, p, static_cast(c)) && + position_matches_char(gp, q, static_cast(c))) { + return true; + } + } + // Non-ASCII: conservative overlap if both positions are non-CHAR types + bool const p_nonascii = + (gp.pos_insts[p].type != CHAR) || (gp.pos_insts[p].u1.c >= GLUSHKOV_ASCII_TABLE_SIZE); + bool const q_nonascii = + (gp.pos_insts[q].type != CHAR) || (gp.pos_insts[q].u1.c >= GLUSHKOV_ASCII_TABLE_SIZE); + if (p_nonascii && q_nonascii) { return true; } + // One side is a non-ASCII CHAR literal; if the other is a wildcard it might match + if (gp.pos_insts[p].type == CHAR && gp.pos_insts[p].u1.c >= GLUSHKOV_ASCII_TABLE_SIZE) { + return (gp.pos_insts[q].type != CHAR); + } + if (gp.pos_insts[q].type == CHAR && gp.pos_insts[q].u1.c >= GLUSHKOV_ASCII_TABLE_SIZE) { + return (gp.pos_insts[p].type != CHAR); + } + return false; +} + +/** + * @brief Returns true if the frontier contains a priority conflict that + * Glushkov's bit-order cannot represent. + * + * Two rules: + * Rule 1 – END before char: an ACCEPT item appears before the first CHAR_POS + * item in Thompson priority order → the pattern can empty-match in a way + * that priority_kill cannot handle correctly. + * Rule 2 – non-monotone gpos + char overlap: two CHAR_POS items appear with + * the higher-priority one at a larger gpos (inverted bit order), AND + * they can match a common character → priority_kill picks the wrong + * alternative when both are active. + */ +bool frontier_has_priority_conflict(std::vector const& items, gkprog const& gp) +{ + // Rule 1: ACCEPT before any CHAR_POS, but only when the frontier also + // contains at least one CHAR_POS. An ACCEPT-only frontier (the normal + // "end of pattern" case) is not a priority conflict. + bool seen_char = false; + bool accept_before_char = false; + for (auto const& item : items) { + if (item.kind == frontier_item::CHAR_POS) { + seen_char = true; + } else if (item.kind == frontier_item::ACCEPT && !seen_char) { + accept_before_char = true; + } + } + if (accept_before_char && seen_char) { return true; } + + // Rule 2: non-monotone gpos pair with character overlap + for (size_t i = 0; i < items.size(); ++i) { + if (items[i].kind != frontier_item::CHAR_POS) continue; + for (size_t j = i + 1; j < items.size(); ++j) { + if (items[j].kind != frontier_item::CHAR_POS) continue; + // items[i] has higher Thompson priority than items[j]. + // If items[i].gpos > items[j].gpos, bit order is inverted. + if (items[i].gpos > items[j].gpos) { + if (positions_chars_overlap( + gp, static_cast(items[i].gpos), static_cast(items[j].gpos))) { + return true; + } + } + } + } + return false; +} + +// --------------------------------------------------------------------------- +// Shift-and mask construction +// --------------------------------------------------------------------------- + +/** + * @brief Build shift masks and exception tables from the follow_table. + * + * For each pair (p, q) where q ∈ follow_table[p]: + * span = q - p + * • 1 ≤ span ≤ 63 (forward): candidate for a shift-mask slot. + * • otherwise : unconditional exception (backward / 0-span / >63). + * + * We select the GLUSHKOV_MAX_SHIFTS most-populated span values as shift slots. + * Any span that does not get a slot is demoted to the exception table. + */ +void build_shift_masks(gkprog& gp) +{ + // Collect per-span source-position bitmasks. Spans are bounded to [1, 63] + // (see the shift-slot check below), so a fixed array indexed by span suffices; + // Index 0 is unused. + std::array span_to_mask{}; + + for (uint32_t p = 0; p < gp.num_states; ++p) { + glushkov_state_t follow = gp.follow_table[p]; + while (follow) { + auto const q = std::countr_zero(follow); + follow &= follow - 1; + + int32_t const span = static_cast(q) - static_cast(p); + if (span >= 1 && span <= 63) { + span_to_mask[span] |= glushkov_state_t(1) << p; + } else { + // Unconditional exception + gp.exception_mask |= glushkov_state_t(1) << p; + gp.exception_successors[p] |= glushkov_state_t(1) << q; + } + } + } + + // Gather populated spans in ascending span order (deterministic input). + std::vector> span_list; + for (int32_t span = 1; span <= 63; ++span) { + if (span_to_mask[span]) { span_list.emplace_back(span, span_to_mask[span]); } + } + + // Sort candidate spans by population (descending) to pick the best slots. + // Break ties by smaller span so the selection is deterministic regardless of + // std::sort's (unspecified) ordering of equal elements. + std::sort(span_list.begin(), span_list.end(), [](auto const& a, auto const& b) { + auto const pa = std::popcount(a.second); + auto const pb = std::popcount(b.second); + return (pa != pb) ? (pa > pb) : (a.first < b.first); + }); + + // Use a local const for the loop bound so GCC's static analyzer can prove + // k < GLUSHKOV_MAX_SHIFTS at each array access (avoids -Wstringop-overflow + // false positive when the function is inlined in release builds). + uint32_t const shift_count = + static_cast(std::min(span_list.size(), GLUSHKOV_MAX_SHIFTS)); + gp.shift_count = shift_count; + + for (uint32_t k = 0; k < shift_count; ++k) { + gp.shift_amounts[k] = static_cast(span_list[k].first); + gp.shift_masks[k] = span_list[k].second; + } + + // Demote overflow spans to exceptions + for (size_t i = gp.shift_count; i < span_list.size(); ++i) { + glushkov_state_t src = span_list[i].second; + int32_t const s = span_list[i].first; + while (src) { + auto const p = std::countr_zero(src); + src &= src - 1; + auto const q = static_cast(static_cast(p) + s); + gp.exception_mask |= glushkov_state_t(1) << p; + gp.exception_successors[p] |= glushkov_state_t(1) << q; + } + } +} + +} // anonymous namespace + +// --------------------------------------------------------------------------- +// Public entry point +// --------------------------------------------------------------------------- + +std::unique_ptr build_glushkov_program(reprog const& prog) +{ + int32_t const num_insts = prog.insts_count(); + + // ---- Step 1: Reject ineligible patterns ---------------------------------- + // Glushkov cannot handle zero-width assertions + for (int32_t i = 0; i < num_insts; ++i) { + if (is_assertion(prog.insts_data()[i].type)) { return nullptr; } + } + + // ---- Step 2: Enumerate all character-consuming instruction IDs ---------- + std::vector char_insts; + char_insts.reserve(static_cast(num_insts)); + for (int32_t i = 0; i < num_insts; ++i) { + if (is_char_consuming(prog.insts_data()[i].type)) { char_insts.push_back(i); } + } + + if (static_cast(char_insts.size()) > GLUSHKOV_MAX_STATES) { return nullptr; } + if (char_insts.empty()) { return nullptr; } // degenerate (e.g. empty pattern) + + // Map instruction ID → Glushkov position index (0-based, left-to-right order) + std::vector inst_to_pos(static_cast(num_insts), -1); + for (int32_t idx = 0; std::cmp_less(idx, char_insts.size()); ++idx) { + inst_to_pos[char_insts[idx]] = idx; + } + + auto gp = std::make_unique(); + gp->num_states = static_cast(char_insts.size()); + + // ---- Step 3: Per-position character-matching descriptors ---------------- + for (uint32_t idx = 0; idx < gp->num_states; ++idx) { + int32_t const inst_id = char_insts[idx]; + auto const& inst = prog.insts_data()[inst_id]; + gp->pos_insts[idx] = inst; + } + + // Copy character class definitions (referenced by CCLASS/NCCLASS positions) + gp->classes.resize(static_cast(prog.classes_count())); + for (int32_t i = 0; i < prog.classes_count(); ++i) { + gp->classes[i] = prog.class_at(i); + } + + // ---- Step 4: first_set = ε_closure(startinst) ∩ {char-consuming} ------ + bool empty_matchable = false; + { + auto const [plist, is_accept, has_assert] = eps_closure_from(prog.get_start_inst(), prog); + if (has_assert) { return nullptr; } + for (int32_t iid : plist) { + int32_t const pos = inst_to_pos[iid]; + if (pos >= 0) { gp->first_set |= glushkov_state_t(1) << pos; } + } + empty_matchable = is_accept; + } + + // Empty-matchable patterns have ε-paths not represented as Glushkov positions. + // When the ε-path is the first alternative (e.g. `(|a)`), Thompson gives it + // highest priority (its END fires first), but Glushkov has no position to + // represent that priority. + if (empty_matchable) { return nullptr; } + + // ---- Step 5: follow_table + accept_mask for each position -------------- + for (uint32_t idx = 0; idx < gp->num_states; ++idx) { + int32_t const inst_id = char_insts[idx]; + int32_t const next_id = prog.insts_data()[inst_id].u2.next_id; + + auto const [follow_insts, is_accept, has_assert] = eps_closure_from(next_id, prog); + if (has_assert) { return nullptr; } + + for (int32_t fiid : follow_insts) { + int32_t const fpos = inst_to_pos[fiid]; + if (fpos >= 0) { gp->follow_table[idx] |= glushkov_state_t(1) << fpos; } + } + if (is_accept) { gp->accept_mask |= glushkov_state_t(1) << idx; } + } + + // ---- Step 5b: Detect Thompson priority conflicts ------------------------ + // Verify that Glushkov's bit-order priority agrees with Thompson's branch- + // priority order for every ε-frontier in the NFA. If a conflict is found + // (Rule 1: accept before char; Rule 2: inverted gpos pair with char overlap), + // the pattern cannot be correctly simulated by Glushkov → fall back. + { + std::unordered_set seen; + std::vector items; + + // Check start frontier + ordered_eps_frontier(prog.get_start_inst(), prog, inst_to_pos, seen, items); + if (frontier_has_priority_conflict(items, *gp)) { return nullptr; } + + // Check follow frontier for each char-consuming position + for (uint32_t idx = 0; idx < gp->num_states; ++idx) { + int32_t const inst_id = char_insts[idx]; + int32_t const next_id = prog.insts_data()[inst_id].u2.next_id; + seen.clear(); + items.clear(); + ordered_eps_frontier(next_id, prog, inst_to_pos, seen, items); + if (frontier_has_priority_conflict(items, *gp)) { return nullptr; } + } + } + + // ---- Step 6: Shift-and masks ------------------------------------------ + build_shift_masks(*gp); + + // ---- Step 7: Precompute ASCII reach table ------------------------------- + // For each ASCII character c (0-127), compute the bitmask of positions that + // match c. This converts the O(num_states) on-the-fly loop in + // glushkov_compute_reach into a single array lookup for ASCII input. + for (int32_t c = 0; c < GLUSHKOV_ASCII_TABLE_SIZE; ++c) { + for (uint32_t p = 0; p < gp->num_states; ++p) { + bool matches = false; + switch (gp->pos_insts[p].type) { + case CHAR: matches = (gp->pos_insts[p].u1.c == static_cast(c)); break; + case ANY: { + // 'N' = EXT_NEWLINE: reject all ASCII newlines (\n, \r) + // default: reject only \n (matches Thompson NFA behaviour) + matches = (gp->pos_insts[p].u1.c == 'N') ? (c != '\n' && c != '\r') : (c != '\n'); + break; + } + case ANYNL: matches = true; break; + case CCLASS: + case NCCLASS: { + bool const m = host_reclass_match_ascii(gp->classes[gp->pos_insts[p].u1.cls_id], + static_cast(c)); + matches = (gp->pos_insts[p].type == CCLASS) ? m : !m; + break; + } + default: break; + } + if (matches) { gp->reach_ascii[c] |= glushkov_state_t(1) << p; } + } + } + + // ---- Step 8: First-character skip optimization ------------------------- + // When every position in first_set is a CHAR instruction for the same + // literal, record that character so glushkov_find can use a tight byte-scan + // (like Thompson NFA's find_char) instead of a reach-table lookup per byte. + if (gp->first_set != 0) { + glushkov_state_t fs = gp->first_set; + char32_t common = 0; + bool all_char = true; + bool seen_first = false; + while (fs) { + auto const p = std::countr_zero(fs); + fs &= fs - 1; + if (gp->pos_insts[p].type != CHAR) { + all_char = false; + break; + } + if (!seen_first) { + common = gp->pos_insts[p].u1.c; + seen_first = true; + } else if (gp->pos_insts[p].u1.c != common) { + all_char = false; + break; + } + } + if (all_char) { gp->startchar = common; } + } + + return gp; +} + +} // namespace detail +} // namespace strings +} // namespace cudf diff --git a/cpp/src/strings/regex/glushkov_regcomp.hpp b/cpp/src/strings/regex/glushkov_regcomp.hpp new file mode 100644 index 000000000000..32c306520c25 --- /dev/null +++ b/cpp/src/strings/regex/glushkov_regcomp.hpp @@ -0,0 +1,113 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include "common.cuh" +#include "regcomp.h" + +#include + +#include +#include +#include +#include + +/** + * @file glushkov_regcomp.h + * @brief Host-side Glushkov NFA construction via ε-elimination from Thompson NFA. + * + * Glushkov's NFA (position automaton) has two key advantages over Thompson's NFA: + * 1. No ε-transitions → no runtime ε-closure overhead per character. + * 2. States are naturally represented as bit-positions → bit-parallel simulation + * on GPU (Hyperscan "shift-and" technique, USENIX ATC'19). + * + * The compiled program uses a 64-bit bitmask to represent the set of active + * positions. Each step costs only a few bitwise operations, compared to the + * list-management overhead of the Thompson NFA. + * + * Limitations (patterns that fall back to Thompson NFA): + * - Zero-width assertions: ^ $ \\b \\B + * - More than GLUSHKOV_MAX_STATES character-consuming positions + * - capture-group extraction (uses Thompson NFA always) + */ + +namespace cudf { +namespace strings { +namespace detail { + +/** + * @brief Host-side compiled Glushkov NFA program. + * + * Produced by @ref build_glushkov_program from an already-compiled + * Thompson NFA (@ref reprog). All arrays are indexed by position ID + * (0 .. num_states-1), which corresponds to the left-to-right order of + * character-consuming instructions in the Thompson NFA. + */ +struct gkprog { + uint32_t num_states{}; ///< Number of character-consuming positions + glushkov_state_t first_set{}; ///< Bitmask: positions reachable before first character + glushkov_state_t accept_mask{}; ///< Bitmask: positions whose match completes the pattern + + /// follow_table[p] = bitmask of positions that can immediately follow position p. + std::array follow_table{}; + + // ---- Per-position character-matching descriptors ---- + std::array pos_insts{}; + + // ---- Shift-and transition optimization (Hyperscan-style) ---- + // + // For each pair (p, q) where q ∈ follow_table[p] and 1 ≤ (q-p) ≤ 63: + // shift span s = q - p + // If s is one of the chosen shift_amounts[k]: set bit p in shift_masks[k] + // + // GPU execution: + // follow(state) ≈ OR_k( (state & shift_masks[k]) << shift_amounts[k] ) + // + exception transitions for states in exception_mask + + /// shift_masks[k]: source positions whose follow includes a span-k transition. + std::array shift_masks{}; + /// The corresponding shift amounts (ascending, 1-based). + std::array shift_amounts{}; + /// Number of valid entries in shift_masks / shift_amounts. + uint32_t shift_count{}; + + /// When true, every position in first_set is a CHAR instruction for the + /// same literal character. Enables a tight first-character skip in + /// glushkov_find (mirrors Thompson NFA's find_char optimization). + cuda::std::optional startchar{}; + + /// Bitmask of positions with at least one "exception" (non-shift) transition. + glushkov_state_t exception_mask{}; + /// exception_successors[p] = union of all non-shift successor positions for position p. + /// Non-zero only when bit p is set in exception_mask. + std::array exception_successors{}; + + /// Character class definitions (copied from the Thompson NFA reprog). + std::vector classes; + + /// Precomputed reach bitmask for each ASCII character (0–127). + /// reach_ascii[c] = bitmask of positions that match ASCII character c. + /// Built by build_glushkov_program after the follow table is complete. + /// Non-ASCII characters are handled on-the-fly at match time. + std::array reach_ascii{}; +}; + +/** + * @brief Attempt to compile a Glushkov NFA from a fully compiled Thompson NFA. + * + * Returns nullptr when the pattern cannot be handled by the Glushkov path: + * - Pattern contains zero-width assertions (BOL, EOL, BOW, NBOW). + * - Pattern has more than GLUSHKOV_MAX_STATES character-consuming positions. + * - Pattern is nullable (matches the empty string): priority semantics cannot + * be faithfully represented without an ε-position for the empty match. + * + * @param prog Compiled Thompson NFA (after reprog::finalize()). + * @return Host-side Glushkov program, or nullptr on failure. + */ +std::unique_ptr build_glushkov_program(reprog const& prog); + +} // namespace detail +} // namespace strings +} // namespace cudf diff --git a/cpp/src/strings/regex/regex.cuh b/cpp/src/strings/regex/regex.cuh index 913aa4b20b2d..d5216200a41b 100644 --- a/cpp/src/strings/regex/regex.cuh +++ b/cpp/src/strings/regex/regex.cuh @@ -1,10 +1,11 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once -#include "strings/regex/regcomp.h" +#include "common.cuh" +#include "regcomp.h" #include #include @@ -12,8 +13,6 @@ #include -#include -#include #include #include @@ -23,37 +22,13 @@ namespace cudf { namespace strings { namespace detail { -/** - * @brief Template type used on `find` to specify desired position values in returned match_result - */ -enum class positional : int8_t { - BEGIN_END = 0, /// both begin and end positions are returned - END_ONLY = 1, /// only the end position is returned -}; - template struct reljunk; -using match_pair = cuda::std::pair; -using match_result = cuda::std::optional; - constexpr int32_t MAX_SHARED_MEM = 2048; ///< Memory size for storing prog instruction data constexpr std::size_t MAX_WORKING_MEM = 0x01'FFFF'FFFF; ///< Memory size for state data constexpr int32_t MINIMUM_THREADS = 256; // Minimum threads for computing working memory -/** - * @brief Regex class stored on the device and executed by reprog_device. - * - * This class holds the unique data for any regex CCLASS instruction. - */ -struct alignas(16) reclass_device { - int32_t builtins{}; - int32_t count{}; - reclass_range const* literals{}; - - __device__ inline bool is_match(char32_t const ch, uint8_t const* flags) const; -}; - class reprog; /** @@ -68,7 +43,7 @@ class reprog; * Once the buffer is allocated, pass the device pointer to the `set_working_memory()` * member function. */ -class reprog_device { +class alignas(16) reprog_device { public: reprog_device() = delete; ~reprog_device() = default; @@ -119,12 +94,14 @@ class reprog_device { } /** - * @brief Returns the size needed for working memory for the given thread count. + * @brief Return the size in bytes needed for working memory to + * execute insts_count instructions in parallel over num_threads threads. * - * @param num_threads Number of threads to be executed in parallel - * @return Size of working memory in bytes + * @param num_threads Number of parallel threads (usually one per string in a strings column) + * @param insts_count Number of instructions from a compiled regex pattern + * @return Number of bytes needed for working memory */ - [[nodiscard]] std::size_t working_memory_size(int32_t num_threads) const; + static std::size_t compute_working_memory_size(int32_t num_threads, int32_t insts_count); /** * @brief Compute working memory for the given thread count with a maximum size. @@ -231,6 +208,11 @@ class reprog_device { */ [[nodiscard]] __device__ inline reclass_device get_class(int32_t id) const; + /** + * @brief Returns the size needed for working memory for the given thread count for this instance + */ + [[nodiscard]] std::size_t working_memory_size(int32_t num_threads) const; + /** * @brief Executes the regex pattern on the given string. */ @@ -265,23 +247,13 @@ class reprog_device { reinst const* _insts{}; // array of regex instructions int32_t const* _startinst_ids{}; // array of start instruction ids reclass_device const* _classes{}; // array of regex classes + bool _empty_match_possible{}; // true if the regex can match an empty string - std::size_t _prog_size{}; // total size of this instance - void* _buffer{}; // working memory buffer - int32_t _thread_count{}; // threads available in working memory - bool _empty_match_possible{}; // true if the regex can match an empty string + std::size_t _prog_size{}; // total size of this instance + void* _buffer{}; // working memory buffer + int32_t _thread_count{}; // threads available in working memory }; -/** - * @brief Return the size in bytes needed for working memory to - * execute insts_count instructions in parallel over num_threads threads. - * - * @param num_threads Number of parallel threads (usually one per string in a strings column) - * @param insts_count Number of instructions from a compiled regex pattern - * @return Number of bytes needed for working memory - */ -std::size_t compute_working_memory_size(int32_t num_threads, int32_t insts_count); - /** * @brief Converts a match_pair from character positions to byte positions */ diff --git a/cpp/src/strings/regex/regex.inl b/cpp/src/strings/regex/regex.inl index dac4a8494ef3..191787ace6ca 100644 --- a/cpp/src/strings/regex/regex.inl +++ b/cpp/src/strings/regex/regex.inl @@ -128,51 +128,6 @@ struct reljunk { } }; -/** - * @brief Check for supported new-line characters - * - * '\n, \r, \u0085, \u2028, or \u2029' - */ -CUDF_HOST_DEVICE constexpr bool is_newline(char32_t const ch) -{ - return (ch == '\n' || ch == '\r' || ch == 0x00c285 || ch == 0x00e280a8 || ch == 0x00e280a9); -} - -/** - * @brief Utility to check a specific character against this class instance. - * - * @param ch A 4-byte UTF-8 character. - * @param codepoint_flags Used for mapping a character to type for builtin classes. - * @return true if the character matches - */ -__device__ __forceinline__ bool reclass_device::is_match(char32_t const ch, - uint8_t const* codepoint_flags) const -{ - for (int i = 0; i < count; ++i) { - auto const literal = literals[i]; - if ((ch >= literal.first) && (ch <= literal.last)) { return true; } - } - - if (!builtins) return false; - uint32_t codept = utf8_to_codepoint(ch); - if (codept > 0x00'FFFF) return false; - int8_t fl = codepoint_flags[codept]; - if ((builtins & CCLASS_W) && ((ch == '_') || IS_ALPHANUM(fl))) // \w - return true; - if ((builtins & CCLASS_S) && IS_SPACE(fl)) // \s - return true; - if ((builtins & CCLASS_D) && IS_DIGIT(fl)) // \d - return true; - if ((builtins & NCCLASS_W) && ((ch != '\n') && (ch != '_') && !IS_ALPHANUM(fl))) // \W - return true; - if ((builtins & NCCLASS_S) && !IS_SPACE(fl)) // \S - return true; - if ((builtins & NCCLASS_D) && ((ch != '\n') && !IS_DIGIT(fl))) // \D - return true; - // - return false; -} - __device__ __forceinline__ reinst reprog_device::get_inst(int32_t id) const { return _insts[id]; } __device__ __forceinline__ reclass_device reprog_device::get_class(int32_t id) const diff --git a/cpp/src/strings/regex/regex_program.cpp b/cpp/src/strings/regex/regex_program.cpp index 1c0dd99acfc6..15827f0d2fbb 100644 --- a/cpp/src/strings/regex/regex_program.cpp +++ b/cpp/src/strings/regex/regex_program.cpp @@ -3,10 +3,13 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include "glushkov_regcomp.hpp" #include "regex_program_impl.h" +#include #include +#include #include #include @@ -31,6 +34,8 @@ regex_program::regex_program(std::string_view pattern, regex_flags flags, captur _impl( std::make_unique(detail::reprog::create_from(pattern, flags, capture))) { + auto const* env = std::getenv("LIBCUDF_DISABLE_GLUSHKOV"); + if (env == nullptr) { _impl->glushkov_prog = detail::build_glushkov_program(_impl->prog); } } std::string regex_program::pattern() const { return _pattern; } @@ -45,7 +50,8 @@ int32_t regex_program::groups_count() const { return _impl->prog.groups_count(); std::size_t regex_program::compute_working_memory_size(int32_t num_strings) const { - return detail::compute_working_memory_size(num_strings, instructions_count()); + if (_impl->glushkov_prog.get() != nullptr) { return 0; } + return detail::reprog_device::compute_working_memory_size(num_strings, instructions_count()); } std::pair regex_program::get_literal_fast_path() const diff --git a/cpp/src/strings/regex/regex_program_impl.h b/cpp/src/strings/regex/regex_program_impl.h index 9a2d7d588607..e3891d0bade1 100644 --- a/cpp/src/strings/regex/regex_program_impl.h +++ b/cpp/src/strings/regex/regex_program_impl.h @@ -1,39 +1,56 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once +#include "glushkov.cuh" +#include "glushkov_regcomp.hpp" #include "regcomp.h" #include "regex.cuh" #include +#include #include +#include + namespace cudf { namespace strings { /** * @brief Implementation object for regex_program * - * It encapsulates internal reprog object used for building its device equivalent + * It encapsulates internal reprog object used for building its device equivalent. */ struct regex_program::regex_program_impl { detail::reprog prog; + std::unique_ptr glushkov_prog; regex_program_impl(detail::reprog const& p) : prog(p) {} - regex_program_impl(detail::reprog&& p) : prog(p) {} + regex_program_impl(detail::reprog&& p) : prog(std::move(p)) {} // TODO: There will be other options added here in the future to handle issues // 10852 and possibly others like 11979 }; struct regex_device_builder { + static bool glushkov_fast_path_supported(regex_program const& p) + { + return p._impl->glushkov_prog.get() != nullptr; + } + static auto create_prog_device(regex_program const& p, rmm::cuda_stream_view stream) { return detail::reprog_device::create(p._impl->prog, stream); } + + static auto create_gkprog_device(regex_program const& p, rmm::cuda_stream_view stream) + { + CUDF_EXPECTS(glushkov_fast_path_supported(p), "fast-path not supported"); + return detail::gkprog_device::create(*p._impl->glushkov_prog, stream); + } }; } // namespace strings diff --git a/cpp/src/strings/regex/regexec.cpp b/cpp/src/strings/regex/regexec.cpp index 85041f32a0a8..8c23c15f8679 100644 --- a/cpp/src/strings/regex/regexec.cpp +++ b/cpp/src/strings/regex/regexec.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -108,7 +108,7 @@ std::unique_ptr> reprog_devic (h_prog.compute_match_flags() == cudf::strings::detail::match_flags::EMPTY_MATCH); // copy flat prog to device memory - cudf::detail::cuda_memcpy(*d_buffer, h_buffer, stream); + cudf::detail::cuda_memcpy_async(*d_buffer, h_buffer, stream); // build deleter to cleanup device memory auto deleter = [d_buffer](reprog_device* t) { @@ -116,6 +116,7 @@ std::unique_ptr> reprog_devic delete d_buffer; }; + stream.synchronize(); // wait for h_buffer to finish copying return std::unique_ptr>(d_prog, deleter); } @@ -155,7 +156,7 @@ int32_t reprog_device::compute_shared_memory_size() const return _prog_size < MAX_SHARED_MEM ? static_cast(_prog_size) : 0; } -std::size_t compute_working_memory_size(int32_t num_threads, int32_t insts_count) +std::size_t reprog_device::compute_working_memory_size(int32_t num_threads, int32_t insts_count) { return relist::alloc_size(insts_count, num_threads) * 2; } diff --git a/cpp/src/strings/regex/utilities.cuh b/cpp/src/strings/regex/utilities.cuh index c6bf18b1dc21..f9f3280e53a4 100644 --- a/cpp/src/strings/regex/utilities.cuh +++ b/cpp/src/strings/regex/utilities.cuh @@ -1,10 +1,11 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once +#include "strings/regex/glushkov.cuh" #include "strings/regex/regex.cuh" #include @@ -29,13 +30,13 @@ namespace detail { constexpr auto regex_launch_kernel_block_size = 256; -template -CUDF_KERNEL void for_each_kernel(ForEachFunction fn, reprog_device const d_prog, size_type size) +template +CUDF_KERNEL void for_each_kernel(ForEachFunction fn, ProgDevice const d_prog, size_type size) { extern __shared__ u_char shmem[]; if (threadIdx.x == 0) { d_prog.store(shmem); } __syncthreads(); - auto const s_prog = reprog_device::load(d_prog, shmem); + auto s_prog = ProgDevice::load(d_prog, shmem); auto const thread_idx = cudf::detail::grid_1d::global_thread_id(); auto const stride = s_prog.thread_count(); @@ -46,9 +47,9 @@ CUDF_KERNEL void for_each_kernel(ForEachFunction fn, reprog_device const d_prog, } } -template +template void launch_for_each_kernel(ForEachFunction fn, - reprog_device& d_prog, + ProgDevice& d_prog, size_type size, rmm::cuda_stream_view stream) { @@ -64,16 +65,16 @@ void launch_for_each_kernel(ForEachFunction fn, CUDF_CUDA_TRY(cudaGetLastError()); } -template +template CUDF_KERNEL void transform_kernel(TransformFunction fn, - reprog_device const d_prog, + ProgDevice const d_prog, OutputType* d_output, size_type size) { extern __shared__ u_char shmem[]; if (threadIdx.x == 0) { d_prog.store(shmem); } __syncthreads(); - auto const s_prog = reprog_device::load(d_prog, shmem); + auto s_prog = ProgDevice::load(d_prog, shmem); auto const thread_idx = cudf::detail::grid_1d::global_thread_id(); auto const stride = s_prog.thread_count(); @@ -84,9 +85,9 @@ CUDF_KERNEL void transform_kernel(TransformFunction fn, } } -template +template void launch_transform_kernel(TransformFunction fn, - reprog_device& d_prog, + ProgDevice& d_prog, OutputType* d_output, size_type size, rmm::cuda_stream_view stream) @@ -103,9 +104,9 @@ void launch_transform_kernel(TransformFunction fn, CUDF_CUDA_TRY(cudaGetLastError()); } -template +template auto make_strings_children(SizeAndExecuteFunction size_and_exec_fn, - reprog_device& d_prog, + ProgDevice& d_prog, size_type strings_count, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) diff --git a/cpp/src/strings/replace/replace_re.cu b/cpp/src/strings/replace/replace_re.cu index d1500bc4a65d..ee3c3b8997f7 100644 --- a/cpp/src/strings/replace/replace_re.cu +++ b/cpp/src/strings/replace/replace_re.cu @@ -37,7 +37,8 @@ struct replace_regex_fn { char* d_chars{}; cudf::detail::input_offsetalator d_offsets; - __device__ void operator()(size_type const idx, reprog_device const prog, int32_t const prog_idx) + template + __device__ void operator()(size_type const idx, ProgDevice const prog, int32_t const prog_idx) { if (d_strings.is_null(idx)) { if (!d_chars) { d_sizes[idx] = 0; } @@ -113,8 +114,17 @@ std::unique_ptr replace_re(strings_column_view const& input, auto const d_prog = regex_device_builder::create_prog_device(prog, stream); auto const d_strings = column_device_view::create(input.parent(), stream); - auto [offsets_column, chars] = make_strings_children( - replace_regex_fn{*d_strings, d_repl, maxrepl}, *d_prog, input.size(), stream, mr); + auto [offsets_column, chars] = [&] { + if (regex_device_builder::glushkov_fast_path_supported(prog)) { + auto d_prog = regex_device_builder::create_gkprog_device(prog, stream); + return make_strings_children( + replace_regex_fn{*d_strings, d_repl, maxrepl}, *d_prog, input.size(), stream, mr); + } else { + auto d_prog = regex_device_builder::create_prog_device(prog, stream); + return make_strings_children( + replace_regex_fn{*d_strings, d_repl, maxrepl}, *d_prog, input.size(), stream, mr); + } + }(); return make_strings_column(input.size(), std::move(offsets_column), diff --git a/cpp/src/strings/search/findall.cu b/cpp/src/strings/search/findall.cu index b9bd2b28a059..f0c8fe2a4fed 100644 --- a/cpp/src/strings/search/findall.cu +++ b/cpp/src/strings/search/findall.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -38,7 +38,8 @@ struct findall_fn { size_type const* d_offsets; string_index_pair* d_indices; - __device__ void operator()(size_type const idx, reprog_device const prog, int32_t const prog_idx) + template + __device__ void operator()(size_type const idx, ProgDevice const prog, int32_t const prog_idx) { if (d_strings.is_null(idx)) { return; } auto const d_str = d_strings.element(idx); @@ -49,7 +50,7 @@ struct findall_fn { auto itr = d_str.begin(); while (itr.position() <= nchars) { - auto const match = prog.find(prog_idx, d_str, itr); + auto const match = prog.template find(prog_idx, d_str, itr); if (!match) { break; } auto const d_result = string_from_match(*match, d_str, itr); @@ -116,24 +117,41 @@ std::unique_ptr findall(strings_column_view const& input, auto const d_strings = column_device_view::create(input.parent(), stream); - // create device object from regex_program - auto d_prog = regex_device_builder::create_prog_device(prog, stream); - - // Create lists offsets column - auto const sizes = count_matches(*d_strings, *d_prog, stream, mr); - auto [offsets, total_matches] = cudf::detail::make_offsets_child_column( - sizes->view().begin(), sizes->view().end(), stream, mr); - auto const d_offsets = offsets->view().data(); + // Builds the lists offsets column and the matches from a single, already-built + // device regex program (avoids a redundant device program build for the count pass). + auto const build_matches = [&](auto& d_prog, auto make_functor) { + auto const sizes = count_matches(*d_strings, d_prog, input.size(), stream, mr); + auto [offsets, total_matches] = + cudf::detail::make_offsets_child_column(sizes->view().template begin(), + sizes->view().template end(), + stream, + mr); + auto const d_offsets = offsets->view().template data(); + + rmm::device_uvector indices(total_matches, stream); + launch_for_each_kernel(make_functor(d_offsets, indices.data()), d_prog, input.size(), stream); + return std::pair(std::move(offsets), std::move(indices)); + }; + + auto [offsets, indices] = [&]() { + if (groups == 1) { + auto d_prog = regex_device_builder::create_prog_device(prog, stream); + return build_matches(*d_prog, [&](size_type const* d_offsets, string_index_pair* d_indices) { + return one_capture_fn{*d_strings, d_offsets, d_indices}; + }); + } + if (regex_device_builder::glushkov_fast_path_supported(prog)) { + auto d_prog = regex_device_builder::create_gkprog_device(prog, stream); + return build_matches(*d_prog, [&](size_type const* d_offsets, string_index_pair* d_indices) { + return findall_fn{*d_strings, d_offsets, d_indices}; + }); + } + auto d_prog = regex_device_builder::create_prog_device(prog, stream); + return build_matches(*d_prog, [&](size_type const* d_offsets, string_index_pair* d_indices) { + return findall_fn{*d_strings, d_offsets, d_indices}; + }); + }(); - // Build strings column of the matches - rmm::device_uvector indices(total_matches, stream); - if (groups == 1) { - launch_for_each_kernel( - one_capture_fn{*d_strings, d_offsets, indices.data()}, *d_prog, input.size(), stream); - } else { - launch_for_each_kernel( - findall_fn{*d_strings, d_offsets, indices.data()}, *d_prog, input.size(), stream); - } auto strings_output = make_strings_column(indices.begin(), indices.end(), stream, mr); // Build the lists column from the offsets and the strings @@ -148,14 +166,15 @@ namespace { struct find_re_fn { column_device_view d_strings; + template __device__ size_type operator()(size_type const idx, - reprog_device const prog, + ProgDevice const prog, int32_t const thread_idx) const { if (d_strings.is_null(idx)) { return 0; } auto const d_str = d_strings.element(idx); - auto const result = prog.find(thread_idx, d_str, d_str.begin()); + auto const result = prog.template find(thread_idx, d_str, d_str.begin()); return result.has_value() ? result.value().first : -1; } }; @@ -175,9 +194,15 @@ std::unique_ptr find_re(strings_column_view const& input, if (input.is_empty()) { return results; } auto d_results = results->mutable_view().data(); - auto d_prog = regex_device_builder::create_prog_device(prog, stream); auto const d_strings = column_device_view::create(input.parent(), stream); - launch_transform_kernel(find_re_fn{*d_strings}, *d_prog, d_results, input.size(), stream); + + if (regex_device_builder::glushkov_fast_path_supported(prog)) { + auto d_prog = regex_device_builder::create_gkprog_device(prog, stream); + launch_transform_kernel(find_re_fn{*d_strings}, *d_prog, d_results, input.size(), stream); + } else { + auto d_prog = regex_device_builder::create_prog_device(prog, stream); + launch_transform_kernel(find_re_fn{*d_strings}, *d_prog, d_results, input.size(), stream); + } return results; } diff --git a/cpp/src/strings/split/split_re.cu b/cpp/src/strings/split/split_re.cu index 8b1d23140ab8..ecbdf3e85840 100644 --- a/cpp/src/strings/split/split_re.cu +++ b/cpp/src/strings/split/split_re.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -50,7 +50,8 @@ struct token_reader_fn { cudf::detail::input_offsetalator const d_token_offsets; string_index_pair* d_tokens; - __device__ void operator()(size_type const idx, reprog_device const prog, int32_t const prog_idx) + template + __device__ void operator()(size_type const idx, ProgDevice const prog, int32_t const prog_idx) { if (d_strings.is_null(idx)) { return; } auto const d_str = d_strings.element(idx); @@ -68,7 +69,7 @@ struct token_reader_fn { auto itr = d_str.begin(); auto last_pos = itr; while (itr.position() <= nchars) { - auto const match = prog.find(prog_idx, d_str, itr); + auto const match = prog.template find(prog_idx, d_str, itr); if (!match) { break; } auto const start_pos = cuda::std::get<0>(match_positions_to_bytes(*match, d_str, last_pos)); @@ -111,24 +112,29 @@ struct token_reader_fn { /** * @brief Call regex to split each input string into tokens. * + * Also computes the match counts (token offsets) against the given, already-built + * device regex program so a second device program build is not needed for this step. + * * @param d_strings Strings to split - * @param d_prog Regex to evaluate against each string + * @param d_prog Device regex program to evaluate against each string * @param direction Whether tokens are generated forwards or backwards. * @param maxsplit The maximum number of tokens for each split. - * @param counts The number of tokens in each string * @param stream CUDA stream used for kernel launches. */ +template std::pair, std::unique_ptr> generate_tokens( column_device_view const& d_strings, - reprog_device& d_prog, + ProgDevice& d_prog, split_direction direction, size_type maxsplit, - column_view const& counts, rmm::cuda_stream_view stream) { auto const strings_count = d_strings.size(); auto const max_tokens = maxsplit > 0 ? maxsplit : std::numeric_limits::max(); - auto const d_counts = counts.data(); + + auto const counts = count_matches( + d_strings, d_prog, strings_count, stream, cudf::get_current_device_resource_ref()); + auto const d_counts = counts->view().template data(); // convert match counts to token offsets auto map_fn = cuda::proclaim_return_type( @@ -152,6 +158,24 @@ std::pair, std::unique_ptr> gener return std::pair(std::move(tokens), std::move(offsets)); } +/** + * @brief Builds the device regex program once and generates the split tokens with it. + */ +std::pair, std::unique_ptr> generate_tokens( + column_device_view const& d_strings, + regex_program const& prog, + split_direction direction, + size_type maxsplit, + rmm::cuda_stream_view stream) +{ + if (regex_device_builder::glushkov_fast_path_supported(prog)) { + auto d_prog = regex_device_builder::create_gkprog_device(prog, stream); + return generate_tokens(d_strings, *d_prog, direction, maxsplit, stream); + } + auto d_prog = regex_device_builder::create_prog_device(prog, stream); + return generate_tokens(d_strings, *d_prog, direction, maxsplit, stream); +} + /** * @brief Returns string pair for the specified column for each string in `d_strings` * @@ -192,19 +216,12 @@ std::unique_ptr split_re(strings_column_view const& input, return std::make_unique
(std::move(results)); } - // create device object from regex_program - auto d_prog = regex_device_builder::create_prog_device(prog, stream); - auto d_strings = column_device_view::create(input.parent(), stream); - // count the number of delimiters matched in each string - auto const counts = - count_matches(*d_strings, *d_prog, stream, cudf::get_current_device_resource_ref()); - - // get the split tokens from the input column; this also converts the counts into offsets - auto [tokens, offsets] = - generate_tokens(*d_strings, *d_prog, direction, maxsplit, counts->view(), stream); - auto const d_offsets = cudf::detail::offsetalator_factory::make_input_iterator(offsets->view()); + // get the split tokens from the input column; this also computes the match counts + // and converts them into offsets, using a single device regex program build + auto [tokens, offsets] = generate_tokens(*d_strings, prog, direction, maxsplit, stream); + auto const d_offsets = cudf::detail::offsetalator_factory::make_input_iterator(offsets->view()); // the output column count is the maximum number of tokens generated for any input string auto const columns_count = thrust::transform_reduce( @@ -256,18 +273,12 @@ std::unique_ptr split_record_re(strings_column_view const& input, CUDF_EXPECTS(!prog.pattern().empty(), "Parameter pattern must not be empty"); auto const strings_count = input.size(); + auto d_strings = column_device_view::create(input.parent(), stream); - // create device object from regex_program - auto d_prog = regex_device_builder::create_prog_device(prog, stream); - - auto d_strings = column_device_view::create(input.parent(), stream); - - // count the number of delimiters matched in each string - auto counts = count_matches(*d_strings, *d_prog, stream, mr); + // get the split tokens from the input column; this also computes the match counts + // and converts them into offsets, using a single device regex program build + auto [tokens, offsets] = generate_tokens(*d_strings, prog, direction, maxsplit, stream); - // get the split tokens from the input column; this also converts the counts into offsets - auto [tokens, offsets] = - generate_tokens(*d_strings, *d_prog, direction, maxsplit, counts->view(), stream); CUDF_EXPECTS(tokens.size() < static_cast(std::numeric_limits::max()), "Size of output exceeds the column size limit", std::overflow_error); diff --git a/cpp/tests/strings/contains_tests.cpp b/cpp/tests/strings/contains_tests.cpp index 6b338ec502d0..bbc285e85add 100644 --- a/cpp/tests/strings/contains_tests.cpp +++ b/cpp/tests/strings/contains_tests.cpp @@ -574,6 +574,32 @@ TEST_F(StringsContainsTests, NestedQuantifier) CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); } +TEST_F(StringsContainsTests, DeeplyNestedNoStackOverflow) +{ + // A pattern with very deep group nesting but only a single character-consuming + // position. It passes the Glushkov state cap (1 position) yet forces the + // compiler's ε-closure traversal to walk a very long ε-chain. + int32_t const depth = 100000; + std::string pattern; + pattern.reserve(static_cast(depth) * 3 + 1 + static_cast(depth)); + for (int32_t i = 0; i < depth; ++i) { + pattern += "(?:"; + } + pattern += "a"; + for (int32_t i = 0; i < depth; ++i) { + pattern += ")"; + } + + auto input = cudf::test::strings_column_wrapper({"a", "b", "aaa", ""}); + auto sv = cudf::strings_column_view(input); + + // The pattern matches any row containing 'a'. + cudf::test::fixed_width_column_wrapper expected({true, false, true, false}); + auto prog = cudf::strings::regex_program::create(pattern); + auto results = cudf::strings::contains_re(sv, *prog); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); +} + TEST_F(StringsContainsTests, QuantifierErrors) { EXPECT_THROW(cudf::strings::regex_program::create("^+"), cudf::logic_error); @@ -1293,3 +1319,25 @@ TEST_F(StringsContainsTests, LazyQuantifiers) CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); } } + +TEST_F(StringsContainsTests, WhitespaceControlCharacters) +{ + // \x1c-\x1f (file/group/record/unit separators) are classified as whitespace + // by cudf's codepoint flags table and should match \s. + auto input = + cudf::test::strings_column_wrapper({"a\x1c" + "b", + "a\x1d" + "b", + "a\x1e" + "b", + "a\x1f" + "b", + "a b", + "ab"}); + auto sv = cudf::strings_column_view(input); + auto prog = cudf::strings::regex_program::create("a\\sb"); + auto results = cudf::strings::contains_re(sv, *prog); + cudf::test::fixed_width_column_wrapper expected({1, 1, 1, 1, 1, 0}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); +}