Skip to content
Merged
Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
5b3e7e7
[FEA] Support dictionary-encoded types in Transforms
lamarrr Feb 25, 2026
ebb4419
[FEA] Enhance dictionary_element with key member and improve document…
lamarrr Feb 25, 2026
eb6b24d
fix style check
lamarrr Feb 25, 2026
74db1a8
code review changes
lamarrr Feb 25, 2026
c298ff5
code review changes
lamarrr Feb 25, 2026
70cc26b
Update cpp/include/cudf/column/column_device_view_base.cuh
lamarrr Feb 25, 2026
15e8cc0
update
lamarrr Feb 27, 2026
3553432
update
lamarrr Feb 27, 2026
7f95088
Merge branch 'main' into transform-dict-support
lamarrr Feb 27, 2026
9c4a2d4
Merge branch 'main' into transform-dict-support
lamarrr Feb 27, 2026
10d2ebd
Update cpp/include/cudf/column/column_device_view_base.cuh
lamarrr Feb 27, 2026
a8215ab
Merge branch 'main' into transform-dict-support
lamarrr Mar 3, 2026
9cea60a
Update cpp/include/cudf/column/column_device_view_base.cuh
lamarrr Mar 4, 2026
352db7f
Merge branch 'main' into transform-dict-support
lamarrr Mar 4, 2026
de00c9a
code review changes
lamarrr Mar 13, 2026
c654a31
code review changes
lamarrr Mar 13, 2026
d9ef52f
Merge remote-tracking branch 'upstream/main' into transform-dict-support
lamarrr May 28, 2026
ea23ea9
Merge remote-tracking branch 'upstream/main' into transform-dict-support
lamarrr Jun 29, 2026
71c227c
rebase on latest cudf + code review changes
lamarrr Jun 30, 2026
6c10d2e
Update cpp/include/cudf/column/column_child_offsets.hpp
lamarrr Jun 30, 2026
98e9e3e
removed dead code for helper function
lamarrr Jun 30, 2026
374efec
code review changes
lamarrr Jun 30, 2026
c92c495
Merge branch 'main' into transform-dict-support
lamarrr Jul 2, 2026
2ad4e8c
Add test for empty column handling in transform_extended function
lamarrr Jul 2, 2026
1746912
Merge branch 'main' into transform-dict-support
lamarrr Jul 2, 2026
66f1002
Fix expected output in transform_extended test case
lamarrr Jul 2, 2026
0d92dda
Add documentation for dictionary indices and keys column indices
lamarrr Jul 3, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions cpp/include/cudf/column/column_child_offsets.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION.
Comment thread
lamarrr marked this conversation as resolved.
Resolved
* SPDX-License-Identifier: Apache-2.0
*/
#pragma once

#include <cudf/types.hpp>
#include <cudf/utilities/export.hpp>

namespace CUDF_EXPORT cudf {

static constexpr size_type offsets_column_index = 0; ///< Child index of the offsets column

static constexpr size_type dictionary_indices_column_index =
0; ///< Child index of the dictionary offsets column

static constexpr size_type dictionary_keys_column_index =
1; ///< Child index of the dictionary key column

} // namespace CUDF_EXPORT cudf
61 changes: 59 additions & 2 deletions cpp/include/cudf/column/column_device_view_base.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
*/
#pragma once

#include <cudf/column/column_child_offsets.h>
#include <cudf/detail/offsets_iterator.cuh>
#include <cudf/fixed_point/fixed_point.hpp>
#include <cudf/strings/string_view.cuh>
Expand Down Expand Up @@ -63,15 +64,45 @@ struct nullate {
};
};

/**
* @brief A type tag to specify that a column should be treated as a dictionary column
*
* @tparam IndexType The type of the dictionary indices
* @tparam KeyType The type of the dictionary keys
*/
template <typename IndexType, typename KeyType>
requires(is_index_type<IndexType>() && is_relationally_comparable<KeyType, KeyType>())
struct dictionary_element {
using index_type = IndexType; ///< The type of the dictionary indices
using key_type = KeyType; ///< The type of the dictionary keys

key_type key{}; ///< The dictionary key for this element
};

/**
* @brief A type trait to determine if a type is a dictionary encoded type.
* @tparam T The type to check
*/
template <typename T>
inline constexpr bool is_dictionary_encoded = false;

/**
* @brief A type trait to determine if a type is a dictionary encoded type.
* @tparam IndexType The type of the dictionary indices
* @tparam KeyType The type of the dictionary keys
*/
template <typename IndexType, typename KeyType>
inline constexpr bool is_dictionary_encoded<dictionary_element<IndexType, KeyType>> = true;

namespace detail {
/**
* @brief An immutable, non-owning view of device data as a column of elements
* that is trivially copyable and usable in CUDA device code.
*/
class alignas(16) column_device_view_base {
public:
// TODO: merge this offsets column index with `strings_column_view::offsets_column_index`
static constexpr size_type offsets_column_index{0}; ///< Child index of the offsets column
static constexpr size_type offsets_column_index =
cudf::offsets_column_index; ///< Child index of the offsets column

column_device_view_base() = delete;
~column_device_view_base() = default;
Expand Down Expand Up @@ -447,6 +478,32 @@ class alignas(16) column_device_view_core : public detail::column_device_view_ba
return T{scaled_integer<rep>{data<rep>()[element_index], scale}};
}

/**
* @brief Returns a decoded copy of the element at the specified index.
*
* If the element at the specified index is NULL, i.e.,
* `is_null(element_index) == true`, then any attempt to use the result will
* lead to undefined behavior.
*
* This function accounts for the offset.
*
* This function does not participate in overload resolution if `is_dictionary_encoded<T>` is
* false.
*
* @tparam T The element type, e.g., `dictionary_element<int32_t, float>` for a dictionary column
* with `int32_t` indices and `float` keys
* @param element_index Position of the desired element
* @return The element at the specified index
*/
template <typename T, CUDF_ENABLE_IF(is_dictionary_encoded<T>)>
[[nodiscard]] __device__ decltype(auto) element(size_type element_index) const noexcept
{
auto const& indices = child(dictionary_indices_column_index);
auto const& keys = child(dictionary_keys_column_index);
auto const index = indices.template element<typename T::index_type>(element_index);
return keys.template element<typename T::key_type>(index);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}

/**
* @brief Returns the specified child
*
Expand Down
7 changes: 4 additions & 3 deletions cpp/include/cudf/dictionary/dictionary_column_view.hpp
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2020-2024, NVIDIA CORPORATION.
* SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION.
* SPDX-License-Identifier: Apache-2.0
*/
#pragma once

#include <cudf/column/column_child_offsets.h>
#include <cudf/column/column_view.hpp>

/**
Expand Down Expand Up @@ -52,9 +53,9 @@ class dictionary_column_view : private column_view {
dictionary_column_view& operator=(dictionary_column_view&&) = default;

/// Index of the indices column of the dictionary column
static constexpr size_type indices_column_index{0};
static constexpr size_type indices_column_index = cudf::dictionary_indices_column_index;
/// Index of the keys column of the dictionary column
static constexpr size_type keys_column_index{1};
static constexpr size_type keys_column_index = cudf::dictionary_keys_column_index;
Comment thread
lamarrr marked this conversation as resolved.
Resolved

using column_view::has_nulls;
using column_view::is_empty;
Expand Down
59 changes: 54 additions & 5 deletions cpp/src/jit/helpers.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,14 @@

#include "helpers.hpp"

#include <cudf/column/column_device_view_base.cuh>
#include <cudf/detail/nvtx/ranges.hpp>
#include <cudf/utilities/type_dispatcher.hpp>

#include <jit/cache.hpp>

#include <format>

namespace cudf {
namespace jit {

Expand Down Expand Up @@ -111,13 +115,58 @@ std::vector<std::string> input_type_names(
return names;
}

input_reflection reflect_input(std::variant<column_view, scalar_column_view> const& input)
namespace {

std::string get_jit_element_type_name_impl(column_view const& view);

struct jit_element_type_name_fn {
template <typename T>
requires(is_fixed_width<T>() || std::is_same_v<T, cudf::string_view>)
std::string operator()(column_view const& view) const
{
return type_to_name(view.type());
}

template <typename T>
requires(std::is_same_v<T, cudf::dictionary32>)
std::string operator()(column_view const& view) const
{
return std::format(
"cudf::dictionary_element<{}, {}>",
get_jit_element_type_name_impl(view.child(cudf::dictionary_indices_column_index)),
get_jit_element_type_name_impl(view.child(cudf::dictionary_keys_column_index)));
}

template <typename T>
requires(!is_fixed_width<T>() && !std::is_same_v<T, cudf::string_view> &&
!std::is_same_v<T, cudf::dictionary32>)
Comment thread
mhaseeb123 marked this conversation as resolved.
Resolved
std::string operator()(column_view const& view) const
{
CUDF_FAIL("Unsupported type for JIT compilation: " + type_to_name(view.type()));
}
Comment thread
lamarrr marked this conversation as resolved.
Resolved
};

std::string get_jit_element_type_name_impl(column_view const& view)
{
auto get_type_name = [](auto const& var) {
return std::visit([](auto& a) { return type_to_name(a.type()); }, var);
};
return cudf::type_dispatcher(view.type(), jit_element_type_name_fn{}, view);
}

std::string get_jit_element_type_name(column_view const& view)
{
return get_jit_element_type_name_impl(view);
}

return input_reflection{get_type_name(input), std::holds_alternative<scalar_column_view>(input)};
std::string get_jit_element_type_name(scalar_column_view const& view)
{
return get_jit_element_type_name(view.as_column_view());
}

} // namespace

input_reflection reflect_input(std::variant<column_view, scalar_column_view> const& input)
{
auto type_name = std::visit([](auto& a) { return get_jit_element_type_name(a); }, input);
return input_reflection{type_name, std::holds_alternative<scalar_column_view>(input)};
}

std::vector<input_reflection> reflect_inputs(
Expand Down
14 changes: 8 additions & 6 deletions cpp/src/transform/transform.cu
Original file line number Diff line number Diff line change
Expand Up @@ -427,12 +427,14 @@ void perform_checks(std::optional<size_type> in_row_size, data_type output_type,
return std::visit([](auto const& col) { return col.type(); }, in);
};

CUDF_EXPECTS(
std::all_of(thrust::make_transform_iterator(inputs.begin(), get_type),
thrust::make_transform_iterator(inputs.end(), get_type),
[](data_type t) { return is_fixed_width(t) || (t.id() == type_id::STRING); }),
"Transforms only support input of fixed-width or string types",
std::invalid_argument);
CUDF_EXPECTS(std::all_of(thrust::make_transform_iterator(inputs.begin(), get_type),
thrust::make_transform_iterator(inputs.end(), get_type),
[](data_type t) {
return is_fixed_width(t) || (t.id() == type_id::STRING) ||
cudf::is_dictionary(t);
Comment thread
lamarrr marked this conversation as resolved.
Resolved
Comment thread
lamarrr marked this conversation as resolved.
Resolved
}),
"Transforms only support input of fixed-width, string, and dictionary types",
std::invalid_argument);

check_row_size(in_row_size, inputs);
}
Expand Down
84 changes: 84 additions & 0 deletions cpp/tests/transform/integration/unary_transform_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
#include <cudf_test/type_lists.hpp>

#include <cudf/detail/iterator.cuh>
#include <cudf/dictionary/encode.hpp>
#include <cudf/transform.hpp>

namespace transformation {
Expand Down Expand Up @@ -313,6 +314,89 @@ __device__ inline void f(cudf::timestamp_us* output, cudf::timestamp_us input)
test_udf<dtype>(cuda, op, data_init, 500, cudf::udf_source_type::CUDA);
}

TEST_F(UnaryOperationIntegrationTest, Transform_DictionaryString)
Comment thread
lamarrr marked this conversation as resolved.
Resolved
Comment thread
lamarrr marked this conversation as resolved.
Resolved
{
std::string const cuda =
R"***(
__device__ inline void decode(cudf::string_view * output, cudf::string_view input){
*output = input;
})***";

// non-nullable
{
auto a =
cudf::test::strings_column_wrapper{
"eee", "aaa", "ddd", "bbb", "ccc", "ccc", "ccc", "eee", "aaa"}
.release();

auto a_encoded = cudf::dictionary::encode(a->view());

cudf::transform_input inputs[] = {*a_encoded};

auto out =
cudf::transform_extended(inputs, cuda, cudf::data_type{cudf::type_id::STRING}, false);
Comment thread
lamarrr marked this conversation as resolved.
Resolved

CUDF_TEST_EXPECT_COLUMNS_EQUAL(out->view(), a->view());
}

// nullable
{
auto a =
cudf::test::strings_column_wrapper{
{"eee", "aaa", "ddd", "bbb", "ccc", "ccc", "ccc", "eee", "aaa"},
{true, true, true, false, true, true, true, true, true}}
.release();

auto a_encoded = cudf::dictionary::encode(a->view());

cudf::transform_input inputs[] = {*a_encoded};

auto out =
cudf::transform_extended(inputs, cuda, cudf::data_type{cudf::type_id::STRING}, false);
Comment thread
lamarrr marked this conversation as resolved.
Resolved

CUDF_TEST_EXPECT_COLUMNS_EQUAL(out->view(), a->view());
}
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

TEST_F(UnaryOperationIntegrationTest, Transform_DictionaryFloat)
Comment thread
lamarrr marked this conversation as resolved.
Resolved
{
std::string const cuda =
R"***(
__device__ inline void decode(float * output, float input){
*output = input;
})***";

// non-nullable
{
auto a = cudf::test::fixed_width_column_wrapper<float>(
{1.0F, 2.0F, 3.0F, 4.0F, 5.0F, 5.0F, 5.0F, 1.0F, 2.0F})
.release();
auto a_encoded = cudf::dictionary::encode(a->view());
cudf::transform_input inputs[] = {*a_encoded};

auto out =
cudf::transform_extended(inputs, cuda, cudf::data_type{cudf::type_id::FLOAT32}, false);

CUDF_TEST_EXPECT_COLUMNS_EQUAL(out->view(), a->view());
}

// nullable
{
auto a = cudf::test::fixed_width_column_wrapper<float>(
{{1.0F, 2.0F, 3.0F, 4.0F, 5.0F, 5.0F, 5.0F, 1.0F, 2.0F},
{true, true, true, true, true, false, true, true, true}})
.release();

auto a_encoded = cudf::dictionary::encode(a->view());
cudf::transform_input inputs[] = {*a_encoded};

auto out =
cudf::transform_extended(inputs, cuda, cudf::data_type{cudf::type_id::FLOAT32}, false);

CUDF_TEST_EXPECT_COLUMNS_EQUAL(out->view(), a->view());
}
}

struct TernaryOperationTest : public cudf::test::BaseFixture {};

TEST_F(TernaryOperationTest, TransformWithScalar)
Expand Down
Loading