From 89197281a1b1469df29efe5cc8403023aea61957 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 20 Apr 2026 23:38:18 +0000 Subject: [PATCH 01/15] support ANSI SQL operators --- cpp/include/cudf/ast/detail/jit_operators.cuh | 1478 +++++++++++++++++ cpp/include/cudf/opcode.hpp | 105 ++ 2 files changed, 1583 insertions(+) create mode 100644 cpp/include/cudf/ast/detail/jit_operators.cuh create mode 100644 cpp/include/cudf/opcode.hpp diff --git a/cpp/include/cudf/ast/detail/jit_operators.cuh b/cpp/include/cudf/ast/detail/jit_operators.cuh new file mode 100644 index 000000000000..819dd0016561 --- /dev/null +++ b/cpp/include/cudf/ast/detail/jit_operators.cuh @@ -0,0 +1,1478 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include + +#include +#include +#include + +namespace CUDF_EXPORT cudf { +namespace jit { + +enum errc : int { + SUCCESS = 0, + ARITHMETIC_OVERFLOW = 1, + ARITHMETIC_UNDERFLOW = 2, + DIVISION_BY_ZERO = 3 +}; + +namespace operators { + +template +using optional = cuda::std::optional; + +template +struct promoted_t; + +template <> +struct promoted_t { + using type = int16_t; +}; + +template <> +struct promoted_t { + using type = uint16_t; +}; + +template <> +struct promoted_t { + using type = int32_t; +}; + +template <> +struct promoted_t { + using type = uint32_t; +}; + +template <> +struct promoted_t { + using type = int64_t; +}; + +template <> +struct promoted_t { + using type = uint64_t; +}; + +template <> +struct promoted_t { + using type = __int128; +}; + +template <> +struct promoted_t { + using type = unsigned __int128; +}; + +template +using promoted = typename promoted_t::type; + +template +__device__ inline errc abs(T* out, T const* a) +{ + *out = (*a < 0) ? -*a : *a; + return errc::SUCCESS; +} + +template +__device__ inline errc abs(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + abs(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc add(T* out, T const* a, T const* b) +{ + *out = (*a + *b); + return errc::SUCCESS; +} + +template +__device__ inline errc add(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + add(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc arccos(T* out, T const* a); + +template <> +__device__ inline errc arccos(float* out, float const* a) +{ + *out = ::acosf(*a); + return errc::SUCCESS; +} + +template <> +__device__ inline errc arccos(double* out, double const* a) +{ + *out = ::acos(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc arccos(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + arccos(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc arccosh(T* out, T const* a); + +template <> +__device__ inline errc arccosh(float* out, float const* a) +{ + *out = ::acoshf(*a); + return errc::SUCCESS; +} + +template <> +__device__ inline errc arccosh(double* out, double const* a) +{ + *out = ::acosh(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc arccosh(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + arccosh(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc arcsin(T* out, T const* a); + +template <> +__device__ inline errc arcsin(float* out, float const* a) +{ + *out = ::asinf(*a); + return errc::SUCCESS; +} + +template <> +__device__ inline errc arcsin(double* out, double const* a) +{ + *out = ::asin(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc arcsin(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + arcsin(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc arcsinh(T* out, T const* a); + +template <> +__device__ inline errc arcsinh(float* out, float const* a) +{ + *out = ::asinhf(*a); + return errc::SUCCESS; +} + +template <> +__device__ inline errc arcsinh(double* out, double const* a) +{ + *out = ::asinh(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc arcsinh(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + arcsinh(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc arctan(T* out, T const* a); + +template <> +__device__ inline errc arctan(float* out, float const* a) +{ + *out = ::atanf(*a); + return errc::SUCCESS; +} + +template <> +__device__ inline errc arctan(double* out, double const* a) +{ + *out = ::atan(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc arctan(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + arctan(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc arctanh(T* out, T const* a); + +template <> +__device__ inline errc arctanh(float* out, float const* a) +{ + *out = ::atanhf(*a); + return errc::SUCCESS; +} + +template <> +__device__ inline errc arctanh(double* out, double const* a) +{ + *out = ::atanh(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc arctanh(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + arctanh(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc bit_and(T* out, T const* a, T const* b) +{ + *out = (*a & *b); + return errc::SUCCESS; +} + +template +__device__ inline errc bit_and(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + bit_and(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc bit_invert(T* out, T const* a) +{ + *out = ~(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc bit_invert(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + bit_invert(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc bit_or(T* out, T const* a, T const* b) +{ + *out = (*a | *b); + return errc::SUCCESS; +} + +template +__device__ inline errc bit_or(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + bit_or(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc bit_xor(T* out, T const* a, T const* b) +{ + *out = (*a ^ *b); + return errc::SUCCESS; +} + +template +__device__ inline errc bit_xor(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + bit_xor(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc cast_to_i32(int32_t* out, T const* a) +{ + *out = static_cast(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc cast_to_i32(optional* out, optional const* a) +{ + if (a->is_valid()) { + int32_t r; + cast_to_i32(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc cast_to_i64(int64_t* out, T const* a) +{ + *out = static_cast(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc cast_to_i64(optional* out, optional const* a) +{ + if (a->is_valid()) { + int64_t r; + cast_to_i64(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc cast_to_u32(uint32_t* out, T const* a) +{ + *out = static_cast(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc cast_to_u32(optional* out, optional const* a) +{ + if (a->is_valid()) { + uint32_t r; + cast_to_u32(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc cast_to_u64(uint64_t* out, T const* a) +{ + *out = static_cast(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc cast_to_u64(optional* out, optional const* a) +{ + if (a->is_valid()) { + uint64_t r; + cast_to_u64(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc cast_to_f32(float* out, T const* a) +{ + *out = static_cast(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc cast_to_f32(optional* out, optional const* a) +{ + if (a->is_valid()) { + float r; + cast_to_f32(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc cast_to_f64(double* out, T const* a) +{ + *out = static_cast(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc cast_to_f64(optional* out, optional const* a) +{ + if (a->is_valid()) { + double r; + cast_to_f64(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc cbrt(T* out, T const* a); + +template <> +__device__ inline errc cbrt(float* out, float const* a) +{ + *out = ::cbrtf(*a); + return errc::SUCCESS; +} + +template <> +__device__ inline errc cbrt(double* out, double const* a) +{ + *out = ::cbrt(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc cbrt(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + cbrt(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc ceil(T* out, T const* a); + +template <> +__device__ inline errc ceil(float* out, float const* a) +{ + *out = ::ceilf(*a); + return errc::SUCCESS; +} + +template <> +__device__ inline errc ceil(double* out, double const* a) +{ + *out = ::ceil(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc ceil(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + ceil(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc cos(T* out, T const* a); + +template <> +__device__ inline errc cos(float* out, float const* a) +{ + *out = ::cosf(*a); + return errc::SUCCESS; +} + +template <> +__device__ inline errc cos(double* out, double const* a) +{ + *out = ::cos(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc cos(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + cos(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc cosh(T* out, T const* a); + +template <> +__device__ inline errc cosh(float* out, float const* a) +{ + *out = ::coshf(*a); + return errc::SUCCESS; +} + +template <> +__device__ inline errc cosh(double* out, double const* a) +{ + *out = ::cosh(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc cosh(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + cosh(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc div(T* out, T const* a, T const* b) +{ + *out = (*a / *b); + return errc::SUCCESS; +} + +template +__device__ inline errc div(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + div(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc equal(bool* out, T const* a, T const* b) +{ + *out = (*a == *b); + return errc::SUCCESS; +} + +template +__device__ inline errc equal(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + bool r; + equal(&r, &a->value(), &b->value()); + *out = r; + } else if (a->is_null() && b->is_null()) { + *out = true; + } else { + *out = false; + } + return errc::SUCCESS; +} + +template +__device__ inline errc exp(T* out, T const* a); + +template <> +__device__ inline errc exp(float* out, float const* a) +{ + *out = ::expf(*a); + return errc::SUCCESS; +} + +template <> +__device__ inline errc exp(double* out, double const* a) +{ + *out = ::exp(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc exp(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + exp(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc floor(T* out, T const* a); + +template <> +__device__ inline errc floor(float* out, float const* a) +{ + *out = ::floorf(*a); + return errc::SUCCESS; +} + +template <> +__device__ inline errc floor(double* out, double const* a) +{ + *out = ::floor(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc floor(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + floor(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc greater(bool* out, T const* a, T const* b) +{ + *out = (*a > *b); + return errc::SUCCESS; +} + +template +__device__ inline errc greater(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + bool r; + greater(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = false; + } + return errc::SUCCESS; +} + +template +__device__ inline errc greater_equal(bool* out, T const* a, T const* b) +{ + *out = (*a >= *b); + return errc::SUCCESS; +} + +template +__device__ inline errc greater_equal(optional* out, + optional const* a, + optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + bool r; + greater_equal(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = false; + } + return errc::SUCCESS; +} + +template +__device__ inline errc identity(T* out, T const* a) +{ + *out = *a; + return errc::SUCCESS; +} + +template +__device__ inline errc identity(optional* out, optional const* a) +{ + *out = *a; + return errc::SUCCESS; +} + +template +__device__ inline errc is_null(bool* out, T const* a) +{ + *out = false; + return errc::SUCCESS; +} + +template +__device__ inline errc is_null(optional* out, optional const* a) +{ + *out = a->is_null(); + return errc::SUCCESS; +} + +template +__device__ inline errc less(bool* out, T const* a, T const* b) +{ + *out = (*a < *b); + return errc::SUCCESS; +} + +template +__device__ inline errc less(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + bool r; + less(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = false; + } + return errc::SUCCESS; +} + +template +__device__ inline errc less_equal(bool* out, T const* a, T const* b) +{ + *out = (*a <= *b); + return errc::SUCCESS; +} + +template +__device__ inline errc less_equal(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + bool r; + less_equal(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = false; + } + return errc::SUCCESS; +} + +template +__device__ inline errc log(T* out, T const* a); + +template <> +__device__ inline errc log(float* out, float const* a) +{ + *out = ::logf(*a); + return errc::SUCCESS; +} + +template <> +__device__ inline errc log(double* out, double const* a) +{ + *out = ::log(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc log(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + log(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc logical_and(T* out, T const* a, T const* b) +{ + *out = (*a && *b); + return errc::SUCCESS; +} + +template +__device__ inline errc logical_and(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + logical_and(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc logical_or(T* out, T const* a, T const* b) +{ + *out = (*a || *b); + return errc::SUCCESS; +} + +template +__device__ inline errc logical_or(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + logical_or(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc mod(T* out, T const* a, T const* b) +{ + *out = (*a % *b); + return errc::SUCCESS; +} + +template <> +__device__ inline errc mod(float* out, float const* a, float const* b) +{ + *out = ::fmodf(*a, *b); + return errc::SUCCESS; +} + +template <> +__device__ inline errc mod(double* out, double const* a, double const* b) +{ + *out = ::fmod(*a, *b); + return errc::SUCCESS; +} + +template +__device__ inline errc mod(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + mod(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc mul(T* out, T const* a, T const* b) +{ + *out = (*a * *b); + return errc::SUCCESS; +} + +template +__device__ inline errc mul(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + mul(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc null_equal(bool* out, T const* a, T const* b) +{ + *out = (*a == *b); + return errc::SUCCESS; +} + +template +__device__ inline errc null_equal(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + *out = (*(*a) == *(*b)); + } else if (a->is_null() && b->is_null()) { + *out = true; + } else { + *out = false; + } + return errc::SUCCESS; +} + +template +__device__ inline errc null_logical_and(T* out, T const* a, T const* b) +{ + *out = (*a && *b); + return errc::SUCCESS; +} + +template +__device__ inline errc null_logical_and(optional* out, + optional const* a, + optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + bool r; + null_logical_and(&r, &a->value(), &b->value()); + *out = r; + } else if (a->is_null() && b->is_null()) { + *out = nullopt; + } else { + if (a->is_valid() ? *(*a) : *(*b)) { + *out = nullopt; + } else { + *out = false; + } + } + return errc::SUCCESS; +} + +template +__device__ inline errc null_logical_or(T* out, T const* a, T const* b) +{ + *out = (*a || *b); + return errc::SUCCESS; +} + +template +__device__ inline errc null_logical_or(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + bool r; + null_logical_or(&r, &a->value(), &b->value()); + *out = r; + } else if (a->is_null() && b->is_null()) { + *out = nullopt; + } else { + if (a->is_valid() ? *(*a) : *(*b)) { + *out = true; + } else { + *out = nullopt; + } + } + return errc::SUCCESS; +} + +template +__device__ inline errc pow(T* out, T const* a, T const* b); + +template <> +__device__ inline errc pow(float* out, float const* a, float const* b) +{ + *out = ::powf(*a, *b); + return errc::SUCCESS; +} + +template <> +__device__ inline errc pow(double* out, double const* a, double const* b) +{ + *out = ::pow(*a, *b); + return errc::SUCCESS; +} + +template +__device__ inline errc pow(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + pow(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc pymod(T* out, T const* a, T const* b) +{ + *out = (*a % *b + *b) % *b; + return errc::SUCCESS; +} + +template <> +__device__ inline errc pymod(float* out, float const* a, float const* b) +{ + *out = ::fmodf(::fmodf(*a, *b) + *b, *b); + return errc::SUCCESS; +} + +template <> +__device__ inline errc pymod(double* out, double const* a, double const* b) +{ + *out = ::fmod(::fmod(*a, *b) + *b, *b); + return errc::SUCCESS; +} + +template +__device__ inline errc pymod(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + pymod(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc rint(T* out, T const* a); + +template <> +__device__ inline errc rint(float* out, float const* a) +{ + *out = ::rintf(*a); + return errc::SUCCESS; +} + +template <> +__device__ inline errc rint(double* out, double const* a) +{ + *out = ::rint(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc rint(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + rint(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc sin(T* out, T const* a); + +template <> +__device__ inline errc sin(float* out, float const* a) +{ + *out = ::sinf(*a); + return errc::SUCCESS; +} + +template <> +__device__ inline errc sin(double* out, double const* a) +{ + *out = ::sin(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc sin(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + sin(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc sinh(T* out, T const* a); + +template <> +__device__ inline errc sinh(float* out, float const* a) +{ + *out = ::sinhf(*a); + return errc::SUCCESS; +} + +template <> +__device__ inline errc sinh(double* out, double const* a) +{ + *out = ::sinh(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc sinh(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + sinh(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc sub(T* out, T const* a, T const* b) +{ + *out = *a - *b; + return errc::SUCCESS; +} + +template +__device__ inline errc sub(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + sub(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc tanh(T* out, T const* a); + +template <> +__device__ inline errc tanh(float* out, float const* a) +{ + *out = ::tanhf(*a); + return errc::SUCCESS; +} + +template <> +__device__ inline errc tanh(double* out, double const* a) +{ + *out = ::tanh(*a); + return errc::SUCCESS; +} + +template +__device__ inline errc tanh(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + tanh(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +template +__device__ inline errc if_else(T* out, + bool const* condition, + T const* true_value, + T const* false_value) +{ + *out = *condition ? *true_value : *false_value; + return errc::SUCCESS; +} + +template +__device__ inline errc if_else(optional* out, + optional const* condition, + optional const* true_value, + optional const* false_value) +{ + if (condition->is_valid() && true_value->is_valid() && false_value->is_valid()) { + if_else(&out->value(), &condition->value(), &true_value->value(), &false_value->value()); + } else { + *out = nullopt; + } + return errc::SUCCESS; +} + +namespace detail { + +template +__device__ inline errc ansi_add_unsigned(T* out, T const* a, T const* b) +{ + using P = promoted; + auto r = static_cast

(*a) + static_cast

(*b); + if (r > static_cast

(cuda::std::numeric_limits::max())) { return errc::ARITHMETIC_OVERFLOW; } + *out = static_cast(r); + return errc::SUCCESS; +} + +template +__device__ inline errc ansi_add_signed(T* out, T const* a, T const* b) +{ + using P = promoted; + auto r = static_cast

(*a) + static_cast

(*b); + if (r > static_cast

(cuda::std::numeric_limits::max()) || + r < static_cast

(cuda::std::numeric_limits::min())) { + return errc::ARITHMETIC_OVERFLOW; + } + *out = static_cast(r); + return errc::SUCCESS; +} + +template +__device__ inline errc ansi_sub_unsigned(T* out, T const* a, T const* b) +{ + if (*a < *b) { return errc::ARITHMETIC_UNDERFLOW; } + auto r = *a - *b; + *out = static_cast(r); + return errc::SUCCESS; +} + +template +__device__ inline errc ansi_sub_signed(T* out, T const* a, T const* b) +{ + using P = promoted; + auto r = static_cast

(*a) - static_cast

(*b); + if (r > static_cast

(cuda::std::numeric_limits::max()) || + r < static_cast

(cuda::std::numeric_limits::min())) { + return errc::ARITHMETIC_UNDERFLOW; + } + *out = static_cast(r); + return errc::SUCCESS; +} + +template +__device__ inline errc ansi_mul_unsigned(T* out, T const* a, T const* b) +{ + using P = promoted; + auto r = static_cast

(*a) * static_cast

(*b); + if (r > static_cast

(cuda::std::numeric_limits::max())) { return errc::ARITHMETIC_OVERFLOW; } + *out = static_cast(r); + return errc::SUCCESS; +} + +template +__device__ inline errc ansi_mul_signed(T* out, T const* a, T const* b) +{ + using P = promoted; + auto r = static_cast

(*a) * static_cast

(*b); + if (r > static_cast

(cuda::std::numeric_limits::max()) || + r < static_cast

(cuda::std::numeric_limits::min())) { + return errc::ARITHMETIC_OVERFLOW; + } + *out = static_cast(r); + return errc::SUCCESS; +} + +template +__device__ inline errc ansi_div_unsigned(T* out, T const* a, T const* b) +{ + if (*b == 0) { return errc::DIVISION_BY_ZERO; } + *out = static_cast(static_cast(*a) / static_cast(*b)); + return errc::SUCCESS; +} + +template +__device__ inline errc ansi_div_signed(T* out, T const* a, T const* b) +{ + if (*b == 0) { return errc::DIVISION_BY_ZERO; } + if (*a == cuda::std::numeric_limits::min() && *b == -1) { return errc::ARITHMETIC_OVERFLOW; } + *out = static_cast(static_cast(*a) / static_cast(*b)); + return errc::SUCCESS; +} + +template +__device__ inline errc ansi_div_float(T* out, T const* a, T const* b) +{ + using P = promoted; + if (*b == 0) { return errc::DIVISION_BY_ZERO; } + auto r = static_cast

(*a) / static_cast

(*b); + if (r > static_cast

(cuda::std::numeric_limits::max()) || + r < static_cast

(cuda::std::numeric_limits::lowest())) { + return errc::ARITHMETIC_OVERFLOW; + } + *out = static_cast(r); + return errc::SUCCESS; +} + +} // namespace detail + +template + requires(cuda::std::is_integral_v && cuda::std::is_unsigned_v) +__device__ inline errc ansi_add(T* out, T const* a, T const* b) +{ + return detail::ansi_add_unsigned(out, a, b); +} + +template + requires(cuda::std::is_integral_v && cuda::std::is_signed_v) +__device__ inline errc ansi_add(T* out, T const* a, T const* b) +{ + return detail::ansi_add_signed(out, a, b); +} + +template + requires(cuda::std::is_floating_point_v) +__device__ inline errc ansi_add(T* out, T const* a, T const* b) +{ + *out = *a + *b; + return errc::SUCCESS; +} + +template + requires(cuda::std::is_integral_v && cuda::std::is_unsigned_v) +__device__ inline errc ansi_sub(T* out, T const* a, T const* b) +{ + return detail::ansi_sub_unsigned(out, a, b); +} + +template + requires(cuda::std::is_integral_v && cuda::std::is_signed_v) +__device__ inline errc ansi_sub(T* out, T const* a, T const* b) +{ + return detail::ansi_sub_signed(out, a, b); +} + +template + requires(cuda::std::is_floating_point_v) +__device__ inline errc ansi_sub(T* out, T const* a, T const* b) +{ + *out = *a - *b; + return errc::SUCCESS; +} + +template + requires(cuda::std::is_integral_v && cuda::std::is_unsigned_v) +__device__ inline errc ansi_mul(T* out, T const* a, T const* b) +{ + return detail::ansi_mul_unsigned(out, a, b); +} + +template + requires(cuda::std::is_integral_v && cuda::std::is_signed_v) +__device__ inline errc ansi_mul(T* out, T const* a, T const* b) +{ + return detail::ansi_mul_signed(out, a, b); +} + +template + requires(cuda::std::is_floating_point_v) +__device__ inline errc ansi_mul(T* out, T const* a, T const* b) +{ + *out = *a * *b; + return errc::SUCCESS; +} + +template + requires(cuda::std::is_integral_v && cuda::std::is_unsigned_v) +__device__ inline errc ansi_div(T* out, T const* a, T const* b) +{ + return detail::ansi_div_unsigned(out, a, b); +} + +template + requires(cuda::std::is_integral_v && cuda::std::is_signed_v) +__device__ inline errc ansi_div(T* out, T const* a, T const* b) +{ + return detail::ansi_div_signed(out, a, b); +} + +template + requires(cuda::std::is_floating_point_v) +__device__ inline errc ansi_div(T* out, T const* a, T const* b) +{ + return detail::ansi_div_float(out, a, b); +} + +template +__device__ inline errc try_add(optional* out, T const* a, T const* b) +{ + auto e = ansi_add(out, a, b); + if (e != errc::SUCCESS) { *out = nullopt; } + return errc::SUCCESS; +} + +template +__device__ inline errc try_sub(optional* out, T const* a, T const* b) +{ + auto e = ansi_sub(out, a, b); + if (e != errc::SUCCESS) { *out = nullopt; } + return errc::SUCCESS; +} + +template +__device__ inline errc try_mul(optional* out, T const* a, T const* b) +{ + auto e = ansi_mul(out, a, b); + if (e != errc::SUCCESS) { *out = nullopt; } + return errc::SUCCESS; +} + +template +__device__ inline errc try_div(optional* out, T const* a, T const* b) +{ + auto e = ansi_div(out, a, b); + if (e != errc::SUCCESS) { *out = nullopt; } + return errc::SUCCESS; +} + +// TODO: overloads for optional + +} // namespace operators +} // namespace jit +} // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/opcode.hpp b/cpp/include/cudf/opcode.hpp new file mode 100644 index 000000000000..c53aa60312f9 --- /dev/null +++ b/cpp/include/cudf/opcode.hpp @@ -0,0 +1,105 @@ + + +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include + +#include + +namespace CUDF_EXPORT cudf { + +/** + * @brief Enum of supported opcodes. + */ +enum class opcode : int32_t { + // Binary operators + ADD = 0, ///< operator + + SUB = 1, ///< operator - + MUL = 2, ///< operator * + DIV = 3, ///< operator / using common type of lhs and rhs + TRUE_DIV = 4, ///< operator / after promoting type to floating point + FLOOR_DIV = 5, ///< operator / after promoting to the common type of lhs and rhs (integral or + ///< floating point), and then flooring the result + MOD = 6, ///< operator % + PYMOD = 7, ///< operator % using Python's sign rules for negatives + POW = 8, ///< lhs ^ rhs + EQUAL = 9, ///< operator == + NULL_EQUAL = + 10, ///< operator == with Spark rules: NULL_EQUAL(null, null) is true, NULL_EQUAL(null, + ///< valid) is false, and + ///< NULL_EQUAL(valid, valid) == EQUAL(valid, valid) + NOT_EQUAL = 11, ///< operator != + LESS = 12, ///< operator < + GREATER = 13, ///< operator > + LESS_EQUAL = 14, ///< operator <= + GREATER_EQUAL = 15, ///< operator >= + BITWISE_AND = 16, ///< operator & + BITWISE_OR = 17, ///< operator | + BITWISE_XOR = 18, ///< operator ^ + LOGICAL_AND = 19, ///< operator && + NULL_LOGICAL_AND = 20, ///< operator && with Spark rules: NULL_LOGICAL_AND(null, null) is null, + ///< NULL_LOGICAL_AND(null, true) is + ///< null, NULL_LOGICAL_AND(null, false) is false, and NULL_LOGICAL_AND(valid, + ///< valid) == LOGICAL_AND(valid, valid) + LOGICAL_OR = 21, ///< operator || + NULL_LOGICAL_OR = 22, ///< operator || with Spark rules: NULL_LOGICAL_OR(null, null) is null, + ///< NULL_LOGICAL_OR(null, true) is true, + ///< NULL_LOGICAL_OR(null, false) is null, and NULL_LOGICAL_OR(valid, valid) == + ///< LOGICAL_OR(valid, valid) + // Unary operators + IDENTITY = 23, ///< Identity function + IS_NULL = 24, ///< Check if operand is null + SIN = 25, ///< Trigonometric sine + COS = 26, ///< Trigonometric cosine + TAN = 27, ///< Trigonometric tangent + ARCSIN = 28, ///< Trigonometric sine inverse + ARCCOS = 29, ///< Trigonometric cosine inverse + ARCTAN = 30, ///< Trigonometric tangent inverse + SINH = 31, ///< Hyperbolic sine + COSH = 32, ///< Hyperbolic cosine + TANH = 33, ///< Hyperbolic tangent + ARCSINH = 34, ///< Hyperbolic sine inverse + ARCCOSH = 35, ///< Hyperbolic cosine inverse + ARCTANH = 36, ///< Hyperbolic tangent inverse + EXP = 37, ///< Exponential (base e, Euler number) + LOG = 38, ///< Natural Logarithm (base e) + SQRT = 39, ///< Square-root (x^0.5) + CBRT = 40, ///< Cube-root (x^(1.0/3)) + CEIL = 41, ///< Smallest integer value not less than arg + FLOOR = 42, ///< largest integer value not greater than arg + ABS = 43, ///< Absolute value + RINT = 44, ///< Rounds the floating-point argument arg to an integer value + BIT_INVERT = 45, ///< Bitwise Not (~) + NOT = 46, ///< Logical Not (!) + CAST_TO_INT64 = 47, ///< Cast value to int64_t + CAST_TO_UINT64 = 48, ///< Cast value to uint64_t + CAST_TO_FLOAT64 = 49, ///< Cast value to double + + ANSI_ADD = 50, ///< operator +, with ANSI SQL semantics (e.g. overflow checking) + ANSI_SUB = 51, ///< operator -, with ANSI SQL semantics (e.g. overflow checking) + ANSI_MUL = 52, ///< operator *, with ANSI SQL semantics (e.g. overflow checking) + ANSI_DIV = 53, ///< operator / using common type of lhs and rhs, with ANSI SQL semantics (e.g. + ///< division by zero checking) + ANSI_ABS = 54, ///< Absolute value, with ANSI SQL semantics (e.g. overflow checking) + ANSI_CAST_TO_INT64 = + 55, ///< Cast value to int64_t, with ANSI SQL semantics (e.g. overflow checking) + ANSI_CAST_TO_UINT64 = + 56, ///< Cast value to uint64_t, with ANSI SQL semantics (e.g. overflow checking) + + TRY_ADD = 57, ///< operator +, with TRY semantics (e.g. returns null on overflow) + TRY_SUB = 58, ///< operator -, with TRY semantics (e.g. returns null on overflow) + TRY_MUL = 59, ///< operator *, with TRY semantics (e.g. returns null on overflow) + TRY_DIV = 60, ///< operator / using common type of lhs and rhs, with TRY semantics (e.g. returns + ///< null on division by zero) + TRY_ABS = 61, ///< Absolute value, with TRY semantics (e.g. returns null on overflow) + TRY_CAST_TO_INT64 = + 62, ///< Cast value to int64_t, with TRY semantics (e.g. returns null on overflow) + TRY_CAST_TO_UINT64 = + 63, ///< Cast value to uint64_t, with TRY semantics (e.g. returns null on overflow) +}; + +} // namespace CUDF_EXPORT cudf From fbf1cecd5864aa42704478029e1ee817ef031059 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 21 Apr 2026 23:12:19 +0000 Subject: [PATCH 02/15] checkpoint --- cpp/include/cudf/ast/detail/jit_operators.cuh | 1478 ----------------- .../cudf/operators/ansi_arithmetic.cuh | 629 +++++++ cpp/include/cudf/operators/arithmetic.cuh | 216 +++ cpp/include/cudf/operators/btiwise.cuh | 94 ++ cpp/include/cudf/operators/casts.cuh | 213 +++ cpp/include/cudf/operators/comparison.cuh | 192 +++ cpp/include/cudf/operators/logic.cuh | 75 + cpp/include/cudf/operators/math.cuh | 258 +++ cpp/include/cudf/operators/optional.cuh | 63 + cpp/include/cudf/operators/trigonometric.cuh | 344 ++++ cpp/include/cudf/operators/types.cuh | 115 ++ 11 files changed, 2199 insertions(+), 1478 deletions(-) delete mode 100644 cpp/include/cudf/ast/detail/jit_operators.cuh create mode 100644 cpp/include/cudf/operators/ansi_arithmetic.cuh create mode 100644 cpp/include/cudf/operators/arithmetic.cuh create mode 100644 cpp/include/cudf/operators/btiwise.cuh create mode 100644 cpp/include/cudf/operators/casts.cuh create mode 100644 cpp/include/cudf/operators/comparison.cuh create mode 100644 cpp/include/cudf/operators/logic.cuh create mode 100644 cpp/include/cudf/operators/math.cuh create mode 100644 cpp/include/cudf/operators/optional.cuh create mode 100644 cpp/include/cudf/operators/trigonometric.cuh create mode 100644 cpp/include/cudf/operators/types.cuh diff --git a/cpp/include/cudf/ast/detail/jit_operators.cuh b/cpp/include/cudf/ast/detail/jit_operators.cuh deleted file mode 100644 index 819dd0016561..000000000000 --- a/cpp/include/cudf/ast/detail/jit_operators.cuh +++ /dev/null @@ -1,1478 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once - -#include - -#include -#include -#include - -namespace CUDF_EXPORT cudf { -namespace jit { - -enum errc : int { - SUCCESS = 0, - ARITHMETIC_OVERFLOW = 1, - ARITHMETIC_UNDERFLOW = 2, - DIVISION_BY_ZERO = 3 -}; - -namespace operators { - -template -using optional = cuda::std::optional; - -template -struct promoted_t; - -template <> -struct promoted_t { - using type = int16_t; -}; - -template <> -struct promoted_t { - using type = uint16_t; -}; - -template <> -struct promoted_t { - using type = int32_t; -}; - -template <> -struct promoted_t { - using type = uint32_t; -}; - -template <> -struct promoted_t { - using type = int64_t; -}; - -template <> -struct promoted_t { - using type = uint64_t; -}; - -template <> -struct promoted_t { - using type = __int128; -}; - -template <> -struct promoted_t { - using type = unsigned __int128; -}; - -template -using promoted = typename promoted_t::type; - -template -__device__ inline errc abs(T* out, T const* a) -{ - *out = (*a < 0) ? -*a : *a; - return errc::SUCCESS; -} - -template -__device__ inline errc abs(optional* out, optional const* a) -{ - if (a->is_valid()) { - T r; - abs(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc add(T* out, T const* a, T const* b) -{ - *out = (*a + *b); - return errc::SUCCESS; -} - -template -__device__ inline errc add(optional* out, optional const* a, optional const* b) -{ - if (a->is_valid() && b->is_valid()) { - T r; - add(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc arccos(T* out, T const* a); - -template <> -__device__ inline errc arccos(float* out, float const* a) -{ - *out = ::acosf(*a); - return errc::SUCCESS; -} - -template <> -__device__ inline errc arccos(double* out, double const* a) -{ - *out = ::acos(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc arccos(optional* out, optional const* a) -{ - if (a->is_valid()) { - T r; - arccos(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc arccosh(T* out, T const* a); - -template <> -__device__ inline errc arccosh(float* out, float const* a) -{ - *out = ::acoshf(*a); - return errc::SUCCESS; -} - -template <> -__device__ inline errc arccosh(double* out, double const* a) -{ - *out = ::acosh(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc arccosh(optional* out, optional const* a) -{ - if (a->is_valid()) { - T r; - arccosh(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc arcsin(T* out, T const* a); - -template <> -__device__ inline errc arcsin(float* out, float const* a) -{ - *out = ::asinf(*a); - return errc::SUCCESS; -} - -template <> -__device__ inline errc arcsin(double* out, double const* a) -{ - *out = ::asin(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc arcsin(optional* out, optional const* a) -{ - if (a->is_valid()) { - T r; - arcsin(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc arcsinh(T* out, T const* a); - -template <> -__device__ inline errc arcsinh(float* out, float const* a) -{ - *out = ::asinhf(*a); - return errc::SUCCESS; -} - -template <> -__device__ inline errc arcsinh(double* out, double const* a) -{ - *out = ::asinh(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc arcsinh(optional* out, optional const* a) -{ - if (a->is_valid()) { - T r; - arcsinh(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc arctan(T* out, T const* a); - -template <> -__device__ inline errc arctan(float* out, float const* a) -{ - *out = ::atanf(*a); - return errc::SUCCESS; -} - -template <> -__device__ inline errc arctan(double* out, double const* a) -{ - *out = ::atan(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc arctan(optional* out, optional const* a) -{ - if (a->is_valid()) { - T r; - arctan(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc arctanh(T* out, T const* a); - -template <> -__device__ inline errc arctanh(float* out, float const* a) -{ - *out = ::atanhf(*a); - return errc::SUCCESS; -} - -template <> -__device__ inline errc arctanh(double* out, double const* a) -{ - *out = ::atanh(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc arctanh(optional* out, optional const* a) -{ - if (a->is_valid()) { - T r; - arctanh(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc bit_and(T* out, T const* a, T const* b) -{ - *out = (*a & *b); - return errc::SUCCESS; -} - -template -__device__ inline errc bit_and(optional* out, optional const* a, optional const* b) -{ - if (a->is_valid() && b->is_valid()) { - T r; - bit_and(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc bit_invert(T* out, T const* a) -{ - *out = ~(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc bit_invert(optional* out, optional const* a) -{ - if (a->is_valid()) { - T r; - bit_invert(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc bit_or(T* out, T const* a, T const* b) -{ - *out = (*a | *b); - return errc::SUCCESS; -} - -template -__device__ inline errc bit_or(optional* out, optional const* a, optional const* b) -{ - if (a->is_valid() && b->is_valid()) { - T r; - bit_or(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc bit_xor(T* out, T const* a, T const* b) -{ - *out = (*a ^ *b); - return errc::SUCCESS; -} - -template -__device__ inline errc bit_xor(optional* out, optional const* a, optional const* b) -{ - if (a->is_valid() && b->is_valid()) { - T r; - bit_xor(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc cast_to_i32(int32_t* out, T const* a) -{ - *out = static_cast(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc cast_to_i32(optional* out, optional const* a) -{ - if (a->is_valid()) { - int32_t r; - cast_to_i32(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc cast_to_i64(int64_t* out, T const* a) -{ - *out = static_cast(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc cast_to_i64(optional* out, optional const* a) -{ - if (a->is_valid()) { - int64_t r; - cast_to_i64(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc cast_to_u32(uint32_t* out, T const* a) -{ - *out = static_cast(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc cast_to_u32(optional* out, optional const* a) -{ - if (a->is_valid()) { - uint32_t r; - cast_to_u32(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc cast_to_u64(uint64_t* out, T const* a) -{ - *out = static_cast(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc cast_to_u64(optional* out, optional const* a) -{ - if (a->is_valid()) { - uint64_t r; - cast_to_u64(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc cast_to_f32(float* out, T const* a) -{ - *out = static_cast(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc cast_to_f32(optional* out, optional const* a) -{ - if (a->is_valid()) { - float r; - cast_to_f32(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc cast_to_f64(double* out, T const* a) -{ - *out = static_cast(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc cast_to_f64(optional* out, optional const* a) -{ - if (a->is_valid()) { - double r; - cast_to_f64(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc cbrt(T* out, T const* a); - -template <> -__device__ inline errc cbrt(float* out, float const* a) -{ - *out = ::cbrtf(*a); - return errc::SUCCESS; -} - -template <> -__device__ inline errc cbrt(double* out, double const* a) -{ - *out = ::cbrt(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc cbrt(optional* out, optional const* a) -{ - if (a->is_valid()) { - T r; - cbrt(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc ceil(T* out, T const* a); - -template <> -__device__ inline errc ceil(float* out, float const* a) -{ - *out = ::ceilf(*a); - return errc::SUCCESS; -} - -template <> -__device__ inline errc ceil(double* out, double const* a) -{ - *out = ::ceil(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc ceil(optional* out, optional const* a) -{ - if (a->is_valid()) { - T r; - ceil(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc cos(T* out, T const* a); - -template <> -__device__ inline errc cos(float* out, float const* a) -{ - *out = ::cosf(*a); - return errc::SUCCESS; -} - -template <> -__device__ inline errc cos(double* out, double const* a) -{ - *out = ::cos(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc cos(optional* out, optional const* a) -{ - if (a->is_valid()) { - T r; - cos(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc cosh(T* out, T const* a); - -template <> -__device__ inline errc cosh(float* out, float const* a) -{ - *out = ::coshf(*a); - return errc::SUCCESS; -} - -template <> -__device__ inline errc cosh(double* out, double const* a) -{ - *out = ::cosh(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc cosh(optional* out, optional const* a) -{ - if (a->is_valid()) { - T r; - cosh(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc div(T* out, T const* a, T const* b) -{ - *out = (*a / *b); - return errc::SUCCESS; -} - -template -__device__ inline errc div(optional* out, optional const* a, optional const* b) -{ - if (a->is_valid() && b->is_valid()) { - T r; - div(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc equal(bool* out, T const* a, T const* b) -{ - *out = (*a == *b); - return errc::SUCCESS; -} - -template -__device__ inline errc equal(optional* out, optional const* a, optional const* b) -{ - if (a->is_valid() && b->is_valid()) { - bool r; - equal(&r, &a->value(), &b->value()); - *out = r; - } else if (a->is_null() && b->is_null()) { - *out = true; - } else { - *out = false; - } - return errc::SUCCESS; -} - -template -__device__ inline errc exp(T* out, T const* a); - -template <> -__device__ inline errc exp(float* out, float const* a) -{ - *out = ::expf(*a); - return errc::SUCCESS; -} - -template <> -__device__ inline errc exp(double* out, double const* a) -{ - *out = ::exp(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc exp(optional* out, optional const* a) -{ - if (a->is_valid()) { - T r; - exp(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc floor(T* out, T const* a); - -template <> -__device__ inline errc floor(float* out, float const* a) -{ - *out = ::floorf(*a); - return errc::SUCCESS; -} - -template <> -__device__ inline errc floor(double* out, double const* a) -{ - *out = ::floor(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc floor(optional* out, optional const* a) -{ - if (a->is_valid()) { - T r; - floor(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc greater(bool* out, T const* a, T const* b) -{ - *out = (*a > *b); - return errc::SUCCESS; -} - -template -__device__ inline errc greater(optional* out, optional const* a, optional const* b) -{ - if (a->is_valid() && b->is_valid()) { - bool r; - greater(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = false; - } - return errc::SUCCESS; -} - -template -__device__ inline errc greater_equal(bool* out, T const* a, T const* b) -{ - *out = (*a >= *b); - return errc::SUCCESS; -} - -template -__device__ inline errc greater_equal(optional* out, - optional const* a, - optional const* b) -{ - if (a->is_valid() && b->is_valid()) { - bool r; - greater_equal(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = false; - } - return errc::SUCCESS; -} - -template -__device__ inline errc identity(T* out, T const* a) -{ - *out = *a; - return errc::SUCCESS; -} - -template -__device__ inline errc identity(optional* out, optional const* a) -{ - *out = *a; - return errc::SUCCESS; -} - -template -__device__ inline errc is_null(bool* out, T const* a) -{ - *out = false; - return errc::SUCCESS; -} - -template -__device__ inline errc is_null(optional* out, optional const* a) -{ - *out = a->is_null(); - return errc::SUCCESS; -} - -template -__device__ inline errc less(bool* out, T const* a, T const* b) -{ - *out = (*a < *b); - return errc::SUCCESS; -} - -template -__device__ inline errc less(optional* out, optional const* a, optional const* b) -{ - if (a->is_valid() && b->is_valid()) { - bool r; - less(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = false; - } - return errc::SUCCESS; -} - -template -__device__ inline errc less_equal(bool* out, T const* a, T const* b) -{ - *out = (*a <= *b); - return errc::SUCCESS; -} - -template -__device__ inline errc less_equal(optional* out, optional const* a, optional const* b) -{ - if (a->is_valid() && b->is_valid()) { - bool r; - less_equal(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = false; - } - return errc::SUCCESS; -} - -template -__device__ inline errc log(T* out, T const* a); - -template <> -__device__ inline errc log(float* out, float const* a) -{ - *out = ::logf(*a); - return errc::SUCCESS; -} - -template <> -__device__ inline errc log(double* out, double const* a) -{ - *out = ::log(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc log(optional* out, optional const* a) -{ - if (a->is_valid()) { - T r; - log(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc logical_and(T* out, T const* a, T const* b) -{ - *out = (*a && *b); - return errc::SUCCESS; -} - -template -__device__ inline errc logical_and(optional* out, optional const* a, optional const* b) -{ - if (a->is_valid() && b->is_valid()) { - T r; - logical_and(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc logical_or(T* out, T const* a, T const* b) -{ - *out = (*a || *b); - return errc::SUCCESS; -} - -template -__device__ inline errc logical_or(optional* out, optional const* a, optional const* b) -{ - if (a->is_valid() && b->is_valid()) { - T r; - logical_or(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc mod(T* out, T const* a, T const* b) -{ - *out = (*a % *b); - return errc::SUCCESS; -} - -template <> -__device__ inline errc mod(float* out, float const* a, float const* b) -{ - *out = ::fmodf(*a, *b); - return errc::SUCCESS; -} - -template <> -__device__ inline errc mod(double* out, double const* a, double const* b) -{ - *out = ::fmod(*a, *b); - return errc::SUCCESS; -} - -template -__device__ inline errc mod(optional* out, optional const* a, optional const* b) -{ - if (a->is_valid() && b->is_valid()) { - T r; - mod(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc mul(T* out, T const* a, T const* b) -{ - *out = (*a * *b); - return errc::SUCCESS; -} - -template -__device__ inline errc mul(optional* out, optional const* a, optional const* b) -{ - if (a->is_valid() && b->is_valid()) { - T r; - mul(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc null_equal(bool* out, T const* a, T const* b) -{ - *out = (*a == *b); - return errc::SUCCESS; -} - -template -__device__ inline errc null_equal(optional* out, optional const* a, optional const* b) -{ - if (a->is_valid() && b->is_valid()) { - *out = (*(*a) == *(*b)); - } else if (a->is_null() && b->is_null()) { - *out = true; - } else { - *out = false; - } - return errc::SUCCESS; -} - -template -__device__ inline errc null_logical_and(T* out, T const* a, T const* b) -{ - *out = (*a && *b); - return errc::SUCCESS; -} - -template -__device__ inline errc null_logical_and(optional* out, - optional const* a, - optional const* b) -{ - if (a->is_valid() && b->is_valid()) { - bool r; - null_logical_and(&r, &a->value(), &b->value()); - *out = r; - } else if (a->is_null() && b->is_null()) { - *out = nullopt; - } else { - if (a->is_valid() ? *(*a) : *(*b)) { - *out = nullopt; - } else { - *out = false; - } - } - return errc::SUCCESS; -} - -template -__device__ inline errc null_logical_or(T* out, T const* a, T const* b) -{ - *out = (*a || *b); - return errc::SUCCESS; -} - -template -__device__ inline errc null_logical_or(optional* out, optional const* a, optional const* b) -{ - if (a->is_valid() && b->is_valid()) { - bool r; - null_logical_or(&r, &a->value(), &b->value()); - *out = r; - } else if (a->is_null() && b->is_null()) { - *out = nullopt; - } else { - if (a->is_valid() ? *(*a) : *(*b)) { - *out = true; - } else { - *out = nullopt; - } - } - return errc::SUCCESS; -} - -template -__device__ inline errc pow(T* out, T const* a, T const* b); - -template <> -__device__ inline errc pow(float* out, float const* a, float const* b) -{ - *out = ::powf(*a, *b); - return errc::SUCCESS; -} - -template <> -__device__ inline errc pow(double* out, double const* a, double const* b) -{ - *out = ::pow(*a, *b); - return errc::SUCCESS; -} - -template -__device__ inline errc pow(optional* out, optional const* a, optional const* b) -{ - if (a->is_valid() && b->is_valid()) { - T r; - pow(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc pymod(T* out, T const* a, T const* b) -{ - *out = (*a % *b + *b) % *b; - return errc::SUCCESS; -} - -template <> -__device__ inline errc pymod(float* out, float const* a, float const* b) -{ - *out = ::fmodf(::fmodf(*a, *b) + *b, *b); - return errc::SUCCESS; -} - -template <> -__device__ inline errc pymod(double* out, double const* a, double const* b) -{ - *out = ::fmod(::fmod(*a, *b) + *b, *b); - return errc::SUCCESS; -} - -template -__device__ inline errc pymod(optional* out, optional const* a, optional const* b) -{ - if (a->is_valid() && b->is_valid()) { - T r; - pymod(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc rint(T* out, T const* a); - -template <> -__device__ inline errc rint(float* out, float const* a) -{ - *out = ::rintf(*a); - return errc::SUCCESS; -} - -template <> -__device__ inline errc rint(double* out, double const* a) -{ - *out = ::rint(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc rint(optional* out, optional const* a) -{ - if (a->is_valid()) { - T r; - rint(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc sin(T* out, T const* a); - -template <> -__device__ inline errc sin(float* out, float const* a) -{ - *out = ::sinf(*a); - return errc::SUCCESS; -} - -template <> -__device__ inline errc sin(double* out, double const* a) -{ - *out = ::sin(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc sin(optional* out, optional const* a) -{ - if (a->is_valid()) { - T r; - sin(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc sinh(T* out, T const* a); - -template <> -__device__ inline errc sinh(float* out, float const* a) -{ - *out = ::sinhf(*a); - return errc::SUCCESS; -} - -template <> -__device__ inline errc sinh(double* out, double const* a) -{ - *out = ::sinh(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc sinh(optional* out, optional const* a) -{ - if (a->is_valid()) { - T r; - sinh(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc sub(T* out, T const* a, T const* b) -{ - *out = *a - *b; - return errc::SUCCESS; -} - -template -__device__ inline errc sub(optional* out, optional const* a, optional const* b) -{ - if (a->is_valid() && b->is_valid()) { - T r; - sub(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc tanh(T* out, T const* a); - -template <> -__device__ inline errc tanh(float* out, float const* a) -{ - *out = ::tanhf(*a); - return errc::SUCCESS; -} - -template <> -__device__ inline errc tanh(double* out, double const* a) -{ - *out = ::tanh(*a); - return errc::SUCCESS; -} - -template -__device__ inline errc tanh(optional* out, optional const* a) -{ - if (a->is_valid()) { - T r; - tanh(&r, &a->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -template -__device__ inline errc if_else(T* out, - bool const* condition, - T const* true_value, - T const* false_value) -{ - *out = *condition ? *true_value : *false_value; - return errc::SUCCESS; -} - -template -__device__ inline errc if_else(optional* out, - optional const* condition, - optional const* true_value, - optional const* false_value) -{ - if (condition->is_valid() && true_value->is_valid() && false_value->is_valid()) { - if_else(&out->value(), &condition->value(), &true_value->value(), &false_value->value()); - } else { - *out = nullopt; - } - return errc::SUCCESS; -} - -namespace detail { - -template -__device__ inline errc ansi_add_unsigned(T* out, T const* a, T const* b) -{ - using P = promoted; - auto r = static_cast

(*a) + static_cast

(*b); - if (r > static_cast

(cuda::std::numeric_limits::max())) { return errc::ARITHMETIC_OVERFLOW; } - *out = static_cast(r); - return errc::SUCCESS; -} - -template -__device__ inline errc ansi_add_signed(T* out, T const* a, T const* b) -{ - using P = promoted; - auto r = static_cast

(*a) + static_cast

(*b); - if (r > static_cast

(cuda::std::numeric_limits::max()) || - r < static_cast

(cuda::std::numeric_limits::min())) { - return errc::ARITHMETIC_OVERFLOW; - } - *out = static_cast(r); - return errc::SUCCESS; -} - -template -__device__ inline errc ansi_sub_unsigned(T* out, T const* a, T const* b) -{ - if (*a < *b) { return errc::ARITHMETIC_UNDERFLOW; } - auto r = *a - *b; - *out = static_cast(r); - return errc::SUCCESS; -} - -template -__device__ inline errc ansi_sub_signed(T* out, T const* a, T const* b) -{ - using P = promoted; - auto r = static_cast

(*a) - static_cast

(*b); - if (r > static_cast

(cuda::std::numeric_limits::max()) || - r < static_cast

(cuda::std::numeric_limits::min())) { - return errc::ARITHMETIC_UNDERFLOW; - } - *out = static_cast(r); - return errc::SUCCESS; -} - -template -__device__ inline errc ansi_mul_unsigned(T* out, T const* a, T const* b) -{ - using P = promoted; - auto r = static_cast

(*a) * static_cast

(*b); - if (r > static_cast

(cuda::std::numeric_limits::max())) { return errc::ARITHMETIC_OVERFLOW; } - *out = static_cast(r); - return errc::SUCCESS; -} - -template -__device__ inline errc ansi_mul_signed(T* out, T const* a, T const* b) -{ - using P = promoted; - auto r = static_cast

(*a) * static_cast

(*b); - if (r > static_cast

(cuda::std::numeric_limits::max()) || - r < static_cast

(cuda::std::numeric_limits::min())) { - return errc::ARITHMETIC_OVERFLOW; - } - *out = static_cast(r); - return errc::SUCCESS; -} - -template -__device__ inline errc ansi_div_unsigned(T* out, T const* a, T const* b) -{ - if (*b == 0) { return errc::DIVISION_BY_ZERO; } - *out = static_cast(static_cast(*a) / static_cast(*b)); - return errc::SUCCESS; -} - -template -__device__ inline errc ansi_div_signed(T* out, T const* a, T const* b) -{ - if (*b == 0) { return errc::DIVISION_BY_ZERO; } - if (*a == cuda::std::numeric_limits::min() && *b == -1) { return errc::ARITHMETIC_OVERFLOW; } - *out = static_cast(static_cast(*a) / static_cast(*b)); - return errc::SUCCESS; -} - -template -__device__ inline errc ansi_div_float(T* out, T const* a, T const* b) -{ - using P = promoted; - if (*b == 0) { return errc::DIVISION_BY_ZERO; } - auto r = static_cast

(*a) / static_cast

(*b); - if (r > static_cast

(cuda::std::numeric_limits::max()) || - r < static_cast

(cuda::std::numeric_limits::lowest())) { - return errc::ARITHMETIC_OVERFLOW; - } - *out = static_cast(r); - return errc::SUCCESS; -} - -} // namespace detail - -template - requires(cuda::std::is_integral_v && cuda::std::is_unsigned_v) -__device__ inline errc ansi_add(T* out, T const* a, T const* b) -{ - return detail::ansi_add_unsigned(out, a, b); -} - -template - requires(cuda::std::is_integral_v && cuda::std::is_signed_v) -__device__ inline errc ansi_add(T* out, T const* a, T const* b) -{ - return detail::ansi_add_signed(out, a, b); -} - -template - requires(cuda::std::is_floating_point_v) -__device__ inline errc ansi_add(T* out, T const* a, T const* b) -{ - *out = *a + *b; - return errc::SUCCESS; -} - -template - requires(cuda::std::is_integral_v && cuda::std::is_unsigned_v) -__device__ inline errc ansi_sub(T* out, T const* a, T const* b) -{ - return detail::ansi_sub_unsigned(out, a, b); -} - -template - requires(cuda::std::is_integral_v && cuda::std::is_signed_v) -__device__ inline errc ansi_sub(T* out, T const* a, T const* b) -{ - return detail::ansi_sub_signed(out, a, b); -} - -template - requires(cuda::std::is_floating_point_v) -__device__ inline errc ansi_sub(T* out, T const* a, T const* b) -{ - *out = *a - *b; - return errc::SUCCESS; -} - -template - requires(cuda::std::is_integral_v && cuda::std::is_unsigned_v) -__device__ inline errc ansi_mul(T* out, T const* a, T const* b) -{ - return detail::ansi_mul_unsigned(out, a, b); -} - -template - requires(cuda::std::is_integral_v && cuda::std::is_signed_v) -__device__ inline errc ansi_mul(T* out, T const* a, T const* b) -{ - return detail::ansi_mul_signed(out, a, b); -} - -template - requires(cuda::std::is_floating_point_v) -__device__ inline errc ansi_mul(T* out, T const* a, T const* b) -{ - *out = *a * *b; - return errc::SUCCESS; -} - -template - requires(cuda::std::is_integral_v && cuda::std::is_unsigned_v) -__device__ inline errc ansi_div(T* out, T const* a, T const* b) -{ - return detail::ansi_div_unsigned(out, a, b); -} - -template - requires(cuda::std::is_integral_v && cuda::std::is_signed_v) -__device__ inline errc ansi_div(T* out, T const* a, T const* b) -{ - return detail::ansi_div_signed(out, a, b); -} - -template - requires(cuda::std::is_floating_point_v) -__device__ inline errc ansi_div(T* out, T const* a, T const* b) -{ - return detail::ansi_div_float(out, a, b); -} - -template -__device__ inline errc try_add(optional* out, T const* a, T const* b) -{ - auto e = ansi_add(out, a, b); - if (e != errc::SUCCESS) { *out = nullopt; } - return errc::SUCCESS; -} - -template -__device__ inline errc try_sub(optional* out, T const* a, T const* b) -{ - auto e = ansi_sub(out, a, b); - if (e != errc::SUCCESS) { *out = nullopt; } - return errc::SUCCESS; -} - -template -__device__ inline errc try_mul(optional* out, T const* a, T const* b) -{ - auto e = ansi_mul(out, a, b); - if (e != errc::SUCCESS) { *out = nullopt; } - return errc::SUCCESS; -} - -template -__device__ inline errc try_div(optional* out, T const* a, T const* b) -{ - auto e = ansi_div(out, a, b); - if (e != errc::SUCCESS) { *out = nullopt; } - return errc::SUCCESS; -} - -// TODO: overloads for optional - -} // namespace operators -} // namespace jit -} // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/operators/ansi_arithmetic.cuh b/cpp/include/cudf/operators/ansi_arithmetic.cuh new file mode 100644 index 000000000000..8ff745a2b5cb --- /dev/null +++ b/cpp/include/cudf/operators/ansi_arithmetic.cuh @@ -0,0 +1,629 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include + +namespace CUDF_EXPORT cudf { + +namespace ops { +namespace detail { + +template +__device__ inline errc ansi_add_unsigned(T* out, T const* a, T const* b) +{ + using P = promoted; + auto r = static_cast

(*a) + static_cast

(*b); + if (r > static_cast

(cuda::std::numeric_limits::max())) { return errc::OVERFLOW; } + *out = static_cast(r); + return errc::OK; +} + +template +__device__ inline errc ansi_add_signed(T* out, T const* a, T const* b) +{ + using P = promoted; + auto r = static_cast

(*a) + static_cast

(*b); + if (r > static_cast

(cuda::std::numeric_limits::max()) || + r < static_cast

(cuda::std::numeric_limits::min())) { + return errc::OVERFLOW; + } + *out = static_cast(r); + return errc::OK; +} + +template +__device__ inline errc ansi_sub_unsigned(T* out, T const* a, T const* b) +{ + if (*a < *b) { return errc::OVERFLOW; } + auto r = *a - *b; + *out = static_cast(r); + return errc::OK; +} + +template +__device__ inline errc ansi_sub_signed(T* out, T const* a, T const* b) +{ + using P = promoted; + auto r = static_cast

(*a) - static_cast

(*b); + if (r > static_cast

(cuda::std::numeric_limits::max()) || + r < static_cast

(cuda::std::numeric_limits::min())) { + return errc::OVERFLOW; + } + *out = static_cast(r); + return errc::OK; +} + +template +__device__ inline errc ansi_mul_unsigned(T* out, T const* a, T const* b) +{ + using P = promoted; + auto r = static_cast

(*a) * static_cast

(*b); + if (r > static_cast

(cuda::std::numeric_limits::max())) { return errc::OVERFLOW; } + *out = static_cast(r); + return errc::OK; +} + +template +__device__ inline errc ansi_mul_signed(T* out, T const* a, T const* b) +{ + using P = promoted; + auto r = static_cast

(*a) * static_cast

(*b); + if (r > static_cast

(cuda::std::numeric_limits::max()) || + r < static_cast

(cuda::std::numeric_limits::min())) { + return errc::OVERFLOW; + } + *out = static_cast(r); + return errc::OK; +} + +template +__device__ inline errc ansi_div_unsigned(T* out, T const* a, T const* b) +{ + if (*b == 0) { return errc::DIVISION_BY_ZERO; } + *out = static_cast(static_cast(*a) / static_cast(*b)); + return errc::OK; +} + +template +__device__ inline errc ansi_div_signed(T* out, T const* a, T const* b) +{ + if (*b == 0) { return errc::DIVISION_BY_ZERO; } + if (*a == cuda::std::numeric_limits::min() && *b == -1) { return errc::OVERFLOW; } + *out = static_cast(static_cast(*a) / static_cast(*b)); + return errc::OK; +} + +template +__device__ inline errc ansi_div_float(T* out, T const* a, T const* b) +{ + using P = promoted; + if (*b == 0) { return errc::DIVISION_BY_ZERO; } + auto r = static_cast

(*a) / static_cast

(*b); + if (r > static_cast

(cuda::std::numeric_limits::max()) || + r < static_cast

(cuda::std::numeric_limits::lowest())) { + return errc::OVERFLOW; + } + *out = static_cast(r); + return errc::OK; +} + +} // namespace detail + +template + requires(cuda::std::is_integral_v && cuda::std::is_unsigned_v) +__device__ inline errc ansi_add(T* out, T const* a, T const* b) +{ + return detail::ansi_add_unsigned(out, a, b); +} + +template + requires(cuda::std::is_integral_v && cuda::std::is_signed_v) +__device__ inline errc ansi_add(T* out, T const* a, T const* b) +{ + return detail::ansi_add_signed(out, a, b); +} + +template + requires(cuda::std::is_floating_point_v) +__device__ inline errc ansi_add(T* out, T const* a, T const* b) +{ + *out = *a + *b; + return errc::OK; +} + +template +__device__ inline errc ansi_add(numeric::fixed_point* out, + numeric::fixed_point const* a, + numeric::fixed_point const* b) +{ + auto scale = cuda::std::min(a->scale(), b->scale()); + auto sum = a->rescaled(scale).value() + b->rescaled(scale).value(); + + if (numeric::addition_overflow(a->rescaled(scale).value(), b->rescaled(scale).value())) { + return errc::OVERFLOW; + } + + *out = numeric::fixed_point{numeric::scaled_integer{sum, scale}}; + return errc::OK; +} + +template +__device__ inline errc ansi_add(cuda::std::chrono::duration* out, + cuda::std::chrono::duration const* a, + cuda::std::chrono::duration const* b) +{ + using P = promoted; + auto r = static_cast

(a->count()) + static_cast

(b->count()); + if (r > static_cast

(cuda::std::numeric_limits::max()) || + r < static_cast

(cuda::std::numeric_limits::min())) { + return errc::OVERFLOW; + } + *out = cuda::std::chrono::duration{static_cast(r)}; + return errc::OK; +} + +template +__device__ inline errc ansi_add(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + if (errc e = ansi_add(&r, &a->value(), &b->value()); e != errc::OK) { + *out = nullopt; + return e; + } + *out = r; + } else { + *out = nullopt; + } + + return errc::OK; +} + +template + requires(cuda::std::is_integral_v && cuda::std::is_unsigned_v) +__device__ inline errc ansi_sub(T* out, T const* a, T const* b) +{ + return detail::ansi_sub_unsigned(out, a, b); +} + +template + requires(cuda::std::is_integral_v && cuda::std::is_signed_v) +__device__ inline errc ansi_sub(T* out, T const* a, T const* b) +{ + return detail::ansi_sub_signed(out, a, b); +} + +template + requires(cuda::std::is_floating_point_v) +__device__ inline errc ansi_sub(T* out, T const* a, T const* b) +{ + *out = *a - *b; + return errc::OK; +} + +template +__device__ inline errc ansi_sub(numeric::fixed_point* out, + numeric::fixed_point const* a, + numeric::fixed_point const* b) +{ + auto scale = cuda::std::min(a->scale(), b->scale()); + auto sum = a->rescaled(scale).value() - b->rescaled(scale).value(); + + if (numeric::subtraction_overflow(a->rescaled(scale).value(), b->rescaled(scale).value())) { + return errc::OVERFLOW; + } + + *out = numeric::fixed_point{numeric::scaled_integer{sum, scale}}; + return errc::OK; +} + +template +__device__ inline errc ansi_sub(cuda::std::chrono::duration* out, + cuda::std::chrono::duration const* a, + cuda::std::chrono::duration const* b) +{ + using P = promoted; + auto r = static_cast

(a->count()) - static_cast

(b->count()); + if (r > static_cast

(cuda::std::numeric_limits::max()) || + r < static_cast

(cuda::std::numeric_limits::min())) { + return errc::OVERFLOW; + } + *out = cuda::std::chrono::duration{static_cast(r)}; + return errc::OK; +} + +template +__device__ inline errc ansi_sub(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + if (errc e = ansi_sub(&r, &a->value(), &b->value()); e != errc::OK) { + *out = nullopt; + return e; + } + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template + requires(cuda::std::is_integral_v && cuda::std::is_unsigned_v) +__device__ inline errc ansi_mul(T* out, T const* a, T const* b) +{ + return detail::ansi_mul_unsigned(out, a, b); +} + +template + requires(cuda::std::is_integral_v && cuda::std::is_signed_v) +__device__ inline errc ansi_mul(T* out, T const* a, T const* b) +{ + return detail::ansi_mul_signed(out, a, b); +} + +template + requires(cuda::std::is_floating_point_v) +__device__ inline errc ansi_mul(T* out, T const* a, T const* b) +{ + *out = *a * *b; + return errc::OK; +} + +template +__device__ inline errc ansi_mul(numeric::fixed_point* out, + numeric::fixed_point const* a, + numeric::fixed_point const* b) +{ + if (numeric::multiplication_overflow(a->value(), b->value())) { return errc::OVERFLOW; } + + *out = numeric::fixed_point{ + numeric::scaled_integer{a->value() * b->value(), a->scale() + b->scale()}}; + return errc::OK; +} + +template +__device__ inline errc ansi_mul(cuda::std::chrono::duration* out, + cuda::std::chrono::duration const* a, + cuda::std::chrono::duration const* b) +{ + using P = promoted; + auto r = static_cast

(a->count()) * static_cast

(b->count()); + if (r > static_cast

(cuda::std::numeric_limits::max()) || + r < static_cast

(cuda::std::numeric_limits::min())) { + return errc::OVERFLOW; + } + *out = cuda::std::chrono::duration{static_cast(r)}; + return errc::OK; +} + +template +__device__ inline errc ansi_mul(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + if (errc e = ansi_mul(&r, &a->value(), &b->value()); e != errc::OK) { + *out = nullopt; + return e; + } + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template + requires(cuda::std::is_integral_v && cuda::std::is_unsigned_v) +__device__ inline errc ansi_div(T* out, T const* a, T const* b) +{ + return detail::ansi_div_unsigned(out, a, b); +} + +template + requires(cuda::std::is_integral_v && cuda::std::is_signed_v) +__device__ inline errc ansi_div(T* out, T const* a, T const* b) +{ + return detail::ansi_div_signed(out, a, b); +} + +template + requires(cuda::std::is_floating_point_v) +__device__ inline errc ansi_div(T* out, T const* a, T const* b) +{ + return detail::ansi_div_float(out, a, b); +} + +template +__device__ inline errc ansi_div(numeric::fixed_point* out, + numeric::fixed_point const* a, + numeric::fixed_point const* b) +{ + if (numeric::division_overflow(a->value(), b->value())) { return errc::OVERFLOW; } + + *out = numeric::fixed_point{ + numeric::scaled_integer{a->value() / b->value(), a->scale() - b->scale()}}; + return errc::OK; +} + +template +__device__ inline errc ansi_div(cuda::std::chrono::duration* out, + cuda::std::chrono::duration const* a, + cuda::std::chrono::duration const* b) +{ + if (b->count() == 0) { return errc::DIVISION_BY_ZERO; } + using P = promoted; + auto r = static_cast

(a->count()) / static_cast

(b->count()); + if (r > static_cast

(cuda::std::numeric_limits::max()) || + r < static_cast

(cuda::std::numeric_limits::min())) { + return errc::OVERFLOW; + } + *out = cuda::std::chrono::duration{static_cast(r)}; + return errc::OK; +} + +template +__device__ inline errc ansi_div(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + if (errc e = ansi_div(&r, &a->value(), &b->value()); e != errc::OK) { + *out = nullopt; + return e; + } + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template + requires(cuda::std::is_integral_v && cuda::std::is_signed_v) +__device__ inline errc ansi_abs(T* out, T const* a) +{ + if (*a == cuda::std::numeric_limits::min()) { return errc::OVERFLOW; } + *out = (*a < 0) ? -(*a) : *a; + return errc::OK; +} + +template + requires(cuda::std::is_integral_v && cuda::std::is_unsigned_v) +__device__ inline errc ansi_abs(T* out, T const* a) +{ + *out = *a; + return errc::OK; +} + +template + requires(cuda::std::is_floating_point_v) +__device__ inline errc ansi_abs(T* out, T const* a) +{ + *out = (*a < 0) ? -(*a) : *a; + return errc::OK; +} + +template +__device__ inline errc ansi_abs(numeric::fixed_point* out, + numeric::fixed_point const* a) +{ + if (a->value() == cuda::std::numeric_limits::min()) { return errc::OVERFLOW; } + out->value() = (a->value() < 0) ? -a->value() : a->value(); + return errc::OK; +} + +template +__device__ inline errc ansi_abs(cuda::std::chrono::duration* out, + cuda::std::chrono::duration const* a) +{ + if (a->count() == cuda::std::numeric_limits::min()) { return errc::OVERFLOW; } + *out = (*a < cuda::std::chrono::duration{0}) ? -(*a) : *a; + return errc::OK; +} + +template +__device__ inline errc ansi_abs(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + if (errc e = ansi_abs(&r, &a->value()); e != errc::OK) { + *out = nullopt; + return e; + } + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template + requires(cuda::std::is_signed_v) +__device__ inline errc ansi_neg(T* out, T const* a) +{ + if (*a == cuda::std::numeric_limits::min()) { return errc::OVERFLOW; } + *out = -(*a); + return errc::OK; +} + +template +__device__ inline errc ansi_neg(numeric::fixed_point* out, + numeric::fixed_point const* a) +{ + if (a->value() == cuda::std::numeric_limits::min()) { return errc::OVERFLOW; } + auto rep = -a->value(); + *out = + numeric::fixed_point{numeric::scaled_integer{rep, a->scale()}}; + return errc::OK; +} + +template +__device__ inline errc ansi_neg(cuda::std::chrono::duration* out, + cuda::std::chrono::duration const* a) +{ + if (a->count() == cuda::std::numeric_limits::min()) { return errc::OVERFLOW; } + auto rep = -a->count(); + *out = cuda::std::chrono::duration{rep}; + return errc::OK; +} + +template +__device__ inline errc ansi_neg(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + if (errc e = ansi_neg(&r, &a->value()); e != errc::OK) { + *out = nullopt; + return e; + } + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc ansi_try_add(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + if (errc e = ansi_add(&r, &a->value(), &b->value()); e != errc::OK) { + *out = nullopt; + } else { + *out = r; + } + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc ansi_try_sub(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + if (errc e = ansi_sub(&r, &a->value(), &b->value()); e != errc::OK) { + *out = nullopt; + } else { + *out = r; + } + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc ansi_try_mul(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + if (errc e = ansi_mul(&r, &a->value(), &b->value()); e != errc::OK) { + *out = nullopt; + } else { + *out = r; + } + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc ansi_try_div(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + if (errc e = ansi_div(&r, &a->value(), &b->value()); e != errc::OK) { + *out = nullopt; + } else { + *out = r; + } + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc ansi_try_abs(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + if (errc e = ansi_abs(&r, a); e != errc::OK) { + *out = nullopt; + } else { + *out = r; + } + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc ansi_try_neg(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + if (errc e = ansi_neg(&r, &a->value()); e != errc::OK) { + *out = nullopt; + } else { + *out = r; + } + } else { + *out = nullopt; + } + return errc::OK; +} + +// TODO: IMPLEMENT +// exponent should not exceed 38 thus 10^exponent should fit in __int128_t + +// TODO: check if a's precision is less than or equal to the provided precision, if not return +// errc::OVERFLOW +template +__device__ inline errc assert_precise(numeric::fixed_point* out, + numeric::fixed_point const* a, + int32_t precision) +{ +} + +template +__device__ inline errc assert_precise(optional* out, optional const* a, int32_t precision) +{ + if (a->is_valid()) { + return assert_precise(&out->value(), &a->value(), precision); + } else { + *out = nullopt; + return errc::OK; + } +} + +template +__device__ inline errc try_precise( + optional>* out, + optional> const* a, + int32_t precision) +{ + if (a->is_valid()) { + if (errc e = assert_precise(&out->value(), &a->value(), precision); e != errc::OK) { + *out = nullopt; + return errc::OK; + } else { + *out = a->value(); + return errc::OK; + } + } else { + *out = nullopt; + return errc::OK; + } +} + +} // namespace ops +} // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/operators/arithmetic.cuh b/cpp/include/cudf/operators/arithmetic.cuh new file mode 100644 index 000000000000..90898a6f5a7c --- /dev/null +++ b/cpp/include/cudf/operators/arithmetic.cuh @@ -0,0 +1,216 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include + +namespace CUDF_EXPORT cudf { + +namespace ops { + +template + requires(cuda::std::is_signed_v || cuda::std::is_floating_point_v) +__device__ inline errc abs(T* out, T const* a) +{ + *out = (*a < 0) ? -*a : *a; + return errc::OK; +} + +template + requires(cuda::std::is_unsigned_v) +__device__ inline errc abs(T* out, T const* a) +{ + *out = *a; + return errc::OK; +} + +template +__device__ inline errc abs(numeric::fixed_point* out, + numeric::fixed_point const* a) +{ + auto rep = a->value() < 0 ? -a->value() : a->value(); + *out = + numeric::fixed_point{numeric::scaled_integer{rep, a->scale()}}; + return errc::OK; +} + +template +__device__ inline errc abs(cuda::std::chrono::duration* out, + cuda::std::chrono::duration const* a) +{ + auto rep = a->count() < 0 ? -a->count() : a->count(); + *out = cuda::std::chrono::duration{rep}; + return errc::OK; +} + +template +__device__ inline errc abs(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + abs(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc add(T* out, T const* a, T const* b) +{ + *out = (*a + *b); + return errc::OK; +} + +template +__device__ inline errc add(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + add(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc div(T* out, T const* a, T const* b) +{ + *out = (*a / *b); + return errc::OK; +} + +template +__device__ inline errc div(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + div(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc mod(T* out, T const* a, T const* b) +{ + *out = (*a % *b); + return errc::OK; +} + +template <> +__device__ inline errc mod(float* out, float const* a, float const* b) +{ + *out = ::fmodf(*a, *b); + return errc::OK; +} + +template <> +__device__ inline errc mod(double* out, double const* a, double const* b) +{ + *out = ::fmod(*a, *b); + return errc::OK; +} + +template +__device__ inline errc mod(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + mod(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc mul(T* out, T const* a, T const* b) +{ + *out = (*a * *b); + return errc::OK; +} + +template +__device__ inline errc mul(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + mul(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template + requires(cuda::std::is_signed_v) +__device__ inline errc neg(T* out, T const* a) +{ + *out = -(*a); + return errc::OK; +} + +template +__device__ inline errc neg(numeric::fixed_point* out, + numeric::fixed_point const* a) +{ + auto rep = -a->value(); + *out = numeric::fixed_point{ + numeric::scaled_integer{rep, a->scale()}}; + return errc::OK; +} + +template +__device__ inline errc neg(cuda::std::chrono::duration* out, + cuda::std::chrono::duration const* a) +{ + auto rep = -a->count(); + *out = cuda::std::chrono::duration{rep}; + return errc::OK; +} + +template +__device__ inline errc neg(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + neg(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc sub(T* out, T const* a, T const* b) +{ + *out = *a - *b; + return errc::OK; +} + +template +__device__ inline errc sub(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + sub(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +} // namespace ops +} // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/operators/btiwise.cuh b/cpp/include/cudf/operators/btiwise.cuh new file mode 100644 index 000000000000..b4f41ad30536 --- /dev/null +++ b/cpp/include/cudf/operators/btiwise.cuh @@ -0,0 +1,94 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include + +namespace CUDF_EXPORT cudf { + +namespace ops { + +template +__device__ inline errc bit_and(T* out, T const* a, T const* b) +{ + *out = (*a & *b); + return errc::OK; +} + +template +__device__ inline errc bit_and(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + bit_and(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc bit_invert(T* out, T const* a) +{ + *out = ~(*a); + return errc::OK; +} + +template +__device__ inline errc bit_invert(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + bit_invert(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc bit_or(T* out, T const* a, T const* b) +{ + *out = (*a | *b); + return errc::OK; +} + +template +__device__ inline errc bit_or(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + bit_or(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc bit_xor(T* out, T const* a, T const* b) +{ + *out = (*a ^ *b); + return errc::OK; +} + +template +__device__ inline errc bit_xor(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + bit_xor(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +} // namespace ops +} // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/operators/casts.cuh b/cpp/include/cudf/operators/casts.cuh new file mode 100644 index 000000000000..578dd99343c5 --- /dev/null +++ b/cpp/include/cudf/operators/casts.cuh @@ -0,0 +1,213 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include + +namespace CUDF_EXPORT cudf { + +namespace ops { + +template +__device__ inline errc cast_to_i32(int32_t* out, T const* a) +{ + *out = static_cast(*a); + return errc::OK; +} + +template +__device__ inline errc cast_to_i32(optional* out, optional const* a) +{ + if (a->is_valid()) { + int32_t r; + cast_to_i32(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc cast_to_i64(int64_t* out, T const* a) +{ + *out = static_cast(*a); + return errc::OK; +} + +template +__device__ inline errc cast_to_i64(optional* out, optional const* a) +{ + if (a->is_valid()) { + int64_t r; + cast_to_i64(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc cast_to_u32(uint32_t* out, T const* a) +{ + *out = static_cast(*a); + return errc::OK; +} + +template +__device__ inline errc cast_to_u32(optional* out, optional const* a) +{ + if (a->is_valid()) { + uint32_t r; + cast_to_u32(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc cast_to_u64(uint64_t* out, T const* a) +{ + *out = static_cast(*a); + return errc::OK; +} + +template +__device__ inline errc cast_to_u64(optional* out, optional const* a) +{ + if (a->is_valid()) { + uint64_t r; + cast_to_u64(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc cast_to_f32(float* out, T const* a) +{ + *out = static_cast(*a); + return errc::OK; +} + +template +__device__ inline errc cast_to_f32(optional* out, optional const* a) +{ + if (a->is_valid()) { + float r; + cast_to_f32(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc cast_to_f64(double* out, T const* a) +{ + *out = static_cast(*a); + return errc::OK; +} + +template +__device__ inline errc cast_to_f64(optional* out, optional const* a) +{ + if (a->is_valid()) { + double r; + cast_to_f64(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +namespace detail { + +template +__device__ inline errc fixed_point_cast(numeric::fixed_point* out, + numeric::fixed_point const* a) +{ + auto rep = static_cast(a->value()); + *out = numeric::fixed_point{numeric::scaled_integer{rep, a->scale()}}; + return errc::OK; +} + +} // namespace detail + +template +__device__ inline errc cast_to_dec32(numeric::decimal32* out, + numeric::fixed_point const* a) +{ + return detail::fixed_point_cast(out, a); +} + +template +__device__ inline errc cast_to_dec32( + optional* out, + optional> const* a) +{ + if (a->is_valid()) { + numeric::decimal32 r; + cast_to_dec32(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc cast_to_dec64(numeric::decimal64* out, + numeric::fixed_point const* a) +{ + return detail::fixed_point_cast(out, a); +} + +template +__device__ inline errc cast_to_dec64( + optional* out, + optional> const* a) +{ + if (a->is_valid()) { + numeric::decimal64 r; + cast_to_dec64(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc cast_to_dec128(numeric::decimal128* out, + numeric::fixed_point const* a) +{ + return detail::fixed_point_cast(out, a); +} + +template +__device__ inline errc cast_to_dec128( + optional* out, + optional> const* a) +{ + if (a->is_valid()) { + numeric::decimal128 r; + cast_to_dec128(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +} // namespace ops +} // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/operators/comparison.cuh b/cpp/include/cudf/operators/comparison.cuh new file mode 100644 index 000000000000..2744e7e8660f --- /dev/null +++ b/cpp/include/cudf/operators/comparison.cuh @@ -0,0 +1,192 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include + +namespace CUDF_EXPORT cudf { + +namespace ops { + +template +__device__ inline errc equal(bool* out, T const* a, T const* b) +{ + *out = (*a == *b); + return errc::OK; +} + +template +__device__ inline errc equal(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + bool r; + equal(&r, &a->value(), &b->value()); + *out = r; + } else if (a->is_null() && b->is_null()) { + *out = true; + } else { + *out = false; + } + return errc::OK; +} + +template +__device__ inline errc greater(bool* out, T const* a, T const* b) +{ + *out = (*a > *b); + return errc::OK; +} + +template +__device__ inline errc greater(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + bool r; + greater(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = false; + } + return errc::OK; +} + +template +__device__ inline errc greater_equal(bool* out, T const* a, T const* b) +{ + *out = (*a >= *b); + return errc::OK; +} + +template +__device__ inline errc greater_equal(optional* out, + optional const* a, + optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + bool r; + greater_equal(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = false; + } + return errc::OK; +} + +template +__device__ inline errc less(bool* out, T const* a, T const* b) +{ + *out = (*a < *b); + return errc::OK; +} + +template +__device__ inline errc less(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + bool r; + less(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = false; + } + return errc::OK; +} + +template +__device__ inline errc less_equal(bool* out, T const* a, T const* b) +{ + *out = (*a <= *b); + return errc::OK; +} + +template +__device__ inline errc less_equal(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + bool r; + less_equal(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = false; + } + return errc::OK; +} + +template +__device__ inline errc null_equal(bool* out, T const* a, T const* b) +{ + *out = (*a == *b); + return errc::OK; +} + +template +__device__ inline errc null_equal(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + *out = (*(*a) == *(*b)); + } else if (a->is_null() && b->is_null()) { + *out = true; + } else { + *out = false; + } + return errc::OK; +} + +template +__device__ inline errc null_logical_and(T* out, T const* a, T const* b) +{ + *out = (*a && *b); + return errc::OK; +} + +template +__device__ inline errc null_logical_and(optional* out, + optional const* a, + optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + bool r; + null_logical_and(&r, &a->value(), &b->value()); + *out = r; + } else if (a->is_null() && b->is_null()) { + *out = nullopt; + } else { + if (a->is_valid() ? *(*a) : *(*b)) { + *out = nullopt; + } else { + *out = false; + } + } + return errc::OK; +} + +template +__device__ inline errc null_logical_or(T* out, T const* a, T const* b) +{ + *out = (*a || *b); + return errc::OK; +} + +template +__device__ inline errc null_logical_or(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + bool r; + null_logical_or(&r, &a->value(), &b->value()); + *out = r; + } else if (a->is_null() && b->is_null()) { + *out = nullopt; + } else { + if (a->is_valid() ? *(*a) : *(*b)) { + *out = true; + } else { + *out = nullopt; + } + } + return errc::OK; +} + +} // namespace ops +} // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/operators/logic.cuh b/cpp/include/cudf/operators/logic.cuh new file mode 100644 index 000000000000..b905ddb38506 --- /dev/null +++ b/cpp/include/cudf/operators/logic.cuh @@ -0,0 +1,75 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include + +namespace CUDF_EXPORT cudf { + +namespace ops { + +template +__device__ inline errc logical_and(T* out, T const* a, T const* b) +{ + *out = (*a && *b); + return errc::OK; +} + +template +__device__ inline errc logical_and(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + logical_and(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc logical_or(T* out, T const* a, T const* b) +{ + *out = (*a || *b); + return errc::OK; +} + +template +__device__ inline errc logical_or(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + logical_or(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc if_else(T* out, bool const* pred, T const* true_value, T const* false_value) +{ + *out = *pred ? *true_value : *false_value; + return errc::OK; +} + +template +__device__ inline errc if_else(optional* out, + optional const* pred, + optional const* true_value, + optional const* false_value) +{ + if (pred->is_valid() && true_value->is_valid() && false_value->is_valid()) { + if_else(&out->value(), &pred->value(), &true_value->value(), &false_value->value()); + } else { + *out = nullopt; + } + return errc::OK; +} + +} // namespace ops +} // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/operators/math.cuh b/cpp/include/cudf/operators/math.cuh new file mode 100644 index 000000000000..17b23dabb11d --- /dev/null +++ b/cpp/include/cudf/operators/math.cuh @@ -0,0 +1,258 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include + +namespace CUDF_EXPORT cudf { + +namespace ops { + +template +__device__ inline errc cbrt(T* out, T const* a); + +template <> +__device__ inline errc cbrt(float* out, float const* a) +{ + *out = ::cbrtf(*a); + return errc::OK; +} + +template <> +__device__ inline errc cbrt(double* out, double const* a) +{ + *out = ::cbrt(*a); + return errc::OK; +} + +template +__device__ inline errc cbrt(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + cbrt(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc ceil(T* out, T const* a); + +template <> +__device__ inline errc ceil(float* out, float const* a) +{ + *out = ::ceilf(*a); + return errc::OK; +} + +template <> +__device__ inline errc ceil(double* out, double const* a) +{ + *out = ::ceil(*a); + return errc::OK; +} + +template +__device__ inline errc ceil(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + ceil(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc exp(T* out, T const* a); + +template <> +__device__ inline errc exp(float* out, float const* a) +{ + *out = ::expf(*a); + return errc::OK; +} + +template <> +__device__ inline errc exp(double* out, double const* a) +{ + *out = ::exp(*a); + return errc::OK; +} + +template +__device__ inline errc exp(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + exp(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc floor(T* out, T const* a); + +template <> +__device__ inline errc floor(float* out, float const* a) +{ + *out = ::floorf(*a); + return errc::OK; +} + +template <> +__device__ inline errc floor(double* out, double const* a) +{ + *out = ::floor(*a); + return errc::OK; +} + +template +__device__ inline errc floor(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + floor(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc log(T* out, T const* a); + +template <> +__device__ inline errc log(float* out, float const* a) +{ + *out = ::logf(*a); + return errc::OK; +} + +template <> +__device__ inline errc log(double* out, double const* a) +{ + *out = ::log(*a); + return errc::OK; +} + +template +__device__ inline errc log(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + log(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc pow(T* out, T const* a, T const* b); + +template <> +__device__ inline errc pow(float* out, float const* a, float const* b) +{ + *out = ::powf(*a, *b); + return errc::OK; +} + +template <> +__device__ inline errc pow(double* out, double const* a, double const* b) +{ + *out = ::pow(*a, *b); + return errc::OK; +} + +template +__device__ inline errc pow(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + pow(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc pymod(T* out, T const* a, T const* b) +{ + *out = (*a % *b + *b) % *b; + return errc::OK; +} + +template <> +__device__ inline errc pymod(float* out, float const* a, float const* b) +{ + *out = ::fmodf(::fmodf(*a, *b) + *b, *b); + return errc::OK; +} + +template <> +__device__ inline errc pymod(double* out, double const* a, double const* b) +{ + *out = ::fmod(::fmod(*a, *b) + *b, *b); + return errc::OK; +} + +template +__device__ inline errc pymod(optional* out, optional const* a, optional const* b) +{ + if (a->is_valid() && b->is_valid()) { + T r; + pymod(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc rint(T* out, T const* a); + +template <> +__device__ inline errc rint(float* out, float const* a) +{ + *out = ::rintf(*a); + return errc::OK; +} + +template <> +__device__ inline errc rint(double* out, double const* a) +{ + *out = ::rint(*a); + return errc::OK; +} + +template +__device__ inline errc rint(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + rint(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +} // namespace ops +} // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/operators/optional.cuh b/cpp/include/cudf/operators/optional.cuh new file mode 100644 index 000000000000..3fd3e311218d --- /dev/null +++ b/cpp/include/cudf/operators/optional.cuh @@ -0,0 +1,63 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include + +namespace CUDF_EXPORT cudf { + +struct nullopt_t {}; + +inline constexpr nullopt_t nullopt; + +template +struct optional { + T _value = {}; + + bool _is_valid = false; + + constexpr optional() = default; + + __device__ constexpr optional(nullopt_t) {} + + template + __device__ constexpr optional(inplace_t, Args&&... args) + : _value{static_cast(args)...}, _is_valid{true} + { + } + + __device__ constexpr optional(T value) : _value{value}, _is_valid{true} {} + + __device__ constexpr bool is_valid() const { return _is_valid; } + + __device__ constexpr bool is_null() const { return !_is_valid; } + + __device__ constexpr void reset() { _is_valid = false; } + + __device__ constexpr T const& get() const { return _value; } + + __device__ constexpr T& get() { return _value; } + + __device__ constexpr T const* operator->() const { return &_value; } + + __device__ constexpr T* operator->() { return &_value; } + + __device__ constexpr T const& operator*() const { return _value; } + + __device__ constexpr T& operator*() { return _value; } + + __device__ constexpr T const& value() const { return _value; } + + __device__ constexpr T& value() { return _value; } + + __device__ constexpr explicit operator bool() const { return _is_valid; } + + __device__ constexpr T value_or(T v) const { return _is_valid ? _value : v; } +}; + +template +optional(T) -> optional; + +} // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/operators/trigonometric.cuh b/cpp/include/cudf/operators/trigonometric.cuh new file mode 100644 index 000000000000..4521c917d013 --- /dev/null +++ b/cpp/include/cudf/operators/trigonometric.cuh @@ -0,0 +1,344 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include + +namespace CUDF_EXPORT cudf { + +namespace ops { + +template +__device__ inline errc arccos(T* out, T const* a); + +template <> +__device__ inline errc arccos(float* out, float const* a) +{ + *out = ::acosf(*a); + return errc::OK; +} + +template <> +__device__ inline errc arccos(double* out, double const* a) +{ + *out = ::acos(*a); + return errc::OK; +} + +template +__device__ inline errc arccos(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + arccos(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc arccosh(T* out, T const* a); + +template <> +__device__ inline errc arccosh(float* out, float const* a) +{ + *out = ::acoshf(*a); + return errc::OK; +} + +template <> +__device__ inline errc arccosh(double* out, double const* a) +{ + *out = ::acosh(*a); + return errc::OK; +} + +template +__device__ inline errc arccosh(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + arccosh(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc arcsin(T* out, T const* a); + +template <> +__device__ inline errc arcsin(float* out, float const* a) +{ + *out = ::asinf(*a); + return errc::OK; +} + +template <> +__device__ inline errc arcsin(double* out, double const* a) +{ + *out = ::asin(*a); + return errc::OK; +} + +template +__device__ inline errc arcsin(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + arcsin(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc arcsinh(T* out, T const* a); + +template <> +__device__ inline errc arcsinh(float* out, float const* a) +{ + *out = ::asinhf(*a); + return errc::OK; +} + +template <> +__device__ inline errc arcsinh(double* out, double const* a) +{ + *out = ::asinh(*a); + return errc::OK; +} + +template +__device__ inline errc arcsinh(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + arcsinh(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc arctan(T* out, T const* a); + +template <> +__device__ inline errc arctan(float* out, float const* a) +{ + *out = ::atanf(*a); + return errc::OK; +} + +template <> +__device__ inline errc arctan(double* out, double const* a) +{ + *out = ::atan(*a); + return errc::OK; +} + +template +__device__ inline errc arctan(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + arctan(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc arctanh(T* out, T const* a); + +template <> +__device__ inline errc arctanh(float* out, float const* a) +{ + *out = ::atanhf(*a); + return errc::OK; +} + +template <> +__device__ inline errc arctanh(double* out, double const* a) +{ + *out = ::atanh(*a); + return errc::OK; +} + +template +__device__ inline errc arctanh(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + arctanh(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc cos(T* out, T const* a); + +template <> +__device__ inline errc cos(float* out, float const* a) +{ + *out = ::cosf(*a); + return errc::OK; +} + +template <> +__device__ inline errc cos(double* out, double const* a) +{ + *out = ::cos(*a); + return errc::OK; +} + +template +__device__ inline errc cos(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + cos(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc cosh(T* out, T const* a); + +template <> +__device__ inline errc cosh(float* out, float const* a) +{ + *out = ::coshf(*a); + return errc::OK; +} + +template <> +__device__ inline errc cosh(double* out, double const* a) +{ + *out = ::cosh(*a); + return errc::OK; +} + +template +__device__ inline errc cosh(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + cosh(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc sin(T* out, T const* a); + +template <> +__device__ inline errc sin(float* out, float const* a) +{ + *out = ::sinf(*a); + return errc::OK; +} + +template <> +__device__ inline errc sin(double* out, double const* a) +{ + *out = ::sin(*a); + return errc::OK; +} + +template +__device__ inline errc sin(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + sin(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc sinh(T* out, T const* a); + +template <> +__device__ inline errc sinh(float* out, float const* a) +{ + *out = ::sinhf(*a); + return errc::OK; +} + +template <> +__device__ inline errc sinh(double* out, double const* a) +{ + *out = ::sinh(*a); + return errc::OK; +} + +template +__device__ inline errc sinh(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + sinh(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc tanh(T* out, T const* a); + +template <> +__device__ inline errc tanh(float* out, float const* a) +{ + *out = ::tanhf(*a); + return errc::OK; +} + +template <> +__device__ inline errc tanh(double* out, double const* a) +{ + *out = ::tanh(*a); + return errc::OK; +} + +template +__device__ inline errc tanh(optional* out, optional const* a) +{ + if (a->is_valid()) { + T r; + tanh(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +} // namespace ops +} // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/operators/types.cuh b/cpp/include/cudf/operators/types.cuh new file mode 100644 index 000000000000..a694dbaa9805 --- /dev/null +++ b/cpp/include/cudf/operators/types.cuh @@ -0,0 +1,115 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace CUDF_EXPORT cudf { + +namespace ops { + +enum errc : int { OK = 0, OVERFLOW = 1, DIVISION_BY_ZERO = 2 }; + +template +struct promoted_t; + +template <> +struct promoted_t { + using type = int16_t; +}; + +template <> +struct promoted_t { + using type = uint16_t; +}; + +template <> +struct promoted_t { + using type = int32_t; +}; + +template <> +struct promoted_t { + using type = uint32_t; +}; + +template <> +struct promoted_t { + using type = int64_t; +}; + +template <> +struct promoted_t { + using type = uint64_t; +}; + +template <> +struct promoted_t { + using type = __int128; +}; + +template <> +struct promoted_t { + using type = unsigned __int128; +}; + +template <> +struct promoted_t<__int128> { + using type = int256_t; +}; + +template <> +struct promoted_t { + using type = uint256_t; +}; + +template +using promoted = typename promoted_t::type; + +template +__device__ inline errc identity(T* out, T const* a) +{ + *out = *a; + return errc::OK; +} + +template +__device__ inline errc identity(optional* out, optional const* a) +{ + *out = *a; + return errc::OK; +} + +template +__device__ inline errc is_null(bool* out, T const* a) +{ + *out = false; + return errc::OK; +} + +template +__device__ inline errc is_null(optional* out, optional const* a) +{ + *out = a->is_null(); + return errc::OK; +} + +// TODO: decimal ansi operators(precision and scale-oriented non-templated arguments) +// TODO: cast operators to match AST +// TODO: decimal cast operators to match AST +// TODO: datetime cast operators & arithmetic +// TODO: decimal ansi cast +// TODO: ansi_mod, div operations for fixed-point and duration types + +} // namespace ops +} // namespace CUDF_EXPORT cudf From 602390f1a2c608e52f72fd18ed2f26dba3c0da38 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 26 Apr 2026 10:21:24 +0000 Subject: [PATCH 03/15] checkpoint Co-authored-by: Copilot --- cpp/include/cudf/ast/expressions.hpp | 62 +- .../cudf/operators/ansi_arithmetic.cuh | 468 +++++++-------- cpp/include/cudf/operators/arithmetic.cuh | 50 +- cpp/include/cudf/operators/btiwise.cuh | 9 +- cpp/include/cudf/operators/casts.cuh | 66 +- cpp/include/cudf/operators/comparison.cuh | 29 +- cpp/include/cudf/operators/logic.cuh | 27 +- cpp/include/cudf/operators/math.cuh | 141 +++-- cpp/include/cudf/operators/null_handling.cuh | 66 ++ cpp/include/cudf/operators/op_attributes.hpp | 562 ++++++++++++++++++ cpp/include/cudf/operators/opcodes.hpp | 107 ++++ cpp/include/cudf/operators/optional.cuh | 63 -- cpp/include/cudf/operators/trigonometric.cuh | 122 +--- cpp/include/cudf/operators/types.cuh | 102 +--- cpp/src/ast/expressions.cpp | 31 +- cpp/src/jit/row_ir.cpp | 490 ++++++--------- cpp/src/jit/row_ir.hpp | 368 +++--------- 17 files changed, 1457 insertions(+), 1306 deletions(-) create mode 100644 cpp/include/cudf/operators/null_handling.cuh create mode 100644 cpp/include/cudf/operators/op_attributes.hpp create mode 100644 cpp/include/cudf/operators/opcodes.hpp delete mode 100644 cpp/include/cudf/operators/optional.cuh diff --git a/cpp/include/cudf/ast/expressions.hpp b/cpp/include/cudf/ast/expressions.hpp index 76fdf2d10120..20eee1741900 100644 --- a/cpp/include/cudf/ast/expressions.hpp +++ b/cpp/include/cudf/ast/expressions.hpp @@ -83,8 +83,7 @@ struct expression { * @param visitor The `row_ir::ast_converter` converting this expression tree * @return The IR node representing this expression */ - [[nodiscard]] virtual std::unique_ptr accept( - cudf::detail::row_ir::ast_converter& visitor) const = 0; + virtual cudf::detail::row_ir::node accept(cudf::detail::row_ir::ast_converter& visitor) const = 0; /** * @brief Returns true if the expression may evaluate to null. @@ -320,8 +319,7 @@ class literal : public expression { /** * @copydoc expression::accept */ - [[nodiscard]] std::unique_ptr accept( - cudf::detail::row_ir::ast_converter& visitor) const override; + cudf::detail::row_ir::node accept(cudf::detail::row_ir::ast_converter& visitor) const override; [[nodiscard]] bool may_evaluate_null(table_view const& left, table_view const& right, @@ -432,8 +430,7 @@ class column_reference : public expression { /** * @copydoc expression::accept */ - [[nodiscard]] std::unique_ptr accept( - cudf::detail::row_ir::ast_converter& visitor) const override; + cudf::detail::row_ir::node accept(cudf::detail::row_ir::ast_converter& visitor) const override; private: cudf::size_type column_index; @@ -504,61 +501,13 @@ class operation : public expression { /** * @copydoc expression::accept */ - [[nodiscard]] std::unique_ptr accept( - cudf::detail::row_ir::ast_converter& visitor) const override; + cudf::detail::row_ir::node accept(cudf::detail::row_ir::ast_converter& visitor) const override; private: ast_operator op; std::vector> operands; }; -namespace detail { - -/// @brief An expression that represents a filter predicate. -/// -/// This is an internal expression used in filter operations. It is not intended to be used by -/// external code and is not a part of the public API. -class filter_predicate : public expression { - public: - /** - * @brief Construct a new filter predicate object - * @param source The source expression from which the predicate value is taken - */ - filter_predicate(expression const& source) : source_{source} {} - - /** - * @copydoc expression::accept - */ - cudf::size_type accept(detail::expression_parser& visitor) const override; - - /** - * @copydoc expression::accept - */ - std::reference_wrapper accept( - detail::expression_transformer& visitor) const override; - - [[nodiscard]] bool may_evaluate_null(table_view const& left, - table_view const& right, - rmm::cuda_stream_view stream) const override; - - /** - * @copydoc expression::accept - */ - [[nodiscard]] std::unique_ptr accept( - cudf::detail::row_ir::ast_converter& visitor) const override; - - /** - * @brief Get the operand expression. - * @return The operand expression - */ - [[nodiscard]] expression const& get_operand() const { return source_; } - - private: - std::reference_wrapper source_; -}; - -} // namespace detail - /** * @brief A expression referring to data from a column in a table. */ @@ -600,8 +549,7 @@ class column_name_reference : public expression { /** * @copydoc expression::accept */ - [[nodiscard]] std::unique_ptr accept( - cudf::detail::row_ir::ast_converter& visitor) const override; + cudf::detail::row_ir::node accept(cudf::detail::row_ir::ast_converter& visitor) const override; private: std::string column_name; diff --git a/cpp/include/cudf/operators/ansi_arithmetic.cuh b/cpp/include/cudf/operators/ansi_arithmetic.cuh index 8ff745a2b5cb..dde6a10b1234 100644 --- a/cpp/include/cudf/operators/ansi_arithmetic.cuh +++ b/cpp/include/cudf/operators/ansi_arithmetic.cuh @@ -4,6 +4,7 @@ */ #pragma once +#include #include namespace CUDF_EXPORT cudf { @@ -12,103 +13,50 @@ namespace ops { namespace detail { template -__device__ inline errc ansi_add_unsigned(T* out, T const* a, T const* b) -{ - using P = promoted; - auto r = static_cast

(*a) + static_cast

(*b); - if (r > static_cast

(cuda::std::numeric_limits::max())) { return errc::OVERFLOW; } - *out = static_cast(r); - return errc::OK; -} +struct promoted_t; -template -__device__ inline errc ansi_add_signed(T* out, T const* a, T const* b) -{ - using P = promoted; - auto r = static_cast

(*a) + static_cast

(*b); - if (r > static_cast

(cuda::std::numeric_limits::max()) || - r < static_cast

(cuda::std::numeric_limits::min())) { - return errc::OVERFLOW; - } - *out = static_cast(r); - return errc::OK; -} +template <> +struct promoted_t { + using type = int16_t; +}; -template -__device__ inline errc ansi_sub_unsigned(T* out, T const* a, T const* b) -{ - if (*a < *b) { return errc::OVERFLOW; } - auto r = *a - *b; - *out = static_cast(r); - return errc::OK; -} +template <> +struct promoted_t { + using type = uint16_t; +}; -template -__device__ inline errc ansi_sub_signed(T* out, T const* a, T const* b) -{ - using P = promoted; - auto r = static_cast

(*a) - static_cast

(*b); - if (r > static_cast

(cuda::std::numeric_limits::max()) || - r < static_cast

(cuda::std::numeric_limits::min())) { - return errc::OVERFLOW; - } - *out = static_cast(r); - return errc::OK; -} +template <> +struct promoted_t { + using type = int32_t; +}; -template -__device__ inline errc ansi_mul_unsigned(T* out, T const* a, T const* b) -{ - using P = promoted; - auto r = static_cast

(*a) * static_cast

(*b); - if (r > static_cast

(cuda::std::numeric_limits::max())) { return errc::OVERFLOW; } - *out = static_cast(r); - return errc::OK; -} +template <> +struct promoted_t { + using type = uint32_t; +}; -template -__device__ inline errc ansi_mul_signed(T* out, T const* a, T const* b) -{ - using P = promoted; - auto r = static_cast

(*a) * static_cast

(*b); - if (r > static_cast

(cuda::std::numeric_limits::max()) || - r < static_cast

(cuda::std::numeric_limits::min())) { - return errc::OVERFLOW; - } - *out = static_cast(r); - return errc::OK; -} +template <> +struct promoted_t { + using type = int64_t; +}; -template -__device__ inline errc ansi_div_unsigned(T* out, T const* a, T const* b) -{ - if (*b == 0) { return errc::DIVISION_BY_ZERO; } - *out = static_cast(static_cast(*a) / static_cast(*b)); - return errc::OK; -} +template <> +struct promoted_t { + using type = uint64_t; +}; -template -__device__ inline errc ansi_div_signed(T* out, T const* a, T const* b) -{ - if (*b == 0) { return errc::DIVISION_BY_ZERO; } - if (*a == cuda::std::numeric_limits::min() && *b == -1) { return errc::OVERFLOW; } - *out = static_cast(static_cast(*a) / static_cast(*b)); - return errc::OK; -} +template <> +struct promoted_t { + using type = __int128; +}; + +template <> +struct promoted_t { + using type = unsigned __int128; +}; template -__device__ inline errc ansi_div_float(T* out, T const* a, T const* b) -{ - using P = promoted; - if (*b == 0) { return errc::DIVISION_BY_ZERO; } - auto r = static_cast

(*a) / static_cast

(*b); - if (r > static_cast

(cuda::std::numeric_limits::max()) || - r < static_cast

(cuda::std::numeric_limits::lowest())) { - return errc::OVERFLOW; - } - *out = static_cast(r); - return errc::OK; -} +using promote = typename promoted_t::type; } // namespace detail @@ -116,14 +64,25 @@ template requires(cuda::std::is_integral_v && cuda::std::is_unsigned_v) __device__ inline errc ansi_add(T* out, T const* a, T const* b) { - return detail::ansi_add_unsigned(out, a, b); + using P = detail::promote; + auto r = static_cast

(*a) + static_cast

(*b); + if (r > static_cast

(cuda::std::numeric_limits::max())) { return errc::OVERFLOW; } + *out = static_cast(r); + return errc::OK; } template requires(cuda::std::is_integral_v && cuda::std::is_signed_v) __device__ inline errc ansi_add(T* out, T const* a, T const* b) { - return detail::ansi_add_signed(out, a, b); + using P = detail::promote; + auto r = static_cast

(*a) + static_cast

(*b); + if (r > static_cast

(cuda::std::numeric_limits::max()) || + r < static_cast

(cuda::std::numeric_limits::min())) { + return errc::OVERFLOW; + } + *out = static_cast(r); + return errc::OK; } template @@ -135,9 +94,7 @@ __device__ inline errc ansi_add(T* out, T const* a, T const* b) } template -__device__ inline errc ansi_add(numeric::fixed_point* out, - numeric::fixed_point const* a, - numeric::fixed_point const* b) +__device__ inline errc ansi_add(decimal* out, decimal const* a, decimal const* b) { auto scale = cuda::std::min(a->scale(), b->scale()); auto sum = a->rescaled(scale).value() + b->rescaled(scale).value(); @@ -146,29 +103,14 @@ __device__ inline errc ansi_add(numeric::fixed_point return errc::OVERFLOW; } - *out = numeric::fixed_point{numeric::scaled_integer{sum, scale}}; - return errc::OK; -} - -template -__device__ inline errc ansi_add(cuda::std::chrono::duration* out, - cuda::std::chrono::duration const* a, - cuda::std::chrono::duration const* b) -{ - using P = promoted; - auto r = static_cast

(a->count()) + static_cast

(b->count()); - if (r > static_cast

(cuda::std::numeric_limits::max()) || - r < static_cast

(cuda::std::numeric_limits::min())) { - return errc::OVERFLOW; - } - *out = cuda::std::chrono::duration{static_cast(r)}; + *out = decimal{numeric::scaled_integer{sum, scale}}; return errc::OK; } template __device__ inline errc ansi_add(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { T r; if (errc e = ansi_add(&r, &a->value(), &b->value()); e != errc::OK) { *out = nullopt; @@ -186,14 +128,24 @@ template requires(cuda::std::is_integral_v && cuda::std::is_unsigned_v) __device__ inline errc ansi_sub(T* out, T const* a, T const* b) { - return detail::ansi_sub_unsigned(out, a, b); + if (*a < *b) { return errc::OVERFLOW; } + auto r = *a - *b; + *out = static_cast(r); + return errc::OK; } template requires(cuda::std::is_integral_v && cuda::std::is_signed_v) __device__ inline errc ansi_sub(T* out, T const* a, T const* b) { - return detail::ansi_sub_signed(out, a, b); + using P = detail::promote; + auto r = static_cast

(*a) - static_cast

(*b); + if (r > static_cast

(cuda::std::numeric_limits::max()) || + r < static_cast

(cuda::std::numeric_limits::min())) { + return errc::OVERFLOW; + } + *out = static_cast(r); + return errc::OK; } template @@ -205,9 +157,7 @@ __device__ inline errc ansi_sub(T* out, T const* a, T const* b) } template -__device__ inline errc ansi_sub(numeric::fixed_point* out, - numeric::fixed_point const* a, - numeric::fixed_point const* b) +__device__ inline errc ansi_sub(decimal* out, decimal const* a, decimal const* b) { auto scale = cuda::std::min(a->scale(), b->scale()); auto sum = a->rescaled(scale).value() - b->rescaled(scale).value(); @@ -216,29 +166,14 @@ __device__ inline errc ansi_sub(numeric::fixed_point return errc::OVERFLOW; } - *out = numeric::fixed_point{numeric::scaled_integer{sum, scale}}; - return errc::OK; -} - -template -__device__ inline errc ansi_sub(cuda::std::chrono::duration* out, - cuda::std::chrono::duration const* a, - cuda::std::chrono::duration const* b) -{ - using P = promoted; - auto r = static_cast

(a->count()) - static_cast

(b->count()); - if (r > static_cast

(cuda::std::numeric_limits::max()) || - r < static_cast

(cuda::std::numeric_limits::min())) { - return errc::OVERFLOW; - } - *out = cuda::std::chrono::duration{static_cast(r)}; + *out = decimal{numeric::scaled_integer{sum, scale}}; return errc::OK; } template __device__ inline errc ansi_sub(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { T r; if (errc e = ansi_sub(&r, &a->value(), &b->value()); e != errc::OK) { *out = nullopt; @@ -255,14 +190,25 @@ template requires(cuda::std::is_integral_v && cuda::std::is_unsigned_v) __device__ inline errc ansi_mul(T* out, T const* a, T const* b) { - return detail::ansi_mul_unsigned(out, a, b); + using P = detail::promote; + auto r = static_cast

(*a) * static_cast

(*b); + if (r > static_cast

(cuda::std::numeric_limits::max())) { return errc::OVERFLOW; } + *out = static_cast(r); + return errc::OK; } template requires(cuda::std::is_integral_v && cuda::std::is_signed_v) __device__ inline errc ansi_mul(T* out, T const* a, T const* b) { - return detail::ansi_mul_signed(out, a, b); + using P = detail::promote; + auto r = static_cast

(*a) * static_cast

(*b); + if (r > static_cast

(cuda::std::numeric_limits::max()) || + r < static_cast

(cuda::std::numeric_limits::min())) { + return errc::OVERFLOW; + } + *out = static_cast(r); + return errc::OK; } template @@ -274,36 +220,18 @@ __device__ inline errc ansi_mul(T* out, T const* a, T const* b) } template -__device__ inline errc ansi_mul(numeric::fixed_point* out, - numeric::fixed_point const* a, - numeric::fixed_point const* b) +__device__ inline errc ansi_mul(decimal* out, decimal const* a, decimal const* b) { if (numeric::multiplication_overflow(a->value(), b->value())) { return errc::OVERFLOW; } - *out = numeric::fixed_point{ - numeric::scaled_integer{a->value() * b->value(), a->scale() + b->scale()}}; - return errc::OK; -} - -template -__device__ inline errc ansi_mul(cuda::std::chrono::duration* out, - cuda::std::chrono::duration const* a, - cuda::std::chrono::duration const* b) -{ - using P = promoted; - auto r = static_cast

(a->count()) * static_cast

(b->count()); - if (r > static_cast

(cuda::std::numeric_limits::max()) || - r < static_cast

(cuda::std::numeric_limits::min())) { - return errc::OVERFLOW; - } - *out = cuda::std::chrono::duration{static_cast(r)}; + *out = decimal{numeric::scaled_integer{a->value() * b->value(), a->scale() + b->scale()}}; return errc::OK; } template __device__ inline errc ansi_mul(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { T r; if (errc e = ansi_mul(&r, &a->value(), &b->value()); e != errc::OK) { *out = nullopt; @@ -320,57 +248,114 @@ template requires(cuda::std::is_integral_v && cuda::std::is_unsigned_v) __device__ inline errc ansi_div(T* out, T const* a, T const* b) { - return detail::ansi_div_unsigned(out, a, b); + if (*b == 0) { return errc::DIVISION_BY_ZERO; } + *out = static_cast(static_cast(*a) / static_cast(*b)); + return errc::OK; } template requires(cuda::std::is_integral_v && cuda::std::is_signed_v) __device__ inline errc ansi_div(T* out, T const* a, T const* b) { - return detail::ansi_div_signed(out, a, b); + if (*b == 0) { return errc::DIVISION_BY_ZERO; } + if (*a == cuda::std::numeric_limits::min() && *b == -1) { return errc::OVERFLOW; } + *out = static_cast(static_cast(*a) / static_cast(*b)); + return errc::OK; } template requires(cuda::std::is_floating_point_v) __device__ inline errc ansi_div(T* out, T const* a, T const* b) { - return detail::ansi_div_float(out, a, b); + using P = detail::promote; + if (*b == 0) { return errc::DIVISION_BY_ZERO; } + auto r = static_cast

(*a) / static_cast

(*b); + if (r > static_cast

(cuda::std::numeric_limits::max()) || + r < static_cast

(cuda::std::numeric_limits::lowest())) { + return errc::OVERFLOW; + } + *out = static_cast(r); + return errc::OK; } template -__device__ inline errc ansi_div(numeric::fixed_point* out, - numeric::fixed_point const* a, - numeric::fixed_point const* b) +__device__ inline errc ansi_div(decimal* out, decimal const* a, decimal const* b) { if (numeric::division_overflow(a->value(), b->value())) { return errc::OVERFLOW; } - *out = numeric::fixed_point{ - numeric::scaled_integer{a->value() / b->value(), a->scale() - b->scale()}}; + *out = decimal{numeric::scaled_integer{a->value() / b->value(), a->scale() - b->scale()}}; return errc::OK; } -template -__device__ inline errc ansi_div(cuda::std::chrono::duration* out, - cuda::std::chrono::duration const* a, - cuda::std::chrono::duration const* b) +template +__device__ inline errc ansi_div(optional* out, optional const* a, optional const* b) { - if (b->count() == 0) { return errc::DIVISION_BY_ZERO; } - using P = promoted; - auto r = static_cast

(a->count()) / static_cast

(b->count()); - if (r > static_cast

(cuda::std::numeric_limits::max()) || - r < static_cast

(cuda::std::numeric_limits::min())) { - return errc::OVERFLOW; + if (a->has_value() && b->has_value()) { + T r; + if (errc e = ansi_div(&r, &a->value(), &b->value()); e != errc::OK) { + *out = nullopt; + return e; + } + *out = r; + } else { + *out = nullopt; } - *out = cuda::std::chrono::duration{static_cast(r)}; return errc::OK; } template -__device__ inline errc ansi_div(optional* out, optional const* a, optional const* b) + requires(cuda::std::is_integral_v && cuda::std::is_signed_v) +__device__ inline errc ansi_mod(T* out, T const* a, T const* b) { - if (a->is_valid() && b->is_valid()) { + if (*b == 0) { return errc::DIVISION_BY_ZERO; } + T r = *a % *b; + if (r != 0 && ((r > 0) != (*b > 0))) { r += *b; } + *out = r; + return errc::OK; +} + +template + requires(cuda::std::is_integral_v && cuda::std::is_unsigned_v) +__device__ inline errc ansi_mod(T* out, T const* a, T const* b) +{ + if (*b == 0) { return errc::DIVISION_BY_ZERO; } + *out = *a % *b; + return errc::OK; +} + +__device__ inline errc ansi_mod(float* out, float const* a, float const* b) +{ + *out = (*a) - (*b) * ::floorf((*a) / (*b)); + return errc::OK; +} + +__device__ inline errc ansi_mod(double* out, double const* a, double const* b) +{ + *out = (*a) - (*b) * ::floor((*a) / (*b)); + return errc::OK; +} + +template +__device__ inline errc ansi_mod(decimal* out, decimal const* a, decimal const* b) +{ + if (b->value() == 0) { return errc::DIVISION_BY_ZERO; } + + decimal div; + + if (errc e = ansi_div(&div, a, b); e != errc::OK) { return e; } + + decimal quotient; + floor("ient, &div); + *out = *a - *b * quotient; + return errc::OK; +} + +template +__device__ inline errc ansi_mod(optional* out, optional const* a, optional const* b) +{ + if (a->has_value() && b->has_value()) { T r; - if (errc e = ansi_div(&r, &a->value(), &b->value()); e != errc::OK) { + if (errc e = ansi_mod(&r, &a->value(), &b->value()); e != errc::OK) { *out = nullopt; return e; } @@ -407,27 +392,17 @@ __device__ inline errc ansi_abs(T* out, T const* a) } template -__device__ inline errc ansi_abs(numeric::fixed_point* out, - numeric::fixed_point const* a) +__device__ inline errc ansi_abs(decimal* out, decimal const* a) { if (a->value() == cuda::std::numeric_limits::min()) { return errc::OVERFLOW; } out->value() = (a->value() < 0) ? -a->value() : a->value(); return errc::OK; } -template -__device__ inline errc ansi_abs(cuda::std::chrono::duration* out, - cuda::std::chrono::duration const* a) -{ - if (a->count() == cuda::std::numeric_limits::min()) { return errc::OVERFLOW; } - *out = (*a < cuda::std::chrono::duration{0}) ? -(*a) : *a; - return errc::OK; -} - template __device__ inline errc ansi_abs(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { T r; if (errc e = ansi_abs(&r, &a->value()); e != errc::OK) { *out = nullopt; @@ -450,30 +425,18 @@ __device__ inline errc ansi_neg(T* out, T const* a) } template -__device__ inline errc ansi_neg(numeric::fixed_point* out, - numeric::fixed_point const* a) +__device__ inline errc ansi_neg(decimal* out, decimal const* a) { if (a->value() == cuda::std::numeric_limits::min()) { return errc::OVERFLOW; } auto rep = -a->value(); - *out = - numeric::fixed_point{numeric::scaled_integer{rep, a->scale()}}; - return errc::OK; -} - -template -__device__ inline errc ansi_neg(cuda::std::chrono::duration* out, - cuda::std::chrono::duration const* a) -{ - if (a->count() == cuda::std::numeric_limits::min()) { return errc::OVERFLOW; } - auto rep = -a->count(); - *out = cuda::std::chrono::duration{rep}; + *out = decimal{numeric::scaled_integer{rep, a->scale()}}; return errc::OK; } template __device__ inline errc ansi_neg(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { T r; if (errc e = ansi_neg(&r, &a->value()); e != errc::OK) { *out = nullopt; @@ -486,10 +449,46 @@ __device__ inline errc ansi_neg(optional* out, optional const* a) return errc::OK; } +template +__device__ inline errc ansi_precision_cast(decimal* out, + decimal const* a, + int32_t const* precision) +{ + auto current_scale = static_cast(a->scale()); + auto allowed_scale = -(*precision); + + if (current_scale >= allowed_scale) { + *out = *a; + return errc::OK; + } + + auto extra_digits = allowed_scale - current_scale; + + auto factor = detail::ipow10(static_cast(extra_digits)); + + if (a->value() % factor != 0) { return errc::OVERFLOW; } + + *out = *a; + return errc::OK; +} + +template +__device__ inline errc ansi_precision_cast(optional* out, + optional const* a, + optional const* precision) +{ + if (a->has_value()) { + return ansi_precision_cast(&out->value(), &a->value(), &precision->value()); + } else { + *out = nullopt; + return errc::OK; + } +} + template __device__ inline errc ansi_try_add(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { T r; if (errc e = ansi_add(&r, &a->value(), &b->value()); e != errc::OK) { *out = nullopt; @@ -505,7 +504,7 @@ __device__ inline errc ansi_try_add(optional* out, optional const* a, opti template __device__ inline errc ansi_try_sub(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { T r; if (errc e = ansi_sub(&r, &a->value(), &b->value()); e != errc::OK) { *out = nullopt; @@ -521,7 +520,7 @@ __device__ inline errc ansi_try_sub(optional* out, optional const* a, opti template __device__ inline errc ansi_try_mul(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { T r; if (errc e = ansi_mul(&r, &a->value(), &b->value()); e != errc::OK) { *out = nullopt; @@ -537,7 +536,7 @@ __device__ inline errc ansi_try_mul(optional* out, optional const* a, opti template __device__ inline errc ansi_try_div(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { T r; if (errc e = ansi_div(&r, &a->value(), &b->value()); e != errc::OK) { *out = nullopt; @@ -551,11 +550,11 @@ __device__ inline errc ansi_try_div(optional* out, optional const* a, opti } template -__device__ inline errc ansi_try_abs(optional* out, optional const* a) +__device__ inline errc ansi_try_mod(optional* out, optional const* a, optional const* b) { - if (a->is_valid()) { + if (a->has_value() && b->has_value()) { T r; - if (errc e = ansi_abs(&r, a); e != errc::OK) { + if (errc e = ansi_mod(&r, &a->value(), &b->value()); e != errc::OK) { *out = nullopt; } else { *out = r; @@ -567,11 +566,11 @@ __device__ inline errc ansi_try_abs(optional* out, optional const* a) } template -__device__ inline errc ansi_try_neg(optional* out, optional const* a) +__device__ inline errc ansi_try_abs(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { T r; - if (errc e = ansi_neg(&r, &a->value()); e != errc::OK) { + if (errc e = ansi_abs(&r, a); e != errc::OK) { *out = nullopt; } else { *out = r; @@ -582,47 +581,38 @@ __device__ inline errc ansi_try_neg(optional* out, optional const* a) return errc::OK; } -// TODO: IMPLEMENT -// exponent should not exceed 38 thus 10^exponent should fit in __int128_t - -// TODO: check if a's precision is less than or equal to the provided precision, if not return -// errc::OVERFLOW -template -__device__ inline errc assert_precise(numeric::fixed_point* out, - numeric::fixed_point const* a, - int32_t precision) -{ -} - template -__device__ inline errc assert_precise(optional* out, optional const* a, int32_t precision) +__device__ inline errc ansi_try_neg(optional* out, optional const* a) { - if (a->is_valid()) { - return assert_precise(&out->value(), &a->value(), precision); + if (a->has_value()) { + T r; + if (errc e = ansi_neg(&r, &a->value()); e != errc::OK) { + *out = nullopt; + } else { + *out = r; + } } else { *out = nullopt; - return errc::OK; } + return errc::OK; } template -__device__ inline errc try_precise( - optional>* out, - optional> const* a, - int32_t precision) +__device__ inline errc ansi_try_precision_cast(optional>* out, + optional> const* a, + optional const* precision) { - if (a->is_valid()) { - if (errc e = assert_precise(&out->value(), &a->value(), precision); e != errc::OK) { + if (a->has_value() && precision->has_value()) { + if (errc e = ansi_precision_cast(&out->value(), &a->value(), &precision->value()); + e != errc::OK) { *out = nullopt; - return errc::OK; } else { *out = a->value(); - return errc::OK; } } else { *out = nullopt; - return errc::OK; } + return errc::OK; } } // namespace ops diff --git a/cpp/include/cudf/operators/arithmetic.cuh b/cpp/include/cudf/operators/arithmetic.cuh index 90898a6f5a7c..c74d392da612 100644 --- a/cpp/include/cudf/operators/arithmetic.cuh +++ b/cpp/include/cudf/operators/arithmetic.cuh @@ -27,28 +27,17 @@ __device__ inline errc abs(T* out, T const* a) } template -__device__ inline errc abs(numeric::fixed_point* out, - numeric::fixed_point const* a) +__device__ inline errc abs(decimal* out, decimal const* a) { auto rep = a->value() < 0 ? -a->value() : a->value(); - *out = - numeric::fixed_point{numeric::scaled_integer{rep, a->scale()}}; - return errc::OK; -} - -template -__device__ inline errc abs(cuda::std::chrono::duration* out, - cuda::std::chrono::duration const* a) -{ - auto rep = a->count() < 0 ? -a->count() : a->count(); - *out = cuda::std::chrono::duration{rep}; + *out = decimal{numeric::scaled_integer{rep, a->scale()}}; return errc::OK; } template __device__ inline errc abs(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { T r; abs(&r, &a->value()); *out = r; @@ -68,7 +57,7 @@ __device__ inline errc add(T* out, T const* a, T const* b) template __device__ inline errc add(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { T r; add(&r, &a->value(), &b->value()); *out = r; @@ -88,7 +77,7 @@ __device__ inline errc div(T* out, T const* a, T const* b) template __device__ inline errc div(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { T r; div(&r, &a->value(), &b->value()); *out = r; @@ -105,15 +94,13 @@ __device__ inline errc mod(T* out, T const* a, T const* b) return errc::OK; } -template <> -__device__ inline errc mod(float* out, float const* a, float const* b) +__device__ inline errc mod(float* out, float const* a, float const* b) { *out = ::fmodf(*a, *b); return errc::OK; } -template <> -__device__ inline errc mod(double* out, double const* a, double const* b) +__device__ inline errc mod(double* out, double const* a, double const* b) { *out = ::fmod(*a, *b); return errc::OK; @@ -122,7 +109,7 @@ __device__ inline errc mod(double* out, double const* a, double const* b template __device__ inline errc mod(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { T r; mod(&r, &a->value(), &b->value()); *out = r; @@ -142,7 +129,7 @@ __device__ inline errc mul(T* out, T const* a, T const* b) template __device__ inline errc mul(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { T r; mul(&r, &a->value(), &b->value()); *out = r; @@ -161,28 +148,17 @@ __device__ inline errc neg(T* out, T const* a) } template -__device__ inline errc neg(numeric::fixed_point* out, - numeric::fixed_point const* a) +__device__ inline errc neg(decimal* out, decimal const* a) { auto rep = -a->value(); - *out = numeric::fixed_point{ - numeric::scaled_integer{rep, a->scale()}}; - return errc::OK; -} - -template -__device__ inline errc neg(cuda::std::chrono::duration* out, - cuda::std::chrono::duration const* a) -{ - auto rep = -a->count(); - *out = cuda::std::chrono::duration{rep}; + *out = decimal{numeric::scaled_integer{rep, a->scale()}}; return errc::OK; } template __device__ inline errc neg(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { T r; neg(&r, &a->value()); *out = r; @@ -202,7 +178,7 @@ __device__ inline errc sub(T* out, T const* a, T const* b) template __device__ inline errc sub(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { T r; sub(&r, &a->value(), &b->value()); *out = r; diff --git a/cpp/include/cudf/operators/btiwise.cuh b/cpp/include/cudf/operators/btiwise.cuh index b4f41ad30536..b87d336538e0 100644 --- a/cpp/include/cudf/operators/btiwise.cuh +++ b/cpp/include/cudf/operators/btiwise.cuh @@ -7,7 +7,6 @@ #include namespace CUDF_EXPORT cudf { - namespace ops { template @@ -20,7 +19,7 @@ __device__ inline errc bit_and(T* out, T const* a, T const* b) template __device__ inline errc bit_and(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { T r; bit_and(&r, &a->value(), &b->value()); *out = r; @@ -40,7 +39,7 @@ __device__ inline errc bit_invert(T* out, T const* a) template __device__ inline errc bit_invert(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { T r; bit_invert(&r, &a->value()); *out = r; @@ -60,7 +59,7 @@ __device__ inline errc bit_or(T* out, T const* a, T const* b) template __device__ inline errc bit_or(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { T r; bit_or(&r, &a->value(), &b->value()); *out = r; @@ -80,7 +79,7 @@ __device__ inline errc bit_xor(T* out, T const* a, T const* b) template __device__ inline errc bit_xor(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { T r; bit_xor(&r, &a->value(), &b->value()); *out = r; diff --git a/cpp/include/cudf/operators/casts.cuh b/cpp/include/cudf/operators/casts.cuh index 578dd99343c5..ad71c19f9f4e 100644 --- a/cpp/include/cudf/operators/casts.cuh +++ b/cpp/include/cudf/operators/casts.cuh @@ -7,7 +7,6 @@ #include namespace CUDF_EXPORT cudf { - namespace ops { template @@ -20,7 +19,7 @@ __device__ inline errc cast_to_i32(int32_t* out, T const* a) template __device__ inline errc cast_to_i32(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { int32_t r; cast_to_i32(&r, &a->value()); *out = r; @@ -40,7 +39,7 @@ __device__ inline errc cast_to_i64(int64_t* out, T const* a) template __device__ inline errc cast_to_i64(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { int64_t r; cast_to_i64(&r, &a->value()); *out = r; @@ -60,7 +59,7 @@ __device__ inline errc cast_to_u32(uint32_t* out, T const* a) template __device__ inline errc cast_to_u32(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { uint32_t r; cast_to_u32(&r, &a->value()); *out = r; @@ -80,7 +79,7 @@ __device__ inline errc cast_to_u64(uint64_t* out, T const* a) template __device__ inline errc cast_to_u64(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { uint64_t r; cast_to_u64(&r, &a->value()); *out = r; @@ -100,7 +99,7 @@ __device__ inline errc cast_to_f32(float* out, T const* a) template __device__ inline errc cast_to_f32(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { float r; cast_to_f32(&r, &a->value()); *out = r; @@ -120,7 +119,7 @@ __device__ inline errc cast_to_f64(double* out, T const* a) template __device__ inline errc cast_to_f64(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { double r; cast_to_f64(&r, &a->value()); *out = r; @@ -133,29 +132,26 @@ __device__ inline errc cast_to_f64(optional* out, optional const* a) namespace detail { template -__device__ inline errc fixed_point_cast(numeric::fixed_point* out, - numeric::fixed_point const* a) +__device__ inline errc decimal_cast(decimal* out, decimal const* a) { auto rep = static_cast(a->value()); - *out = numeric::fixed_point{numeric::scaled_integer{rep, a->scale()}}; + *out = decimal{numeric::scaled_integer{rep, a->scale()}}; return errc::OK; } } // namespace detail -template -__device__ inline errc cast_to_dec32(numeric::decimal32* out, - numeric::fixed_point const* a) +template +__device__ inline errc cast_to_dec32(numeric::decimal32* out, decimal const* a) { - return detail::fixed_point_cast(out, a); + return detail::decimal_cast(out, a); } -template -__device__ inline errc cast_to_dec32( - optional* out, - optional> const* a) +template +__device__ inline errc cast_to_dec32(optional* out, + optional> const* a) { - if (a->is_valid()) { + if (a->has_value()) { numeric::decimal32 r; cast_to_dec32(&r, &a->value()); *out = r; @@ -165,19 +161,17 @@ __device__ inline errc cast_to_dec32( return errc::OK; } -template -__device__ inline errc cast_to_dec64(numeric::decimal64* out, - numeric::fixed_point const* a) +template +__device__ inline errc cast_to_dec64(numeric::decimal64* out, decimal const* a) { - return detail::fixed_point_cast(out, a); + return detail::decimal_cast(out, a); } -template -__device__ inline errc cast_to_dec64( - optional* out, - optional> const* a) +template +__device__ inline errc cast_to_dec64(optional* out, + optional> const* a) { - if (a->is_valid()) { + if (a->has_value()) { numeric::decimal64 r; cast_to_dec64(&r, &a->value()); *out = r; @@ -187,19 +181,17 @@ __device__ inline errc cast_to_dec64( return errc::OK; } -template -__device__ inline errc cast_to_dec128(numeric::decimal128* out, - numeric::fixed_point const* a) +template +__device__ inline errc cast_to_dec128(numeric::decimal128* out, decimal const* a) { - return detail::fixed_point_cast(out, a); + return detail::decimal_cast(out, a); } -template -__device__ inline errc cast_to_dec128( - optional* out, - optional> const* a) +template +__device__ inline errc cast_to_dec128(optional* out, + optional> const* a) { - if (a->is_valid()) { + if (a->has_value()) { numeric::decimal128 r; cast_to_dec128(&r, &a->value()); *out = r; diff --git a/cpp/include/cudf/operators/comparison.cuh b/cpp/include/cudf/operators/comparison.cuh index 2744e7e8660f..509b8e86bd24 100644 --- a/cpp/include/cudf/operators/comparison.cuh +++ b/cpp/include/cudf/operators/comparison.cuh @@ -7,7 +7,6 @@ #include namespace CUDF_EXPORT cudf { - namespace ops { template @@ -20,11 +19,11 @@ __device__ inline errc equal(bool* out, T const* a, T const* b) template __device__ inline errc equal(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { bool r; equal(&r, &a->value(), &b->value()); *out = r; - } else if (a->is_null() && b->is_null()) { + } else if (!a->has_value() && !b->has_value()) { *out = true; } else { *out = false; @@ -42,7 +41,7 @@ __device__ inline errc greater(bool* out, T const* a, T const* b) template __device__ inline errc greater(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { bool r; greater(&r, &a->value(), &b->value()); *out = r; @@ -64,7 +63,7 @@ __device__ inline errc greater_equal(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { bool r; greater_equal(&r, &a->value(), &b->value()); *out = r; @@ -84,7 +83,7 @@ __device__ inline errc less(bool* out, T const* a, T const* b) template __device__ inline errc less(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { bool r; less(&r, &a->value(), &b->value()); *out = r; @@ -104,7 +103,7 @@ __device__ inline errc less_equal(bool* out, T const* a, T const* b) template __device__ inline errc less_equal(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { bool r; less_equal(&r, &a->value(), &b->value()); *out = r; @@ -124,9 +123,9 @@ __device__ inline errc null_equal(bool* out, T const* a, T const* b) template __device__ inline errc null_equal(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { *out = (*(*a) == *(*b)); - } else if (a->is_null() && b->is_null()) { + } else if (!a->has_value() && !b->has_value()) { *out = true; } else { *out = false; @@ -146,14 +145,14 @@ __device__ inline errc null_logical_and(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { bool r; null_logical_and(&r, &a->value(), &b->value()); *out = r; - } else if (a->is_null() && b->is_null()) { + } else if (!a->has_value() && !b->has_value()) { *out = nullopt; } else { - if (a->is_valid() ? *(*a) : *(*b)) { + if (a->has_value() ? *(*a) : *(*b)) { *out = nullopt; } else { *out = false; @@ -172,14 +171,14 @@ __device__ inline errc null_logical_or(T* out, T const* a, T const* b) template __device__ inline errc null_logical_or(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { bool r; null_logical_or(&r, &a->value(), &b->value()); *out = r; - } else if (a->is_null() && b->is_null()) { + } else if (!a->has_value() && !b->has_value()) { *out = nullopt; } else { - if (a->is_valid() ? *(*a) : *(*b)) { + if (a->has_value() ? *(*a) : *(*b)) { *out = true; } else { *out = nullopt; diff --git a/cpp/include/cudf/operators/logic.cuh b/cpp/include/cudf/operators/logic.cuh index b905ddb38506..e70f70aeb183 100644 --- a/cpp/include/cudf/operators/logic.cuh +++ b/cpp/include/cudf/operators/logic.cuh @@ -7,7 +7,6 @@ #include namespace CUDF_EXPORT cudf { - namespace ops { template @@ -20,7 +19,7 @@ __device__ inline errc logical_and(T* out, T const* a, T const* b) template __device__ inline errc logical_and(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { T r; logical_and(&r, &a->value(), &b->value()); *out = r; @@ -40,7 +39,7 @@ __device__ inline errc logical_or(T* out, T const* a, T const* b) template __device__ inline errc logical_or(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { T r; logical_or(&r, &a->value(), &b->value()); *out = r; @@ -50,6 +49,26 @@ __device__ inline errc logical_or(optional* out, optional const* a, option return errc::OK; } +template +__device__ inline errc logical_not(T* out, T const* a) +{ + *out = !(*a); + return errc::OK; +} + +template +__device__ inline errc logical_not(optional* out, optional const* a) +{ + if (a->has_value()) { + T r; + logical_not(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + template __device__ inline errc if_else(T* out, bool const* pred, T const* true_value, T const* false_value) { @@ -63,7 +82,7 @@ __device__ inline errc if_else(optional* out, optional const* true_value, optional const* false_value) { - if (pred->is_valid() && true_value->is_valid() && false_value->is_valid()) { + if (pred->has_value() && true_value->has_value() && false_value->has_value()) { if_else(&out->value(), &pred->value(), &true_value->value(), &false_value->value()); } else { *out = nullopt; diff --git a/cpp/include/cudf/operators/math.cuh b/cpp/include/cudf/operators/math.cuh index 17b23dabb11d..dcc1c0e596e9 100644 --- a/cpp/include/cudf/operators/math.cuh +++ b/cpp/include/cudf/operators/math.cuh @@ -7,21 +7,15 @@ #include namespace CUDF_EXPORT cudf { - namespace ops { -template -__device__ inline errc cbrt(T* out, T const* a); - -template <> -__device__ inline errc cbrt(float* out, float const* a) +__device__ inline errc cbrt(float* out, float const* a) { *out = ::cbrtf(*a); return errc::OK; } -template <> -__device__ inline errc cbrt(double* out, double const* a) +__device__ inline errc cbrt(double* out, double const* a) { *out = ::cbrt(*a); return errc::OK; @@ -30,7 +24,7 @@ __device__ inline errc cbrt(double* out, double const* a) template __device__ inline errc cbrt(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { T r; cbrt(&r, &a->value()); *out = r; @@ -40,27 +34,37 @@ __device__ inline errc cbrt(optional* out, optional const* a) return errc::OK; } -template -__device__ inline errc ceil(T* out, T const* a); - -template <> -__device__ inline errc ceil(float* out, float const* a) +__device__ inline errc ceil(float* out, float const* a) { *out = ::ceilf(*a); return errc::OK; } -template <> -__device__ inline errc ceil(double* out, double const* a) +__device__ inline errc ceil(double* out, double const* a) { *out = ::ceil(*a); return errc::OK; } +template +__device__ inline errc ceil(decimal* out, decimal const* a) +{ + auto factor = detail::ipow10(static_cast(a->scale())); + auto div = a->value() / factor; + auto rem = a->value() % factor; + if (rem == 0) { + *out = *a; + } else { + auto val = a->value() > 0 ? (div + 1) : div; + *out = decimal{numeric::scaled_integer{val, a->scale()}}; + } + return errc::OK; +} + template __device__ inline errc ceil(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { T r; ceil(&r, &a->value()); *out = r; @@ -70,18 +74,13 @@ __device__ inline errc ceil(optional* out, optional const* a) return errc::OK; } -template -__device__ inline errc exp(T* out, T const* a); - -template <> -__device__ inline errc exp(float* out, float const* a) +__device__ inline errc exp(float* out, float const* a) { *out = ::expf(*a); return errc::OK; } -template <> -__device__ inline errc exp(double* out, double const* a) +__device__ inline errc exp(double* out, double const* a) { *out = ::exp(*a); return errc::OK; @@ -90,7 +89,7 @@ __device__ inline errc exp(double* out, double const* a) template __device__ inline errc exp(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { T r; exp(&r, &a->value()); *out = r; @@ -100,27 +99,37 @@ __device__ inline errc exp(optional* out, optional const* a) return errc::OK; } -template -__device__ inline errc floor(T* out, T const* a); - -template <> -__device__ inline errc floor(float* out, float const* a) +__device__ inline errc floor(float* out, float const* a) { *out = ::floorf(*a); return errc::OK; } -template <> -__device__ inline errc floor(double* out, double const* a) +__device__ inline errc floor(double* out, double const* a) { *out = ::floor(*a); return errc::OK; } +template +__device__ inline errc floor(decimal* out, decimal const* a) +{ + auto factor = detail::ipow10(static_cast(a->scale())); + auto div = a->value() / factor; + auto rem = a->value() % factor; + if (rem == 0) { + *out = *a; + } else { + auto val = a->value() > 0 ? div : (div - 1); + *out = decimal{numeric::scaled_integer{val, a->scale()}}; + } + return errc::OK; +} + template __device__ inline errc floor(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { T r; floor(&r, &a->value()); *out = r; @@ -130,18 +139,13 @@ __device__ inline errc floor(optional* out, optional const* a) return errc::OK; } -template -__device__ inline errc log(T* out, T const* a); - -template <> -__device__ inline errc log(float* out, float const* a) +__device__ inline errc log(float* out, float const* a) { *out = ::logf(*a); return errc::OK; } -template <> -__device__ inline errc log(double* out, double const* a) +__device__ inline errc log(double* out, double const* a) { *out = ::log(*a); return errc::OK; @@ -150,7 +154,7 @@ __device__ inline errc log(double* out, double const* a) template __device__ inline errc log(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { T r; log(&r, &a->value()); *out = r; @@ -160,18 +164,13 @@ __device__ inline errc log(optional* out, optional const* a) return errc::OK; } -template -__device__ inline errc pow(T* out, T const* a, T const* b); - -template <> -__device__ inline errc pow(float* out, float const* a, float const* b) +__device__ inline errc pow(float* out, float const* a, float const* b) { *out = ::powf(*a, *b); return errc::OK; } -template <> -__device__ inline errc pow(double* out, double const* a, double const* b) +__device__ inline errc pow(double* out, double const* a, double const* b) { *out = ::pow(*a, *b); return errc::OK; @@ -180,7 +179,7 @@ __device__ inline errc pow(double* out, double const* a, double const* b template __device__ inline errc pow(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { T r; pow(&r, &a->value(), &b->value()); *out = r; @@ -197,15 +196,13 @@ __device__ inline errc pymod(T* out, T const* a, T const* b) return errc::OK; } -template <> -__device__ inline errc pymod(float* out, float const* a, float const* b) +__device__ inline errc pymod(float* out, float const* a, float const* b) { *out = ::fmodf(::fmodf(*a, *b) + *b, *b); return errc::OK; } -template <> -__device__ inline errc pymod(double* out, double const* a, double const* b) +__device__ inline errc pymod(double* out, double const* a, double const* b) { *out = ::fmod(::fmod(*a, *b) + *b, *b); return errc::OK; @@ -214,7 +211,7 @@ __device__ inline errc pymod(double* out, double const* a, double const* template __device__ inline errc pymod(optional* out, optional const* a, optional const* b) { - if (a->is_valid() && b->is_valid()) { + if (a->has_value() && b->has_value()) { T r; pymod(&r, &a->value(), &b->value()); *out = r; @@ -224,18 +221,13 @@ __device__ inline errc pymod(optional* out, optional const* a, optional return errc::OK; } -template -__device__ inline errc rint(T* out, T const* a); - -template <> -__device__ inline errc rint(float* out, float const* a) +__device__ inline errc rint(float* out, float const* a) { *out = ::rintf(*a); return errc::OK; } -template <> -__device__ inline errc rint(double* out, double const* a) +__device__ inline errc rint(double* out, double const* a) { *out = ::rint(*a); return errc::OK; @@ -244,7 +236,7 @@ __device__ inline errc rint(double* out, double const* a) template __device__ inline errc rint(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { T r; rint(&r, &a->value()); *out = r; @@ -254,5 +246,30 @@ __device__ inline errc rint(optional* out, optional const* a) return errc::OK; } +__device__ inline errc sqrt(float* out, float const* a) +{ + *out = ::sqrtf(*a); + return errc::OK; +} + +__device__ inline errc sqrt(double* out, double const* a) +{ + *out = ::sqrt(*a); + return errc::OK; +} + +template +__device__ inline errc sqrt(optional* out, optional const* a) +{ + if (a->has_value()) { + T r; + sqrt(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + } // namespace ops } // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/operators/null_handling.cuh b/cpp/include/cudf/operators/null_handling.cuh new file mode 100644 index 000000000000..fd04b35d596d --- /dev/null +++ b/cpp/include/cudf/operators/null_handling.cuh @@ -0,0 +1,66 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once +#include + +#include + +namespace CUDF_EXPORT cudf { +namespace ops { + +template +__device__ inline errc is_null(bool* out, T const* a) +{ + *out = false; + return errc::OK; +} + +template +__device__ inline errc is_null(optional* out, optional const* a) +{ + *out = !a->has_value(); + return errc::OK; +} + +template +__device__ inline errc nullify_if(optional* out, + optional const* condition, + optional const* a) +{ + if (condition->has_value() && a->has_value()) { + if (condition->value()) { + *out = nullopt; + } else { + *out = a->value(); + } + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc coalesce(T* out, T const* a, T const* b) +{ + *out = *a; + return errc::OK; +} + +template +__device__ inline errc coalesce(optional* out, optional const* a, optional const* b) +{ + if (a->has_value()) { + *out = a->value(); + } else if (b->has_value()) { + *out = b->value(); + } else { + *out = nullopt; + } + return errc::OK; +} + +} // namespace ops +} // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/operators/op_attributes.hpp b/cpp/include/cudf/operators/op_attributes.hpp new file mode 100644 index 000000000000..765e82d0b99f --- /dev/null +++ b/cpp/include/cudf/operators/op_attributes.hpp @@ -0,0 +1,562 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once +#include + +namespace cudf::detail::row_ir { + +enum typing : uint64_t { + NONE = 0x0, + BOOL8 = 0x1, + INT8 = 0x2, + INT16 = 0x4, + INT32 = 0x8, + INT64 = 0x10, + UINT8 = 0x20, + UINT16 = 0x40, + UINT32 = 0x80, + UINT64 = 0x100, + FLOAT32 = 0x200, + FLOAT64 = 0x400, + DECIMAL32 = 0x800, + DECIMAL64 = 0x1000, + DECIMAL128 = 0x2000, + TIMESTAMP_DAYS = 0x4000, + TIMESTAMP_SECONDS = 0x8000, + TIMESTAMP_MILLISECONDS = 0x10000, + TIMESTAMP_MICROSECONDS = 0x20000, + TIMESTAMP_NANOSECONDS = 0x40000, + DURATION_DAYS = 0x80000, + DURATION_SECONDS = 0x100000, + DURATION_MILLISECONDS = 0x200000, + DURATION_MICROSECONDS = 0x400000, + DURATION_NANOSECONDS = 0x800000, + STRING = 0x1000000, + INTEGERS = INT8 | INT16 | INT32 | INT64 | UINT8 | UINT16 | UINT32 | UINT64, + SIGNED_INTEGERS = INT8 | INT16 | INT32 | INT64, + UNSIGNED_INTEGERS = UINT8 | UINT16 | UINT32 | UINT64, + FLOATS = FLOAT32 | FLOAT64, + DECIMALS = DECIMAL32 | DECIMAL64 | DECIMAL128, + ARITHMETIC = SIGNED_INTEGERS | UNSIGNED_INTEGERS | FLOATS | DECIMALS, + SIGNED_ARITHMETIC = SIGNED_INTEGERS | FLOATS | DECIMALS, + ALL = 0x0FFFFFFF, + ARG0 = 0x10000000, + ARG1 = 0x10000001, + ARG2 = 0x10000002, + INPUT = 0x20000000, +}; + +struct op_typing { + typing output = typing::NONE; + typing arg0 = typing::NONE; + typing arg1 = typing::NONE; + typing arg2 = typing::NONE; +}; + +/** + * @brief Indicates how an operator propagates null values + */ +enum class null_output : uint8_t { + PROPAGATE = 0, + ALWAYS_VALID = 1, + ALWAYS_NULLABLE = 2, +}; + +inline std::string_view get_op_name(opcode op) +{ + switch (op) { + case opcode::GET_INPUT: return "get_input"; + case opcode::SET_OUTPUT: return "set_output"; + case opcode::IS_NULL: return "is_null"; + case opcode::NULLIFY_IF: return "nullify_if"; + case opcode::COALESCE: return "coalesce"; + case opcode::ABS: return "abs"; + case opcode::ADD: return "add"; + case opcode::DIV: return "div"; + case opcode::MOD: return "mod"; + case opcode::MUL: return "mul"; + case opcode::NEG: return "neg"; + case opcode::SUB: return "sub"; + case opcode::ANSI_ADD: return "ansi_add"; + case opcode::ANSI_SUB: return "ansi_sub"; + case opcode::ANSI_MUL: return "ansi_mul"; + case opcode::ANSI_DIV: return "ansi_div"; + case opcode::ANSI_MOD: return "ansi_mod"; + case opcode::ANSI_ABS: return "ansi_abs"; + case opcode::ANSI_NEG: return "ansi_neg"; + case opcode::ANSI_PRECISION_CAST: return "ansi_precision_cast"; + case opcode::ANSI_TRY_ADD: return "ansi_try_add"; + case opcode::ANSI_TRY_SUB: return "ansi_try_sub"; + case opcode::ANSI_TRY_MUL: return "ansi_try_mul"; + case opcode::ANSI_TRY_DIV: return "ansi_try_div"; + case opcode::ANSI_TRY_MOD: return "ansi_try_mod"; + case opcode::ANSI_TRY_ABS: return "ansi_try_abs"; + case opcode::ANSI_TRY_NEG: return "ansi_try_neg"; + case opcode::ANSI_TRY_PRECISION_CAST: return "ansi_try_precision_cast"; + case opcode::BIT_AND: return "bit_and"; + case opcode::BIT_INVERT: return "bit_invert"; + case opcode::BIT_OR: return "bit_or"; + case opcode::BIT_XOR: return "bit_xor"; + case opcode::CAST_TO_I32: return "cast_to_i32"; + case opcode::CAST_TO_I64: return "cast_to_i64"; + case opcode::CAST_TO_U32: return "cast_to_u32"; + case opcode::CAST_TO_U64: return "cast_to_u64"; + case opcode::CAST_TO_F32: return "cast_to_f32"; + case opcode::CAST_TO_F64: return "cast_to_f64"; + case opcode::CAST_TO_DEC32: return "cast_to_dec32"; + case opcode::CAST_TO_DEC64: return "cast_to_dec64"; + case opcode::CAST_TO_DEC128: return "cast_to_dec128"; + case opcode::EQUAL: return "equal"; + case opcode::GREATER: return "greater"; + case opcode::GREATER_EQUAL: return "greater_equal"; + case opcode::LESS: return "less"; + case opcode::LESS_EQUAL: return "less_equal"; + case opcode::NULL_EQUAL: return "null_equal"; + case opcode::NULL_LOGICAL_AND: return "null_logical_and"; + case opcode::NULL_LOGICAL_OR: return "null_logical_or"; + case opcode::LOGICAL_NOT: return "logical_not"; + case opcode::IF_ELSE: return "if_else"; + case opcode::CBRT: return "cbrt"; + case opcode::CEIL: return "ceil"; + case opcode::FLOOR: return "floor"; + case opcode::SQRT: return "sqrt"; + case opcode::POW: return "pow"; + case opcode::EXP: return "exp"; + case opcode::LOG: return "log"; + case opcode::ARCCOS: return "arccos"; + case opcode::ARCCOSH: return "arccosh"; + case opcode::ARCSIN: return "arcsin"; + case opcode::ARCSINH: return "arcsinh"; + case opcode::ARCTAN: return "arctan"; + case opcode::ARCTANH: return "arctanh"; + case opcode::COS: return "cos"; + case opcode::COSH: return "cosh"; + case opcode::SIN: return "sin"; + case opcode::SINH: return "sinh"; + case opcode::TAN: return "tan"; + case opcode::TANH: return "tanh"; + } +} + +inline null_output get_op_null_output(opcode op) +{ + switch (op) { + case opcode::IS_NULL: + case opcode::NULL_EQUAL: return null_output::ALWAYS_VALID; + + case opcode::GET_INPUT: + case opcode::SET_OUTPUT: + case opcode::LOGICAL_NOT: + case opcode::ABS: + case opcode::ADD: + case opcode::DIV: + case opcode::MOD: + case opcode::MUL: + case opcode::NEG: + case opcode::SUB: + case opcode::ANSI_ADD: + case opcode::ANSI_SUB: + case opcode::ANSI_MUL: + case opcode::ANSI_DIV: + case opcode::ANSI_MOD: + case opcode::ANSI_ABS: + case opcode::ANSI_NEG: + case opcode::ANSI_PRECISION_CAST: + case opcode::IF_ELSE: + case opcode::CBRT: + case opcode::CEIL: + case opcode::FLOOR: + case opcode::SQRT: + case opcode::POW: + case opcode::EXP: + case opcode::LOG: + case opcode::ARCCOS: + case opcode::ARCCOSH: + case opcode::ARCSIN: + case opcode::ARCSINH: + case opcode::ARCTAN: + case opcode::ARCTANH: + case opcode::COS: + case opcode::COSH: + case opcode::SIN: + case opcode::SINH: + case opcode::TAN: + case opcode::BIT_AND: + case opcode::BIT_INVERT: + case opcode::BIT_OR: + case opcode::BIT_XOR: + case opcode::CAST_TO_I32: + case opcode::CAST_TO_I64: + case opcode::CAST_TO_U32: + case opcode::CAST_TO_U64: + case opcode::CAST_TO_F32: + case opcode::CAST_TO_F64: + case opcode::CAST_TO_DEC32: + case opcode::CAST_TO_DEC64: + case opcode::CAST_TO_DEC128: + case opcode::EQUAL: + case opcode::GREATER: + case opcode::GREATER_EQUAL: + case opcode::LESS: + case opcode::LESS_EQUAL: + case opcode::TANH: return null_output::PROPAGATE; + + case opcode::NULLIFY_IF: + case opcode::COALESCE: + case opcode::ANSI_TRY_ADD: + case opcode::ANSI_TRY_SUB: + case opcode::ANSI_TRY_MUL: + case opcode::ANSI_TRY_DIV: + case opcode::ANSI_TRY_MOD: + case opcode::ANSI_TRY_ABS: + case opcode::ANSI_TRY_NEG: + case opcode::ANSI_TRY_PRECISION_CAST: + case opcode::NULL_LOGICAL_AND: + case opcode::NULL_LOGICAL_OR: return null_output::ALWAYS_NULLABLE; + } +} + +/** + * @brief Indicates whether the output of the operator will be different when it is called with or + * without the null-ness of a value. + */ +inline bool get_op_requires_nulls(opcode op) +{ + switch (op) { + case opcode::GET_INPUT: + case opcode::SET_OUTPUT: + case opcode::NULLIFY_IF: + case opcode::ABS: + case opcode::ADD: + case opcode::DIV: + case opcode::MOD: + case opcode::MUL: + case opcode::NEG: + case opcode::SUB: + case opcode::ANSI_ADD: + case opcode::ANSI_SUB: + case opcode::ANSI_MUL: + case opcode::ANSI_DIV: + case opcode::ANSI_MOD: + case opcode::ANSI_ABS: + case opcode::ANSI_NEG: + case opcode::ANSI_PRECISION_CAST: + case opcode::ANSI_TRY_ADD: + case opcode::ANSI_TRY_SUB: + case opcode::ANSI_TRY_MUL: + case opcode::ANSI_TRY_DIV: + case opcode::ANSI_TRY_MOD: + case opcode::ANSI_TRY_ABS: + case opcode::ANSI_TRY_NEG: + case opcode::ANSI_TRY_PRECISION_CAST: + case opcode::BIT_AND: + case opcode::BIT_INVERT: + case opcode::BIT_OR: + case opcode::BIT_XOR: + case opcode::CAST_TO_I32: + case opcode::CAST_TO_I64: + case opcode::CAST_TO_U32: + case opcode::CAST_TO_U64: + case opcode::CAST_TO_F32: + case opcode::CAST_TO_F64: + case opcode::CAST_TO_DEC32: + case opcode::CAST_TO_DEC64: + case opcode::CAST_TO_DEC128: + case opcode::EQUAL: + case opcode::GREATER: + case opcode::GREATER_EQUAL: + case opcode::LESS: + case opcode::LESS_EQUAL: + case opcode::LOGICAL_NOT: + case opcode::IF_ELSE: + case opcode::CBRT: + case opcode::CEIL: + case opcode::FLOOR: + case opcode::SQRT: + case opcode::POW: + case opcode::EXP: + case opcode::LOG: + case opcode::ARCCOS: + case opcode::ARCCOSH: + case opcode::ARCSIN: + case opcode::ARCSINH: + case opcode::ARCTAN: + case opcode::ARCTANH: + case opcode::COS: + case opcode::COSH: + case opcode::SIN: + case opcode::SINH: + case opcode::TAN: + case opcode::TANH: return false; + + case opcode::COALESCE: + case opcode::IS_NULL: + case opcode::NULL_EQUAL: + case opcode::NULL_LOGICAL_AND: + case opcode::NULL_LOGICAL_OR: return true; + } +} + +inline bool get_op_is_fallible(opcode op) +{ + switch (op) { + case opcode::ANSI_ADD: + case opcode::ANSI_SUB: + case opcode::ANSI_MUL: + case opcode::ANSI_DIV: + case opcode::ANSI_MOD: + case opcode::ANSI_ABS: + case opcode::ANSI_NEG: + case opcode::ANSI_PRECISION_CAST: return true; + + case opcode::GET_INPUT: + case opcode::SET_OUTPUT: + case opcode::IS_NULL: + case opcode::NULLIFY_IF: + case opcode::COALESCE: + case opcode::ABS: + case opcode::ADD: + case opcode::DIV: + case opcode::MOD: + case opcode::MUL: + case opcode::NEG: + case opcode::SUB: + case opcode::ANSI_TRY_ADD: + case opcode::ANSI_TRY_SUB: + case opcode::ANSI_TRY_MUL: + case opcode::ANSI_TRY_DIV: + case opcode::ANSI_TRY_MOD: + case opcode::ANSI_TRY_ABS: + case opcode::ANSI_TRY_NEG: + case opcode::ANSI_TRY_PRECISION_CAST: + case opcode::BIT_AND: + case opcode::BIT_INVERT: + case opcode::BIT_OR: + case opcode::BIT_XOR: + case opcode::CAST_TO_I32: + case opcode::CAST_TO_I64: + case opcode::CAST_TO_U32: + case opcode::CAST_TO_U64: + case opcode::CAST_TO_F32: + case opcode::CAST_TO_F64: + case opcode::CAST_TO_DEC32: + case opcode::CAST_TO_DEC64: + case opcode::CAST_TO_DEC128: + case opcode::EQUAL: + case opcode::GREATER: + case opcode::GREATER_EQUAL: + case opcode::LESS: + case opcode::LESS_EQUAL: + case opcode::NULL_EQUAL: + case opcode::NULL_LOGICAL_AND: + case opcode::NULL_LOGICAL_OR: + case opcode::LOGICAL_NOT: + case opcode::IF_ELSE: + case opcode::CBRT: + case opcode::CEIL: + case opcode::FLOOR: + case opcode::SQRT: + case opcode::POW: + case opcode::EXP: + case opcode::LOG: + case opcode::ARCCOS: + case opcode::ARCCOSH: + case opcode::ARCSIN: + case opcode::ARCSINH: + case opcode::ARCTAN: + case opcode::ARCTANH: + case opcode::COS: + case opcode::COSH: + case opcode::SIN: + case opcode::SINH: + case opcode::TAN: + case opcode::TANH: return false; + } +} + +inline constexpr int32_t get_op_arity(opcode op) +{ + switch (op) { + case opcode::GET_INPUT: return 0; + + case opcode::SET_OUTPUT: + case opcode::IS_NULL: + case opcode::NULLIFY_IF: + case opcode::ABS: + case opcode::NEG: + case opcode::ANSI_ABS: + case opcode::ANSI_NEG: + case opcode::ANSI_TRY_ABS: + case opcode::ANSI_TRY_NEG: + case opcode::BIT_INVERT: + case opcode::CAST_TO_I32: + case opcode::CAST_TO_I64: + case opcode::CAST_TO_U32: + case opcode::CAST_TO_U64: + case opcode::CAST_TO_F32: + case opcode::CAST_TO_F64: + case opcode::CAST_TO_DEC32: + case opcode::CAST_TO_DEC64: + case opcode::CAST_TO_DEC128: + case opcode::LOGICAL_NOT: + case opcode::CBRT: + case opcode::CEIL: + case opcode::FLOOR: + case opcode::SQRT: + case opcode::EXP: + case opcode::LOG: + case opcode::ARCCOS: + case opcode::ARCCOSH: + case opcode::ARCSIN: + case opcode::ARCSINH: + case opcode::ARCTAN: + case opcode::ARCTANH: + case opcode::COS: + case opcode::COSH: + case opcode::SIN: + case opcode::SINH: + case opcode::TAN: + case opcode::TANH: return 1; + + case opcode::COALESCE: + case opcode::ADD: + case opcode::DIV: + case opcode::MOD: + case opcode::MUL: + case opcode::SUB: + case opcode::ANSI_ADD: + case opcode::ANSI_SUB: + case opcode::ANSI_MUL: + case opcode::ANSI_DIV: + case opcode::ANSI_MOD: + case opcode::ANSI_TRY_ADD: + case opcode::ANSI_TRY_SUB: + case opcode::ANSI_TRY_MUL: + case opcode::ANSI_TRY_DIV: + case opcode::ANSI_TRY_MOD: + case opcode::ANSI_PRECISION_CAST: + case opcode::ANSI_TRY_PRECISION_CAST: + case opcode::BIT_AND: + case opcode::BIT_OR: + case opcode::BIT_XOR: + case opcode::EQUAL: + case opcode::GREATER: + case opcode::GREATER_EQUAL: + case opcode::LESS: + case opcode::LESS_EQUAL: + case opcode::NULL_EQUAL: + case opcode::NULL_LOGICAL_OR: + case opcode::NULL_LOGICAL_AND: + case opcode::POW: return 2; + + case opcode::IF_ELSE: return 3; + } +} + +/** + * @brief Get the typing information for a given operator + * This function returns the expected input and output types for a given operator. The typing + * information can be used for type checking and inference when constructing expression trees. + * @param op The operator for which to get the typing information + * @return An `op_typing` struct containing the expected output type and input types for the + * operator + */ +inline op_typing get_op_typing(opcode op) +{ + switch (op) { + case opcode::GET_INPUT: return {typing::INPUT, typing::NONE, typing::NONE, typing::NONE}; + + case opcode::SET_OUTPUT: return {typing::NONE, typing::ALL, typing::NONE, typing::NONE}; + + case opcode::IS_NULL: return {typing::ARG0, typing::ALL, typing::NONE, typing::NONE}; + + case opcode::NULLIFY_IF: return {typing::ARG1, typing::BOOL8, typing::ALL, typing::NONE}; + case opcode::COALESCE: return {typing::ARG0, typing::ALL, typing::ARG0, typing::NONE}; + + case opcode::ABS: + case opcode::NEG: + case opcode::ANSI_ABS: + case opcode::ANSI_NEG: + case opcode::ANSI_TRY_NEG: + case opcode::ANSI_TRY_ABS: + return {typing::ARG0, typing::ARITHMETIC, typing::NONE, typing::NONE}; + + case opcode::ADD: + case opcode::DIV: + case opcode::MOD: + case opcode::MUL: + case opcode::SUB: + case opcode::ANSI_ADD: + case opcode::ANSI_SUB: + case opcode::ANSI_MUL: + case opcode::ANSI_DIV: + case opcode::ANSI_MOD: + case opcode::ANSI_TRY_ADD: + case opcode::ANSI_TRY_SUB: + case opcode::ANSI_TRY_MUL: + case opcode::ANSI_TRY_DIV: + case opcode::ANSI_TRY_MOD: + return {typing::ARG0, typing::ARITHMETIC, typing::ARG0, typing::NONE}; + + case opcode::ANSI_PRECISION_CAST: + case opcode::ANSI_TRY_PRECISION_CAST: + return {typing::ARG0, typing::DECIMALS, typing::INT32, typing::NONE}; + + case opcode::BIT_AND: + case opcode::BIT_INVERT: + case opcode::BIT_OR: + case opcode::BIT_XOR: return {typing::ARG0, typing::INTEGERS, typing::ARG0, typing::NONE}; + + case opcode::CAST_TO_I32: + case opcode::CAST_TO_I64: + case opcode::CAST_TO_U32: + case opcode::CAST_TO_U64: + case opcode::CAST_TO_F32: + case opcode::CAST_TO_F64: + return {typing::ARG0, typing{typing::INTEGERS | typing::FLOATS}, typing::NONE, typing::NONE}; + + case opcode::CAST_TO_DEC32: + case opcode::CAST_TO_DEC64: + case opcode::CAST_TO_DEC128: + return {typing::ARG0, typing::DECIMALS, typing::NONE, typing::NONE}; + + case opcode::EQUAL: + case opcode::GREATER: + case opcode::GREATER_EQUAL: + case opcode::LESS: + case opcode::LESS_EQUAL: return {typing::BOOL8, typing::ALL, typing::ARG0, typing::NONE}; + + case opcode::NULL_EQUAL: + case opcode::NULL_LOGICAL_AND: + case opcode::NULL_LOGICAL_OR: return {typing::BOOL8, typing::BOOL8, typing::ARG0, typing::NONE}; + + case opcode::LOGICAL_NOT: return {typing::ARG0, typing::BOOL8, typing::NONE, typing::NONE}; + + case opcode::IF_ELSE: return {typing::ARG1, typing::BOOL8, typing::ALL, typing::ARG0}; + + case opcode::CBRT: + case opcode::CEIL: + case opcode::FLOOR: + case opcode::SQRT: + case opcode::POW: + case opcode::EXP: + case opcode::LOG: + case opcode::ARCCOS: + case opcode::ARCCOSH: + case opcode::ARCSIN: + case opcode::ARCSINH: + case opcode::ARCTAN: + case opcode::ARCTANH: + case opcode::COS: + case opcode::COSH: + case opcode::SIN: + case opcode::SINH: + case opcode::TAN: + case opcode::TANH: return {typing::ARG0, typing::FLOATS, typing::NONE, typing::NONE}; + } +} + +} // namespace cudf::detail::row_ir diff --git a/cpp/include/cudf/operators/opcodes.hpp b/cpp/include/cudf/operators/opcodes.hpp new file mode 100644 index 000000000000..8b8a0c9d7dbb --- /dev/null +++ b/cpp/include/cudf/operators/opcodes.hpp @@ -0,0 +1,107 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once +#include +#include + +namespace cudf { +namespace detail { +namespace row_ir { + +enum class [[nodiscard]] opcode : int32_t { + GET_INPUT, + SET_OUTPUT, + + // Null handling operators + IS_NULL, + NULLIFY_IF, + COALESCE, + + /// Arithmetic operators + ABS, + ADD, + DIV, + MOD, + MUL, + NEG, + SUB, + + /// ANSI Arithmetic functions. raise errors on overflow, division by zero, etc. + ANSI_ADD, + ANSI_SUB, + ANSI_MUL, + ANSI_DIV, + ANSI_MOD, + ANSI_ABS, + ANSI_NEG, + ANSI_PRECISION_CAST, + + /// ANSI TRY arithmetic functions. return NULL instead of raising errors + ANSI_TRY_ADD, + ANSI_TRY_SUB, + ANSI_TRY_MUL, + ANSI_TRY_DIV, + ANSI_TRY_MOD, + ANSI_TRY_ABS, + ANSI_TRY_NEG, + ANSI_TRY_PRECISION_CAST, + + /// Bitwise operators + BIT_AND, + BIT_INVERT, + BIT_OR, + BIT_XOR, + + /// Type conversion operators + CAST_TO_I32, + CAST_TO_I64, + CAST_TO_U32, + CAST_TO_U64, + CAST_TO_F32, + CAST_TO_F64, + CAST_TO_DEC32, + CAST_TO_DEC64, + CAST_TO_DEC128, + + /// Comparison & Logical operators + EQUAL, + GREATER, + GREATER_EQUAL, + LESS, + LESS_EQUAL, + NULL_EQUAL, + NULL_LOGICAL_AND, + NULL_LOGICAL_OR, + LOGICAL_NOT, + IF_ELSE, + + /// Mathematical operators + CBRT, + CEIL, + FLOOR, + SQRT, + POW, + EXP, + LOG, + + /// Trigonometric operators + ARCCOS, + ARCCOSH, + ARCSIN, + ARCSINH, + ARCTAN, + ARCTANH, + COS, + COSH, + SIN, + SINH, + TAN, + TANH, +}; + +} // namespace row_ir +} // namespace detail +} // namespace cudf diff --git a/cpp/include/cudf/operators/optional.cuh b/cpp/include/cudf/operators/optional.cuh deleted file mode 100644 index 3fd3e311218d..000000000000 --- a/cpp/include/cudf/operators/optional.cuh +++ /dev/null @@ -1,63 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once - -#include - -namespace CUDF_EXPORT cudf { - -struct nullopt_t {}; - -inline constexpr nullopt_t nullopt; - -template -struct optional { - T _value = {}; - - bool _is_valid = false; - - constexpr optional() = default; - - __device__ constexpr optional(nullopt_t) {} - - template - __device__ constexpr optional(inplace_t, Args&&... args) - : _value{static_cast(args)...}, _is_valid{true} - { - } - - __device__ constexpr optional(T value) : _value{value}, _is_valid{true} {} - - __device__ constexpr bool is_valid() const { return _is_valid; } - - __device__ constexpr bool is_null() const { return !_is_valid; } - - __device__ constexpr void reset() { _is_valid = false; } - - __device__ constexpr T const& get() const { return _value; } - - __device__ constexpr T& get() { return _value; } - - __device__ constexpr T const* operator->() const { return &_value; } - - __device__ constexpr T* operator->() { return &_value; } - - __device__ constexpr T const& operator*() const { return _value; } - - __device__ constexpr T& operator*() { return _value; } - - __device__ constexpr T const& value() const { return _value; } - - __device__ constexpr T& value() { return _value; } - - __device__ constexpr explicit operator bool() const { return _is_valid; } - - __device__ constexpr T value_or(T v) const { return _is_valid ? _value : v; } -}; - -template -optional(T) -> optional; - -} // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/operators/trigonometric.cuh b/cpp/include/cudf/operators/trigonometric.cuh index 4521c917d013..50dd6f269f10 100644 --- a/cpp/include/cudf/operators/trigonometric.cuh +++ b/cpp/include/cudf/operators/trigonometric.cuh @@ -7,21 +7,15 @@ #include namespace CUDF_EXPORT cudf { - namespace ops { -template -__device__ inline errc arccos(T* out, T const* a); - -template <> -__device__ inline errc arccos(float* out, float const* a) +__device__ inline errc arccos(float* out, float const* a) { *out = ::acosf(*a); return errc::OK; } -template <> -__device__ inline errc arccos(double* out, double const* a) +__device__ inline errc arccos(double* out, double const* a) { *out = ::acos(*a); return errc::OK; @@ -30,7 +24,7 @@ __device__ inline errc arccos(double* out, double const* a) template __device__ inline errc arccos(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { T r; arccos(&r, &a->value()); *out = r; @@ -40,18 +34,13 @@ __device__ inline errc arccos(optional* out, optional const* a) return errc::OK; } -template -__device__ inline errc arccosh(T* out, T const* a); - -template <> -__device__ inline errc arccosh(float* out, float const* a) +__device__ inline errc arccosh(float* out, float const* a) { *out = ::acoshf(*a); return errc::OK; } -template <> -__device__ inline errc arccosh(double* out, double const* a) +__device__ inline errc arccosh(double* out, double const* a) { *out = ::acosh(*a); return errc::OK; @@ -60,7 +49,7 @@ __device__ inline errc arccosh(double* out, double const* a) template __device__ inline errc arccosh(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { T r; arccosh(&r, &a->value()); *out = r; @@ -70,18 +59,13 @@ __device__ inline errc arccosh(optional* out, optional const* a) return errc::OK; } -template -__device__ inline errc arcsin(T* out, T const* a); - -template <> -__device__ inline errc arcsin(float* out, float const* a) +__device__ inline errc arcsin(float* out, float const* a) { *out = ::asinf(*a); return errc::OK; } -template <> -__device__ inline errc arcsin(double* out, double const* a) +__device__ inline errc arcsin(double* out, double const* a) { *out = ::asin(*a); return errc::OK; @@ -90,7 +74,7 @@ __device__ inline errc arcsin(double* out, double const* a) template __device__ inline errc arcsin(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { T r; arcsin(&r, &a->value()); *out = r; @@ -100,18 +84,13 @@ __device__ inline errc arcsin(optional* out, optional const* a) return errc::OK; } -template -__device__ inline errc arcsinh(T* out, T const* a); - -template <> -__device__ inline errc arcsinh(float* out, float const* a) +__device__ inline errc arcsinh(float* out, float const* a) { *out = ::asinhf(*a); return errc::OK; } -template <> -__device__ inline errc arcsinh(double* out, double const* a) +__device__ inline errc arcsinh(double* out, double const* a) { *out = ::asinh(*a); return errc::OK; @@ -120,7 +99,7 @@ __device__ inline errc arcsinh(double* out, double const* a) template __device__ inline errc arcsinh(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { T r; arcsinh(&r, &a->value()); *out = r; @@ -130,18 +109,13 @@ __device__ inline errc arcsinh(optional* out, optional const* a) return errc::OK; } -template -__device__ inline errc arctan(T* out, T const* a); - -template <> -__device__ inline errc arctan(float* out, float const* a) +__device__ inline errc arctan(float* out, float const* a) { *out = ::atanf(*a); return errc::OK; } -template <> -__device__ inline errc arctan(double* out, double const* a) +__device__ inline errc arctan(double* out, double const* a) { *out = ::atan(*a); return errc::OK; @@ -150,7 +124,7 @@ __device__ inline errc arctan(double* out, double const* a) template __device__ inline errc arctan(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { T r; arctan(&r, &a->value()); *out = r; @@ -160,18 +134,13 @@ __device__ inline errc arctan(optional* out, optional const* a) return errc::OK; } -template -__device__ inline errc arctanh(T* out, T const* a); - -template <> -__device__ inline errc arctanh(float* out, float const* a) +__device__ inline errc arctanh(float* out, float const* a) { *out = ::atanhf(*a); return errc::OK; } -template <> -__device__ inline errc arctanh(double* out, double const* a) +__device__ inline errc arctanh(double* out, double const* a) { *out = ::atanh(*a); return errc::OK; @@ -180,7 +149,7 @@ __device__ inline errc arctanh(double* out, double const* a) template __device__ inline errc arctanh(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { T r; arctanh(&r, &a->value()); *out = r; @@ -190,18 +159,13 @@ __device__ inline errc arctanh(optional* out, optional const* a) return errc::OK; } -template -__device__ inline errc cos(T* out, T const* a); - -template <> -__device__ inline errc cos(float* out, float const* a) +__device__ inline errc cos(float* out, float const* a) { *out = ::cosf(*a); return errc::OK; } -template <> -__device__ inline errc cos(double* out, double const* a) +__device__ inline errc cos(double* out, double const* a) { *out = ::cos(*a); return errc::OK; @@ -210,7 +174,7 @@ __device__ inline errc cos(double* out, double const* a) template __device__ inline errc cos(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { T r; cos(&r, &a->value()); *out = r; @@ -220,18 +184,13 @@ __device__ inline errc cos(optional* out, optional const* a) return errc::OK; } -template -__device__ inline errc cosh(T* out, T const* a); - -template <> -__device__ inline errc cosh(float* out, float const* a) +__device__ inline errc cosh(float* out, float const* a) { *out = ::coshf(*a); return errc::OK; } -template <> -__device__ inline errc cosh(double* out, double const* a) +__device__ inline errc cosh(double* out, double const* a) { *out = ::cosh(*a); return errc::OK; @@ -240,7 +199,7 @@ __device__ inline errc cosh(double* out, double const* a) template __device__ inline errc cosh(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { T r; cosh(&r, &a->value()); *out = r; @@ -250,18 +209,13 @@ __device__ inline errc cosh(optional* out, optional const* a) return errc::OK; } -template -__device__ inline errc sin(T* out, T const* a); - -template <> -__device__ inline errc sin(float* out, float const* a) +__device__ inline errc sin(float* out, float const* a) { *out = ::sinf(*a); return errc::OK; } -template <> -__device__ inline errc sin(double* out, double const* a) +__device__ inline errc sin(double* out, double const* a) { *out = ::sin(*a); return errc::OK; @@ -270,7 +224,7 @@ __device__ inline errc sin(double* out, double const* a) template __device__ inline errc sin(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { T r; sin(&r, &a->value()); *out = r; @@ -280,18 +234,13 @@ __device__ inline errc sin(optional* out, optional const* a) return errc::OK; } -template -__device__ inline errc sinh(T* out, T const* a); - -template <> -__device__ inline errc sinh(float* out, float const* a) +__device__ inline errc sinh(float* out, float const* a) { *out = ::sinhf(*a); return errc::OK; } -template <> -__device__ inline errc sinh(double* out, double const* a) +__device__ inline errc sinh(double* out, double const* a) { *out = ::sinh(*a); return errc::OK; @@ -300,7 +249,7 @@ __device__ inline errc sinh(double* out, double const* a) template __device__ inline errc sinh(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { T r; sinh(&r, &a->value()); *out = r; @@ -310,18 +259,13 @@ __device__ inline errc sinh(optional* out, optional const* a) return errc::OK; } -template -__device__ inline errc tanh(T* out, T const* a); - -template <> -__device__ inline errc tanh(float* out, float const* a) +__device__ inline errc tanh(float* out, float const* a) { *out = ::tanhf(*a); return errc::OK; } -template <> -__device__ inline errc tanh(double* out, double const* a) +__device__ inline errc tanh(double* out, double const* a) { *out = ::tanh(*a); return errc::OK; @@ -330,7 +274,7 @@ __device__ inline errc tanh(double* out, double const* a) template __device__ inline errc tanh(optional* out, optional const* a) { - if (a->is_valid()) { + if (a->has_value()) { T r; tanh(&r, &a->value()); *out = r; diff --git a/cpp/include/cudf/operators/types.cuh b/cpp/include/cudf/operators/types.cuh index a694dbaa9805..47194add243d 100644 --- a/cpp/include/cudf/operators/types.cuh +++ b/cpp/include/cudf/operators/types.cuh @@ -5,111 +5,51 @@ #pragma once #include -#include -#include #include #include +#include #include #include +#include #include namespace CUDF_EXPORT cudf { - namespace ops { enum errc : int { OK = 0, OVERFLOW = 1, DIVISION_BY_ZERO = 2 }; template -struct promoted_t; - -template <> -struct promoted_t { - using type = int16_t; -}; - -template <> -struct promoted_t { - using type = uint16_t; -}; - -template <> -struct promoted_t { - using type = int32_t; -}; - -template <> -struct promoted_t { - using type = uint32_t; -}; +using optional = cuda::std::optional; -template <> -struct promoted_t { - using type = int64_t; -}; +inline constexpr auto nullopt = cuda::std::nullopt; -template <> -struct promoted_t { - using type = uint64_t; -}; +template +using decimal = numeric::fixed_point; -template <> -struct promoted_t { - using type = __int128; -}; +template +using duration = cuda::std::chrono::duration; -template <> -struct promoted_t { - using type = unsigned __int128; -}; - -template <> -struct promoted_t<__int128> { - using type = int256_t; -}; - -template <> -struct promoted_t { - using type = uint256_t; -}; +namespace detail { template -using promoted = typename promoted_t::type; - -template -__device__ inline errc identity(T* out, T const* a) +__device__ constexpr T ipow10(T exponent) { - *out = *a; - return errc::OK; -} + if (exponent == 0) { return 1; } -template -__device__ inline errc identity(optional* out, optional const* a) -{ - *out = *a; - return errc::OK; -} + T extra = 1; + T square = 10; + T n = exponent; -template -__device__ inline errc is_null(bool* out, T const* a) -{ - *out = false; - return errc::OK; -} + while (n > 1) { + if ((n & 1) == 1) { extra *= square; } + n >>= 1; + square *= square; + } -template -__device__ inline errc is_null(optional* out, optional const* a) -{ - *out = a->is_null(); - return errc::OK; + return square * extra; } -// TODO: decimal ansi operators(precision and scale-oriented non-templated arguments) -// TODO: cast operators to match AST -// TODO: decimal cast operators to match AST -// TODO: datetime cast operators & arithmetic -// TODO: decimal ansi cast -// TODO: ansi_mod, div operations for fixed-point and duration types - +} // namespace detail } // namespace ops } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/ast/expressions.cpp b/cpp/src/ast/expressions.cpp index d51d3f323498..24c1679bc30f 100644 --- a/cpp/src/ast/expressions.cpp +++ b/cpp/src/ast/expressions.cpp @@ -79,29 +79,7 @@ bool operation::may_evaluate_null(table_view const& left, return subexpr.get().may_evaluate_null(left, right, stream); }); }; - -cudf::size_type detail::filter_predicate::accept(detail::expression_parser& visitor) const -{ - CUDF_FAIL( - "filter_predicate is an internal expression and should not be visited by expression_parser", - std::invalid_argument); -} - -std::reference_wrapper detail::filter_predicate::accept( - detail::expression_transformer& visitor) const -{ - CUDF_FAIL( - "filter_predicate is an internal expression and should not be visited by " - "expression_transformer", - std::invalid_argument); -} - -bool detail::filter_predicate::may_evaluate_null(table_view const& left, - table_view const& right, - rmm::cuda_stream_view stream) const -{ - return false; -} + auto column_name_reference::accept(detail::expression_transformer& visitor) const -> decltype(visitor.visit(*this)) @@ -134,12 +112,7 @@ std::unique_ptr column_name_reference::accept( "column_name_reference is not supported in row_ir. row_ir only supports resolved expressions", std::invalid_argument); } - -std::unique_ptr detail::filter_predicate::accept( - cudf::detail::row_ir::ast_converter& converter) const -{ - return converter.add_ir_node(*this); -} + } // namespace ast } // namespace cudf diff --git a/cpp/src/jit/row_ir.cpp b/cpp/src/jit/row_ir.cpp index 82cd308c2f5d..5d66dd0e6304 100644 --- a/cpp/src/jit/row_ir.cpp +++ b/cpp/src/jit/row_ir.cpp @@ -18,17 +18,7 @@ #include #include -namespace cudf { - -namespace detail { - -namespace row_ir { - -std::string cuda_type(cudf::data_type type, bool nullable) -{ - auto name = type_to_name(type); - return nullable ? std::format("cuda::std::optional<{}>", name) : name; -} +namespace cudf::detail::row_ir { std::string instance_context::make_tmp_id() { @@ -39,323 +29,230 @@ bool instance_context::has_nulls() const { return has_nulls_; } void instance_context::set_has_nulls(bool has_nulls) { has_nulls_ = has_nulls; } -get_input::get_input(int32_t input) : id_(), input_(input), type_() {} - -std::string_view get_input::get_id() { return id_; } - -data_type get_input::get_type() { return type_; } - -bool get_input::is_null_aware() { return false; } - -bool get_input::is_always_valid() { return false; } - -void get_input::instantiate(instance_context& ctx, instance_info const& info) +node::node(opcode op, std::vector args) : op_{op}, args_{std::move(args)} { - id_ = ctx.make_tmp_id(); - auto const& input = info.inputs[input_]; - type_ = input.type; + CUDF_EXPECTS(op != opcode::GET_INPUT && op != opcode::SET_OUTPUT, + std::format("Invalid opcode `{}` for operation node.", get_op_name(op))); + CUDF_EXPECTS(args_.size() == get_op_arity(op), + std::format("Invalid number of arguments for operator `{}`. Expected {}, Got {}.", + get_op_name(op), + get_op_arity(op), + args_.size())); + // TODO: check argument types, this will be after resolving types } -std::string get_input::generate_code(instance_context& ctx, - target_info const& info, - instance_info const& instance) -{ - switch (info.id) { - case target::CUDA: { - return std::format( - "{} {} = {};", cuda_type(type_, ctx.has_nulls()), id_, instance.inputs[input_].id); - } - default: - CUDF_FAIL("Unsupported target: " + std::to_string(static_cast(info.id)), - std::invalid_argument); - } -} +node::node(input_reference input) : reference_{input}, op_{opcode::SET_OUTPUT} {} -set_output::set_output(int32_t output, std::unique_ptr source) - : id_(), output_(output), source_(std::move(source)), type_(), output_id_() +node::node(output_reference reference, node arg) + : reference_{reference}, op_{opcode::SET_OUTPUT}, args_{std::move(arg)} { } -std::string_view set_output::get_id() { return id_; } - -data_type set_output::get_type() { return type_; } +std::string_view node::get_id() const { return id_; } -bool set_output::is_null_aware() { return source_->is_null_aware(); } +data_type node::get_type() const { return type_; } -bool set_output::is_always_valid() { return source_->is_always_valid(); } +opcode node::get_opcode() const { return op_; } -node& set_output::get_source() { return *source_; } +std::span node::get_args() const { return args_; } -void set_output::instantiate(instance_context& ctx, instance_info const& info) +bool node::is_null_aware() const { - source_->instantiate(ctx, info); - id_ = ctx.make_tmp_id(); - auto source_type = source_->get_type(); - type_ = source_type; - output_id_ = info.outputs[output_].id; + return get_op_null_output(op_) == + null_output::ALWAYS_NULLABLE || // to emit nulls for always-nullable operators, we need + // to mark them as null-aware + get_op_requires_nulls(op_) || + std::any_of(args_.begin(), args_.end(), [](auto& a) { return a.is_null_aware(); }); } -std::string set_output::generate_code(instance_context& ctx, - target_info const& info, - instance_info const& instance) +bool node::is_always_valid() const { - switch (info.id) { - case target::CUDA: { - auto source_code = source_->generate_code(ctx, info, instance); - return std::format( - "{}\n" - "{} {} = {};\n" - "*{} = {};", - source_code, - cuda_type(type_, ctx.has_nulls()), - id_, - source_->get_id(), - output_id_, - id_); - } - default: - CUDF_FAIL("Unsupported target: " + std::to_string(static_cast(info.id)), - std::invalid_argument); - } + return get_op_null_output(op_) == null_output::ALWAYS_VALID || + std::all_of(args_.begin(), args_.end(), [](auto& a) { return a.is_always_valid(); }); } -operation::operation(opcode op, std::unique_ptr* move_begin, std::unique_ptr* move_end) - : id_(), op_(op), operands_(), type_() +bool node::is_fallible() const { - operands_.insert( - operands_.begin(), std::make_move_iterator(move_begin), std::make_move_iterator(move_end)); - CUDF_EXPECTS(static_cast(operands_.size()) == ast::detail::ast_operator_arity(op), - "Invalid number of arguments for operator.", - std::invalid_argument); - CUDF_EXPECTS( - operands_.size() > 0, "Operator must have at least one operand", std::invalid_argument); + return get_op_is_fallible(op_) || + std::any_of(args_.begin(), args_.end(), [](auto& a) { return a.is_fallible(); }); } -operation::operation(opcode op, std::vector> operands) - : operation(op, operands.data(), operands.data() + operands.size()) +row_ir::typing as_typing(data_type type) { -} - -std::string_view operation::get_id() { return id_; } - -data_type operation::get_type() { return type_; } - -inline bool is_operator_null_aware(opcode op) -{ - switch (op) { - case ast::ast_operator::IS_NULL: - case ast::ast_operator::NULL_EQUAL: - case ast::ast_operator::NULL_LOGICAL_AND: - case ast::ast_operator::NULL_LOGICAL_OR: return true; - - case ast::ast_operator::ADD: - case ast::ast_operator::SUB: - case ast::ast_operator::MUL: - case ast::ast_operator::DIV: - case ast::ast_operator::TRUE_DIV: - case ast::ast_operator::FLOOR_DIV: - case ast::ast_operator::MOD: - case ast::ast_operator::PYMOD: - case ast::ast_operator::POW: - case ast::ast_operator::NOT_EQUAL: - case ast::ast_operator::EQUAL: - case ast::ast_operator::LESS: - case ast::ast_operator::GREATER: - case ast::ast_operator::LESS_EQUAL: - case ast::ast_operator::GREATER_EQUAL: - case ast::ast_operator::BITWISE_AND: - case ast::ast_operator::BITWISE_OR: - case ast::ast_operator::BITWISE_XOR: - case ast::ast_operator::LOGICAL_AND: - case ast::ast_operator::LOGICAL_OR: - case ast::ast_operator::IDENTITY: - case ast::ast_operator::SIN: - case ast::ast_operator::COS: - case ast::ast_operator::TAN: - case ast::ast_operator::ARCSIN: - case ast::ast_operator::ARCCOS: - case ast::ast_operator::ARCTAN: - case ast::ast_operator::SINH: - case ast::ast_operator::COSH: - case ast::ast_operator::TANH: - case ast::ast_operator::ARCSINH: - case ast::ast_operator::ARCCOSH: - case ast::ast_operator::ARCTANH: - case ast::ast_operator::EXP: - case ast::ast_operator::LOG: - case ast::ast_operator::SQRT: - case ast::ast_operator::CBRT: - case ast::ast_operator::CEIL: - case ast::ast_operator::FLOOR: - case ast::ast_operator::ABS: - case ast::ast_operator::RINT: - case ast::ast_operator::BIT_INVERT: - case ast::ast_operator::NOT: - case ast::ast_operator::CAST_TO_INT64: - case ast::ast_operator::CAST_TO_UINT64: - case ast::ast_operator::CAST_TO_FLOAT64: return false; - - default: CUDF_UNREACHABLE("Unrecognized operator type."); + switch (type.id()) { + case type_id::BOOL8: return typing::BOOL8; + case type_id::INT8: return typing::INT8; + case type_id::INT16: return typing::INT16; + case type_id::INT32: return typing::INT32; + case type_id::INT64: return typing::INT64; + case type_id::UINT8: return typing::UINT8; + case type_id::UINT16: return typing::UINT16; + case type_id::UINT32: return typing::UINT32; + case type_id::UINT64: return typing::UINT64; + case type_id::FLOAT32: return typing::FLOAT32; + case type_id::FLOAT64: return typing::FLOAT64; + case type_id::DECIMAL32: return typing::DECIMAL32; + case type_id::DECIMAL64: return typing::DECIMAL64; + case type_id::DECIMAL128: return typing::DECIMAL128; + case type_id::TIMESTAMP_DAYS: return typing::TIMESTAMP_DAYS; + case type_id::TIMESTAMP_SECONDS: return typing::TIMESTAMP_SECONDS; + case type_id::TIMESTAMP_MILLISECONDS: return typing::TIMESTAMP_MILLISECONDS; + case type_id::TIMESTAMP_MICROSECONDS: return typing::TIMESTAMP_MICROSECONDS; + case type_id::TIMESTAMP_NANOSECONDS: return typing::TIMESTAMP_NANOSECONDS; + case type_id::DURATION_DAYS: return typing::DURATION_DAYS; + case type_id::DURATION_SECONDS: return typing::DURATION_SECONDS; + case type_id::DURATION_MILLISECONDS: return typing::DURATION_MILLISECONDS; + case type_id::DURATION_MICROSECONDS: return typing::DURATION_MICROSECONDS; + case type_id::DURATION_NANOSECONDS: return typing::DURATION_NANOSECONDS; + case type_id::STRING: return typing::STRING; + default: + CUDF_FAIL(std::format("Unsupported data type for Row IR: {}", type_to_name(type)), + std::invalid_argument); } } -bool operation::is_null_aware() +data_type get_return_type(opcode op, std::span args) { - return is_operator_null_aware(op_) || - std::any_of( - operands_.begin(), operands_.end(), [](auto& op) { return op->is_null_aware(); }); -} + std::vector typings; -inline bool is_operator_always_valid(opcode op) -{ - switch (op) { - case ast::ast_operator::IS_NULL: - case ast::ast_operator::NULL_EQUAL: return true; - - case ast::ast_operator::NULL_LOGICAL_AND: - case ast::ast_operator::NULL_LOGICAL_OR: - case ast::ast_operator::ADD: - case ast::ast_operator::SUB: - case ast::ast_operator::MUL: - case ast::ast_operator::DIV: - case ast::ast_operator::TRUE_DIV: - case ast::ast_operator::FLOOR_DIV: - case ast::ast_operator::MOD: - case ast::ast_operator::PYMOD: - case ast::ast_operator::POW: - case ast::ast_operator::NOT_EQUAL: - case ast::ast_operator::EQUAL: - case ast::ast_operator::LESS: - case ast::ast_operator::GREATER: - case ast::ast_operator::LESS_EQUAL: - case ast::ast_operator::GREATER_EQUAL: - case ast::ast_operator::BITWISE_AND: - case ast::ast_operator::BITWISE_OR: - case ast::ast_operator::BITWISE_XOR: - case ast::ast_operator::LOGICAL_AND: - case ast::ast_operator::LOGICAL_OR: - case ast::ast_operator::IDENTITY: - case ast::ast_operator::SIN: - case ast::ast_operator::COS: - case ast::ast_operator::TAN: - case ast::ast_operator::ARCSIN: - case ast::ast_operator::ARCCOS: - case ast::ast_operator::ARCTAN: - case ast::ast_operator::SINH: - case ast::ast_operator::COSH: - case ast::ast_operator::TANH: - case ast::ast_operator::ARCSINH: - case ast::ast_operator::ARCCOSH: - case ast::ast_operator::ARCTANH: - case ast::ast_operator::EXP: - case ast::ast_operator::LOG: - case ast::ast_operator::SQRT: - case ast::ast_operator::CBRT: - case ast::ast_operator::CEIL: - case ast::ast_operator::FLOOR: - case ast::ast_operator::ABS: - case ast::ast_operator::RINT: - case ast::ast_operator::BIT_INVERT: - case ast::ast_operator::NOT: - case ast::ast_operator::CAST_TO_INT64: - case ast::ast_operator::CAST_TO_UINT64: - case ast::ast_operator::CAST_TO_FLOAT64: return false; - - default: CUDF_UNREACHABLE("Unrecognized operator type."); + for (auto& type : args) { + typings.push_back(as_typing(type)); } -} - -bool operation::is_always_valid() -{ - return is_operator_always_valid(op_) || - std::all_of( - operands_.begin(), operands_.end(), [](auto& op) { return op->is_always_valid(); }); -} -opcode operation::get_opcode() const { return op_; } + auto op_type_match = get_op_typing(op); + // TODO: match typing and get return type + // + // + // TODO(lamarrr): figure out scale propagation rules and creation/assignment rules + // + // TODO(lamarrr): implement filter_predicate to return false on nulls + // + // TODO: scale-propagation rules + // TODO: decimal ansi operators(precision and scale-oriented non-templated arguments) + // TODO: cast operators to match AST + // TODO: decimal cast operators to match AST + // TODO: datetime cast operators & arithmetic + // TODO: decimal ansi cast + // TODO: ansi_mod, div operations for fixed-point and duration types -std::span const> operation::get_operands() const { return operands_; } + data_type return_type; +} -void operation::instantiate(instance_context& ctx, instance_info const& info) +void node::instantiate(instance_context& ctx, instance_info const& info) { - for (auto& arg : operands_) { - arg->instantiate(ctx, info); - } - id_ = ctx.make_tmp_id(); - std::vector operand_types; - for (auto& arg : operands_) { - operand_types.emplace_back(arg->get_type()); + for (auto& arg : args_) { + arg.instantiate(ctx, info); } - type_ = ast::detail::ast_operator_return_type(op_, operand_types); + switch (op_) { + case opcode::GET_INPUT: { + type_ = info.inputs[std::get(reference_).index].type; + } break; + case opcode::SET_OUTPUT: { + type_ = args_[0].get_type(); + } break; + default: { + std::vector arg_types; + for (auto& arg : args_) { + arg_types.emplace_back(arg.get_type()); + } + type_ = get_return_type(op_, arg_types); + } break; + } } -std::string operation::generate_code(instance_context& ctx, - target_info const& info, - instance_info const& instance) +void node::emit_code(instance_context& ctx, + target_info const& info, + instance_info const& instance, + code_sink& sink) const { - std::string operands_code; + auto to_cuda_type = [](cudf::data_type type, bool nullable) { + auto name = type_to_name(type); + return nullable ? std::format("cuda::std::optional<{}>", name) : name; + }; - for (auto& arg : operands_) { - operands_code = - std::format("{}{}{}", operands_code, arg->generate_code(ctx, info, instance), "\n"); + for (auto& arg : args_) { + arg.emit_code(ctx, info, instance, sink); } - auto operation_code = [&]() { - switch (info.id) { - case target::CUDA: { - auto first_operand = operands_[0]->get_id(); - auto operands_str = (operands_.size() == 1) - ? std::string{first_operand} - : std::accumulate(operands_.begin() + 1, - operands_.end(), - std::string{first_operand}, - [](auto const& a, auto& node) { - return std::format("{}, {}", a, node->get_id()); - }); - - auto cuda = std::format( - "{} {} = cudf::ast::detail::operator_functor{{}}({});", - cuda_type(type_, ctx.has_nulls()), - id_, - ast::detail::ast_operator_string(op_), - ctx.has_nulls(), - operands_str); - return cuda; + switch (info.id) { + case target::CUDA: { + auto type = to_cuda_type(type_, ctx.has_nulls()); + + switch (op_) { + case opcode::GET_INPUT: { + sink.emit(std::format("{} {} = {};", + type, + id_, + instance.inputs[std::get(reference_).index].id)); + } break; + + case opcode::SET_OUTPUT: { + sink.emit(std::format( + R"**({} {} = {}; +*{} = {}; +)**", + type, + id_, + args_[0].get_id(), + instance.outputs[std::get(reference_).index].id, + id_)); + } break; + + default: { + auto first_arg = args_[0].get_id(); + auto args_str = (args_.size() == 1) + ? std::string{first_arg} + : std::accumulate(args_.begin() + 1, + args_.end(), + std::string{first_arg}, + [](auto const& a, auto& node) { + return std::format("{}, &{}", a, node.get_id()); + }); + + bool fallible = get_op_is_fallible(op_); + auto op_name = get_op_name(op_); + + if (!fallible) { + sink.emit(std::format( + R"***({} {}; +cudf::ops::{}(&{}, {}); + )***", + type, + id_, + op_name, + id_, + args_str)); + } else { + sink.emit(std::format( + R"***({} {}; +if(cudf::ops::errc e = cudf::ops::{}(&{}, {}); e != cudf::ops::errc::SUCCESS) {{ + return e; +}} + )***", + type, + id_, + op_name, + id_, + args_str)); + } + } break; } - default: - CUDF_FAIL("Unsupported target: " + std::to_string(static_cast(info.id)), - std::invalid_argument); - } - }(); - - return operands_code + operation_code; -} + } break; -filter_predicate::filter_predicate(std::unique_ptr source) : id_(), source_(std::move(source)) -{ + default: + CUDF_FAIL(std::format("Unsupported target: {}", static_cast(info.id)), + std::invalid_argument); + } } -std::string_view filter_predicate::get_id() { return id_; } - -data_type filter_predicate::get_type() { return data_type{type_id::BOOL8}; } - -bool filter_predicate::is_null_aware() { return source_->is_null_aware(); } - -bool filter_predicate::is_always_valid() { return true; } - -node& filter_predicate::get_source() { return *source_; } +/* +// TODO: transitive null-ness -void filter_predicate::instantiate(instance_context& ctx, instance_info const& info) +filter_predicate::filter_predicate(std::unique_ptr source) : id_(), source_(std::move(source)) { - source_->instantiate(ctx, info); - CUDF_EXPECTS(source_->get_type().id() == type_id::BOOL8, - "Filter predicate source must be boolean.", - std::invalid_argument); - id_ = ctx.make_tmp_id(); } [[nodiscard]] std::string filter_predicate::generate_code(instance_context& ctx, @@ -377,6 +274,7 @@ void filter_predicate::instantiate(instance_context& ctx, instance_info const& i std::invalid_argument); } } + */ std::span ast_converter::get_input_specs() const { return input_specs_; } @@ -387,35 +285,35 @@ int32_t ast_converter::add_ast_input(ast_input_spec in) return id; } -std::unique_ptr ast_converter::add_ir_node(ast::literal const& expr) +row_ir::node ast_converter::add_ir_node(ast::literal const& expr) { auto index = add_ast_input( ast_scalar_input_spec{expr.get_scalar(), expr.get_value(), make_column_from_scalar(expr.get_scalar(), 1, stream_, mr_)}); - return std::make_unique(index); + return row_ir::node(input_reference{index}); } -std::unique_ptr ast_converter::add_ir_node(ast::column_reference const& expr) +row_ir::node ast_converter::add_ir_node(ast::column_reference const& expr) { auto index = add_ast_input(ast_column_input_spec{expr.get_table_source(), expr.get_column_index()}); - return std::make_unique(index); + return row_ir::node(input_reference{index}); } -std::unique_ptr ast_converter::add_ir_node(ast::operation const& expr) +row_ir::node ast_converter::add_ir_node(ast::operation const& expr) { - std::vector> operands; + std::vector operands; for (auto const& operand : expr.get_operands()) { operands.push_back(operand.get().accept(*this)); } - return std::make_unique(expr.get_operator(), std::move(operands)); + return row_ir::node(row_ir::operation{expr.get_operator(), std::move(operands)}); } -std::unique_ptr ast_converter::add_ir_node(ast::detail::filter_predicate const& expr) +row_ir::node ast_converter::add_ir_node(ast::detail::filter_predicate const& expr) { auto operand = expr.get_operand().accept(*this); - return std::make_unique(std::move(operand)); + return row_ir::node(row_ir::filter_predicate{std::move(operand)}); } // Resolve the table for a column input spec, preferring left_table/right_table for join cases, @@ -430,7 +328,6 @@ table_view const& resolve_table(ast_column_input_spec const& in, ast_args const& void ast_converter::add_input_var(ast_column_input_spec const& in, ast_args const& args) { - // TODO(lamarrr): consider mangling column name to make debugging easier auto id = std::format("in_{}", input_vars_.size()); auto type = resolve_table(in, args).column(in.column).type(); input_vars_.emplace_back(std::move(id), type); @@ -478,8 +375,7 @@ std::tuple ast_converter::generate_code(target t ast::expression const& expr, ast_args const& args) { - auto output_expr_ir = expr.accept(*this); - output_irs_.emplace_back(std::make_unique(0, std::move(output_expr_ir))); + output_irs_.emplace_back(std::make_unique(0, expr.accept(*this))); // resolve the flattened input references into IR input variables for (auto const& input : input_specs_) { @@ -572,10 +468,10 @@ std::tuple ast_converter::generate_code(target t code_ = std::format( R"***( -__device__ void expression({}) +__device__ cudf::ops::errc expression({}) {{ {} -return; +return cudf::ops::errc::SUCCESS; }} )***", params_decl, @@ -674,6 +570,4 @@ filter_args ast_converter::filter(target target_id, return result; } -} // namespace row_ir -} // namespace detail -} // namespace cudf +} // namespace cudf::detail::row_ir diff --git a/cpp/src/jit/row_ir.hpp b/cpp/src/jit/row_ir.hpp index 5e3cd2633e68..f231181bb9f4 100644 --- a/cpp/src/jit/row_ir.hpp +++ b/cpp/src/jit/row_ir.hpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -16,7 +17,6 @@ #include #include -#include #include #include #include @@ -116,336 +116,127 @@ struct [[nodiscard]] instance_context { void set_has_nulls(bool has_nulls); }; -struct [[nodiscard]] node { - /** - * @brief Get the identifier of the IR node - * @return The identifier of the IR node - */ - virtual std::string_view get_id() = 0; - - /** - * @brief Get the type info of the IR node - * @return The type information of the IR node - */ - [[nodiscard]] virtual data_type get_type() = 0; - - /** - * @brief Returns `false` if this node forwards nulls from its inputs to its output. - * e.g., `ADD` operator is not null-aware because if any of its inputs is null, the output is - * null. but `NULL_EQUAL` operator is null-aware because it can produce a non-null output even if - * its inputs are null. - */ - [[nodiscard]] virtual bool is_null_aware() = 0; - - /** - * @brief Returns `true` if this node always produces a valid output even if its inputs are - * nullable, e.g., `IS_NULL` operator produces a valid boolean output regardless of the - * nullability of its input. - */ - [[nodiscard]] virtual bool is_always_valid() = 0; - - /** - * @brief Instantiate the IR node with the given context and instance information, setting up any - * necessary state and preprocessing needed for code generation. - * @param ctx The context within which the IR is instantiated - * @param info The instance information - */ - virtual void instantiate(instance_context& ctx, instance_info const& info) = 0; - - /** - * @brief Generate the code for the IR node based on the instance context and target information. - * @param ctx The context within which the IR is instantiated - * @param info The target information - * @param instance The instance information - * @return The generated code for the IR node - */ - [[nodiscard]] virtual std::string generate_code(instance_context& ctx, - target_info const& info, - instance_info const& instance) = 0; - - virtual ~node() = default; +struct [[nodiscard]] code_sink { + void emit(std::string_view code); }; -/** - * @brief The operation code used in the IR nodes. - */ -using opcode = ast::ast_operator; - -/** - * @brief An IR node that retrieves an input variable by its index. - * This node is used to access input variables in the IR. - */ -struct [[nodiscard]] get_input final : node { - private: - std::string id_; ///< The identifier of the IR node - int32_t input_; ///< The index of the input variable - data_type type_; ///< The type information of the IR node - - public: - /** - * @brief Construct a new get_input IR node - * @param input The index of the input variable - */ - get_input(int32_t input); - - get_input(get_input const&) = delete; - - get_input& operator=(get_input const&) = delete; - - get_input(get_input&&) = default; ///< Move constructor - - get_input& operator=(get_input&&) = default; ///< Move assignment operator - - ~get_input() override = default; ///< Destructor - - /** - * @copydoc node::get_id - */ - [[nodiscard]] std::string_view get_id() override; - - /** - * @copydoc node::get_type - */ - [[nodiscard]] data_type get_type() override; - - /** - * @copydoc node::is_null_aware - */ - [[nodiscard]] bool is_null_aware() override; - - /** - * @copydoc node::is_always_valid - */ - [[nodiscard]] bool is_always_valid() override; - - /** - * @copydoc node::instantiate - */ - void instantiate(instance_context& ctx, instance_info const& info) override; - - /** - * @copydoc node::generate_code - */ - [[nodiscard]] std::string generate_code(instance_context& ctx, - target_info const& info, - instance_info const& instance) override; +struct [[nodiscard]] input_reference { + int32_t index = 0; ///< The index of the input variable }; -/** - * @brief An IR node that sets the output variable to the value of a source IR node. - */ -struct [[nodiscard]] set_output final : node { - private: - std::string id_; ///< The identifier of the IR node - int32_t output_; ///< The index of the output variable - std::unique_ptr source_; ///< The source IR node from which the value is taken - data_type type_; ///< The type information of the IR node - std::string output_id_; ///< The identifier of the output variable - - public: - /** - * @brief Construct a new set_output IR node - * @param output The index of the output variable - * @param source The source IR node from which the value is taken - */ - set_output(int32_t output, std::unique_ptr source); - - set_output(set_output const&) = delete; - - set_output& operator=(set_output const&) = delete; - - set_output(set_output&&) = default; ///< Move constructor - - set_output& operator=(set_output&&) = default; ///< Move assignment operator - - ~set_output() override = default; ///< Destructor - - /** - * @copydoc node::get_id - */ - [[nodiscard]] std::string_view get_id() override; - - /** - * @copydoc node::get_type - */ - [[nodiscard]] data_type get_type() override; - - /** - * @copydoc node::is_null_aware - */ - [[nodiscard]] bool is_null_aware() override; - - /** - * @copydoc node::is_always_valid - */ - [[nodiscard]] bool is_always_valid() override; - - /** - * @brief Get the source IR node from which the value is taken - */ - [[nodiscard]] node& get_source(); - - /** - * @copydoc node::instantiate - */ - void instantiate(instance_context& ctx, instance_info const& info) override; - - /** - * @copydoc node::generate_code - */ - [[nodiscard]] std::string generate_code(instance_context& ctx, - target_info const& info, - instance_info const& instance) override; +struct [[nodiscard]] output_reference { + int32_t index = 0; ///< The index of the output variable }; -/** - * @brief An IR node that represents an operation with zero or more operands. - */ -struct [[nodiscard]] operation final : node { - private: - std::string id_; ///< The identifier of the IR node - opcode op_; ///< The operation code - std::vector> operands_; ///< The operands of the operation - data_type type_; ///< The type information of the IR node +struct [[nodiscard]] node { + std::variant reference_ = + std::monostate{}; ///< The index of the input/output variable + opcode op_ = opcode::GET_INPUT; ///< The operation code + std::vector args_ = {}; ///< The arguments of the operation - operation(opcode op, std::unique_ptr* move_begin, std::unique_ptr* move_end); + data_type type_ = {}; ///< The resolved type information of the IR node - public: - /** - * @brief Create a set of operand IR nodes - */ - template - requires(std::is_base_of_v && ...) - static std::array, sizeof...(T)> operands(T&&... args) - { - return {std::make_unique(std::forward(args))...}; - } + std::string id_ = {}; ///< The identifier of the IR node /** - * @brief Create a set of operand IR nodes from existing unique pointers + * @brief Create a set of argument IR nodes */ template - requires(std::is_base_of_v && ...) - static std::array, sizeof...(T)> operands(std::unique_ptr&&... args) + requires(std::is_same_v && ...) + static std::array arguments(T&&... args) { - return {std::move(args)...}; + return {std::forward(args)...}; } /** * @brief Construct a new operation IR node * @param op The operation code - * @param operands The operands of the operation + * @param args The arguments of the operation */ - operation(opcode op, std::vector> operands); - - template - operation(opcode op, std::array, N> operands) - : operation{op, operands.data(), operands.data() + N} - { - } - - operation(operation const&) = delete; - - operation& operator=(operation const&) = delete; - - operation(operation&&) = default; ///< Move constructor - - operation& operator=(operation&&) = default; ///< Move assignment operator - - ~operation() override = default; ///< Destructor + node(opcode op, std::vector args); /** - * @copydoc node::get_id + * @brief Construct a new input reference IR node + * @param input The index of the input variable */ - [[nodiscard]] std::string_view get_id() override; + node(input_reference input); /** - * @copydoc node::get_type + * @brief Construct a new output reference IR node + * @param output The index of the output variable + * @param arg The argument node that produces the value to be set to the output variable */ - [[nodiscard]] data_type get_type() override; + node(output_reference reference, node arg); + + node(node const& other) = default; ///< Copy constructor + node(node&& other) = default; ///< Move constructor + node& operator=(node const& other) = default; ///< Copy assignment operator + node& operator=(node&& other) = default; ///< Move assignment operator + ~node() = default; ///< Destructor /** - * @copydoc node::is_null_aware + * @brief Get the identifier of the IR node + * @return The identifier of the IR node */ - [[nodiscard]] bool is_null_aware() override; + [[nodiscard]] std::string_view get_id() const; /** - * @copydoc node::is_always_valid + * @brief Get the type info of the IR node + * @return The type information of the IR node */ - [[nodiscard]] bool is_always_valid() override; + [[nodiscard]] data_type get_type() const; /** * @brief Get the operation code of the operation * @return The operation code of the operation */ - [[nodiscard]] opcode get_opcode() const; + opcode get_opcode() const; - /** @brief Get the operands of the operation - * @return A span of unique pointers to the operands of the operation + /** @brief Get the arguments of the operation + * @return A span of unique pointers to the arguments of the operation */ - [[nodiscard]] std::span const> get_operands() const; + [[nodiscard]] std::span get_args() const; /** - * @copydoc node::instantiate - */ - void instantiate(instance_context& ctx, instance_info const& info) override; - - /** - * @copydoc node::generate_code - */ - [[nodiscard]] std::string generate_code(instance_context& ctx, - target_info const& info, - instance_info const& instance) override; -}; - -/** - * @brief An IR node that flattens a boolean predicate to be used in a filter operation. - * This node replaces null values with false. - */ -struct [[nodiscard]] filter_predicate final : node { - private: - std::string id_; ///< The identifier of the IR node - std::unique_ptr source_; ///< The source IR node from which the predicate value is taken - - public: - filter_predicate(std::unique_ptr source); - - /** - * @copydoc node::get_id - */ - [[nodiscard]] std::string_view get_id() override; - - /** - * @copydoc node::get_type - */ - [[nodiscard]] data_type get_type() override; - - /** - * @copydoc node::is_null_aware + * @brief Returns `false` if this node forwards nulls from its inputs to its output. + * e.g., `ADD` operator is not null-aware because if any of its inputs is null, the output is + * null. but `NULL_EQUAL` operator is null-aware because it can produce a non-null output even if + * its inputs are null. */ - [[nodiscard]] bool is_null_aware() override; + [[nodiscard]] bool is_null_aware() const; /** - * @copydoc node::is_always_valid + * @brief Returns `true` if this node always produces a valid output even if its inputs are + * nullable, e.g., `IS_NULL` operator produces a valid boolean output regardless of the + * nullability of its input. */ - [[nodiscard]] bool is_always_valid() override; + [[nodiscard]] bool is_always_valid() const; /** - * @brief Get the source IR node from which the value is taken + * @brief Get if the IR node can raise an error during evaluation. + * @return `true` if the IR node can raise an error during evaluation, `false` otherwise */ - [[nodiscard]] node& get_source(); + [[nodiscard]] bool is_fallible() const; /** - * @copydoc node::instantiate + * @brief Instantiate the IR node with the given context and instance information, setting up any + * necessary state and preprocessing needed for code generation. + * @param ctx The context within which the IR is instantiated + * @param info The instance information */ - void instantiate(instance_context& ctx, instance_info const& info) override; + void instantiate(instance_context& ctx, instance_info const& info); /** - * @copydoc node::generate_code + * @brief Generate the code for the IR node based on the instance context and target information. + * @param ctx The context within which the IR is instantiated + * @param info The target information + * @param instance The instance information + * @param sink The code sink to which the generated code is emitted */ - [[nodiscard]] std::string generate_code(instance_context& ctx, - target_info const& info, - instance_info const& instance) override; + void emit_code(instance_context& ctx, + target_info const& info, + instance_info const& instance, + code_sink& sink) const; }; /** @@ -521,11 +312,11 @@ struct ast_args { */ struct [[nodiscard]] ast_converter { private: - std::vector input_specs_; ///< The input specs for the AST - std::vector input_vars_; ///< The input variables for the IR - std::vector output_vars_; ///< The output variables for the IR - std::vector> output_irs_; ///< The output IR nodes - std::string code_; ///< The generated code for the IR + std::vector input_specs_; ///< The input specs for the AST + std::vector input_vars_; ///< The input variables for the IR + std::vector output_vars_; ///< The output variables for the IR + std::vector output_irs_; ///< The output IR nodes + std::string code_; ///< The generated code for the IR rmm::cuda_stream_view stream_; ///< CUDA stream used for device memory operations and kernel launches. rmm::device_async_resource_ref @@ -555,17 +346,14 @@ struct [[nodiscard]] ast_converter { friend class ast::literal; friend class ast::column_reference; friend class ast::operation; - friend class ast::column_name_reference; - friend class ast::detail::filter_predicate; + friend class ast::column_name_reference; - [[nodiscard]] std::unique_ptr add_ir_node(ast::literal const& expr); + row_ir::node add_ir_node(ast::literal const& expr); - [[nodiscard]] std::unique_ptr add_ir_node(ast::column_reference const& expr); + row_ir::node add_ir_node(ast::column_reference const& expr); - [[nodiscard]] std::unique_ptr add_ir_node(ast::operation const& expr); + row_ir::node add_ir_node(ast::operation const& expr); - [[nodiscard]] std::unique_ptr add_ir_node( - ast::detail::filter_predicate const& expr); [[nodiscard]] std::span get_input_specs() const; From c52c6b88df99cf64282dcf9c08df30d43a6e15b0 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 26 Apr 2026 10:25:53 +0000 Subject: [PATCH 04/15] Remove deprecated opcode definitions from opcode.hpp --- cpp/include/cudf/opcode.hpp | 105 ------------------------------------ 1 file changed, 105 deletions(-) delete mode 100644 cpp/include/cudf/opcode.hpp diff --git a/cpp/include/cudf/opcode.hpp b/cpp/include/cudf/opcode.hpp deleted file mode 100644 index c53aa60312f9..000000000000 --- a/cpp/include/cudf/opcode.hpp +++ /dev/null @@ -1,105 +0,0 @@ - - -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ -#pragma once - -#include - -#include - -namespace CUDF_EXPORT cudf { - -/** - * @brief Enum of supported opcodes. - */ -enum class opcode : int32_t { - // Binary operators - ADD = 0, ///< operator + - SUB = 1, ///< operator - - MUL = 2, ///< operator * - DIV = 3, ///< operator / using common type of lhs and rhs - TRUE_DIV = 4, ///< operator / after promoting type to floating point - FLOOR_DIV = 5, ///< operator / after promoting to the common type of lhs and rhs (integral or - ///< floating point), and then flooring the result - MOD = 6, ///< operator % - PYMOD = 7, ///< operator % using Python's sign rules for negatives - POW = 8, ///< lhs ^ rhs - EQUAL = 9, ///< operator == - NULL_EQUAL = - 10, ///< operator == with Spark rules: NULL_EQUAL(null, null) is true, NULL_EQUAL(null, - ///< valid) is false, and - ///< NULL_EQUAL(valid, valid) == EQUAL(valid, valid) - NOT_EQUAL = 11, ///< operator != - LESS = 12, ///< operator < - GREATER = 13, ///< operator > - LESS_EQUAL = 14, ///< operator <= - GREATER_EQUAL = 15, ///< operator >= - BITWISE_AND = 16, ///< operator & - BITWISE_OR = 17, ///< operator | - BITWISE_XOR = 18, ///< operator ^ - LOGICAL_AND = 19, ///< operator && - NULL_LOGICAL_AND = 20, ///< operator && with Spark rules: NULL_LOGICAL_AND(null, null) is null, - ///< NULL_LOGICAL_AND(null, true) is - ///< null, NULL_LOGICAL_AND(null, false) is false, and NULL_LOGICAL_AND(valid, - ///< valid) == LOGICAL_AND(valid, valid) - LOGICAL_OR = 21, ///< operator || - NULL_LOGICAL_OR = 22, ///< operator || with Spark rules: NULL_LOGICAL_OR(null, null) is null, - ///< NULL_LOGICAL_OR(null, true) is true, - ///< NULL_LOGICAL_OR(null, false) is null, and NULL_LOGICAL_OR(valid, valid) == - ///< LOGICAL_OR(valid, valid) - // Unary operators - IDENTITY = 23, ///< Identity function - IS_NULL = 24, ///< Check if operand is null - SIN = 25, ///< Trigonometric sine - COS = 26, ///< Trigonometric cosine - TAN = 27, ///< Trigonometric tangent - ARCSIN = 28, ///< Trigonometric sine inverse - ARCCOS = 29, ///< Trigonometric cosine inverse - ARCTAN = 30, ///< Trigonometric tangent inverse - SINH = 31, ///< Hyperbolic sine - COSH = 32, ///< Hyperbolic cosine - TANH = 33, ///< Hyperbolic tangent - ARCSINH = 34, ///< Hyperbolic sine inverse - ARCCOSH = 35, ///< Hyperbolic cosine inverse - ARCTANH = 36, ///< Hyperbolic tangent inverse - EXP = 37, ///< Exponential (base e, Euler number) - LOG = 38, ///< Natural Logarithm (base e) - SQRT = 39, ///< Square-root (x^0.5) - CBRT = 40, ///< Cube-root (x^(1.0/3)) - CEIL = 41, ///< Smallest integer value not less than arg - FLOOR = 42, ///< largest integer value not greater than arg - ABS = 43, ///< Absolute value - RINT = 44, ///< Rounds the floating-point argument arg to an integer value - BIT_INVERT = 45, ///< Bitwise Not (~) - NOT = 46, ///< Logical Not (!) - CAST_TO_INT64 = 47, ///< Cast value to int64_t - CAST_TO_UINT64 = 48, ///< Cast value to uint64_t - CAST_TO_FLOAT64 = 49, ///< Cast value to double - - ANSI_ADD = 50, ///< operator +, with ANSI SQL semantics (e.g. overflow checking) - ANSI_SUB = 51, ///< operator -, with ANSI SQL semantics (e.g. overflow checking) - ANSI_MUL = 52, ///< operator *, with ANSI SQL semantics (e.g. overflow checking) - ANSI_DIV = 53, ///< operator / using common type of lhs and rhs, with ANSI SQL semantics (e.g. - ///< division by zero checking) - ANSI_ABS = 54, ///< Absolute value, with ANSI SQL semantics (e.g. overflow checking) - ANSI_CAST_TO_INT64 = - 55, ///< Cast value to int64_t, with ANSI SQL semantics (e.g. overflow checking) - ANSI_CAST_TO_UINT64 = - 56, ///< Cast value to uint64_t, with ANSI SQL semantics (e.g. overflow checking) - - TRY_ADD = 57, ///< operator +, with TRY semantics (e.g. returns null on overflow) - TRY_SUB = 58, ///< operator -, with TRY semantics (e.g. returns null on overflow) - TRY_MUL = 59, ///< operator *, with TRY semantics (e.g. returns null on overflow) - TRY_DIV = 60, ///< operator / using common type of lhs and rhs, with TRY semantics (e.g. returns - ///< null on division by zero) - TRY_ABS = 61, ///< Absolute value, with TRY semantics (e.g. returns null on overflow) - TRY_CAST_TO_INT64 = - 62, ///< Cast value to int64_t, with TRY semantics (e.g. returns null on overflow) - TRY_CAST_TO_UINT64 = - 63, ///< Cast value to uint64_t, with TRY semantics (e.g. returns null on overflow) -}; - -} // namespace CUDF_EXPORT cudf From 87b1d7a20cd98af00011bba4479c7c8fb55c7955 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 27 Apr 2026 17:28:57 +0000 Subject: [PATCH 05/15] update Co-authored-by: Copilot --- .../cudf/operators/ansi_arithmetic.cuh | 1 - cpp/include/cudf/operators/arithmetic.cuh | 67 +++- .../operators/{btiwise.cuh => bitwise.cuh} | 0 cpp/include/cudf/operators/comparison.cuh | 76 ++-- cpp/include/cudf/operators/logic.cuh | 56 +++ cpp/include/cudf/operators/math.cuh | 32 -- cpp/include/cudf/operators/null_handling.cuh | 20 +- cpp/include/cudf/operators/op_attributes.hpp | 144 +++++--- cpp/include/cudf/operators/opcodes.hpp | 10 + cpp/include/cudf/operators/types.cuh | 15 + cpp/src/ast/expressions.cpp | 13 +- cpp/src/jit/row_ir.cpp | 344 +++++++++++------- cpp/src/jit/row_ir.hpp | 13 +- cpp/src/transform/jit/kernel.cu | 9 + 14 files changed, 513 insertions(+), 287 deletions(-) rename cpp/include/cudf/operators/{btiwise.cuh => bitwise.cuh} (100%) diff --git a/cpp/include/cudf/operators/ansi_arithmetic.cuh b/cpp/include/cudf/operators/ansi_arithmetic.cuh index dde6a10b1234..e0f226a0e443 100644 --- a/cpp/include/cudf/operators/ansi_arithmetic.cuh +++ b/cpp/include/cudf/operators/ansi_arithmetic.cuh @@ -8,7 +8,6 @@ #include namespace CUDF_EXPORT cudf { - namespace ops { namespace detail { diff --git a/cpp/include/cudf/operators/arithmetic.cuh b/cpp/include/cudf/operators/arithmetic.cuh index c74d392da612..095b526cfae4 100644 --- a/cpp/include/cudf/operators/arithmetic.cuh +++ b/cpp/include/cudf/operators/arithmetic.cuh @@ -4,10 +4,10 @@ */ #pragma once +#include #include namespace CUDF_EXPORT cudf { - namespace ops { template @@ -87,6 +87,39 @@ __device__ inline errc div(optional* out, optional const* a, optional c return errc::OK; } +template + requires(cuda::std::is_integral_v) +__device__ inline errc floor_div(T* out, T const* a, T const* b) +{ + *out = cudf::detail::integral_floor_div(*a, *b); + return errc::OK; +} + +__device__ inline errc floor_div(float* out, float const* a, float const* b) +{ + *out = ::floorf(*a / *b); + return errc::OK; +} + +__device__ inline errc floor_div(double* out, double const* a, double const* b) +{ + *out = ::floor(*a / *b); + return errc::OK; +} + +template +__device__ inline errc floor_div(optional* out, optional const* a, optional const* b) +{ + if (a->has_value() && b->has_value()) { + T r; + floor_div(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + template __device__ inline errc mod(T* out, T const* a, T const* b) { @@ -119,6 +152,38 @@ __device__ inline errc mod(optional* out, optional const* a, optional c return errc::OK; } +template +__device__ inline errc pymod(T* out, T const* a, T const* b) +{ + *out = (*a % *b + *b) % *b; + return errc::OK; +} + +__device__ inline errc pymod(float* out, float const* a, float const* b) +{ + *out = ::fmodf(::fmodf(*a, *b) + *b, *b); + return errc::OK; +} + +__device__ inline errc pymod(double* out, double const* a, double const* b) +{ + *out = ::fmod(::fmod(*a, *b) + *b, *b); + return errc::OK; +} + +template +__device__ inline errc pymod(optional* out, optional const* a, optional const* b) +{ + if (a->has_value() && b->has_value()) { + T r; + pymod(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + template __device__ inline errc mul(T* out, T const* a, T const* b) { diff --git a/cpp/include/cudf/operators/btiwise.cuh b/cpp/include/cudf/operators/bitwise.cuh similarity index 100% rename from cpp/include/cudf/operators/btiwise.cuh rename to cpp/include/cudf/operators/bitwise.cuh diff --git a/cpp/include/cudf/operators/comparison.cuh b/cpp/include/cudf/operators/comparison.cuh index 509b8e86bd24..39ed1145eaf8 100644 --- a/cpp/include/cudf/operators/comparison.cuh +++ b/cpp/include/cudf/operators/comparison.cuh @@ -31,6 +31,28 @@ __device__ inline errc equal(optional* out, optional const* a, optional return errc::OK; } +template +__device__ inline errc not_equal(bool* out, T const* a, T const* b) +{ + *out = (*a != *b); + return errc::OK; +} + +template +__device__ inline errc not_equal(optional* out, optional const* a, optional const* b) +{ + if (a->has_value() && b->has_value()) { + bool r; + not_equal(&r, &a->value(), &b->value()); + *out = r; + } else if (!a->has_value() && !b->has_value()) { + *out = false; + } else { + *out = true; + } + return errc::OK; +} + template __device__ inline errc greater(bool* out, T const* a, T const* b) { @@ -133,59 +155,5 @@ __device__ inline errc null_equal(optional* out, optional const* a, opt return errc::OK; } -template -__device__ inline errc null_logical_and(T* out, T const* a, T const* b) -{ - *out = (*a && *b); - return errc::OK; -} - -template -__device__ inline errc null_logical_and(optional* out, - optional const* a, - optional const* b) -{ - if (a->has_value() && b->has_value()) { - bool r; - null_logical_and(&r, &a->value(), &b->value()); - *out = r; - } else if (!a->has_value() && !b->has_value()) { - *out = nullopt; - } else { - if (a->has_value() ? *(*a) : *(*b)) { - *out = nullopt; - } else { - *out = false; - } - } - return errc::OK; -} - -template -__device__ inline errc null_logical_or(T* out, T const* a, T const* b) -{ - *out = (*a || *b); - return errc::OK; -} - -template -__device__ inline errc null_logical_or(optional* out, optional const* a, optional const* b) -{ - if (a->has_value() && b->has_value()) { - bool r; - null_logical_or(&r, &a->value(), &b->value()); - *out = r; - } else if (!a->has_value() && !b->has_value()) { - *out = nullopt; - } else { - if (a->has_value() ? *(*a) : *(*b)) { - *out = true; - } else { - *out = nullopt; - } - } - return errc::OK; -} - } // namespace ops } // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/operators/logic.cuh b/cpp/include/cudf/operators/logic.cuh index e70f70aeb183..d99879939e0d 100644 --- a/cpp/include/cudf/operators/logic.cuh +++ b/cpp/include/cudf/operators/logic.cuh @@ -9,6 +9,62 @@ namespace CUDF_EXPORT cudf { namespace ops { + +template +__device__ inline errc null_logical_and(T* out, T const* a, T const* b) +{ + *out = (*a && *b); + return errc::OK; +} + +template +__device__ inline errc null_logical_and(optional* out, + optional const* a, + optional const* b) +{ + if (a->has_value() && b->has_value()) { + bool r; + null_logical_and(&r, &a->value(), &b->value()); + *out = r; + } else if (!a->has_value() && !b->has_value()) { + *out = nullopt; + } else { + if (a->has_value() ? *(*a) : *(*b)) { + *out = nullopt; + } else { + *out = false; + } + } + return errc::OK; +} + +template +__device__ inline errc null_logical_or(T* out, T const* a, T const* b) +{ + *out = (*a || *b); + return errc::OK; +} + +template +__device__ inline errc null_logical_or(optional* out, optional const* a, optional const* b) +{ + if (a->has_value() && b->has_value()) { + bool r; + null_logical_or(&r, &a->value(), &b->value()); + *out = r; + } else if (!a->has_value() && !b->has_value()) { + *out = nullopt; + } else { + if (a->has_value() ? *(*a) : *(*b)) { + *out = true; + } else { + *out = nullopt; + } + } + return errc::OK; +} + + template __device__ inline errc logical_and(T* out, T const* a, T const* b) { diff --git a/cpp/include/cudf/operators/math.cuh b/cpp/include/cudf/operators/math.cuh index dcc1c0e596e9..30f759b3d012 100644 --- a/cpp/include/cudf/operators/math.cuh +++ b/cpp/include/cudf/operators/math.cuh @@ -189,38 +189,6 @@ __device__ inline errc pow(optional* out, optional const* a, optional c return errc::OK; } -template -__device__ inline errc pymod(T* out, T const* a, T const* b) -{ - *out = (*a % *b + *b) % *b; - return errc::OK; -} - -__device__ inline errc pymod(float* out, float const* a, float const* b) -{ - *out = ::fmodf(::fmodf(*a, *b) + *b, *b); - return errc::OK; -} - -__device__ inline errc pymod(double* out, double const* a, double const* b) -{ - *out = ::fmod(::fmod(*a, *b) + *b, *b); - return errc::OK; -} - -template -__device__ inline errc pymod(optional* out, optional const* a, optional const* b) -{ - if (a->has_value() && b->has_value()) { - T r; - pymod(&r, &a->value(), &b->value()); - *out = r; - } else { - *out = nullopt; - } - return errc::OK; -} - __device__ inline errc rint(float* out, float const* a) { *out = ::rintf(*a); diff --git a/cpp/include/cudf/operators/null_handling.cuh b/cpp/include/cudf/operators/null_handling.cuh index fd04b35d596d..573e7ee88455 100644 --- a/cpp/include/cudf/operators/null_handling.cuh +++ b/cpp/include/cudf/operators/null_handling.cuh @@ -6,8 +6,6 @@ #pragma once #include -#include - namespace CUDF_EXPORT cudf { namespace ops { @@ -62,5 +60,23 @@ __device__ inline errc coalesce(optional* out, optional const* a, optional return errc::OK; } +template +__device__ inline errc replace_nulls(T* out, T const* a, [[maybe_unused]] T const* replacement) +{ + *out = *a; + return errc::OK; +} + +template +__device__ inline errc replace_nulls(optional* out, optional const* a, T const* replacement) +{ + if (a->has_value()) { + *out = a->value(); + } else { + *out = *replacement; + } + return errc::OK; +} + } // namespace ops } // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/operators/op_attributes.hpp b/cpp/include/cudf/operators/op_attributes.hpp index 765e82d0b99f..1d0ed611e5f3 100644 --- a/cpp/include/cudf/operators/op_attributes.hpp +++ b/cpp/include/cudf/operators/op_attributes.hpp @@ -6,9 +6,11 @@ #pragma once #include +#include + namespace cudf::detail::row_ir { -enum typing : uint64_t { +enum [[nodiscard]] typing : uint64_t { NONE = 0x0, BOOL8 = 0x1, INT8 = 0x2, @@ -43,40 +45,44 @@ enum typing : uint64_t { ARITHMETIC = SIGNED_INTEGERS | UNSIGNED_INTEGERS | FLOATS | DECIMALS, SIGNED_ARITHMETIC = SIGNED_INTEGERS | FLOATS | DECIMALS, ALL = 0x0FFFFFFF, + ARG_MASK = 0x10000000, ARG0 = 0x10000000, ARG1 = 0x10000001, ARG2 = 0x10000002, + ARG3 = 0x10000003, INPUT = 0x20000000, }; -struct op_typing { - typing output = typing::NONE; - typing arg0 = typing::NONE; - typing arg1 = typing::NONE; - typing arg2 = typing::NONE; +struct [[nodiscard]] op_typing { + typing output = typing::NONE; + std::array args = {typing::NONE, typing::NONE, typing::NONE}; }; /** * @brief Indicates how an operator propagates null values */ -enum class null_output : uint8_t { +enum class [[nodiscard]] null_output : uint8_t { PROPAGATE = 0, ALWAYS_VALID = 1, ALWAYS_NULLABLE = 2, }; -inline std::string_view get_op_name(opcode op) +[[nodiscard]] inline std::string_view get_op_name(opcode op) { switch (op) { case opcode::GET_INPUT: return "get_input"; case opcode::SET_OUTPUT: return "set_output"; + case opcode::IDENTITY: return "identity"; case opcode::IS_NULL: return "is_null"; case opcode::NULLIFY_IF: return "nullify_if"; case opcode::COALESCE: return "coalesce"; + case opcode::REPLACE_NULLS: return "replace_nulls"; case opcode::ABS: return "abs"; case opcode::ADD: return "add"; case opcode::DIV: return "div"; + case opcode::FLOOR_DIV: return "floor_div"; case opcode::MOD: return "mod"; + case opcode::PYMOD: return "pymod"; case opcode::MUL: return "mul"; case opcode::NEG: return "neg"; case opcode::SUB: return "sub"; @@ -110,6 +116,7 @@ inline std::string_view get_op_name(opcode op) case opcode::CAST_TO_DEC64: return "cast_to_dec64"; case opcode::CAST_TO_DEC128: return "cast_to_dec128"; case opcode::EQUAL: return "equal"; + case opcode::NOT_EQUAL: return "not_equal"; case opcode::GREATER: return "greater"; case opcode::GREATER_EQUAL: return "greater_equal"; case opcode::LESS: return "less"; @@ -117,11 +124,14 @@ inline std::string_view get_op_name(opcode op) case opcode::NULL_EQUAL: return "null_equal"; case opcode::NULL_LOGICAL_AND: return "null_logical_and"; case opcode::NULL_LOGICAL_OR: return "null_logical_or"; + case opcode::LOGICAL_AND: return "logical_and"; + case opcode::LOGICAL_OR: return "logical_or"; case opcode::LOGICAL_NOT: return "logical_not"; case opcode::IF_ELSE: return "if_else"; case opcode::CBRT: return "cbrt"; case opcode::CEIL: return "ceil"; case opcode::FLOOR: return "floor"; + case opcode::RINT: return "rint"; case opcode::SQRT: return "sqrt"; case opcode::POW: return "pow"; case opcode::EXP: return "exp"; @@ -138,22 +148,29 @@ inline std::string_view get_op_name(opcode op) case opcode::SINH: return "sinh"; case opcode::TAN: return "tan"; case opcode::TANH: return "tanh"; + default: CUDF_UNREACHABLE("Invalid opcode"); } } -inline null_output get_op_null_output(opcode op) +[[nodiscard]] inline null_output get_op_null_output(opcode op) { switch (op) { case opcode::IS_NULL: - case opcode::NULL_EQUAL: return null_output::ALWAYS_VALID; + case opcode::NULL_EQUAL: + case opcode::REPLACE_NULLS: return null_output::ALWAYS_VALID; case opcode::GET_INPUT: case opcode::SET_OUTPUT: + case opcode::IDENTITY: case opcode::LOGICAL_NOT: + case opcode::LOGICAL_AND: + case opcode::LOGICAL_OR: case opcode::ABS: case opcode::ADD: case opcode::DIV: + case opcode::FLOOR_DIV: case opcode::MOD: + case opcode::PYMOD: case opcode::MUL: case opcode::NEG: case opcode::SUB: @@ -169,6 +186,7 @@ inline null_output get_op_null_output(opcode op) case opcode::CBRT: case opcode::CEIL: case opcode::FLOOR: + case opcode::RINT: case opcode::SQRT: case opcode::POW: case opcode::EXP: @@ -198,6 +216,7 @@ inline null_output get_op_null_output(opcode op) case opcode::CAST_TO_DEC64: case opcode::CAST_TO_DEC128: case opcode::EQUAL: + case opcode::NOT_EQUAL: case opcode::GREATER: case opcode::GREATER_EQUAL: case opcode::LESS: @@ -216,6 +235,8 @@ inline null_output get_op_null_output(opcode op) case opcode::ANSI_TRY_PRECISION_CAST: case opcode::NULL_LOGICAL_AND: case opcode::NULL_LOGICAL_OR: return null_output::ALWAYS_NULLABLE; + + default: CUDF_UNREACHABLE("Invalid opcode"); } } @@ -223,16 +244,19 @@ inline null_output get_op_null_output(opcode op) * @brief Indicates whether the output of the operator will be different when it is called with or * without the null-ness of a value. */ -inline bool get_op_requires_nulls(opcode op) +[[nodiscard]] inline bool get_op_requires_nulls(opcode op) { switch (op) { case opcode::GET_INPUT: case opcode::SET_OUTPUT: case opcode::NULLIFY_IF: + case opcode::IDENTITY: case opcode::ABS: case opcode::ADD: case opcode::DIV: + case opcode::FLOOR_DIV: case opcode::MOD: + case opcode::PYMOD: case opcode::MUL: case opcode::NEG: case opcode::SUB: @@ -266,15 +290,19 @@ inline bool get_op_requires_nulls(opcode op) case opcode::CAST_TO_DEC64: case opcode::CAST_TO_DEC128: case opcode::EQUAL: + case opcode::NOT_EQUAL: case opcode::GREATER: case opcode::GREATER_EQUAL: case opcode::LESS: case opcode::LESS_EQUAL: case opcode::LOGICAL_NOT: + case opcode::LOGICAL_AND: + case opcode::LOGICAL_OR: case opcode::IF_ELSE: case opcode::CBRT: case opcode::CEIL: case opcode::FLOOR: + case opcode::RINT: case opcode::SQRT: case opcode::POW: case opcode::EXP: @@ -296,11 +324,14 @@ inline bool get_op_requires_nulls(opcode op) case opcode::IS_NULL: case opcode::NULL_EQUAL: case opcode::NULL_LOGICAL_AND: - case opcode::NULL_LOGICAL_OR: return true; + case opcode::NULL_LOGICAL_OR: + case opcode::REPLACE_NULLS: return true; + + default: CUDF_UNREACHABLE("Invalid opcode"); } } -inline bool get_op_is_fallible(opcode op) +[[nodiscard]] inline bool get_op_is_fallible(opcode op) { switch (op) { case opcode::ANSI_ADD: @@ -315,12 +346,16 @@ inline bool get_op_is_fallible(opcode op) case opcode::GET_INPUT: case opcode::SET_OUTPUT: case opcode::IS_NULL: + case opcode::IDENTITY: case opcode::NULLIFY_IF: case opcode::COALESCE: + case opcode::REPLACE_NULLS: case opcode::ABS: case opcode::ADD: case opcode::DIV: + case opcode::FLOOR_DIV: case opcode::MOD: + case opcode::PYMOD: case opcode::MUL: case opcode::NEG: case opcode::SUB: @@ -346,6 +381,7 @@ inline bool get_op_is_fallible(opcode op) case opcode::CAST_TO_DEC64: case opcode::CAST_TO_DEC128: case opcode::EQUAL: + case opcode::NOT_EQUAL: case opcode::GREATER: case opcode::GREATER_EQUAL: case opcode::LESS: @@ -353,11 +389,14 @@ inline bool get_op_is_fallible(opcode op) case opcode::NULL_EQUAL: case opcode::NULL_LOGICAL_AND: case opcode::NULL_LOGICAL_OR: + case opcode::LOGICAL_AND: + case opcode::LOGICAL_OR: case opcode::LOGICAL_NOT: case opcode::IF_ELSE: case opcode::CBRT: case opcode::CEIL: case opcode::FLOOR: + case opcode::RINT: case opcode::SQRT: case opcode::POW: case opcode::EXP: @@ -374,10 +413,12 @@ inline bool get_op_is_fallible(opcode op) case opcode::SINH: case opcode::TAN: case opcode::TANH: return false; + + default: CUDF_UNREACHABLE("Invalid opcode"); } } -inline constexpr int32_t get_op_arity(opcode op) +[[nodiscard]] inline constexpr int32_t get_op_arity(opcode op) { switch (op) { case opcode::GET_INPUT: return 0; @@ -385,6 +426,7 @@ inline constexpr int32_t get_op_arity(opcode op) case opcode::SET_OUTPUT: case opcode::IS_NULL: case opcode::NULLIFY_IF: + case opcode::IDENTITY: case opcode::ABS: case opcode::NEG: case opcode::ANSI_ABS: @@ -405,6 +447,7 @@ inline constexpr int32_t get_op_arity(opcode op) case opcode::CBRT: case opcode::CEIL: case opcode::FLOOR: + case opcode::RINT: case opcode::SQRT: case opcode::EXP: case opcode::LOG: @@ -422,9 +465,12 @@ inline constexpr int32_t get_op_arity(opcode op) case opcode::TANH: return 1; case opcode::COALESCE: + case opcode::REPLACE_NULLS: case opcode::ADD: case opcode::DIV: + case opcode::FLOOR_DIV: case opcode::MOD: + case opcode::PYMOD: case opcode::MUL: case opcode::SUB: case opcode::ANSI_ADD: @@ -443,6 +489,7 @@ inline constexpr int32_t get_op_arity(opcode op) case opcode::BIT_OR: case opcode::BIT_XOR: case opcode::EQUAL: + case opcode::NOT_EQUAL: case opcode::GREATER: case opcode::GREATER_EQUAL: case opcode::LESS: @@ -450,9 +497,13 @@ inline constexpr int32_t get_op_arity(opcode op) case opcode::NULL_EQUAL: case opcode::NULL_LOGICAL_OR: case opcode::NULL_LOGICAL_AND: + case opcode::LOGICAL_AND: + case opcode::LOGICAL_OR: case opcode::POW: return 2; case opcode::IF_ELSE: return 3; + + default: CUDF_UNREACHABLE("Invalid opcode"); } } @@ -464,29 +515,28 @@ inline constexpr int32_t get_op_arity(opcode op) * @return An `op_typing` struct containing the expected output type and input types for the * operator */ -inline op_typing get_op_typing(opcode op) +[[nodiscard]] inline op_typing get_op_typing(opcode op) { switch (op) { - case opcode::GET_INPUT: return {typing::INPUT, typing::NONE, typing::NONE, typing::NONE}; - - case opcode::SET_OUTPUT: return {typing::NONE, typing::ALL, typing::NONE, typing::NONE}; - - case opcode::IS_NULL: return {typing::ARG0, typing::ALL, typing::NONE, typing::NONE}; - - case opcode::NULLIFY_IF: return {typing::ARG1, typing::BOOL8, typing::ALL, typing::NONE}; - case opcode::COALESCE: return {typing::ARG0, typing::ALL, typing::ARG0, typing::NONE}; - + case opcode::GET_INPUT: return {typing::INPUT, {}}; + case opcode::SET_OUTPUT: return {typing::NONE, {typing::ALL}}; + case opcode::IDENTITY: return {typing::ARG0, {typing::ALL}}; + case opcode::IS_NULL: return {typing::BOOL8, {typing::ALL}}; + case opcode::NULLIFY_IF: return {typing::ARG1, {typing::BOOL8, typing::ALL}}; + case opcode::COALESCE: return {typing::ARG0, {typing::ALL, typing::ARG0}}; + case opcode::REPLACE_NULLS: return {typing::ARG0, {typing::ALL, typing::ARG0}}; case opcode::ABS: case opcode::NEG: case opcode::ANSI_ABS: case opcode::ANSI_NEG: case opcode::ANSI_TRY_NEG: - case opcode::ANSI_TRY_ABS: - return {typing::ARG0, typing::ARITHMETIC, typing::NONE, typing::NONE}; - + case opcode::ANSI_TRY_ABS: return {typing::ARG0, {typing::ARITHMETIC}}; + case opcode::FLOOR_DIV: + return {typing::ARG0, {typing{typing::FLOATS | typing::INTEGERS}, typing::ARG0}}; case opcode::ADD: case opcode::DIV: case opcode::MOD: + case opcode::PYMOD: case opcode::MUL: case opcode::SUB: case opcode::ANSI_ADD: @@ -498,48 +548,39 @@ inline op_typing get_op_typing(opcode op) case opcode::ANSI_TRY_SUB: case opcode::ANSI_TRY_MUL: case opcode::ANSI_TRY_DIV: - case opcode::ANSI_TRY_MOD: - return {typing::ARG0, typing::ARITHMETIC, typing::ARG0, typing::NONE}; - + case opcode::ANSI_TRY_MOD: return {typing::ARG0, {typing::ARITHMETIC, typing::ARG0}}; case opcode::ANSI_PRECISION_CAST: - case opcode::ANSI_TRY_PRECISION_CAST: - return {typing::ARG0, typing::DECIMALS, typing::INT32, typing::NONE}; - + case opcode::ANSI_TRY_PRECISION_CAST: return {typing::ARG0, {typing::DECIMALS, typing::INT32}}; case opcode::BIT_AND: case opcode::BIT_INVERT: case opcode::BIT_OR: - case opcode::BIT_XOR: return {typing::ARG0, typing::INTEGERS, typing::ARG0, typing::NONE}; - + case opcode::BIT_XOR: return {typing::ARG0, {typing::INTEGERS, typing::ARG0}}; case opcode::CAST_TO_I32: case opcode::CAST_TO_I64: case opcode::CAST_TO_U32: case opcode::CAST_TO_U64: case opcode::CAST_TO_F32: - case opcode::CAST_TO_F64: - return {typing::ARG0, typing{typing::INTEGERS | typing::FLOATS}, typing::NONE, typing::NONE}; - + case opcode::CAST_TO_F64: return {typing::ARG0, {typing{typing::INTEGERS | typing::FLOATS}}}; case opcode::CAST_TO_DEC32: case opcode::CAST_TO_DEC64: - case opcode::CAST_TO_DEC128: - return {typing::ARG0, typing::DECIMALS, typing::NONE, typing::NONE}; - + case opcode::CAST_TO_DEC128: return {typing::ARG0, {typing::DECIMALS}}; case opcode::EQUAL: case opcode::GREATER: case opcode::GREATER_EQUAL: case opcode::LESS: - case opcode::LESS_EQUAL: return {typing::BOOL8, typing::ALL, typing::ARG0, typing::NONE}; - - case opcode::NULL_EQUAL: + case opcode::LESS_EQUAL: return {typing::BOOL8, {typing::ALL, typing::ARG0}}; + case opcode::NOT_EQUAL: + case opcode::NULL_EQUAL: return {typing::BOOL8, {typing::ALL, typing::ARG0}}; case opcode::NULL_LOGICAL_AND: - case opcode::NULL_LOGICAL_OR: return {typing::BOOL8, typing::BOOL8, typing::ARG0, typing::NONE}; - - case opcode::LOGICAL_NOT: return {typing::ARG0, typing::BOOL8, typing::NONE, typing::NONE}; - - case opcode::IF_ELSE: return {typing::ARG1, typing::BOOL8, typing::ALL, typing::ARG0}; - + case opcode::NULL_LOGICAL_OR: + case opcode::LOGICAL_AND: + case opcode::LOGICAL_OR: return {typing::BOOL8, {typing::BOOL8, typing::ARG0}}; + case opcode::LOGICAL_NOT: return {typing::ARG0, {typing::BOOL8}}; + case opcode::IF_ELSE: return {typing::ARG1, {typing::BOOL8, typing::ALL, typing::ARG0}}; case opcode::CBRT: case opcode::CEIL: case opcode::FLOOR: + case opcode::RINT: case opcode::SQRT: case opcode::POW: case opcode::EXP: @@ -555,7 +596,8 @@ inline op_typing get_op_typing(opcode op) case opcode::SIN: case opcode::SINH: case opcode::TAN: - case opcode::TANH: return {typing::ARG0, typing::FLOATS, typing::NONE, typing::NONE}; + case opcode::TANH: return {typing::ARG0, {typing::FLOATS}}; + default: CUDF_UNREACHABLE("Invalid opcode"); } } diff --git a/cpp/include/cudf/operators/opcodes.hpp b/cpp/include/cudf/operators/opcodes.hpp index 8b8a0c9d7dbb..b46db033bd2a 100644 --- a/cpp/include/cudf/operators/opcodes.hpp +++ b/cpp/include/cudf/operators/opcodes.hpp @@ -15,16 +15,22 @@ enum class [[nodiscard]] opcode : int32_t { GET_INPUT, SET_OUTPUT, + // Identity operators + IDENTITY, + // Null handling operators IS_NULL, NULLIFY_IF, COALESCE, + REPLACE_NULLS, /// Arithmetic operators ABS, ADD, DIV, + FLOOR_DIV, MOD, + PYMOD, MUL, NEG, SUB, @@ -68,6 +74,7 @@ enum class [[nodiscard]] opcode : int32_t { /// Comparison & Logical operators EQUAL, + NOT_EQUAL, GREATER, GREATER_EQUAL, LESS, @@ -75,6 +82,8 @@ enum class [[nodiscard]] opcode : int32_t { NULL_EQUAL, NULL_LOGICAL_AND, NULL_LOGICAL_OR, + LOGICAL_AND, + LOGICAL_OR, LOGICAL_NOT, IF_ELSE, @@ -82,6 +91,7 @@ enum class [[nodiscard]] opcode : int32_t { CBRT, CEIL, FLOOR, + RINT, SQRT, POW, EXP, diff --git a/cpp/include/cudf/operators/types.cuh b/cpp/include/cudf/operators/types.cuh index 47194add243d..9be973656c78 100644 --- a/cpp/include/cudf/operators/types.cuh +++ b/cpp/include/cudf/operators/types.cuh @@ -51,5 +51,20 @@ __device__ constexpr T ipow10(T exponent) } } // namespace detail + +template +__device__ inline errc identity(T* out, T const* a) +{ + *out = *a; + return errc::OK; +} + +template +__device__ inline errc identity(optional* out, optional const* a) +{ + *out = *a; + return errc::OK; +} + } // namespace ops } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/ast/expressions.cpp b/cpp/src/ast/expressions.cpp index 24c1679bc30f..b80fd5555bae 100644 --- a/cpp/src/ast/expressions.cpp +++ b/cpp/src/ast/expressions.cpp @@ -79,7 +79,6 @@ bool operation::may_evaluate_null(table_view const& left, return subexpr.get().may_evaluate_null(left, right, stream); }); }; - auto column_name_reference::accept(detail::expression_transformer& visitor) const -> decltype(visitor.visit(*this)) @@ -87,32 +86,28 @@ auto column_name_reference::accept(detail::expression_transformer& visitor) cons return visitor.visit(*this); } -std::unique_ptr literal::accept( - cudf::detail::row_ir::ast_converter& converter) const +cudf::detail::row_ir::node literal::accept(cudf::detail::row_ir::ast_converter& converter) const { return converter.add_ir_node(*this); } -std::unique_ptr column_reference::accept( +cudf::detail::row_ir::node column_reference::accept( cudf::detail::row_ir::ast_converter& converter) const { return converter.add_ir_node(*this); } -std::unique_ptr operation::accept( - cudf::detail::row_ir::ast_converter& converter) const +cudf::detail::row_ir::node operation::accept(cudf::detail::row_ir::ast_converter& converter) const { return converter.add_ir_node(*this); } -std::unique_ptr column_name_reference::accept( - cudf::detail::row_ir::ast_converter&) const +cudf::detail::row_ir::node column_name_reference::accept(cudf::detail::row_ir::ast_converter&) const { CUDF_FAIL( "column_name_reference is not supported in row_ir. row_ir only supports resolved expressions", std::invalid_argument); } - } // namespace ast } // namespace cudf diff --git a/cpp/src/jit/row_ir.cpp b/cpp/src/jit/row_ir.cpp index 5d66dd0e6304..c05f30488a57 100644 --- a/cpp/src/jit/row_ir.cpp +++ b/cpp/src/jit/row_ir.cpp @@ -33,12 +33,11 @@ node::node(opcode op, std::vector args) : op_{op}, args_{std::move(args)} { CUDF_EXPECTS(op != opcode::GET_INPUT && op != opcode::SET_OUTPUT, std::format("Invalid opcode `{}` for operation node.", get_op_name(op))); - CUDF_EXPECTS(args_.size() == get_op_arity(op), + CUDF_EXPECTS(args_.size() == static_cast(get_op_arity(op)), std::format("Invalid number of arguments for operator `{}`. Expected {}, Got {}.", get_op_name(op), get_op_arity(op), args_.size())); - // TODO: check argument types, this will be after resolving types } node::node(input_reference input) : reference_{input}, op_{opcode::SET_OUTPUT} {} @@ -111,31 +110,159 @@ row_ir::typing as_typing(data_type type) } } -data_type get_return_type(opcode op, std::span args) +data_type as_data_type(typing type, int32_t scale) { - std::vector typings; + switch (type) { + case typing::BOOL8: return data_type{type_id::BOOL8, 0}; + case typing::INT8: return data_type{type_id::INT8, 0}; + case typing::INT16: return data_type{type_id::INT16, 0}; + case typing::INT32: return data_type{type_id::INT32, 0}; + case typing::INT64: return data_type{type_id::INT64, 0}; + case typing::UINT8: return data_type{type_id::UINT8, 0}; + case typing::UINT16: return data_type{type_id::UINT16, 0}; + case typing::UINT32: return data_type{type_id::UINT32, 0}; + case typing::UINT64: return data_type{type_id::UINT64, 0}; + case typing::FLOAT32: return data_type{type_id::FLOAT32, 0}; + case typing::FLOAT64: return data_type{type_id::FLOAT64, 0}; + case typing::DECIMAL32: return data_type{type_id::DECIMAL32, scale}; + case typing::DECIMAL64: return data_type{type_id::DECIMAL64, scale}; + case typing::DECIMAL128: return data_type{type_id::DECIMAL128, scale}; + case typing::TIMESTAMP_DAYS: return data_type{type_id::TIMESTAMP_DAYS, 0}; + case typing::TIMESTAMP_SECONDS: return data_type{type_id::TIMESTAMP_SECONDS, 0}; + case typing::TIMESTAMP_MILLISECONDS: return data_type{type_id::TIMESTAMP_MILLISECONDS, 0}; + case typing::TIMESTAMP_MICROSECONDS: return data_type{type_id::TIMESTAMP_MICROSECONDS, 0}; + case typing::TIMESTAMP_NANOSECONDS: return data_type{type_id::TIMESTAMP_NANOSECONDS, 0}; + case typing::DURATION_DAYS: return data_type{type_id::DURATION_DAYS, 0}; + case typing::DURATION_SECONDS: return data_type{type_id::DURATION_SECONDS, 0}; + case typing::DURATION_MILLISECONDS: return data_type{type_id::DURATION_MILLISECONDS, 0}; + case typing::DURATION_MICROSECONDS: return data_type{type_id::DURATION_MICROSECONDS, 0}; + case typing::DURATION_NANOSECONDS: return data_type{type_id::DURATION_NANOSECONDS, 0}; + case typing::STRING: return data_type{type_id::STRING, 0}; + default: + CUDF_FAIL(std::format("Invalid typing for {}: {}", __FUNCTION__, static_cast(type)), + std::invalid_argument); + } +} + +opcode as_opcode(ast::ast_operator op) +{ + switch (op) { + case ast::ast_operator::ADD: return opcode::ADD; + case ast::ast_operator::SUB: return opcode::SUB; + case ast::ast_operator::MUL: return opcode::MUL; + case ast::ast_operator::DIV: return opcode::DIV; + case ast::ast_operator::TRUE_DIV: return opcode::DIV; + case ast::ast_operator::FLOOR_DIV: return opcode::FLOOR_DIV; + case ast::ast_operator::MOD: return opcode::MOD; + case ast::ast_operator::PYMOD: return opcode::PYMOD; + case ast::ast_operator::POW: return opcode::POW; + case ast::ast_operator::EQUAL: return opcode::EQUAL; + case ast::ast_operator::NULL_EQUAL: return opcode::NULL_EQUAL; + case ast::ast_operator::NOT_EQUAL: return opcode::NOT_EQUAL; + case ast::ast_operator::LESS: return opcode::LESS; + case ast::ast_operator::GREATER: return opcode::GREATER; + case ast::ast_operator::LESS_EQUAL: return opcode::LESS_EQUAL; + case ast::ast_operator::GREATER_EQUAL: return opcode::GREATER_EQUAL; + case ast::ast_operator::BITWISE_AND: return opcode::BIT_AND; + case ast::ast_operator::BITWISE_OR: return opcode::BIT_OR; + case ast::ast_operator::BITWISE_XOR: return opcode::BIT_XOR; + case ast::ast_operator::LOGICAL_AND: return opcode::LOGICAL_AND; + case ast::ast_operator::NULL_LOGICAL_AND: return opcode::NULL_LOGICAL_AND; + case ast::ast_operator::LOGICAL_OR: return opcode::LOGICAL_OR; + case ast::ast_operator::NULL_LOGICAL_OR: return opcode::NULL_LOGICAL_OR; + case ast::ast_operator::IDENTITY: return opcode::IDENTITY; + case ast::ast_operator::IS_NULL: return opcode::IS_NULL; + case ast::ast_operator::SIN: return opcode::SIN; + case ast::ast_operator::COS: return opcode::COS; + case ast::ast_operator::TAN: return opcode::TAN; + case ast::ast_operator::ARCSIN: return opcode::ARCSIN; + case ast::ast_operator::ARCCOS: return opcode::ARCCOS; + case ast::ast_operator::ARCTAN: return opcode::ARCTAN; + case ast::ast_operator::SINH: return opcode::SINH; + case ast::ast_operator::COSH: return opcode::COSH; + case ast::ast_operator::TANH: return opcode::TANH; + case ast::ast_operator::ARCSINH: return opcode::ARCSINH; + case ast::ast_operator::ARCCOSH: return opcode::ARCCOSH; + case ast::ast_operator::ARCTANH: return opcode::ARCTANH; + case ast::ast_operator::EXP: return opcode::EXP; + case ast::ast_operator::LOG: return opcode::LOG; + case ast::ast_operator::SQRT: return opcode::SQRT; + case ast::ast_operator::CBRT: return opcode::CBRT; + case ast::ast_operator::CEIL: return opcode::CEIL; + case ast::ast_operator::FLOOR: return opcode::FLOOR; + case ast::ast_operator::ABS: return opcode::ABS; + case ast::ast_operator::RINT: return opcode::RINT; + case ast::ast_operator::BIT_INVERT: return opcode::BIT_INVERT; + case ast::ast_operator::NOT: return opcode::LOGICAL_NOT; + case ast::ast_operator::CAST_TO_INT64: return opcode::CAST_TO_I64; + case ast::ast_operator::CAST_TO_UINT64: return opcode::CAST_TO_U64; + case ast::ast_operator::CAST_TO_FLOAT64: return opcode::CAST_TO_F64; + default: CUDF_UNREACHABLE("Invalid opcode"); + } +} + +std::string to_cuda_type(cudf::data_type type, bool nullable) +{ + auto name = type_to_name(type); + return nullable ? std::format("cuda::std::optional<{}>", name) : name; +} + +data_type get_return_type(opcode op, + std::span args, + std::optional target_scale) +{ + std::vector arg_types; for (auto& type : args) { - typings.push_back(as_typing(type)); + arg_types.emplace_back(as_typing(type)); } + // TODO: ideally, we'd want to have rules for null propagation and checking + // i.e. the REPLACE_NULLS has the requirement that the second argument is non-nullable but + // that is presently only implied by the name. + auto op_type_match = get_op_typing(op); - // TODO: match typing and get return type - // - // - // TODO(lamarrr): figure out scale propagation rules and creation/assignment rules - // - // TODO(lamarrr): implement filter_predicate to return false on nulls - // - // TODO: scale-propagation rules - // TODO: decimal ansi operators(precision and scale-oriented non-templated arguments) - // TODO: cast operators to match AST - // TODO: decimal cast operators to match AST - // TODO: datetime cast operators & arithmetic - // TODO: decimal ansi cast - // TODO: ansi_mod, div operations for fixed-point and duration types - - data_type return_type; + + for (size_t i = 0; i < args.size(); ++i) { + auto type = op_type_match.args[i]; + if (type == typing::NONE) { continue; } + + if ((type & typing::ARG_MASK) != typing::NONE) { + CUDF_EXPECTS((arg_types[i] & type) != 0, + std::format("Argument {} of operator `{}` does not match expected types. Got {}", + i, + get_op_name(op), + type_to_name(args[i]))); + } else { + auto src_index = static_cast(type & ~typing::ARG_MASK); + CUDF_EXPECTS( + src_index < i, + std::format("Invalid type match rule for operator `{}` at argument {}", get_op_name(op), i), + std::runtime_error); + CUDF_EXPECTS(arg_types[i] == arg_types[src_index], + std::format("Argument {} of operator `{}` does not match type of argument " + "`{}`. Got `{}`, expected `{}`", + i, + get_op_name(op), + src_index, + type_to_name(args[i]), + type_to_name(args[src_index]))); + } + } + + // TODO: implement filter_predicate to return false on nulls + + if ((op_type_match.output & typing::ARG_MASK) != typing::NONE) { + auto arg_index = static_cast(op_type_match.output & ~typing::ARG_MASK); + return args[arg_index]; + } else { + CUDF_EXPECTS( + op_type_match.output != typing::NONE && + (op_type_match.output & typing::DECIMALS) == typing::NONE, + std::format("Invalid type match rule for operator `{}` return type", get_op_name(op)), + std::runtime_error); + return as_data_type(op_type_match.output, target_scale.value_or(0)); + } } void node::instantiate(instance_context& ctx, instance_info const& info) @@ -158,7 +285,7 @@ void node::instantiate(instance_context& ctx, instance_info const& info) for (auto& arg : args_) { arg_types.emplace_back(arg.get_type()); } - type_ = get_return_type(op_, arg_types); + type_ = get_return_type(op_, arg_types, std::nullopt); } break; } } @@ -168,11 +295,6 @@ void node::emit_code(instance_context& ctx, instance_info const& instance, code_sink& sink) const { - auto to_cuda_type = [](cudf::data_type type, bool nullable) { - auto name = type_to_name(type); - return nullable ? std::format("cuda::std::optional<{}>", name) : name; - }; - for (auto& arg : args_) { arg.emit_code(ctx, info, instance, sink); } @@ -248,40 +370,12 @@ if(cudf::ops::errc e = cudf::ops::{}(&{}, {}); e != cudf::ops::errc::SUCCESS) {{ } } -/* -// TODO: transitive null-ness - -filter_predicate::filter_predicate(std::unique_ptr source) : id_(), source_(std::move(source)) -{ -} - -[[nodiscard]] std::string filter_predicate::generate_code(instance_context& ctx, - target_info const& info, - instance_info const& instance) -{ - switch (info.id) { - case target::CUDA: { - auto source_code = source_->generate_code(ctx, info, instance); - return std::format( - "{}\n" - "bool {} = cudf::ast::detail::flatten_predicate({});\n", - source_code, - id_, - source_->get_id()); - } - default: - CUDF_FAIL("Unsupported target: " + std::to_string(static_cast(info.id)), - std::invalid_argument); - } -} - */ - std::span ast_converter::get_input_specs() const { return input_specs_; } int32_t ast_converter::add_ast_input(ast_input_spec in) { auto id = static_cast(input_specs_.size()); - input_specs_.push_back(std::move(in)); + input_specs_.emplace_back(std::move(in)); return id; } @@ -305,15 +399,9 @@ row_ir::node ast_converter::add_ir_node(ast::operation const& expr) { std::vector operands; for (auto const& operand : expr.get_operands()) { - operands.push_back(operand.get().accept(*this)); + operands.emplace_back(operand.get().accept(*this)); } - return row_ir::node(row_ir::operation{expr.get_operator(), std::move(operands)}); -} - -row_ir::node ast_converter::add_ir_node(ast::detail::filter_predicate const& expr) -{ - auto operand = expr.get_operand().accept(*this); - return row_ir::node(row_ir::filter_predicate{std::move(operand)}); + return row_ir::node(as_opcode(expr.get_operator()), std::move(operands)); } // Resolve the table for a column input spec, preferring left_table/right_table for join cases, @@ -371,11 +459,12 @@ std::variant get_column_view(ast_scalar_input_s return scalar_column_view{spec.broadcast_column->view()}; } -std::tuple ast_converter::generate_code(target target_id, - ast::expression const& expr, - ast_args const& args) +std::tuple ast_converter::generate_code( + target target_id, ast::expression const& expr, ast_args const& args) { - output_irs_.emplace_back(std::make_unique(0, expr.accept(*this))); + output_irs_.emplace_back(output_reference{0}, expr.accept(*this)); + + // TODO: return fallible or not // resolve the flattened input references into IR input variables for (auto const& input : input_specs_) { @@ -401,90 +490,76 @@ std::tuple ast_converter::generate_code(target t auto is_null_aware = std::any_of( - output_irs_.cbegin(), output_irs_.cend(), [](auto& ir) { return ir->is_null_aware(); }) + output_irs_.cbegin(), output_irs_.cend(), [](auto& ir) { return ir.is_null_aware(); }) ? null_aware::YES : null_aware::NO; bool output_is_always_valid = std::all_of( - output_irs_.cbegin(), output_irs_.cend(), [](auto& ir) { return ir->is_always_valid(); }); + output_irs_.cbegin(), output_irs_.cend(), [](auto& ir) { return ir.is_always_valid(); }); bool may_evaluate_null = !output_is_always_valid && has_nullable_inputs; auto null_policy = may_evaluate_null ? output_nullability::PRESERVE : output_nullability::ALL_VALID; + auto is_fallible = std::any_of( + output_irs_.cbegin(), output_irs_.cend(), [](auto& ir) { return ir.is_fallible(); }); + instance_ctx.set_has_nulls(is_null_aware == null_aware::YES); // instantiate the IR nodes for (auto& ir : output_irs_) { - ir->instantiate(instance_ctx, instance); + ir.instantiate(instance_ctx, instance); } target_info target{target_id}; - std::string body; + CUDF_EXPECTS( + target.id == target::CUDA, "Unsupported target for code generation", std::invalid_argument); - for (auto& ir : output_irs_) { - body = std::format("{}{}{}", body, ir->generate_code(instance_ctx, target, instance), "\n"); + auto output_decl = [&](size_t i) { + auto& var = output_vars_[i]; + auto& ir = output_irs_[i]; + return std::format("{}* {}", to_cuda_type(ir.get_type(), instance_ctx.has_nulls()), var.id); + }; + + auto input_decl = [&](size_t i) { + auto& var = input_vars_[i]; + return std::format("{} {}", to_cuda_type(var.type, instance_ctx.has_nulls()), var.id); + }; + + std::vector arg_decls; + + for (size_t i = 0; i < output_vars_.size(); ++i) { + arg_decls.emplace_back(output_decl(i)); } - switch (target.id) { - case target::CUDA: { - { - auto output_decl = [&](size_t i) { - auto const& var = output_vars_[i]; - auto const& ir = output_irs_[i]; - auto output_type = ir->get_type(); - return std::format("{}* {}", cuda_type(output_type, instance_ctx.has_nulls()), var.id); - }; - - auto input_decl = [&](size_t i) { - auto const& var = input_vars_[i]; - return std::format("{} {}", cuda_type(var.type, instance_ctx.has_nulls()), var.id); - }; - - std::vector params_decls; - - for (size_t i = 0; i < output_vars_.size(); ++i) { - params_decls.push_back(output_decl(i)); - } - - for (size_t i = 0; i < input_vars_.size(); ++i) { - params_decls.push_back(input_decl(i)); - } - - auto params_decl = [&] { - if (params_decls.empty()) { - return std::string{}; - } else if (params_decls.size() == 1) { - return params_decls[0]; - } else { - return std::accumulate( - params_decls.begin() + 1, - params_decls.end(), - params_decls[0], - [](auto const& a, auto const& b) { return std::format("{}, {}", a, b); }); - } - }(); - - code_ = std::format( - R"***( -__device__ cudf::ops::errc expression({}) -{{ -{} -return cudf::ops::errc::SUCCESS; -}} -)***", - params_decl, - body); + for (size_t i = 0; i < input_vars_.size(); ++i) { + arg_decls.emplace_back(input_decl(i)); + } - return {is_null_aware, null_policy}; - } - break; + auto args_decl = [&] { + if (arg_decls.empty()) { + return std::string{}; + } else if (arg_decls.size() == 1) { + return arg_decls[0]; + } else { + return std::accumulate( + arg_decls.begin() + 1, arg_decls.end(), arg_decls[0], [](auto const& a, auto const& b) { + return std::format("{}, {}", a, b); + }); } - default: - CUDF_FAIL("Unsupported target: " + std::to_string(static_cast(target.id)), - std::invalid_argument); + }(); + + code_sink sink; + sink.emit("__device__ cudf::ops::errc expression("); + sink.emit(args_decl); + sink.emit(")\n{"); + for (auto& ir : output_irs_) { + ir.emit_code(instance_ctx, target, instance, sink); } + sink.emit(" return cudf::ops::errc::SUCCESS;\n}"); + + return {is_null_aware, null_policy, is_fallible}; } // Due to the AST expression tree structure, we can't generate the IR without the target @@ -500,7 +575,8 @@ transform_args ast_converter::compute_column(target target_id, // TODO(lamarrr): consider deduplicating ast expression's input column references. See // TransformTest/1.DeeplyNestedArithmeticLogicalExpression for reference - auto [is_null_aware, output_nullability] = converter.generate_code(target_id, expr, args); + auto [is_null_aware, output_nullability, is_fallible] = + converter.generate_code(target_id, expr, args); std::vector> inputs; std::vector> scalar_columns; @@ -512,12 +588,12 @@ transform_args ast_converter::compute_column(target target_id, if (std::holds_alternative(input)) { auto& scalar_input = std::get(input); - scalar_columns.push_back(std::move(scalar_input.broadcast_column)); + scalar_columns.emplace_back(std::move(scalar_input.broadcast_column)); } } auto& out = converter.output_irs_[0]; - auto output_column_type = out->get_type(); + auto output_column_type = out.get_type(); auto result = transform_args{.scalar_columns = std::move(scalar_columns), .inputs = inputs, @@ -544,7 +620,7 @@ filter_args ast_converter::filter(target target_id, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - auto filter = ast::detail::filter_predicate{expr}; + /* auto filter = ast::detail::filter_predicate{expr}; auto transform = compute_column(target_id, filter, args, stream, mr); CUDF_EXPECTS(transform.output_type.id() == type_id::BOOL8, @@ -568,6 +644,8 @@ filter_args ast_converter::filter(target target_id, .input_specs = std::move(transform.input_specs)}; return result; + */ + CUDF_FAIL("Filtering is not yet implemented", std::runtime_error); } } // namespace cudf::detail::row_ir diff --git a/cpp/src/jit/row_ir.hpp b/cpp/src/jit/row_ir.hpp index f231181bb9f4..ff2dab8a4fbb 100644 --- a/cpp/src/jit/row_ir.hpp +++ b/cpp/src/jit/row_ir.hpp @@ -117,7 +117,13 @@ struct [[nodiscard]] instance_context { }; struct [[nodiscard]] code_sink { - void emit(std::string_view code); + private: + std::string code_; + + public: + void emit(std::string_view code) { code_ += code; } + + [[nodiscard]] std::string_view get_code() const { return code_; } }; struct [[nodiscard]] input_reference { @@ -346,7 +352,7 @@ struct [[nodiscard]] ast_converter { friend class ast::literal; friend class ast::column_reference; friend class ast::operation; - friend class ast::column_name_reference; + friend class ast::column_name_reference; row_ir::node add_ir_node(ast::literal const& expr); @@ -354,7 +360,6 @@ struct [[nodiscard]] ast_converter { row_ir::node add_ir_node(ast::operation const& expr); - [[nodiscard]] std::span get_input_specs() const; /** @@ -368,7 +373,7 @@ struct [[nodiscard]] ast_converter { void add_output_var(); - [[nodiscard]] std::tuple generate_code( + [[nodiscard]] std::tuple generate_code( target target, ast::expression const& expr, ast_args const& args); public: diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index 4a7cb9c9af12..e417e1bcdbb5 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -6,6 +6,15 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include From b2c3ac8e301bd0886a14ba4f56d43257da1dbe28 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Wed, 29 Apr 2026 17:06:18 +0000 Subject: [PATCH 06/15] add decimal rescale implementation --- cpp/include/cudf/ast/expressions.hpp | 62 ++- cpp/include/cudf/ast/jit_expressions.hpp | 23 + .../cudf/operators/ansi_arithmetic.cuh | 36 +- cpp/include/cudf/operators/arithmetic.cuh | 21 + cpp/include/cudf/operators/bitwise.cuh | 40 ++ cpp/include/cudf/operators/casts.cuh | 122 +++++ cpp/include/cudf/operators/error.hpp | 28 ++ cpp/include/cudf/operators/logic.cuh | 8 +- cpp/include/cudf/operators/null_handling.cuh | 12 +- .../{op_attributes.hpp => op_traits.hpp} | 397 +++++----------- cpp/include/cudf/operators/opcodes.hpp | 18 +- cpp/include/cudf/operators/trigonometric.cuh | 25 + cpp/include/cudf/operators/types.cuh | 3 +- cpp/include/cudf/transform.hpp | 3 + cpp/src/ast/expressions.cpp | 39 +- cpp/src/jit/error_sink.cuh | 38 ++ cpp/src/jit/row_ir.cpp | 448 ++++++++++-------- cpp/src/jit/row_ir.hpp | 241 ++++++---- cpp/src/transform/jit/kernel.cu | 44 +- cpp/src/transform/transform.cu | 70 ++- cpp/tests/jit/row_ir.cpp | 141 +++--- 21 files changed, 1103 insertions(+), 716 deletions(-) create mode 100644 cpp/include/cudf/ast/jit_expressions.hpp create mode 100644 cpp/include/cudf/operators/error.hpp rename cpp/include/cudf/operators/{op_attributes.hpp => op_traits.hpp} (55%) create mode 100644 cpp/src/jit/error_sink.cuh diff --git a/cpp/include/cudf/ast/expressions.hpp b/cpp/include/cudf/ast/expressions.hpp index 20eee1741900..71590f593a01 100644 --- a/cpp/include/cudf/ast/expressions.hpp +++ b/cpp/include/cudf/ast/expressions.hpp @@ -83,7 +83,8 @@ struct expression { * @param visitor The `row_ir::ast_converter` converting this expression tree * @return The IR node representing this expression */ - virtual cudf::detail::row_ir::node accept(cudf::detail::row_ir::ast_converter& visitor) const = 0; + virtual std::unique_ptr accept( + cudf::detail::row_ir::ast_converter& visitor) const = 0; /** * @brief Returns true if the expression may evaluate to null. @@ -319,7 +320,8 @@ class literal : public expression { /** * @copydoc expression::accept */ - cudf::detail::row_ir::node accept(cudf::detail::row_ir::ast_converter& visitor) const override; + [[nodiscard]] std::unique_ptr accept( + cudf::detail::row_ir::ast_converter& visitor) const override; [[nodiscard]] bool may_evaluate_null(table_view const& left, table_view const& right, @@ -430,7 +432,8 @@ class column_reference : public expression { /** * @copydoc expression::accept */ - cudf::detail::row_ir::node accept(cudf::detail::row_ir::ast_converter& visitor) const override; + [[nodiscard]] std::unique_ptr accept( + cudf::detail::row_ir::ast_converter& visitor) const override; private: cudf::size_type column_index; @@ -501,13 +504,61 @@ class operation : public expression { /** * @copydoc expression::accept */ - cudf::detail::row_ir::node accept(cudf::detail::row_ir::ast_converter& visitor) const override; + [[nodiscard]] std::unique_ptr accept( + cudf::detail::row_ir::ast_converter& visitor) const override; private: ast_operator op; std::vector> operands; }; +namespace detail { + +/// @brief An expression that represents a predicate. +/// +/// This is an internal expression used in filter operations. It is not intended to be used by +/// external code and is not a part of the public API. +class predicate : public expression { + public: + /** + * @brief Construct a new filter predicate object + * @param source The source expression from which the predicate value is taken + */ + predicate(expression const& source) : source_{source} {} + + /** + * @copydoc expression::accept + */ + cudf::size_type accept(detail::expression_parser& visitor) const override; + + /** + * @copydoc expression::accept + */ + std::reference_wrapper accept( + detail::expression_transformer& visitor) const override; + + [[nodiscard]] bool may_evaluate_null(table_view const& left, + table_view const& right, + rmm::cuda_stream_view stream) const override; + + /** + * @copydoc expression::accept + */ + [[nodiscard]] std::unique_ptr accept( + cudf::detail::row_ir::ast_converter& visitor) const override; + + /** + * @brief Get the operand expression. + * @return The operand expression + */ + [[nodiscard]] expression const& get_operand() const { return source_; } + + private: + std::reference_wrapper source_; +}; + +} // namespace detail + /** * @brief A expression referring to data from a column in a table. */ @@ -549,7 +600,8 @@ class column_name_reference : public expression { /** * @copydoc expression::accept */ - cudf::detail::row_ir::node accept(cudf::detail::row_ir::ast_converter& visitor) const override; + [[nodiscard]] std::unique_ptr accept( + cudf::detail::row_ir::ast_converter& visitor) const override; private: std::string column_name; diff --git a/cpp/include/cudf/ast/jit_expressions.hpp b/cpp/include/cudf/ast/jit_expressions.hpp new file mode 100644 index 000000000000..f051618fe8f9 --- /dev/null +++ b/cpp/include/cudf/ast/jit_expressions.hpp @@ -0,0 +1,23 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include + +namespace CUDF_EXPORT cudf { + +namespace ast { +/** + * @addtogroup expressions + * @{ + * @file + */ + +namespace jit { + +} + +} // namespace ast +} // namespace CUDF_EXPORT cudf \ No newline at end of file diff --git a/cpp/include/cudf/operators/ansi_arithmetic.cuh b/cpp/include/cudf/operators/ansi_arithmetic.cuh index e0f226a0e443..9bfac77021b1 100644 --- a/cpp/include/cudf/operators/ansi_arithmetic.cuh +++ b/cpp/include/cudf/operators/ansi_arithmetic.cuh @@ -449,35 +449,27 @@ __device__ inline errc ansi_neg(optional* out, optional const* a) } template -__device__ inline errc ansi_precision_cast(decimal* out, - decimal const* a, - int32_t const* precision) +__device__ inline errc ansi_precision_check(decimal* out, + decimal const* a, + int32_t const* precision) { - auto current_scale = static_cast(a->scale()); - auto allowed_scale = -(*precision); + if (*precision <= 0) { return errc::OVERFLOW; } - if (current_scale >= allowed_scale) { - *out = *a; - return errc::OK; - } - - auto extra_digits = allowed_scale - current_scale; - - auto factor = detail::ipow10(static_cast(extra_digits)); + if (a->value() == cuda::std::numeric_limits::min()) { return errc::OVERFLOW; } - if (a->value() % factor != 0) { return errc::OVERFLOW; } + if (::abs(a->value()) >= detail::ipow10(static_cast(*precision))) { return errc::OVERFLOW; } *out = *a; return errc::OK; } template -__device__ inline errc ansi_precision_cast(optional* out, - optional const* a, - optional const* precision) +__device__ inline errc ansi_precision_check(optional* out, + optional const* a, + optional const* precision) { if (a->has_value()) { - return ansi_precision_cast(&out->value(), &a->value(), &precision->value()); + return ansi_precision_check(&out->value(), &a->value(), &precision->value()); } else { *out = nullopt; return errc::OK; @@ -597,12 +589,12 @@ __device__ inline errc ansi_try_neg(optional* out, optional const* a) } template -__device__ inline errc ansi_try_precision_cast(optional>* out, - optional> const* a, - optional const* precision) +__device__ inline errc ansi_try_precision_check(optional>* out, + optional> const* a, + optional const* precision) { if (a->has_value() && precision->has_value()) { - if (errc e = ansi_precision_cast(&out->value(), &a->value(), &precision->value()); + if (errc e = ansi_precision_check(&out->value(), &a->value(), &precision->value()); e != errc::OK) { *out = nullopt; } else { diff --git a/cpp/include/cudf/operators/arithmetic.cuh b/cpp/include/cudf/operators/arithmetic.cuh index 095b526cfae4..4163328385b4 100644 --- a/cpp/include/cudf/operators/arithmetic.cuh +++ b/cpp/include/cudf/operators/arithmetic.cuh @@ -253,5 +253,26 @@ __device__ inline errc sub(optional* out, optional const* a, optional c return errc::OK; } +template + requires(cuda::std::is_floating_point_v || cuda::std::is_integral_v) +__device__ inline errc true_div(double* out, T const* a, T const* b) +{ + *out = static_cast(*a) / static_cast(*b); + return errc::OK; +} + +template +__device__ inline errc true_div(optional* out, optional const* a, optional const* b) +{ + if (a->has_value() && b->has_value()) { + double r; + true_div(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + } // namespace ops } // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/operators/bitwise.cuh b/cpp/include/cudf/operators/bitwise.cuh index b87d336538e0..c9b76518bbfe 100644 --- a/cpp/include/cudf/operators/bitwise.cuh +++ b/cpp/include/cudf/operators/bitwise.cuh @@ -89,5 +89,45 @@ __device__ inline errc bit_xor(optional* out, optional const* a, optional< return errc::OK; } +template +__device__ inline errc shift_left(T* out, T const* a, T const* b) +{ + *out = (*a << *b); + return errc::OK; +} + +template +__device__ inline errc shift_left(optional* out, optional const* a, optional const* b) +{ + if (a->has_value() && b->has_value()) { + T r; + shift_left(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc shift_right(T* out, T const* a, T const* b) +{ + *out = (*a >> *b); + return errc::OK; +} + +template +__device__ inline errc shift_right(optional* out, optional const* a, optional const* b) +{ + if (a->has_value() && b->has_value()) { + T r; + shift_right(&r, &a->value(), &b->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + } // namespace ops } // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/operators/casts.cuh b/cpp/include/cudf/operators/casts.cuh index ad71c19f9f4e..a4363ac44e6a 100644 --- a/cpp/include/cudf/operators/casts.cuh +++ b/cpp/include/cudf/operators/casts.cuh @@ -9,6 +9,66 @@ namespace CUDF_EXPORT cudf { namespace ops { +template +__device__ inline errc cast_to_b8(bool* out, T const* a) +{ + *out = static_cast(*a); + return errc::OK; +} + +template +__device__ inline errc cast_to_b8(optional* out, optional const* a) +{ + if (a->has_value()) { + bool r; + cast_to_b8(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc cast_to_i8(int8_t* out, T const* a) +{ + *out = static_cast(*a); + return errc::OK; +} + +template +__device__ inline errc cast_to_i8(optional* out, optional const* a) +{ + if (a->has_value()) { + int8_t r; + cast_to_i8(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc cast_to_i16(int16_t* out, T const* a) +{ + *out = static_cast(*a); + return errc::OK; +} + +template +__device__ inline errc cast_to_i16(optional* out, optional const* a) +{ + if (a->has_value()) { + int16_t r; + cast_to_i16(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + template __device__ inline errc cast_to_i32(int32_t* out, T const* a) { @@ -49,6 +109,46 @@ __device__ inline errc cast_to_i64(optional* out, optional const* a) return errc::OK; } +template +__device__ inline errc cast_to_u8(uint8_t* out, T const* a) +{ + *out = static_cast(*a); + return errc::OK; +} + +template +__device__ inline errc cast_to_u8(optional* out, optional const* a) +{ + if (a->has_value()) { + uint8_t r; + cast_to_u8(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + +template +__device__ inline errc cast_to_u16(uint16_t* out, T const* a) +{ + *out = static_cast(*a); + return errc::OK; +} + +template +__device__ inline errc cast_to_u16(optional* out, optional const* a) +{ + if (a->has_value()) { + uint16_t r; + cast_to_u16(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + template __device__ inline errc cast_to_u32(uint32_t* out, T const* a) { @@ -201,5 +301,27 @@ __device__ inline errc cast_to_dec128(optional* out, return errc::OK; } +template +__device__ inline errc rescale(decimal* out, decimal const* a, int32_t const* new_scale) +{ + *out = a->rescaled(new_scale); + return errc::OK; +} + +template +__device__ inline errc rescale(optional>* out, + optional> const* a, + optional const* new_scale) +{ + if (a->has_value() && new_scale->has_value()) { + decimal r; + rescale(&r, &a->value(), new_scale->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + } // namespace ops } // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/operators/error.hpp b/cpp/include/cudf/operators/error.hpp new file mode 100644 index 000000000000..52e6465a1e47 --- /dev/null +++ b/cpp/include/cudf/operators/error.hpp @@ -0,0 +1,28 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include + +namespace CUDF_EXPORT cudf { +namespace ops { + +enum errc : int { OK = 0, OVERFLOW = 1, DIVISION_BY_ZERO = 2 }; + +inline char const* to_string(errc error_code) +{ + switch (error_code) { + case errc::OK: return "cudf::ops::errc::OK"; + case errc::OVERFLOW: return "cudf::ops::errc::OVERFLOW"; + case errc::DIVISION_BY_ZERO: return "cudf::ops::errc::DIVISION_BY_ZERO"; + default: return "UNKNOWN_ERROR"; + } +} + +enum class error_mode : char { IGNORE = 0, ANY_ROW = 1 }; + +} // namespace ops +} // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/operators/logic.cuh b/cpp/include/cudf/operators/logic.cuh index d99879939e0d..c674e4285434 100644 --- a/cpp/include/cudf/operators/logic.cuh +++ b/cpp/include/cudf/operators/logic.cuh @@ -9,7 +9,6 @@ namespace CUDF_EXPORT cudf { namespace ops { - template __device__ inline errc null_logical_and(T* out, T const* a, T const* b) { @@ -64,7 +63,6 @@ __device__ inline errc null_logical_or(optional* out, optional const* a, o return errc::OK; } - template __device__ inline errc logical_and(T* out, T const* a, T const* b) { @@ -126,7 +124,7 @@ __device__ inline errc logical_not(optional* out, optional const* a) } template -__device__ inline errc if_else(T* out, bool const* pred, T const* true_value, T const* false_value) +__device__ inline errc if_else(T* out, T const* true_value, T const* false_value, bool const* pred) { *out = *pred ? *true_value : *false_value; return errc::OK; @@ -134,9 +132,9 @@ __device__ inline errc if_else(T* out, bool const* pred, T const* true_value, T template __device__ inline errc if_else(optional* out, - optional const* pred, optional const* true_value, - optional const* false_value) + optional const* false_value, + optional const* pred) { if (pred->has_value() && true_value->has_value() && false_value->has_value()) { if_else(&out->value(), &pred->value(), &true_value->value(), &false_value->value()); diff --git a/cpp/include/cudf/operators/null_handling.cuh b/cpp/include/cudf/operators/null_handling.cuh index 573e7ee88455..c107e9283d13 100644 --- a/cpp/include/cudf/operators/null_handling.cuh +++ b/cpp/include/cudf/operators/null_handling.cuh @@ -25,8 +25,8 @@ __device__ inline errc is_null(optional* out, optional const* a) template __device__ inline errc nullify_if(optional* out, - optional const* condition, - optional const* a) + optional const* a, + optional const* condition) { if (condition->has_value() && a->has_value()) { if (condition->value()) { @@ -60,20 +60,18 @@ __device__ inline errc coalesce(optional* out, optional const* a, optional return errc::OK; } -template -__device__ inline errc replace_nulls(T* out, T const* a, [[maybe_unused]] T const* replacement) +__device__ inline errc predicate(bool* out, bool const* a) { *out = *a; return errc::OK; } -template -__device__ inline errc replace_nulls(optional* out, optional const* a, T const* replacement) +__device__ inline errc predicate(optional* out, optional const* a) { if (a->has_value()) { *out = a->value(); } else { - *out = *replacement; + *out = false; } return errc::OK; } diff --git a/cpp/include/cudf/operators/op_attributes.hpp b/cpp/include/cudf/operators/op_traits.hpp similarity index 55% rename from cpp/include/cudf/operators/op_attributes.hpp rename to cpp/include/cudf/operators/op_traits.hpp index 1d0ed611e5f3..1948ef038e97 100644 --- a/cpp/include/cudf/operators/op_attributes.hpp +++ b/cpp/include/cudf/operators/op_traits.hpp @@ -6,11 +6,11 @@ #pragma once #include -#include +#include namespace cudf::detail::row_ir { -enum [[nodiscard]] typing : uint64_t { +enum [[nodiscard]] type : uint64_t { NONE = 0x0, BOOL8 = 0x1, INT8 = 0x2, @@ -53,9 +53,9 @@ enum [[nodiscard]] typing : uint64_t { INPUT = 0x20000000, }; -struct [[nodiscard]] op_typing { - typing output = typing::NONE; - std::array args = {typing::NONE, typing::NONE, typing::NONE}; +struct [[nodiscard]] op_type { + type output = type::NONE; + std::vector args = {}; }; /** @@ -76,10 +76,11 @@ enum class [[nodiscard]] null_output : uint8_t { case opcode::IS_NULL: return "is_null"; case opcode::NULLIFY_IF: return "nullify_if"; case opcode::COALESCE: return "coalesce"; - case opcode::REPLACE_NULLS: return "replace_nulls"; + case opcode::PREDICATE: return "predicate"; case opcode::ABS: return "abs"; case opcode::ADD: return "add"; case opcode::DIV: return "div"; + case opcode::TRUE_DIV: return "true_div"; case opcode::FLOOR_DIV: return "floor_div"; case opcode::MOD: return "mod"; case opcode::PYMOD: return "pymod"; @@ -93,7 +94,7 @@ enum class [[nodiscard]] null_output : uint8_t { case opcode::ANSI_MOD: return "ansi_mod"; case opcode::ANSI_ABS: return "ansi_abs"; case opcode::ANSI_NEG: return "ansi_neg"; - case opcode::ANSI_PRECISION_CAST: return "ansi_precision_cast"; + case opcode::ANSI_PRECISION_CHECK: return "ansi_precision_check"; case opcode::ANSI_TRY_ADD: return "ansi_try_add"; case opcode::ANSI_TRY_SUB: return "ansi_try_sub"; case opcode::ANSI_TRY_MUL: return "ansi_try_mul"; @@ -101,13 +102,20 @@ enum class [[nodiscard]] null_output : uint8_t { case opcode::ANSI_TRY_MOD: return "ansi_try_mod"; case opcode::ANSI_TRY_ABS: return "ansi_try_abs"; case opcode::ANSI_TRY_NEG: return "ansi_try_neg"; - case opcode::ANSI_TRY_PRECISION_CAST: return "ansi_try_precision_cast"; + case opcode::ANSI_TRY_PRECISION_CHECK: return "ansi_try_precision_check"; case opcode::BIT_AND: return "bit_and"; case opcode::BIT_INVERT: return "bit_invert"; case opcode::BIT_OR: return "bit_or"; case opcode::BIT_XOR: return "bit_xor"; + case opcode::SHIFT_LEFT: return "shift_left"; + case opcode::SHIFT_RIGHT: return "shift_right"; + case opcode::CAST_TO_B8: return "cast_to_b8"; + case opcode::CAST_TO_I8: return "cast_to_i8"; + case opcode::CAST_TO_I16: return "cast_to_i16"; case opcode::CAST_TO_I32: return "cast_to_i32"; case opcode::CAST_TO_I64: return "cast_to_i64"; + case opcode::CAST_TO_U8: return "cast_to_u8"; + case opcode::CAST_TO_U16: return "cast_to_u16"; case opcode::CAST_TO_U32: return "cast_to_u32"; case opcode::CAST_TO_U64: return "cast_to_u64"; case opcode::CAST_TO_F32: return "cast_to_f32"; @@ -115,6 +123,7 @@ enum class [[nodiscard]] null_output : uint8_t { case opcode::CAST_TO_DEC32: return "cast_to_dec32"; case opcode::CAST_TO_DEC64: return "cast_to_dec64"; case opcode::CAST_TO_DEC128: return "cast_to_dec128"; + case opcode::RESCALE: return "rescale"; case opcode::EQUAL: return "equal"; case opcode::NOT_EQUAL: return "not_equal"; case opcode::GREATER: return "greater"; @@ -157,71 +166,7 @@ enum class [[nodiscard]] null_output : uint8_t { switch (op) { case opcode::IS_NULL: case opcode::NULL_EQUAL: - case opcode::REPLACE_NULLS: return null_output::ALWAYS_VALID; - - case opcode::GET_INPUT: - case opcode::SET_OUTPUT: - case opcode::IDENTITY: - case opcode::LOGICAL_NOT: - case opcode::LOGICAL_AND: - case opcode::LOGICAL_OR: - case opcode::ABS: - case opcode::ADD: - case opcode::DIV: - case opcode::FLOOR_DIV: - case opcode::MOD: - case opcode::PYMOD: - case opcode::MUL: - case opcode::NEG: - case opcode::SUB: - case opcode::ANSI_ADD: - case opcode::ANSI_SUB: - case opcode::ANSI_MUL: - case opcode::ANSI_DIV: - case opcode::ANSI_MOD: - case opcode::ANSI_ABS: - case opcode::ANSI_NEG: - case opcode::ANSI_PRECISION_CAST: - case opcode::IF_ELSE: - case opcode::CBRT: - case opcode::CEIL: - case opcode::FLOOR: - case opcode::RINT: - case opcode::SQRT: - case opcode::POW: - case opcode::EXP: - case opcode::LOG: - case opcode::ARCCOS: - case opcode::ARCCOSH: - case opcode::ARCSIN: - case opcode::ARCSINH: - case opcode::ARCTAN: - case opcode::ARCTANH: - case opcode::COS: - case opcode::COSH: - case opcode::SIN: - case opcode::SINH: - case opcode::TAN: - case opcode::BIT_AND: - case opcode::BIT_INVERT: - case opcode::BIT_OR: - case opcode::BIT_XOR: - case opcode::CAST_TO_I32: - case opcode::CAST_TO_I64: - case opcode::CAST_TO_U32: - case opcode::CAST_TO_U64: - case opcode::CAST_TO_F32: - case opcode::CAST_TO_F64: - case opcode::CAST_TO_DEC32: - case opcode::CAST_TO_DEC64: - case opcode::CAST_TO_DEC128: - case opcode::EQUAL: - case opcode::NOT_EQUAL: - case opcode::GREATER: - case opcode::GREATER_EQUAL: - case opcode::LESS: - case opcode::LESS_EQUAL: - case opcode::TANH: return null_output::PROPAGATE; + case opcode::PREDICATE: return null_output::ALWAYS_VALID; case opcode::NULLIFY_IF: case opcode::COALESCE: @@ -232,11 +177,11 @@ enum class [[nodiscard]] null_output : uint8_t { case opcode::ANSI_TRY_MOD: case opcode::ANSI_TRY_ABS: case opcode::ANSI_TRY_NEG: - case opcode::ANSI_TRY_PRECISION_CAST: + case opcode::ANSI_TRY_PRECISION_CHECK: case opcode::NULL_LOGICAL_AND: case opcode::NULL_LOGICAL_OR: return null_output::ALWAYS_NULLABLE; - default: CUDF_UNREACHABLE("Invalid opcode"); + default: return null_output::PROPAGATE; } } @@ -247,87 +192,14 @@ enum class [[nodiscard]] null_output : uint8_t { [[nodiscard]] inline bool get_op_requires_nulls(opcode op) { switch (op) { - case opcode::GET_INPUT: - case opcode::SET_OUTPUT: - case opcode::NULLIFY_IF: - case opcode::IDENTITY: - case opcode::ABS: - case opcode::ADD: - case opcode::DIV: - case opcode::FLOOR_DIV: - case opcode::MOD: - case opcode::PYMOD: - case opcode::MUL: - case opcode::NEG: - case opcode::SUB: - case opcode::ANSI_ADD: - case opcode::ANSI_SUB: - case opcode::ANSI_MUL: - case opcode::ANSI_DIV: - case opcode::ANSI_MOD: - case opcode::ANSI_ABS: - case opcode::ANSI_NEG: - case opcode::ANSI_PRECISION_CAST: - case opcode::ANSI_TRY_ADD: - case opcode::ANSI_TRY_SUB: - case opcode::ANSI_TRY_MUL: - case opcode::ANSI_TRY_DIV: - case opcode::ANSI_TRY_MOD: - case opcode::ANSI_TRY_ABS: - case opcode::ANSI_TRY_NEG: - case opcode::ANSI_TRY_PRECISION_CAST: - case opcode::BIT_AND: - case opcode::BIT_INVERT: - case opcode::BIT_OR: - case opcode::BIT_XOR: - case opcode::CAST_TO_I32: - case opcode::CAST_TO_I64: - case opcode::CAST_TO_U32: - case opcode::CAST_TO_U64: - case opcode::CAST_TO_F32: - case opcode::CAST_TO_F64: - case opcode::CAST_TO_DEC32: - case opcode::CAST_TO_DEC64: - case opcode::CAST_TO_DEC128: - case opcode::EQUAL: - case opcode::NOT_EQUAL: - case opcode::GREATER: - case opcode::GREATER_EQUAL: - case opcode::LESS: - case opcode::LESS_EQUAL: - case opcode::LOGICAL_NOT: - case opcode::LOGICAL_AND: - case opcode::LOGICAL_OR: - case opcode::IF_ELSE: - case opcode::CBRT: - case opcode::CEIL: - case opcode::FLOOR: - case opcode::RINT: - case opcode::SQRT: - case opcode::POW: - case opcode::EXP: - case opcode::LOG: - case opcode::ARCCOS: - case opcode::ARCCOSH: - case opcode::ARCSIN: - case opcode::ARCSINH: - case opcode::ARCTAN: - case opcode::ARCTANH: - case opcode::COS: - case opcode::COSH: - case opcode::SIN: - case opcode::SINH: - case opcode::TAN: - case opcode::TANH: return false; - case opcode::COALESCE: case opcode::IS_NULL: case opcode::NULL_EQUAL: case opcode::NULL_LOGICAL_AND: case opcode::NULL_LOGICAL_OR: - case opcode::REPLACE_NULLS: return true; + case opcode::PREDICATE: return true; - default: CUDF_UNREACHABLE("Invalid opcode"); + default: return false; } } @@ -341,38 +213,37 @@ enum class [[nodiscard]] null_output : uint8_t { case opcode::ANSI_MOD: case opcode::ANSI_ABS: case opcode::ANSI_NEG: - case opcode::ANSI_PRECISION_CAST: return true; + case opcode::ANSI_PRECISION_CHECK: return true; + + default: return false; + } +} +[[nodiscard]] inline int32_t get_output_decimal_scale(opcode op, + std::span arg_scales, + std::optional output_scale) +{ + // TODO: finish up + switch (op) { case opcode::GET_INPUT: case opcode::SET_OUTPUT: - case opcode::IS_NULL: case opcode::IDENTITY: - case opcode::NULLIFY_IF: case opcode::COALESCE: - case opcode::REPLACE_NULLS: + case opcode::PREDICATE: + case opcode::IS_NULL: case opcode::ABS: - case opcode::ADD: - case opcode::DIV: - case opcode::FLOOR_DIV: - case opcode::MOD: - case opcode::PYMOD: - case opcode::MUL: case opcode::NEG: - case opcode::SUB: - case opcode::ANSI_TRY_ADD: - case opcode::ANSI_TRY_SUB: - case opcode::ANSI_TRY_MUL: - case opcode::ANSI_TRY_DIV: - case opcode::ANSI_TRY_MOD: - case opcode::ANSI_TRY_ABS: + case opcode::ANSI_ABS: + case opcode::ANSI_NEG: case opcode::ANSI_TRY_NEG: - case opcode::ANSI_TRY_PRECISION_CAST: - case opcode::BIT_AND: - case opcode::BIT_INVERT: - case opcode::BIT_OR: - case opcode::BIT_XOR: + case opcode::ANSI_TRY_ABS: + case opcode::CAST_TO_B8: + case opcode::CAST_TO_I8: + case opcode::CAST_TO_I16: case opcode::CAST_TO_I32: case opcode::CAST_TO_I64: + case opcode::CAST_TO_U8: + case opcode::CAST_TO_U16: case opcode::CAST_TO_U32: case opcode::CAST_TO_U64: case opcode::CAST_TO_F32: @@ -380,12 +251,13 @@ enum class [[nodiscard]] null_output : uint8_t { case opcode::CAST_TO_DEC32: case opcode::CAST_TO_DEC64: case opcode::CAST_TO_DEC128: + case opcode::NULLIFY_IF: case opcode::EQUAL: - case opcode::NOT_EQUAL: case opcode::GREATER: case opcode::GREATER_EQUAL: case opcode::LESS: case opcode::LESS_EQUAL: + case opcode::NOT_EQUAL: case opcode::NULL_EQUAL: case opcode::NULL_LOGICAL_AND: case opcode::NULL_LOGICAL_OR: @@ -393,14 +265,6 @@ enum class [[nodiscard]] null_output : uint8_t { case opcode::LOGICAL_OR: case opcode::LOGICAL_NOT: case opcode::IF_ELSE: - case opcode::CBRT: - case opcode::CEIL: - case opcode::FLOOR: - case opcode::RINT: - case opcode::SQRT: - case opcode::POW: - case opcode::EXP: - case opcode::LOG: case opcode::ARCCOS: case opcode::ARCCOSH: case opcode::ARCSIN: @@ -412,97 +276,42 @@ enum class [[nodiscard]] null_output : uint8_t { case opcode::SIN: case opcode::SINH: case opcode::TAN: - case opcode::TANH: return false; - - default: CUDF_UNREACHABLE("Invalid opcode"); - } -} - -[[nodiscard]] inline constexpr int32_t get_op_arity(opcode op) -{ - switch (op) { - case opcode::GET_INPUT: return 0; - - case opcode::SET_OUTPUT: - case opcode::IS_NULL: - case opcode::NULLIFY_IF: - case opcode::IDENTITY: - case opcode::ABS: - case opcode::NEG: - case opcode::ANSI_ABS: - case opcode::ANSI_NEG: - case opcode::ANSI_TRY_ABS: - case opcode::ANSI_TRY_NEG: + case opcode::TANH: + case opcode::ANSI_PRECISION_CHECK: + case opcode::ANSI_TRY_PRECISION_CHECK: + case opcode::BIT_AND: case opcode::BIT_INVERT: - case opcode::CAST_TO_I32: - case opcode::CAST_TO_I64: - case opcode::CAST_TO_U32: - case opcode::CAST_TO_U64: - case opcode::CAST_TO_F32: - case opcode::CAST_TO_F64: - case opcode::CAST_TO_DEC32: - case opcode::CAST_TO_DEC64: - case opcode::CAST_TO_DEC128: - case opcode::LOGICAL_NOT: + case opcode::BIT_OR: + case opcode::BIT_XOR: + case opcode::SHIFT_LEFT: + case opcode::SHIFT_RIGHT: case opcode::CBRT: case opcode::CEIL: case opcode::FLOOR: case opcode::RINT: case opcode::SQRT: + case opcode::POW: case opcode::EXP: - case opcode::LOG: - case opcode::ARCCOS: - case opcode::ARCCOSH: - case opcode::ARCSIN: - case opcode::ARCSINH: - case opcode::ARCTAN: - case opcode::ARCTANH: - case opcode::COS: - case opcode::COSH: - case opcode::SIN: - case opcode::SINH: - case opcode::TAN: - case opcode::TANH: return 1; - - case opcode::COALESCE: - case opcode::REPLACE_NULLS: - case opcode::ADD: - case opcode::DIV: + case opcode::TRUE_DIV: + case opcode::LOG: return arg_scales[0]; case opcode::FLOOR_DIV: - case opcode::MOD: - case opcode::PYMOD: - case opcode::MUL: + case opcode::ANSI_DIV: + case opcode::DIV: + case opcode::ANSI_TRY_DIV: return arg_scales[0] - arg_scales[1]; + case opcode::ADD: case opcode::SUB: case opcode::ANSI_ADD: case opcode::ANSI_SUB: - case opcode::ANSI_MUL: - case opcode::ANSI_DIV: - case opcode::ANSI_MOD: case opcode::ANSI_TRY_ADD: case opcode::ANSI_TRY_SUB: - case opcode::ANSI_TRY_MUL: - case opcode::ANSI_TRY_DIV: + case opcode::MOD: + case opcode::ANSI_MOD: case opcode::ANSI_TRY_MOD: - case opcode::ANSI_PRECISION_CAST: - case opcode::ANSI_TRY_PRECISION_CAST: - case opcode::BIT_AND: - case opcode::BIT_OR: - case opcode::BIT_XOR: - case opcode::EQUAL: - case opcode::NOT_EQUAL: - case opcode::GREATER: - case opcode::GREATER_EQUAL: - case opcode::LESS: - case opcode::LESS_EQUAL: - case opcode::NULL_EQUAL: - case opcode::NULL_LOGICAL_OR: - case opcode::NULL_LOGICAL_AND: - case opcode::LOGICAL_AND: - case opcode::LOGICAL_OR: - case opcode::POW: return 2; - - case opcode::IF_ELSE: return 3; - + case opcode::PYMOD: return std::min(arg_scales[0], arg_scales[1]); + case opcode::MUL: + case opcode::ANSI_MUL: + case opcode::ANSI_TRY_MUL: return arg_scales[0] + arg_scales[1]; + case opcode::RESCALE: return output_scale.value(); default: CUDF_UNREACHABLE("Invalid opcode"); } } @@ -515,24 +324,25 @@ enum class [[nodiscard]] null_output : uint8_t { * @return An `op_typing` struct containing the expected output type and input types for the * operator */ -[[nodiscard]] inline op_typing get_op_typing(opcode op) +[[nodiscard]] inline op_type get_op_typing(opcode op) { + // TODO: finish up switch (op) { - case opcode::GET_INPUT: return {typing::INPUT, {}}; - case opcode::SET_OUTPUT: return {typing::NONE, {typing::ALL}}; - case opcode::IDENTITY: return {typing::ARG0, {typing::ALL}}; - case opcode::IS_NULL: return {typing::BOOL8, {typing::ALL}}; - case opcode::NULLIFY_IF: return {typing::ARG1, {typing::BOOL8, typing::ALL}}; - case opcode::COALESCE: return {typing::ARG0, {typing::ALL, typing::ARG0}}; - case opcode::REPLACE_NULLS: return {typing::ARG0, {typing::ALL, typing::ARG0}}; + case opcode::GET_INPUT: return {type::INPUT, {}}; + case opcode::SET_OUTPUT: return {type::NONE, {type::ALL}}; + case opcode::IDENTITY: return {type::ARG0, {type::ALL}}; + case opcode::IS_NULL: return {type::BOOL8, {type::ALL}}; + case opcode::NULLIFY_IF: return {type::ARG0, {type::ALL, type::BOOL8}}; + case opcode::COALESCE: return {type::ARG0, {type::ALL, type::ARG0}}; + case opcode::PREDICATE: return {type::ARG0, {type::BOOL8}}; case opcode::ABS: case opcode::NEG: case opcode::ANSI_ABS: case opcode::ANSI_NEG: case opcode::ANSI_TRY_NEG: - case opcode::ANSI_TRY_ABS: return {typing::ARG0, {typing::ARITHMETIC}}; + case opcode::ANSI_TRY_ABS: return {type::ARG0, {type::ARITHMETIC}}; case opcode::FLOOR_DIV: - return {typing::ARG0, {typing{typing::FLOATS | typing::INTEGERS}, typing::ARG0}}; + case opcode::TRUE_DIV: return {type::ARG0, {type{type::FLOATS | type::INTEGERS}, type::ARG0}}; case opcode::ADD: case opcode::DIV: case opcode::MOD: @@ -548,35 +358,43 @@ enum class [[nodiscard]] null_output : uint8_t { case opcode::ANSI_TRY_SUB: case opcode::ANSI_TRY_MUL: case opcode::ANSI_TRY_DIV: - case opcode::ANSI_TRY_MOD: return {typing::ARG0, {typing::ARITHMETIC, typing::ARG0}}; - case opcode::ANSI_PRECISION_CAST: - case opcode::ANSI_TRY_PRECISION_CAST: return {typing::ARG0, {typing::DECIMALS, typing::INT32}}; + case opcode::ANSI_TRY_MOD: return {type::ARG0, {type::ARITHMETIC, type::ARG0}}; + case opcode::ANSI_PRECISION_CHECK: + case opcode::ANSI_TRY_PRECISION_CHECK: return {type::ARG0, {type::DECIMALS, type::INT32}}; case opcode::BIT_AND: case opcode::BIT_INVERT: case opcode::BIT_OR: - case opcode::BIT_XOR: return {typing::ARG0, {typing::INTEGERS, typing::ARG0}}; - case opcode::CAST_TO_I32: - case opcode::CAST_TO_I64: - case opcode::CAST_TO_U32: - case opcode::CAST_TO_U64: - case opcode::CAST_TO_F32: - case opcode::CAST_TO_F64: return {typing::ARG0, {typing{typing::INTEGERS | typing::FLOATS}}}; - case opcode::CAST_TO_DEC32: - case opcode::CAST_TO_DEC64: - case opcode::CAST_TO_DEC128: return {typing::ARG0, {typing::DECIMALS}}; + case opcode::BIT_XOR: + case opcode::SHIFT_LEFT: + case opcode::SHIFT_RIGHT: return {type::ARG0, {type::INTEGERS, type::ARG0}}; + case opcode::CAST_TO_B8: return {type::BOOL8, {type{type::INTEGERS | type::FLOATS}}}; + case opcode::CAST_TO_I8: return {type::INT8, {type{type::INTEGERS | type::FLOATS}}}; + case opcode::CAST_TO_I16: return {type::INT16, {type{type::INTEGERS | type::FLOATS}}}; + case opcode::CAST_TO_I32: return {type::INT32, {type{type::INTEGERS | type::FLOATS}}}; + case opcode::CAST_TO_I64: return {type::INT64, {type{type::INTEGERS | type::FLOATS}}}; + case opcode::CAST_TO_U8: return {type::UINT8, {type{type::INTEGERS | type::FLOATS}}}; + case opcode::CAST_TO_U16: return {type::UINT16, {type{type::INTEGERS | type::FLOATS}}}; + case opcode::CAST_TO_U32: return {type::UINT32, {type{type::INTEGERS | type::FLOATS}}}; + case opcode::CAST_TO_U64: return {type::UINT64, {type{type::INTEGERS | type::FLOATS}}}; + case opcode::CAST_TO_F32: return {type::FLOAT32, {type{type::INTEGERS | type::FLOATS}}}; + case opcode::CAST_TO_F64: return {type::FLOAT64, {type{type::INTEGERS | type::FLOATS}}}; + case opcode::CAST_TO_DEC32: return {type::DECIMAL32, {type::DECIMALS}}; + case opcode::CAST_TO_DEC64: return {type::DECIMAL64, {type::DECIMALS}}; + case opcode::CAST_TO_DEC128: return {type::DECIMAL128, {type::DECIMALS}}; + case opcode::RESCALE: return {type::ARG0, {type::DECIMALS, type::INT32}}; case opcode::EQUAL: case opcode::GREATER: case opcode::GREATER_EQUAL: case opcode::LESS: - case opcode::LESS_EQUAL: return {typing::BOOL8, {typing::ALL, typing::ARG0}}; + case opcode::LESS_EQUAL: return {type::BOOL8, {type::ALL, type::ARG0}}; case opcode::NOT_EQUAL: - case opcode::NULL_EQUAL: return {typing::BOOL8, {typing::ALL, typing::ARG0}}; + case opcode::NULL_EQUAL: return {type::BOOL8, {type::ALL, type::ARG0}}; case opcode::NULL_LOGICAL_AND: case opcode::NULL_LOGICAL_OR: case opcode::LOGICAL_AND: - case opcode::LOGICAL_OR: return {typing::BOOL8, {typing::BOOL8, typing::ARG0}}; - case opcode::LOGICAL_NOT: return {typing::ARG0, {typing::BOOL8}}; - case opcode::IF_ELSE: return {typing::ARG1, {typing::BOOL8, typing::ALL, typing::ARG0}}; + case opcode::LOGICAL_OR: return {type::BOOL8, {type::BOOL8, type::ARG0}}; + case opcode::LOGICAL_NOT: return {type::ARG0, {type::BOOL8}}; + case opcode::IF_ELSE: return {type::ARG0, {type::ALL, type::ARG0, type::BOOL8}}; case opcode::CBRT: case opcode::CEIL: case opcode::FLOOR: @@ -596,9 +414,14 @@ enum class [[nodiscard]] null_output : uint8_t { case opcode::SIN: case opcode::SINH: case opcode::TAN: - case opcode::TANH: return {typing::ARG0, {typing::FLOATS}}; + case opcode::TANH: return {type::ARG0, {type::FLOATS}}; default: CUDF_UNREACHABLE("Invalid opcode"); } } +[[nodiscard]] inline int32_t get_op_arity(opcode op) +{ + return static_cast(get_op_typing(op).args.size()); +} + } // namespace cudf::detail::row_ir diff --git a/cpp/include/cudf/operators/opcodes.hpp b/cpp/include/cudf/operators/opcodes.hpp index b46db033bd2a..36c4a6eb3cca 100644 --- a/cpp/include/cudf/operators/opcodes.hpp +++ b/cpp/include/cudf/operators/opcodes.hpp @@ -21,13 +21,15 @@ enum class [[nodiscard]] opcode : int32_t { // Null handling operators IS_NULL, NULLIFY_IF, + COALESCE, - REPLACE_NULLS, + PREDICATE, /// Arithmetic operators ABS, ADD, DIV, + TRUE_DIV, FLOOR_DIV, MOD, PYMOD, @@ -43,7 +45,7 @@ enum class [[nodiscard]] opcode : int32_t { ANSI_MOD, ANSI_ABS, ANSI_NEG, - ANSI_PRECISION_CAST, + ANSI_PRECISION_CHECK, /// ANSI TRY arithmetic functions. return NULL instead of raising errors ANSI_TRY_ADD, @@ -53,17 +55,24 @@ enum class [[nodiscard]] opcode : int32_t { ANSI_TRY_MOD, ANSI_TRY_ABS, ANSI_TRY_NEG, - ANSI_TRY_PRECISION_CAST, + ANSI_TRY_PRECISION_CHECK, /// Bitwise operators BIT_AND, BIT_INVERT, BIT_OR, BIT_XOR, + SHIFT_LEFT, + SHIFT_RIGHT, /// Type conversion operators + CAST_TO_B8, + CAST_TO_I8, + CAST_TO_I16, CAST_TO_I32, CAST_TO_I64, + CAST_TO_U8, + CAST_TO_U16, CAST_TO_U32, CAST_TO_U64, CAST_TO_F32, @@ -71,8 +80,9 @@ enum class [[nodiscard]] opcode : int32_t { CAST_TO_DEC32, CAST_TO_DEC64, CAST_TO_DEC128, + RESCALE, - /// Comparison & Logical operators + /// Comparison & Logic operators EQUAL, NOT_EQUAL, GREATER, diff --git a/cpp/include/cudf/operators/trigonometric.cuh b/cpp/include/cudf/operators/trigonometric.cuh index 50dd6f269f10..1156c3eba2e3 100644 --- a/cpp/include/cudf/operators/trigonometric.cuh +++ b/cpp/include/cudf/operators/trigonometric.cuh @@ -259,6 +259,31 @@ __device__ inline errc sinh(optional* out, optional const* a) return errc::OK; } +__device__ inline errc tan(float* out, float const* a) +{ + *out = ::tanf(*a); + return errc::OK; +} + +__device__ inline errc tan(double* out, double const* a) +{ + *out = ::tan(*a); + return errc::OK; +} + +template +__device__ inline errc tan(optional* out, optional const* a) +{ + if (a->has_value()) { + T r; + tan(&r, &a->value()); + *out = r; + } else { + *out = nullopt; + } + return errc::OK; +} + __device__ inline errc tanh(float* out, float const* a) { *out = ::tanhf(*a); diff --git a/cpp/include/cudf/operators/types.cuh b/cpp/include/cudf/operators/types.cuh index 9be973656c78..673a6b883016 100644 --- a/cpp/include/cudf/operators/types.cuh +++ b/cpp/include/cudf/operators/types.cuh @@ -5,6 +5,7 @@ #pragma once #include +#include #include #include #include @@ -17,8 +18,6 @@ namespace CUDF_EXPORT cudf { namespace ops { -enum errc : int { OK = 0, OVERFLOW = 1, DIVISION_BY_ZERO = 2 }; - template using optional = cuda::std::optional; diff --git a/cpp/include/cudf/transform.hpp b/cpp/include/cudf/transform.hpp index a2a99ed0c23b..1936c5284b2e 100644 --- a/cpp/include/cudf/transform.hpp +++ b/cpp/include/cudf/transform.hpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -163,6 +164,7 @@ std::unique_ptr transform_extended( * input columns. * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Device memory resource used to allocate the returned column's device memory + * @param error_mode The error handling mode to use during the transform * @return A table containing the columns resulting from applying the transform * function to every element of the input according to the output specifications * @@ -176,6 +178,7 @@ std::unique_ptr multi_transform( std::span outputs, std::vector>&& string_offsets, std::optional row_size, + ops::error_mode error_mode = ops::error_mode::IGNORE, rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); diff --git a/cpp/src/ast/expressions.cpp b/cpp/src/ast/expressions.cpp index b80fd5555bae..9e6b8cf3ce8c 100644 --- a/cpp/src/ast/expressions.cpp +++ b/cpp/src/ast/expressions.cpp @@ -80,34 +80,65 @@ bool operation::may_evaluate_null(table_view const& left, }); }; +cudf::size_type detail::predicate::accept(detail::expression_parser& visitor) const +{ + CUDF_FAIL("predicate is an internal expression and should not be visited by expression_parser", + std::invalid_argument); +} + +std::reference_wrapper detail::predicate::accept( + detail::expression_transformer& visitor) const +{ + CUDF_FAIL( + "predicate is an internal expression and should not be visited by " + "expression_transformer", + std::invalid_argument); +} + +bool detail::predicate::may_evaluate_null(table_view const& left, + table_view const& right, + rmm::cuda_stream_view stream) const +{ + return false; +} + auto column_name_reference::accept(detail::expression_transformer& visitor) const -> decltype(visitor.visit(*this)) { return visitor.visit(*this); } -cudf::detail::row_ir::node literal::accept(cudf::detail::row_ir::ast_converter& converter) const +std::unique_ptr literal::accept( + cudf::detail::row_ir::ast_converter& converter) const { return converter.add_ir_node(*this); } -cudf::detail::row_ir::node column_reference::accept( +std::unique_ptr column_reference::accept( cudf::detail::row_ir::ast_converter& converter) const { return converter.add_ir_node(*this); } -cudf::detail::row_ir::node operation::accept(cudf::detail::row_ir::ast_converter& converter) const +std::unique_ptr operation::accept( + cudf::detail::row_ir::ast_converter& converter) const { return converter.add_ir_node(*this); } -cudf::detail::row_ir::node column_name_reference::accept(cudf::detail::row_ir::ast_converter&) const +std::unique_ptr column_name_reference::accept( + cudf::detail::row_ir::ast_converter&) const { CUDF_FAIL( "column_name_reference is not supported in row_ir. row_ir only supports resolved expressions", std::invalid_argument); } +std::unique_ptr detail::predicate::accept( + cudf::detail::row_ir::ast_converter& converter) const +{ + return converter.add_ir_node(*this); +} + } // namespace ast } // namespace cudf diff --git a/cpp/src/jit/error_sink.cuh b/cpp/src/jit/error_sink.cuh new file mode 100644 index 000000000000..9063663d9615 --- /dev/null +++ b/cpp/src/jit/error_sink.cuh @@ -0,0 +1,38 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +#include + +namespace cudf { +namespace jit { + +struct error_sink { + ops::errc any_error_ = ops::errc::OK; + + constexpr error_sink() = default; + + template + __device__ void report(ops::errc error) + { + if constexpr (mode == ops::error_mode::IGNORE) { + return; + } else { + if (error != ops::errc::OK) [[unlikely]] { + cuda::std::atomic_ref any_error_ref{any_error_}; + any_error_ref.store(error, cuda::std::memory_order_relaxed); + } + } + } + + [[nodiscard]] __host__ __device__ constexpr ops::errc any_error() const { return any_error_; } +}; + +} // namespace jit +} // namespace cudf diff --git a/cpp/src/jit/row_ir.cpp b/cpp/src/jit/row_ir.cpp index c05f30488a57..02435c479df8 100644 --- a/cpp/src/jit/row_ir.cpp +++ b/cpp/src/jit/row_ir.cpp @@ -20,6 +20,44 @@ namespace cudf::detail::row_ir { +// Resolve the table for a column input spec, preferring left_table/right_table for join cases, +// falling back to args.table for the single-table case. +table_view const& resolve_table(ast_column_input_spec const& in, ast_args const& args) +{ + if (in.table == ast::table_reference::LEFT) { + return args.left_table.num_columns() > 0 ? args.left_table : args.table; + } + return args.right_table; +} + +void instance_context::add_input_var(ast_column_input_spec const& in, ast_args const& args) +{ + auto id = std::format("in_{}", input_vars_.size()); + auto type = resolve_table(in, args).column(in.column).type(); + input_vars_.emplace_back(std::move(id), type); +} + +void instance_context::add_input_var(ast_scalar_input_spec const& in, + [[maybe_unused]] ast_args const& args) +{ + auto id = std::format("in_{}", input_vars_.size()); + auto type = in.scalar_column->type(); + input_vars_.emplace_back(std::move(id), type); +} + +void instance_context::add_output_var() +{ + auto id = std::format("out_{}", output_vars_.size()); + output_vars_.emplace_back(std::move(id)); +} + +int32_t instance_context::add_ast_input(ast_input_spec in) +{ + auto id = static_cast(input_specs_.size()); + input_specs_.emplace_back(std::move(in)); + return id; +} + std::string instance_context::make_tmp_id() { return std::format("{}{}", tmp_prefix_, num_tmp_vars_++); @@ -29,7 +67,20 @@ bool instance_context::has_nulls() const { return has_nulls_; } void instance_context::set_has_nulls(bool has_nulls) { has_nulls_ = has_nulls; } -node::node(opcode op, std::vector args) : op_{op}, args_{std::move(args)} +std::span instance_context::get_input_specs() const { return input_specs_; } + +std::span instance_context::get_inputs() const { return input_vars_; } + +std::span instance_context::get_outputs() const { return output_vars_; } + +int32_t instance_context::add_constant(cudf::scalar const& value) +{ + auto scalar_column = make_column_from_scalar(value, 1, stream_, mr_); + return add_ast_input(ast_scalar_input_spec{.scalar_column = std::move(scalar_column)}); +} + +node::node(opcode op, std::optional target_scale, std::vector> args) + : op_{op}, target_scale_{target_scale}, args_{std::move(args)} { CUDF_EXPECTS(op != opcode::GET_INPUT && op != opcode::SET_OUTPUT, std::format("Invalid opcode `{}` for operation node.", get_op_name(op))); @@ -38,22 +89,28 @@ node::node(opcode op, std::vector args) : op_{op}, args_{std::move(args)} get_op_name(op), get_op_arity(op), args_.size())); + CUDF_EXPECTS(target_scale_.has_value() == (op == opcode::RESCALE), + std::format("Target scale must be provided for RESCALE operator and must be nullopt " + "for other operators.")); } -node::node(input_reference input) : reference_{input}, op_{opcode::SET_OUTPUT} {} +node::node(input_reference input) : reference_{input}, op_{opcode::GET_INPUT} {} -node::node(output_reference reference, node arg) - : reference_{reference}, op_{opcode::SET_OUTPUT}, args_{std::move(arg)} +node::node(output_reference reference, std::unique_ptr arg) + : reference_{reference}, op_{opcode::SET_OUTPUT} { + args_.emplace_back(std::move(arg)); } std::string_view node::get_id() const { return id_; } data_type node::get_type() const { return type_; } +std::optional node::get_target_scale() const { return target_scale_; } + opcode node::get_opcode() const { return op_; } -std::span node::get_args() const { return args_; } +std::span const> node::get_args() const { return args_; } bool node::is_null_aware() const { @@ -61,83 +118,83 @@ bool node::is_null_aware() const null_output::ALWAYS_NULLABLE || // to emit nulls for always-nullable operators, we need // to mark them as null-aware get_op_requires_nulls(op_) || - std::any_of(args_.begin(), args_.end(), [](auto& a) { return a.is_null_aware(); }); + std::any_of(args_.begin(), args_.end(), [](auto& a) { return a->is_null_aware(); }); } bool node::is_always_valid() const { return get_op_null_output(op_) == null_output::ALWAYS_VALID || - std::all_of(args_.begin(), args_.end(), [](auto& a) { return a.is_always_valid(); }); + std::all_of(args_.begin(), args_.end(), [](auto& a) { return a->is_always_valid(); }); } bool node::is_fallible() const { return get_op_is_fallible(op_) || - std::any_of(args_.begin(), args_.end(), [](auto& a) { return a.is_fallible(); }); + std::any_of(args_.begin(), args_.end(), [](auto& a) { return a->is_fallible(); }); } -row_ir::typing as_typing(data_type type) +row_ir::type as_typing(data_type type) { switch (type.id()) { - case type_id::BOOL8: return typing::BOOL8; - case type_id::INT8: return typing::INT8; - case type_id::INT16: return typing::INT16; - case type_id::INT32: return typing::INT32; - case type_id::INT64: return typing::INT64; - case type_id::UINT8: return typing::UINT8; - case type_id::UINT16: return typing::UINT16; - case type_id::UINT32: return typing::UINT32; - case type_id::UINT64: return typing::UINT64; - case type_id::FLOAT32: return typing::FLOAT32; - case type_id::FLOAT64: return typing::FLOAT64; - case type_id::DECIMAL32: return typing::DECIMAL32; - case type_id::DECIMAL64: return typing::DECIMAL64; - case type_id::DECIMAL128: return typing::DECIMAL128; - case type_id::TIMESTAMP_DAYS: return typing::TIMESTAMP_DAYS; - case type_id::TIMESTAMP_SECONDS: return typing::TIMESTAMP_SECONDS; - case type_id::TIMESTAMP_MILLISECONDS: return typing::TIMESTAMP_MILLISECONDS; - case type_id::TIMESTAMP_MICROSECONDS: return typing::TIMESTAMP_MICROSECONDS; - case type_id::TIMESTAMP_NANOSECONDS: return typing::TIMESTAMP_NANOSECONDS; - case type_id::DURATION_DAYS: return typing::DURATION_DAYS; - case type_id::DURATION_SECONDS: return typing::DURATION_SECONDS; - case type_id::DURATION_MILLISECONDS: return typing::DURATION_MILLISECONDS; - case type_id::DURATION_MICROSECONDS: return typing::DURATION_MICROSECONDS; - case type_id::DURATION_NANOSECONDS: return typing::DURATION_NANOSECONDS; - case type_id::STRING: return typing::STRING; + case type_id::BOOL8: return type::BOOL8; + case type_id::INT8: return type::INT8; + case type_id::INT16: return type::INT16; + case type_id::INT32: return type::INT32; + case type_id::INT64: return type::INT64; + case type_id::UINT8: return type::UINT8; + case type_id::UINT16: return type::UINT16; + case type_id::UINT32: return type::UINT32; + case type_id::UINT64: return type::UINT64; + case type_id::FLOAT32: return type::FLOAT32; + case type_id::FLOAT64: return type::FLOAT64; + case type_id::DECIMAL32: return type::DECIMAL32; + case type_id::DECIMAL64: return type::DECIMAL64; + case type_id::DECIMAL128: return type::DECIMAL128; + case type_id::TIMESTAMP_DAYS: return type::TIMESTAMP_DAYS; + case type_id::TIMESTAMP_SECONDS: return type::TIMESTAMP_SECONDS; + case type_id::TIMESTAMP_MILLISECONDS: return type::TIMESTAMP_MILLISECONDS; + case type_id::TIMESTAMP_MICROSECONDS: return type::TIMESTAMP_MICROSECONDS; + case type_id::TIMESTAMP_NANOSECONDS: return type::TIMESTAMP_NANOSECONDS; + case type_id::DURATION_DAYS: return type::DURATION_DAYS; + case type_id::DURATION_SECONDS: return type::DURATION_SECONDS; + case type_id::DURATION_MILLISECONDS: return type::DURATION_MILLISECONDS; + case type_id::DURATION_MICROSECONDS: return type::DURATION_MICROSECONDS; + case type_id::DURATION_NANOSECONDS: return type::DURATION_NANOSECONDS; + case type_id::STRING: return type::STRING; default: CUDF_FAIL(std::format("Unsupported data type for Row IR: {}", type_to_name(type)), std::invalid_argument); } } -data_type as_data_type(typing type, int32_t scale) +type_id as_type_id(type type) { switch (type) { - case typing::BOOL8: return data_type{type_id::BOOL8, 0}; - case typing::INT8: return data_type{type_id::INT8, 0}; - case typing::INT16: return data_type{type_id::INT16, 0}; - case typing::INT32: return data_type{type_id::INT32, 0}; - case typing::INT64: return data_type{type_id::INT64, 0}; - case typing::UINT8: return data_type{type_id::UINT8, 0}; - case typing::UINT16: return data_type{type_id::UINT16, 0}; - case typing::UINT32: return data_type{type_id::UINT32, 0}; - case typing::UINT64: return data_type{type_id::UINT64, 0}; - case typing::FLOAT32: return data_type{type_id::FLOAT32, 0}; - case typing::FLOAT64: return data_type{type_id::FLOAT64, 0}; - case typing::DECIMAL32: return data_type{type_id::DECIMAL32, scale}; - case typing::DECIMAL64: return data_type{type_id::DECIMAL64, scale}; - case typing::DECIMAL128: return data_type{type_id::DECIMAL128, scale}; - case typing::TIMESTAMP_DAYS: return data_type{type_id::TIMESTAMP_DAYS, 0}; - case typing::TIMESTAMP_SECONDS: return data_type{type_id::TIMESTAMP_SECONDS, 0}; - case typing::TIMESTAMP_MILLISECONDS: return data_type{type_id::TIMESTAMP_MILLISECONDS, 0}; - case typing::TIMESTAMP_MICROSECONDS: return data_type{type_id::TIMESTAMP_MICROSECONDS, 0}; - case typing::TIMESTAMP_NANOSECONDS: return data_type{type_id::TIMESTAMP_NANOSECONDS, 0}; - case typing::DURATION_DAYS: return data_type{type_id::DURATION_DAYS, 0}; - case typing::DURATION_SECONDS: return data_type{type_id::DURATION_SECONDS, 0}; - case typing::DURATION_MILLISECONDS: return data_type{type_id::DURATION_MILLISECONDS, 0}; - case typing::DURATION_MICROSECONDS: return data_type{type_id::DURATION_MICROSECONDS, 0}; - case typing::DURATION_NANOSECONDS: return data_type{type_id::DURATION_NANOSECONDS, 0}; - case typing::STRING: return data_type{type_id::STRING, 0}; + case type::BOOL8: return type_id::BOOL8; + case type::INT8: return type_id::INT8; + case type::INT16: return type_id::INT16; + case type::INT32: return type_id::INT32; + case type::INT64: return type_id::INT64; + case type::UINT8: return type_id::UINT8; + case type::UINT16: return type_id::UINT16; + case type::UINT32: return type_id::UINT32; + case type::UINT64: return type_id::UINT64; + case type::FLOAT32: return type_id::FLOAT32; + case type::FLOAT64: return type_id::FLOAT64; + case type::DECIMAL32: return type_id::DECIMAL32; + case type::DECIMAL64: return type_id::DECIMAL64; + case type::DECIMAL128: return type_id::DECIMAL128; + case type::TIMESTAMP_DAYS: return type_id::TIMESTAMP_DAYS; + case type::TIMESTAMP_SECONDS: return type_id::TIMESTAMP_SECONDS; + case type::TIMESTAMP_MILLISECONDS: return type_id::TIMESTAMP_MILLISECONDS; + case type::TIMESTAMP_MICROSECONDS: return type_id::TIMESTAMP_MICROSECONDS; + case type::TIMESTAMP_NANOSECONDS: return type_id::TIMESTAMP_NANOSECONDS; + case type::DURATION_DAYS: return type_id::DURATION_DAYS; + case type::DURATION_SECONDS: return type_id::DURATION_SECONDS; + case type::DURATION_MILLISECONDS: return type_id::DURATION_MILLISECONDS; + case type::DURATION_MICROSECONDS: return type_id::DURATION_MICROSECONDS; + case type::DURATION_NANOSECONDS: return type_id::DURATION_NANOSECONDS; + case type::STRING: return type_id::STRING; default: CUDF_FAIL(std::format("Invalid typing for {}: {}", __FUNCTION__, static_cast(type)), std::invalid_argument); @@ -151,7 +208,7 @@ opcode as_opcode(ast::ast_operator op) case ast::ast_operator::SUB: return opcode::SUB; case ast::ast_operator::MUL: return opcode::MUL; case ast::ast_operator::DIV: return opcode::DIV; - case ast::ast_operator::TRUE_DIV: return opcode::DIV; + case ast::ast_operator::TRUE_DIV: return opcode::TRUE_DIV; case ast::ast_operator::FLOOR_DIV: return opcode::FLOOR_DIV; case ast::ast_operator::MOD: return opcode::MOD; case ast::ast_operator::PYMOD: return opcode::PYMOD; @@ -207,39 +264,31 @@ std::string to_cuda_type(cudf::data_type type, bool nullable) return nullable ? std::format("cuda::std::optional<{}>", name) : name; } +// TODO: implicit casts; what level should this be handled at? AST, IR, or codegen?; AST certainly, +// IR maps to functions with specific type signatures data_type get_return_type(opcode op, std::span args, std::optional target_scale) { - std::vector arg_types; + std::vector arg_types; for (auto& type : args) { arg_types.emplace_back(as_typing(type)); } - // TODO: ideally, we'd want to have rules for null propagation and checking - // i.e. the REPLACE_NULLS has the requirement that the second argument is non-nullable but - // that is presently only implied by the name. - auto op_type_match = get_op_typing(op); for (size_t i = 0; i < args.size(); ++i) { - auto type = op_type_match.args[i]; - if (type == typing::NONE) { continue; } + auto required_type = op_type_match.args[i]; + auto arg_type = arg_types[i]; - if ((type & typing::ARG_MASK) != typing::NONE) { - CUDF_EXPECTS((arg_types[i] & type) != 0, - std::format("Argument {} of operator `{}` does not match expected types. Got {}", - i, - get_op_name(op), - type_to_name(args[i]))); - } else { - auto src_index = static_cast(type & ~typing::ARG_MASK); + if ((required_type & type::ARG_MASK) != type::NONE) { + auto src_index = static_cast(required_type & ~type::ARG_MASK); CUDF_EXPECTS( src_index < i, std::format("Invalid type match rule for operator `{}` at argument {}", get_op_name(op), i), std::runtime_error); - CUDF_EXPECTS(arg_types[i] == arg_types[src_index], + CUDF_EXPECTS(args[i].id() == args[src_index].id(), std::format("Argument {} of operator `{}` does not match type of argument " "`{}`. Got `{}`, expected `{}`", i, @@ -247,93 +296,111 @@ data_type get_return_type(opcode op, src_index, type_to_name(args[i]), type_to_name(args[src_index]))); + } else { + CUDF_EXPECTS((arg_type & required_type) != 0, + std::format("Argument {} of operator `{}` does not match expected types. Got {}", + i, + get_op_name(op), + type_to_name(args[i]))); } } - // TODO: implement filter_predicate to return false on nulls - - if ((op_type_match.output & typing::ARG_MASK) != typing::NONE) { - auto arg_index = static_cast(op_type_match.output & ~typing::ARG_MASK); - return args[arg_index]; + if ((op_type_match.output & type::ARG_MASK) != type::NONE) { + auto arg_index = static_cast(op_type_match.output & ~type::ARG_MASK); + auto type = args[arg_index]; + if (target_scale.has_value()) { + type = data_type{type.id(), numeric::scale_type{target_scale.value()}}; + } + return type; } else { CUDF_EXPECTS( - op_type_match.output != typing::NONE && - (op_type_match.output & typing::DECIMALS) == typing::NONE, + op_type_match.output != type::NONE && (op_type_match.output & type::DECIMALS) == type::NONE, std::format("Invalid type match rule for operator `{}` return type", get_op_name(op)), std::runtime_error); - return as_data_type(op_type_match.output, target_scale.value_or(0)); + return data_type{as_type_id(op_type_match.output), + numeric::scale_type{target_scale.value_or(0)}}; } } -void node::instantiate(instance_context& ctx, instance_info const& info) +void node::instantiate(instance_context& ctx) { id_ = ctx.make_tmp_id(); for (auto& arg : args_) { - arg.instantiate(ctx, info); + arg->instantiate(ctx); } switch (op_) { case opcode::GET_INPUT: { - type_ = info.inputs[std::get(reference_).index].type; + type_ = ctx.get_inputs()[std::get(reference_).index].type; } break; case opcode::SET_OUTPUT: { - type_ = args_[0].get_type(); + type_ = args_[0]->get_type(); } break; default: { std::vector arg_types; for (auto& arg : args_) { - arg_types.emplace_back(arg.get_type()); + arg_types.emplace_back(arg->get_type()); + } + + if (op_ == opcode::RESCALE) { + scale_reference_ = scalar_refernce{ + ctx.add_constant(cudf::numeric_scalar{target_scale_.value_or(0)})}; } - type_ = get_return_type(op_, arg_types, std::nullopt); + + type_ = get_return_type(op_, arg_types, target_scale_); } break; } } -void node::emit_code(instance_context& ctx, - target_info const& info, - instance_info const& instance, - code_sink& sink) const +void node::emit_code(instance_context& instance, target_info const& info, code_sink& sink) const { for (auto& arg : args_) { - arg.emit_code(ctx, info, instance, sink); + arg->emit_code(instance, info, sink); } switch (info.id) { case target::CUDA: { - auto type = to_cuda_type(type_, ctx.has_nulls()); + auto type = to_cuda_type(type_, instance.has_nulls()); switch (op_) { case opcode::GET_INPUT: { - sink.emit(std::format("{} {} = {};", - type, - id_, - instance.inputs[std::get(reference_).index].id)); + sink.emit( + std::format(R"***({} {} = {}; +)***", + type, + id_, + instance.get_inputs()[std::get(reference_).index].id)); } break; case opcode::SET_OUTPUT: { sink.emit(std::format( - R"**({} {} = {}; + R"***({} {} = {}; *{} = {}; -)**", +)***", type, id_, - args_[0].get_id(), - instance.outputs[std::get(reference_).index].id, + args_[0]->get_id(), + instance.get_outputs()[std::get(reference_).index].id, id_)); } break; default: { - auto first_arg = args_[0].get_id(); + auto first_arg = std::format("&{}", args_[0]->get_id()); auto args_str = (args_.size() == 1) ? std::string{first_arg} : std::accumulate(args_.begin() + 1, args_.end(), std::string{first_arg}, [](auto const& a, auto& node) { - return std::format("{}, &{}", a, node.get_id()); + return std::format("{}, &{}", a, node->get_id()); }); + if (op_ == opcode::RESCALE) { + args_str = + std::format("{}, &{}", args_str, instance.get_inputs()[scale_reference_.index].id); + } + bool fallible = get_op_is_fallible(op_); auto op_name = get_op_name(op_); @@ -341,7 +408,7 @@ void node::emit_code(instance_context& ctx, sink.emit(std::format( R"***({} {}; cudf::ops::{}(&{}, {}); - )***", +)***", type, id_, op_name, @@ -353,7 +420,7 @@ cudf::ops::{}(&{}, {}); if(cudf::ops::errc e = cudf::ops::{}(&{}, {}); e != cudf::ops::errc::SUCCESS) {{ return e; }} - )***", +)***", type, id_, op_name, @@ -370,69 +437,34 @@ if(cudf::ops::errc e = cudf::ops::{}(&{}, {}); e != cudf::ops::errc::SUCCESS) {{ } } -std::span ast_converter::get_input_specs() const { return input_specs_; } - -int32_t ast_converter::add_ast_input(ast_input_spec in) -{ - auto id = static_cast(input_specs_.size()); - input_specs_.emplace_back(std::move(in)); - return id; -} - -row_ir::node ast_converter::add_ir_node(ast::literal const& expr) -{ - auto index = add_ast_input( - ast_scalar_input_spec{expr.get_scalar(), - expr.get_value(), - make_column_from_scalar(expr.get_scalar(), 1, stream_, mr_)}); - return row_ir::node(input_reference{index}); -} - -row_ir::node ast_converter::add_ir_node(ast::column_reference const& expr) +std::unique_ptr ast_converter::add_ir_node(ast::literal const& expr) { - auto index = - add_ast_input(ast_column_input_spec{expr.get_table_source(), expr.get_column_index()}); - return row_ir::node(input_reference{index}); + auto index = instance_.add_ast_input( + ast_scalar_input_spec{make_column_from_scalar(expr.get_scalar(), 1, stream_, mr_)}); + return std::make_unique(input_reference{index}); } -row_ir::node ast_converter::add_ir_node(ast::operation const& expr) +std::unique_ptr ast_converter::add_ir_node(ast::column_reference const& expr) { - std::vector operands; - for (auto const& operand : expr.get_operands()) { - operands.emplace_back(operand.get().accept(*this)); - } - return row_ir::node(as_opcode(expr.get_operator()), std::move(operands)); + auto index = instance_.add_ast_input( + ast_column_input_spec{expr.get_table_source(), expr.get_column_index()}); + return std::make_unique(input_reference{index}); } -// Resolve the table for a column input spec, preferring left_table/right_table for join cases, -// falling back to args.table for the single-table case. -table_view const& resolve_table(ast_column_input_spec const& in, ast_args const& args) +std::unique_ptr ast_converter::add_ir_node(ast::operation const& expr) { - if (in.table == ast::table_reference::LEFT) { - return args.left_table.num_columns() > 0 ? args.left_table : args.table; + std::vector> args; + for (auto& operand : expr.get_operands()) { + args.emplace_back(operand.get().accept(*this)); } - return args.right_table; -} - -void ast_converter::add_input_var(ast_column_input_spec const& in, ast_args const& args) -{ - auto id = std::format("in_{}", input_vars_.size()); - auto type = resolve_table(in, args).column(in.column).type(); - input_vars_.emplace_back(std::move(id), type); + return std::make_unique( + as_opcode(expr.get_operator()), std::nullopt, std::move(args)); } -void ast_converter::add_input_var(ast_scalar_input_spec const& in, - [[maybe_unused]] ast_args const& args) +std::unique_ptr ast_converter::add_ir_node(ast::detail::predicate const& expr) { - auto id = std::format("in_{}", input_vars_.size()); - auto type = in.ref.get().type(); - input_vars_.emplace_back(std::move(id), type); -} - -void ast_converter::add_output_var() -{ - auto id = std::format("out_{}", output_vars_.size()); - output_vars_.emplace_back(std::move(id)); + return std::make_unique( + row_ir::opcode::PREDICATE, std::nullopt, expr.get_operand().accept(*this)); } template @@ -456,23 +488,21 @@ std::variant get_column_view(ast_column_input_s std::variant get_column_view(ast_scalar_input_spec const& spec, ast_args const& args) { - return scalar_column_view{spec.broadcast_column->view()}; + return scalar_column_view{spec.scalar_column->view()}; } -std::tuple ast_converter::generate_code( +std::tuple ast_converter::generate_code( target target_id, ast::expression const& expr, ast_args const& args) { - output_irs_.emplace_back(output_reference{0}, expr.accept(*this)); - - // TODO: return fallible or not + output_irs_.emplace_back(std::make_unique(output_reference{0}, expr.accept(*this))); // resolve the flattened input references into IR input variables - for (auto const& input : input_specs_) { - dispatch_input_spec(input, [this](auto&... args) { add_input_var(args...); }, args); + for (auto& input : instance_.input_specs_) { + dispatch_input_spec(input, [&](auto&... args) { instance_.add_input_var(args...); }, args); } - bool has_nullable_inputs = - std::any_of(input_specs_.begin(), input_specs_.end(), [&](auto const& input) { + bool has_nullable_inputs = std::any_of( + instance_.input_specs_.begin(), instance_.input_specs_.end(), [&](auto const& input) { return dispatch_input_spec( input, [](auto&... args) { @@ -483,32 +513,29 @@ std::tuple ast_converter::generate_code( }); // add 1 auto-deduced output variable - add_output_var(); - - instance_context instance_ctx; - instance_info instance{input_vars_, output_vars_}; + instance_.add_output_var(); auto is_null_aware = std::any_of( - output_irs_.cbegin(), output_irs_.cend(), [](auto& ir) { return ir.is_null_aware(); }) + output_irs_.cbegin(), output_irs_.cend(), [](auto& ir) { return ir->is_null_aware(); }) ? null_aware::YES : null_aware::NO; bool output_is_always_valid = std::all_of( - output_irs_.cbegin(), output_irs_.cend(), [](auto& ir) { return ir.is_always_valid(); }); + output_irs_.cbegin(), output_irs_.cend(), [](auto& ir) { return ir->is_always_valid(); }); - bool may_evaluate_null = !output_is_always_valid && has_nullable_inputs; + bool may_evaluate_null = !output_is_always_valid || has_nullable_inputs; auto null_policy = may_evaluate_null ? output_nullability::PRESERVE : output_nullability::ALL_VALID; auto is_fallible = std::any_of( - output_irs_.cbegin(), output_irs_.cend(), [](auto& ir) { return ir.is_fallible(); }); + output_irs_.cbegin(), output_irs_.cend(), [](auto& ir) { return ir->is_fallible(); }); - instance_ctx.set_has_nulls(is_null_aware == null_aware::YES); + instance_.set_has_nulls(is_null_aware == null_aware::YES); // instantiate the IR nodes for (auto& ir : output_irs_) { - ir.instantiate(instance_ctx, instance); + ir->instantiate(instance_); } target_info target{target_id}; @@ -516,24 +543,24 @@ std::tuple ast_converter::generate_code( CUDF_EXPECTS( target.id == target::CUDA, "Unsupported target for code generation", std::invalid_argument); - auto output_decl = [&](size_t i) { - auto& var = output_vars_[i]; + auto output_decl = [&](auto i) { + auto& var = instance_.output_vars_[i]; auto& ir = output_irs_[i]; - return std::format("{}* {}", to_cuda_type(ir.get_type(), instance_ctx.has_nulls()), var.id); + return std::format("{}* {}", to_cuda_type(ir->get_type(), instance_.has_nulls()), var.id); }; - auto input_decl = [&](size_t i) { - auto& var = input_vars_[i]; - return std::format("{} {}", to_cuda_type(var.type, instance_ctx.has_nulls()), var.id); + auto input_decl = [&](auto i) { + auto& var = instance_.input_vars_[i]; + return std::format("{} {}", to_cuda_type(var.type, instance_.has_nulls()), var.id); }; std::vector arg_decls; - for (size_t i = 0; i < output_vars_.size(); ++i) { + for (size_t i = 0; i < instance_.output_vars_.size(); ++i) { arg_decls.emplace_back(output_decl(i)); } - for (size_t i = 0; i < input_vars_.size(); ++i) { + for (size_t i = 0; i < instance_.input_vars_.size(); ++i) { arg_decls.emplace_back(input_decl(i)); } @@ -553,13 +580,13 @@ std::tuple ast_converter::generate_code( code_sink sink; sink.emit("__device__ cudf::ops::errc expression("); sink.emit(args_decl); - sink.emit(")\n{"); + sink.emit(")\n{\n"); for (auto& ir : output_irs_) { - ir.emit_code(instance_ctx, target, instance, sink); + ir->emit_code(instance_, target, sink); } - sink.emit(" return cudf::ops::errc::SUCCESS;\n}"); + sink.emit("return cudf::ops::errc::OK;\n}"); - return {is_null_aware, null_policy, is_fallible}; + return {std::string{sink.get_code()}, is_null_aware, null_policy, is_fallible}; } // Due to the AST expression tree structure, we can't generate the IR without the target @@ -575,36 +602,38 @@ transform_args ast_converter::compute_column(target target_id, // TODO(lamarrr): consider deduplicating ast expression's input column references. See // TransformTest/1.DeeplyNestedArithmeticLogicalExpression for reference - auto [is_null_aware, output_nullability, is_fallible] = + auto [code, is_null_aware, output_nullability, is_fallible] = converter.generate_code(target_id, expr, args); std::vector> inputs; std::vector> scalar_columns; - for (auto& input : converter.input_specs_) { + for (auto& input : converter.instance_.input_specs_) { auto column_view = dispatch_input_spec(input, [](auto&... args) { return get_column_view(args...); }, args); inputs.emplace_back(column_view); if (std::holds_alternative(input)) { auto& scalar_input = std::get(input); - scalar_columns.emplace_back(std::move(scalar_input.broadcast_column)); + scalar_columns.emplace_back(std::move(scalar_input.scalar_column)); } } auto& out = converter.output_irs_[0]; - auto output_column_type = out.get_type(); - - auto result = transform_args{.scalar_columns = std::move(scalar_columns), - .inputs = inputs, - .udf = std::move(converter.code_), - .output_type = output_column_type, - .source_type = cudf::udf_source_type::CUDA, - .user_data = std::nullopt, - .is_null_aware = is_null_aware, - .null_policy = output_nullability, - .row_size = args.table.num_rows(), - .input_specs = std::move(converter.input_specs_)}; + auto output_column_type = out->get_type(); + + auto result = + transform_args{.scalar_columns = std::move(scalar_columns), + .inputs = inputs, + .udf = std::move(code), + .output_type = output_column_type, + .source_type = cudf::udf_source_type::CUDA, + .user_data = std::nullopt, + .is_null_aware = is_null_aware, + .null_policy = output_nullability, + .row_size = args.table.num_rows(), + .error_mode = is_fallible ? ops::error_mode::ANY_ROW : ops::error_mode::IGNORE, + .input_specs = std::move(converter.instance_.input_specs_)}; if (get_context().dump_codegen()) { std::cout << "Generated code for transform: " << result.udf << std::endl; @@ -620,7 +649,7 @@ filter_args ast_converter::filter(target target_id, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - /* auto filter = ast::detail::filter_predicate{expr}; + auto filter = ast::detail::predicate{expr}; auto transform = compute_column(target_id, filter, args, stream, mr); CUDF_EXPECTS(transform.output_type.id() == type_id::BOOL8, @@ -641,11 +670,10 @@ filter_args ast_converter::filter(target target_id, .user_data = transform.user_data, .is_null_aware = transform.is_null_aware, .predicate_nullability = transform.null_policy, + .error_mode = transform.error_mode, .input_specs = std::move(transform.input_specs)}; return result; - */ - CUDF_FAIL("Filtering is not yet implemented", std::runtime_error); } } // namespace cudf::detail::row_ir diff --git a/cpp/src/jit/row_ir.hpp b/cpp/src/jit/row_ir.hpp index ff2dab8a4fbb..b0a27888b513 100644 --- a/cpp/src/jit/row_ir.hpp +++ b/cpp/src/jit/row_ir.hpp @@ -7,7 +7,8 @@ #include #include #include -#include +#include +#include #include #include #include @@ -16,7 +17,6 @@ #include #include -#include #include #include #include @@ -60,20 +60,41 @@ struct untyped_var_info { }; /** - * @brief The information needed to instantiate the IR nodes + * @brief The information about the target for which the IR is generated. */ -struct instance_info { - std::span inputs; ///< The input variables - std::span outputs; ///< The output variables +struct target_info { + target id = target::CUDA; ///< The target identifier }; /** - * @brief The information about the target for which the IR is generated. + * @brief A specification of an input column to the AST */ -struct target_info { - target id = target::CUDA; ///< The target identifier +struct ast_column_input_spec { + ast::table_reference table = {}; ///< The table reference (LEFT or RIGHT) + int32_t column = 0; ///< The column index in the referenced table }; +/** + * @brief A specification of an input scalar to the AST + */ +struct ast_scalar_input_spec { + std::unique_ptr scalar_column = nullptr; ///< The broadcasted column, a column of size 1 +}; + +/** + * @brief The AST input column arguments used to resolve the column expressions + */ +struct ast_args { + table_view table = {}; ///< The table view containing the columns (single-table case) + table_view left_table = {}; ///< The left table for join predicates + table_view right_table = {}; ///< The right table for join predicates +}; + +/** + * @brief An input specification for the AST + */ +using ast_input_spec = std::variant; + /** * @brief The context within which the IR is instantiated. * This context is used to generate temporary variable identifiers and any state setup needed for @@ -81,12 +102,33 @@ struct target_info { */ struct [[nodiscard]] instance_context { private: - int32_t num_tmp_vars_ = 0; ///< The number of temporary variables generated - std::string tmp_prefix_ = "tmp_"; ///< The prefix for temporary variable identifiers - bool has_nulls_ = false; ///< If expressions involve null values + int32_t num_tmp_vars_ = 0; ///< The number of temporary variables generated + std::string tmp_prefix_ = "tmp_"; ///< The prefix for temporary variable identifiers + bool has_nulls_ = false; ///< If expressions involve null values + std::vector input_specs_; ///< The input specs for the AST + std::vector input_vars_; ///< The input variables for the IR + std::vector output_vars_; ///< The output variables for the IR + rmm::cuda_stream_view + stream_; ///< The CUDA stream for any device operations during IR generation + rmm::device_async_resource_ref + mr_; ///< The device memory resource for any device memory allocation during IR generation + + private: + void add_input_var(ast_column_input_spec const& in, ast_args const& args); + + void add_input_var(ast_scalar_input_spec const& in, ast_args const& args); + + void add_output_var(); + + [[nodiscard]] int32_t add_ast_input(ast_input_spec in); public: - instance_context() = default; ///< Default constructor + friend struct ast_converter; + + instance_context(rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) + : stream_(stream), mr_(mr) + { + } instance_context(instance_context const&) = delete; @@ -114,6 +156,31 @@ struct [[nodiscard]] instance_context { * @param has_nulls True if expressions involve null values */ void set_has_nulls(bool has_nulls); + + /** + * @brief Get the input specifications for the AST + * @return A span of AST input specifications + */ + [[nodiscard]] std::span get_input_specs() const; + + /** + * @brief Get the input variables for the IR + * @return A span of input variable information + */ + [[nodiscard]] std::span get_inputs() const; + + /** + * @brief Get the output variables for the IR + * @return A span of output variable information + */ + [[nodiscard]] std::span get_outputs() const; + + /** + * @brief Add a constant scalar value to the IR + * @param value The scalar value to add + * @return The identifier of the constant variable + */ + [[nodiscard]] int32_t add_constant(cudf::scalar const& value); }; struct [[nodiscard]] code_sink { @@ -134,24 +201,66 @@ struct [[nodiscard]] output_reference { int32_t index = 0; ///< The index of the output variable }; +struct [[nodiscard]] scalar_refernce { + int32_t index = 0; ///< The index of the scalar variable +}; + struct [[nodiscard]] node { + private: std::variant reference_ = - std::monostate{}; ///< The index of the input/output variable - opcode op_ = opcode::GET_INPUT; ///< The operation code - std::vector args_ = {}; ///< The arguments of the operation + std::monostate{}; ///< The index of the input/output variable + opcode op_ = opcode::GET_INPUT; ///< The operation code + std::optional target_scale_ = std::nullopt; ///< The target scale for decimal + std::vector> args_ = {}; ///< The arguments of the operation data_type type_ = {}; ///< The resolved type information of the IR node std::string id_ = {}; ///< The identifier of the IR node + scalar_refernce + scale_reference_; ///< The index of the scale variable for decimal rescaling if applicable /** * @brief Create a set of argument IR nodes */ template requires(std::is_same_v && ...) - static std::array arguments(T&&... args) + static std::vector> arguments(T... args) + { + std::vector> result; + (result.emplace_back(std::make_unique(std::move(args))), ...); + return result; + } + + /** + * @brief Create a set of argument IR nodes + */ + template + requires(std::is_same_v, T> && ...) + static std::vector> arguments(T... args) + { + std::vector> result; + (result.emplace_back(std::move(args)), ...); + return result; + } + + public: + /** + * @brief Construct a new operation IR node + * @param op The operation code + * @param args The arguments of the operation + */ + node(opcode op, std::optional target_scale, std::vector> args); + + /** + * @brief Construct a new operation IR node + * @param op The operation code + * @param args The arguments of the operation + */ + template + requires(std::is_same_v && ...) + node(opcode op, std::optional target_scale, T... args) + : node(op, target_scale, arguments(std::move(args)...)) { - return {std::forward(args)...}; } /** @@ -159,7 +268,12 @@ struct [[nodiscard]] node { * @param op The operation code * @param args The arguments of the operation */ - node(opcode op, std::vector args); + template + requires(std::is_same_v && ...) + node(opcode op, std::optional target_scale, std::unique_ptr... args) + : node(op, target_scale, arguments(std::move(args)...)) + { + } /** * @brief Construct a new input reference IR node @@ -172,11 +286,11 @@ struct [[nodiscard]] node { * @param output The index of the output variable * @param arg The argument node that produces the value to be set to the output variable */ - node(output_reference reference, node arg); + node(output_reference reference, std::unique_ptr arg); - node(node const& other) = default; ///< Copy constructor + node(node const& other) = delete; node(node&& other) = default; ///< Move constructor - node& operator=(node const& other) = default; ///< Copy assignment operator + node& operator=(node const& other) = delete; node& operator=(node&& other) = default; ///< Move assignment operator ~node() = default; ///< Destructor @@ -192,6 +306,12 @@ struct [[nodiscard]] node { */ [[nodiscard]] data_type get_type() const; + /** + * @brief Get the target scale for decimal rescaling if applicable + * @return The target scale for decimal rescaling if applicable, std::nullopt otherwise + */ + [[nodiscard]] std::optional get_target_scale() const; + /** * @brief Get the operation code of the operation * @return The operation code of the operation @@ -201,7 +321,7 @@ struct [[nodiscard]] node { /** @brief Get the arguments of the operation * @return A span of unique pointers to the arguments of the operation */ - [[nodiscard]] std::span get_args() const; + [[nodiscard]] std::span const> get_args() const; /** * @brief Returns `false` if this node forwards nulls from its inputs to its output. @@ -230,7 +350,7 @@ struct [[nodiscard]] node { * @param ctx The context within which the IR is instantiated * @param info The instance information */ - void instantiate(instance_context& ctx, instance_info const& info); + void instantiate(instance_context& ctx); /** * @brief Generate the code for the IR node based on the instance context and target information. @@ -239,35 +359,9 @@ struct [[nodiscard]] node { * @param instance The instance information * @param sink The code sink to which the generated code is emitted */ - void emit_code(instance_context& ctx, - target_info const& info, - instance_info const& instance, - code_sink& sink) const; -}; - -/** - * @brief A specification of an input column to the AST - */ -struct ast_column_input_spec { - ast::table_reference table = {}; ///< The table reference (LEFT or RIGHT) - int32_t column = 0; ///< The column index in the referenced table -}; - -/** - * @brief A specification of an input scalar to the AST - */ -struct ast_scalar_input_spec { - std::reference_wrapper ref; ///< The scalar value - ast::generic_scalar_device_view view; ///< The device view of the scalar value - std::unique_ptr broadcast_column = - nullptr; ///< The broadcasted column, a column of size 1 + void emit_code(instance_context& ctx, target_info const& info, code_sink& sink) const; }; -/** - * @brief An input specification for the AST - */ -using ast_input_spec = std::variant; - /** * @brief The arguments needed to invoke a `cudf::transform` */ @@ -283,7 +377,9 @@ struct [[nodiscard]] transform_args { null_aware is_null_aware = null_aware::NO; ///< Whether the transform is null-aware output_nullability null_policy = output_nullability::PRESERVE; ///< Null-transformation policy std::optional row_size = std::nullopt; ///< The row size of the transform operation - std::vector input_specs = {}; ///< The input specs (table ref + column index) + ops::error_mode error_mode = + ops::error_mode::IGNORE; ///< The error handling mode for the transform + std::vector input_specs = {}; ///< The input specs (table ref + column index) }; /** @@ -301,32 +397,21 @@ struct [[nodiscard]] filter_args { null_aware is_null_aware = null_aware::NO; ///< Whether the filter is null-aware output_nullability predicate_nullability = output_nullability::PRESERVE; ///< Null-transformation policy for the predicate output + ops::error_mode error_mode = ops::error_mode::IGNORE; ///< The error handling mode for the filter std::vector input_specs = {}; ///< The input specs (table ref + column index) }; -/** - * @brief The AST input column arguments used to resolve the column expressions - */ -struct ast_args { - table_view table = {}; ///< The table view containing the columns (single-table case) - table_view left_table = {}; ///< The left table for join predicates - table_view right_table = {}; ///< The right table for join predicates -}; - /** * @brief AST Converter is a class for converting AST expressions to codegen targets, ie. CUDA. */ struct [[nodiscard]] ast_converter { private: - std::vector input_specs_; ///< The input specs for the AST - std::vector input_vars_; ///< The input variables for the IR - std::vector output_vars_; ///< The output variables for the IR - std::vector output_irs_; ///< The output IR nodes - std::string code_; ///< The generated code for the IR + std::vector> output_irs_; ///< The output IR nodes rmm::cuda_stream_view stream_; ///< CUDA stream used for device memory operations and kernel launches. rmm::device_async_resource_ref mr_; ///< Device memory resource used to allocate the returned table's device memory + instance_context instance_; ///< The instance context used during the IR generation public: /** @@ -335,7 +420,7 @@ struct [[nodiscard]] ast_converter { * @param mr Device memory resource used to allocate the returned table's device memory */ ast_converter(rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) - : stream_(std::move(stream)), mr_(std::move(mr)) + : stream_(std::move(stream)), mr_(std::move(mr)), instance_(stream_, mr_) { } @@ -353,27 +438,17 @@ struct [[nodiscard]] ast_converter { friend class ast::column_reference; friend class ast::operation; friend class ast::column_name_reference; + friend class ast::detail::predicate; - row_ir::node add_ir_node(ast::literal const& expr); - - row_ir::node add_ir_node(ast::column_reference const& expr); + [[nodiscard]] std::unique_ptr add_ir_node(ast::literal const& expr); - row_ir::node add_ir_node(ast::operation const& expr); + [[nodiscard]] std::unique_ptr add_ir_node(ast::column_reference const& expr); - [[nodiscard]] std::span get_input_specs() const; + [[nodiscard]] std::unique_ptr add_ir_node(ast::operation const& expr); - /** - * @brief add an AST input/input_reference and return its reference index - */ - [[nodiscard]] int32_t add_ast_input(ast_input_spec in); - - void add_input_var(ast_column_input_spec const& in, ast_args const& args); - - void add_input_var(ast_scalar_input_spec const& in, ast_args const& args); - - void add_output_var(); + [[nodiscard]] std::unique_ptr add_ir_node(ast::detail::predicate const& expr); - [[nodiscard]] std::tuple generate_code( + [[nodiscard]] std::tuple generate_code( target target, ast::expression const& expr, ast_args const& args); public: diff --git a/cpp/src/transform/jit/kernel.cu b/cpp/src/transform/jit/kernel.cu index 33975c8cd68b..f95bb0cfcda3 100644 --- a/cpp/src/transform/jit/kernel.cu +++ b/cpp/src/transform/jit/kernel.cu @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -27,6 +28,7 @@ #include #include +#include #include #include @@ -43,20 +45,39 @@ namespace cudf { namespace jit { -template -__device__ void execute_transform_op(void* user_data, size_type element_idx, Args args) +template +__device__ void execute_transform_op(error_sink* __restrict__ error_sink, + void* user_data, + size_type element_idx, + Args args) { // TODO: static assert invocable if constexpr (has_user_data) { - cuda::std::apply([&](auto... a) { GENERIC_TRANSFORM_OP(a...); }, - cuda::std::tuple_cat(cuda::std::tuple{user_data, element_idx}, args)); + cuda::std::apply( + [&](auto... a) { + if constexpr (mode == ops::error_mode::IGNORE) { + GENERIC_TRANSFORM_OP(a...); + } else { + error_sink->report(GENERIC_TRANSFORM_OP(a...)); + } + }, + cuda::std::tuple_cat(cuda::std::tuple{user_data, element_idx}, args)); } else { - cuda::std::apply([&](auto... a) { GENERIC_TRANSFORM_OP(a...); }, args); + cuda::std::apply( + [&](auto... a) { + if constexpr (mode == ops::error_mode::IGNORE) { + GENERIC_TRANSFORM_OP(a...); + } else { + error_sink->report(GENERIC_TRANSFORM_OP(a...)); + } + }, + args); } } /// @brief The generic transform kernel. Supports all types and nullability combinations. -template @@ -64,7 +85,8 @@ CUDF_KERNEL void transform_kernel(size_type row_size, bitmask_type const* __restrict__ stencil, void* __restrict__ user_data, column_device_view_core const* __restrict__ input_cols, - mutable_column_device_view_core const* __restrict__ output_cols) + mutable_column_device_view_core const* __restrict__ output_cols, + error_sink* __restrict__ error_sink) { // TODO: ensure block size is a multiple of warp size for correct warp-synchronous behavior auto start = detail::grid_1d::global_thread_id(); @@ -84,8 +106,8 @@ CUDF_KERNEL void transform_kernel(size_type row_size, auto out_ptrs = cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); - execute_transform_op( - user_data, element_idx, cuda::std::tuple_cat(out_ptrs, ins)); + execute_transform_op( + error_sink, user_data, element_idx, cuda::std::tuple_cat(out_ptrs, ins)); OutputAccessors::map([&]() { (A::assign(output_cols, element_idx, cuda::std::get(outs)), ...); @@ -105,8 +127,8 @@ CUDF_KERNEL void transform_kernel(size_type row_size, auto out_ptrs = cuda::std::apply([&](auto&... args) { return cuda::std::tuple{&args...}; }, outs); - execute_transform_op( - user_data, element_idx, cuda::std::tuple_cat(out_ptrs, ins)); + execute_transform_op( + error_sink, user_data, element_idx, cuda::std::tuple_cat(out_ptrs, ins)); OutputAccessors::map([&]() { (A::assign(output_cols, element_idx, *cuda::std::get(outs)), ...); diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 436316241432..35a698c82208 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -158,7 +159,8 @@ using handle = std::variant< namespace jit_transform { -jitify2::Kernel instantiate(null_aware is_null_aware, +jitify2::Kernel instantiate(ops::error_mode error_handling_mode, + null_aware is_null_aware, bool has_user_data, std::string const& ins, std::string const& outs, @@ -176,7 +178,7 @@ jitify2::Kernel instantiate(null_aware is_null_aware, : jit::parse_single_function_cuda(udf, "GENERIC_TRANSFORM_OP"); auto kernel = jitify2::reflection::Template("cudf::jit::transform_kernel") - .instantiate(is_null_aware, has_user_data, ins, outs); + .instantiate(error_handling_mode, is_null_aware, has_user_data, ins, outs); return jit::get_udf_kernel( *transform_jit_kernel_cu_jit, kernel, cuda_source, {"-restrict", "--dopt=on"}); @@ -188,10 +190,11 @@ void launch(jitify2::Kernel const& kernel, void* user_data, column_device_view_core const* input_cols, mutable_column_device_view_core const* output_cols, + jit::error_sink* error_sink, rmm::cuda_stream_view stream) { CUDF_FUNC_RANGE(); - void* args[] = {&row_size, &stencil, &user_data, &input_cols, &output_cols}; + void* args[] = {&row_size, &stencil, &user_data, &input_cols, &output_cols, &error_sink}; kernel->configure_1d_max_occupancy(0, 0, nullptr, stream.value())->launch_raw(args); } @@ -319,20 +322,23 @@ auto to_args(std::span inputs, return std::make_tuple(std::move(d_args), std::move(handles)); } -void run(null_aware is_null_aware, +void run(ops::error_mode error_handling_mode, + null_aware is_null_aware, bool has_user_data, size_type row_size, bitmask_type const* d_stencil, void* user_data, std::span inputs, std::span outputs, + jit::error_sink* d_error_sink, std::string const& udf, udf_source_type source_type, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { auto [in_types, out_types, ptx_in_types, ptx_out_types] = reflect(source_type, inputs, outputs); - auto kernel = instantiate(is_null_aware, + auto kernel = instantiate(error_handling_mode, + is_null_aware, has_user_data, in_types, out_types, @@ -344,7 +350,8 @@ void run(null_aware is_null_aware, auto* input_cols = reinterpret_cast(cols.data()); auto* output_cols = reinterpret_cast(input_cols + inputs.size()); - return launch(kernel, row_size, d_stencil, user_data, input_cols, output_cols, stream); + return launch( + kernel, row_size, d_stencil, user_data, input_cols, output_cols, d_error_sink, stream); } } // namespace jit_transform @@ -797,6 +804,7 @@ auto finalize_outputs(null_aware is_null_aware, std::unique_ptr
execute_transform(std::string const& udf, udf_source_type source_type, + ops::error_mode error_handling_mode, null_aware is_null_aware, std::optional in_row_size, std::optional user_data, @@ -819,39 +827,67 @@ std::unique_ptr
execute_transform(std::string const& udf, auto stencil_arg = stencil.has_value() ? stencil->first : nullptr; auto stencil_has_nulls = stencil.has_value() ? (stencil->second > 0) : false; - jit_transform::run(is_null_aware, + + std::optional> d_error_sink = std::nullopt; + + switch (error_handling_mode) { + case ops::error_mode::IGNORE: break; + case ops::error_mode::ANY_ROW: + d_error_sink = rmm::device_scalar(jit::error_sink{}, stream); + break; + } + + jit_transform::run(error_handling_mode, + is_null_aware, user_data.has_value(), row_size, stencil_has_nulls ? stencil_arg : nullptr, user_data.value_or(nullptr), inputs, output_columns, + d_error_sink.has_value() ? d_error_sink->data() : nullptr, udf, source_type, stream, mr); auto finalized = finalize_outputs(is_null_aware, row_size, std::move(output_columns), stream, mr); + + switch (error_handling_mode) { + case ops::error_mode::IGNORE: { + } break; + case ops::error_mode::ANY_ROW: { + auto sink = d_error_sink->value(stream).any_error(); + auto err = sink->error(); + CUDF_EXPECTS(err == ops::errc::OK, + std::format("Error `{}` in transform UDF", ops::to_string(err)), + std::runtime_error); + } break; + } + return std::make_unique
(std::move(finalized)); } } // namespace -std::unique_ptr
multi_transform(std::string const& udf, - udf_source_type source_type, - null_aware is_null_aware, - std::optional user_data, - std::span inputs, - std::span outputs, - std::vector>&& string_offsets, - std::optional row_size, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +std::unique_ptr
multi_transform_extended( + std::string const& udf, + udf_source_type source_type, + null_aware is_null_aware, + std::optional user_data, + std::span inputs, + std::span outputs, + std::vector>&& string_offsets, + std::optional row_size, + ops::error_mode error_handling_mode, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); perform_checks(source_type, is_null_aware, row_size, inputs, outputs, string_offsets); return execute_transform(udf, source_type, + error_handling_mode, is_null_aware, row_size, user_data, diff --git a/cpp/tests/jit/row_ir.cpp b/cpp/tests/jit/row_ir.cpp index 32022af56ddf..c69daecb0a11 100644 --- a/cpp/tests/jit/row_ir.cpp +++ b/cpp/tests/jit/row_ir.cpp @@ -33,24 +33,26 @@ TEST_F(RowIRCudaCodeGenTest, GetInput) { row_ir::instance_context ctx{}; - row_ir::get_input get_input_0{0}; + row_ir::code_sink sink; + row_ir::node get_input_0{row_ir::input_reference{0}}; get_input_0.instantiate(ctx, info); - auto code = get_input_0.generate_code(ctx, target_info, info); + get_input_0.emit_code(ctx, target_info, info, sink); auto expected_code = "int32_t tmp_0 = in_0;"; - EXPECT_EQ(code, expected_code); + EXPECT_EQ(sink.get_code(), expected_code); } { row_ir::instance_context ctx{}; - row_ir::get_input get_input_1{1}; + row_ir::code_sink sink; + row_ir::node get_input_1{row_ir::input_reference{1}}; get_input_1.instantiate(ctx, info); - auto null_code = get_input_1.generate_code(ctx, target_info, info); + get_input_1.emit_code(ctx, target_info, info, sink); auto expected_null_code = "float tmp_0 = in_1;"; - EXPECT_EQ(null_code, expected_null_code); + EXPECT_EQ(sink.get_code(), expected_null_code); } } @@ -67,30 +69,34 @@ TEST_F(RowIRCudaCodeGenTest, SetOutput) { row_ir::instance_context ctx{}; - row_ir::set_output set_output_0{0, std::make_unique(0)}; + row_ir::code_sink sink; + row_ir::node set_output_0{row_ir::output_reference{0}, + std::make_unique(row_ir::input_reference{0})}; set_output_0.instantiate(ctx, info); - auto code = set_output_0.generate_code(ctx, target_info, info); + set_output_0.emit_code(ctx, target_info, info, sink); auto expected_code = R"***(int32_t tmp_0 = in_0; int32_t tmp_1 = tmp_0; *out_0 = tmp_1;)***"; - EXPECT_EQ(code, expected_code); + EXPECT_EQ(sink.get_code(), expected_code); } { row_ir::instance_context ctx{}; - row_ir::set_output set_output_1{1, std::make_unique(1)}; + row_ir::code_sink sink; + row_ir::node set_output_1{row_ir::output_reference{1}, + std::make_unique(row_ir::input_reference{1})}; set_output_1.instantiate(ctx, info); - auto code = set_output_1.generate_code(ctx, target_info, info); + set_output_1.emit_code(ctx, target_info, info, sink); auto expected_code = R"***(float tmp_0 = in_1; float tmp_1 = tmp_0; *out_1 = tmp_1;)***"; - EXPECT_EQ(code, expected_code); + EXPECT_EQ(sink.get_code(), expected_code); } } @@ -107,30 +113,32 @@ TEST_F(RowIRCudaCodeGenTest, UnaryOperation) { row_ir::instance_context ctx{}; - row_ir::operation op{row_ir::opcode::IDENTITY, - row_ir::operation::operands(row_ir::get_input(0))}; + row_ir::code_sink sink; + row_ir::node op{row_ir::opcode::IDENTITY, row_ir::node{row_ir::input_reference{0}}}; op.instantiate(ctx, info); - auto code = op.generate_code(ctx, target_info, info); + op.emit_code(ctx, target_info, info, sink); auto expected_code = R"***(int32_t tmp_0 = in_0; -int32_t tmp_1 = cudf::ast::detail::operator_functor{}(tmp_0);)***"; +int32_t tmp_1; +cudf::ops::identity(&tmp_1, &tmp_0);)***"; - EXPECT_EQ(code, expected_code); + EXPECT_EQ(sink.get_code(), expected_code); } { row_ir::instance_context ctx{}; - row_ir::operation op{row_ir::opcode::IDENTITY, - row_ir::operation::operands(row_ir::get_input(1))}; + row_ir::code_sink sink; + row_ir::node op{row_ir::opcode::IDENTITY, row_ir::node{row_ir::input_reference{1}}}; op.instantiate(ctx, info); - auto null_code = op.generate_code(ctx, target_info, info); + op.emit_code(ctx, target_info, info, sink); auto expected_null_code = R"***(numeric::decimal32 tmp_0 = in_1; -numeric::decimal32 tmp_1 = cudf::ast::detail::operator_functor{}(tmp_0);)***"; +numeric::decimal32 tmp_1; +cudf::ops::identity(&tmp_1, &tmp_0);)***"; - EXPECT_EQ(null_code, expected_null_code); + EXPECT_EQ(sink.get_code(), expected_null_code); } } @@ -147,32 +155,38 @@ TEST_F(RowIRCudaCodeGenTest, BinaryOperation) { row_ir::instance_context ctx{}; - row_ir::operation op{row_ir::opcode::ADD, - row_ir::operation::operands(row_ir::get_input(0), row_ir::get_input(0))}; + row_ir::code_sink sink; + row_ir::node op{row_ir::opcode::ADD, + row_ir::node{row_ir::input_reference{0}}, + row_ir::node{row_ir::input_reference{0}}}; op.instantiate(ctx, info); - auto code = op.generate_code(ctx, target_info, info); + op.emit_code(ctx, target_info, info, sink); auto expected_code = R"***(int32_t tmp_0 = in_0; int32_t tmp_1 = in_0; -int32_t tmp_2 = cudf::ast::detail::operator_functor{}(tmp_0, tmp_1);)***"; +int32_t tmp_2; +cudf::ops::add(&tmp_2, &tmp_0, &tmp_1);)***"; - EXPECT_EQ(code, expected_code); + EXPECT_EQ(sink.get_code(), expected_code); } { row_ir::instance_context ctx{}; - row_ir::operation op{row_ir::opcode::ADD, - row_ir::operation::operands(row_ir::get_input(1), row_ir::get_input(1))}; + row_ir::code_sink sink; + row_ir::node op{row_ir::opcode::ADD, + row_ir::node{row_ir::input_reference{1}}, + row_ir::node{row_ir::input_reference{1}}}; op.instantiate(ctx, info); - auto null_code = op.generate_code(ctx, target_info, info); + op.emit_code(ctx, target_info, info, sink); auto expected_null_code = R"***(numeric::decimal32 tmp_0 = in_1; numeric::decimal32 tmp_1 = in_1; -numeric::decimal32 tmp_2 = cudf::ast::detail::operator_functor{}(tmp_0, tmp_1);)***"; +numeric::decimal32 tmp_2; +cudf::ops::add(&tmp_2, &tmp_0, &tmp_1);)***"; - EXPECT_EQ(null_code, expected_null_code); + EXPECT_EQ(sink.get_code(), expected_null_code); } } @@ -195,44 +209,46 @@ TEST_F(RowIRCudaCodeGenTest, VectorLengthOperation) // This function generates the IR for the vector length operation: // length(v) = sqrt(x^2 + y^2) // where v = (x, y) and v is a 2D vector. - auto x2 = std::make_unique( - row_ir::opcode::MUL, - row_ir::operation::operands(row_ir::get_input(input0), row_ir::get_input(input0))); + auto x2 = row_ir::node(row_ir::opcode::MUL, + row_ir::node{row_ir::input_reference{input0}}, + row_ir::node{row_ir::input_reference{input0}}); - auto y2 = std::make_unique( - row_ir::opcode::MUL, - row_ir::operation::operands(row_ir::get_input(input1), row_ir::get_input(input1))); + auto y2 = row_ir::node(row_ir::opcode::MUL, + row_ir::node{row_ir::input_reference{input1}}, + row_ir::node{row_ir::input_reference{input1}}); - auto sum = std::make_unique( - row_ir::opcode::ADD, row_ir::operation::operands(std::move(x2), std::move(y2))); + auto sum = row_ir::node(row_ir::opcode::ADD, std::move(x2), std::move(y2)); - auto length = std::make_unique(row_ir::opcode::SQRT, - row_ir::operation::operands(std::move(sum))); + auto length = row_ir::node(row_ir::opcode::SQRT, std::move(sum)); - return std::make_unique(output, std::move(length)); + return std::make_unique(row_ir::opcode::SET_OUTPUT, std::move(length)); }; { row_ir::instance_context ctx{}; + row_ir::code_sink sink; auto expr_ir = length_operation(0, 1, 0); expr_ir->instantiate(ctx, info); - - auto code = expr_ir->generate_code(ctx, target_info, info); + expr_ir->emit_code(ctx, target_info, info, sink); auto expected_code = R"***(double tmp_0 = in_0; double tmp_1 = in_0; -double tmp_2 = cudf::ast::detail::operator_functor{}(tmp_0, tmp_1); +double tmp_2; +cudf::ops::mul(&tmp_2, &tmp_0, &tmp_1); double tmp_3 = in_1; double tmp_4 = in_1; -double tmp_5 = cudf::ast::detail::operator_functor{}(tmp_3, tmp_4); -double tmp_6 = cudf::ast::detail::operator_functor{}(tmp_2, tmp_5); -double tmp_7 = cudf::ast::detail::operator_functor{}(tmp_6); +double tmp_5; +cudf::ops::mul(&tmp_5, &tmp_3, &tmp_4); +double tmp_6; +cudf::ops::add(&tmp_6, &tmp_2, &tmp_5); +double tmp_7; +cudf::ops::sqrt(&tmp_7, &tmp_6); double tmp_8 = tmp_7; *out_0 = tmp_8;)***"; - EXPECT_EQ(code, expected_code); + EXPECT_EQ(sink.get_code(), expected_code); } } @@ -287,7 +303,8 @@ __device__ void expression(int32_t* out_0, int32_t in_0, int32_t in_1) { int32_t tmp_0 = in_0; int32_t tmp_1 = in_1; -int32_t tmp_2 = cudf::ast::detail::operator_functor{}(tmp_0, tmp_1); +int32_t tmp_2; +cudf::ops::add(&tmp_2, &tmp_0, &tmp_1); int32_t tmp_3 = tmp_2; *out_0 = tmp_3; @@ -319,29 +336,35 @@ TEST_F(RowIRCudaCodeGenTest, FilterPredicate) { row_ir::instance_context ctx{}; - row_ir::filter_predicate filter_predicate(std::make_unique(0)); + row_ir::code_sink sink; + row_ir::node filter_predicate(row_ir::opcode::PREDICATE, + std::make_unique(row_ir::input_reference{0})); filter_predicate.instantiate(ctx, info); - auto code = filter_predicate.generate_code(ctx, target_info, info); + filter_predicate.emit_code(ctx, target_info, info, sink); auto expected_code = R"***(bool tmp_0 = in_0; -bool tmp_1 = cudf::ast::detail::flatten_predicate(tmp_0); +bool tmp_1; +cudf::ops::predicate(&tmp_1, &tmp_0); )***"; - EXPECT_EQ(code, expected_code); + EXPECT_EQ(sink.get_code(), expected_code); } { row_ir::instance_context ctx{}; - row_ir::filter_predicate filter_predicate(std::make_unique(0)); + row_ir::code_sink sink; + row_ir::node filter_predicate(row_ir::opcode::PREDICATE, + std::make_unique(row_ir::input_reference{0})); ctx.set_has_nulls(true); filter_predicate.instantiate(ctx, info); - auto null_code = filter_predicate.generate_code(ctx, target_info, info); + filter_predicate.emit_code(ctx, target_info, info, sink); auto expected_code = R"***(cuda::std::optional tmp_0 = in_0; -bool tmp_1 = cudf::ast::detail::flatten_predicate(tmp_0); +cuda::std::optional tmp_1; +cudf::ops::predicate(&tmp_1, &tmp_0); )***"; - EXPECT_EQ(null_code, expected_code); + EXPECT_EQ(sink.get_code(), expected_code); } } From a62cdfd145eafa39ec90cc00a90ffcac3e6c02d5 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 1 May 2026 17:34:56 +0000 Subject: [PATCH 07/15] update --- .../cudf/ast/detail/operator_functor.cuh | 7 - cpp/include/cudf/ast/expressions.hpp | 2 +- cpp/include/cudf/ast/jit_expressions.hpp | 4 +- cpp/include/cudf/operators/casts.cuh | 20 +- cpp/include/cudf/operators/logic.cuh | 26 +- cpp/include/cudf/operators/op_traits.hpp | 63 +-- cpp/include/cudf/operators/opcodes.hpp | 4 +- cpp/src/jit/column_accessor.cuh | 11 +- cpp/src/jit/join_column_accessor.cuh | 129 ------ cpp/src/jit/row_ir.cpp | 304 +++++++------- cpp/src/jit/row_ir.hpp | 188 ++++----- cpp/src/join/filter_join_indices_jit.cu | 203 ++++----- cpp/src/join/jit/filter_join_kernel.cu | 75 ++-- cpp/src/join/jit/filter_join_kernel.cuh | 21 +- cpp/src/stream_compaction/filter/filter.cu | 106 ++--- cpp/src/transform/transform.cu | 81 ++-- cpp/tests/jit/row_ir.cpp | 384 +++++++++++------- 17 files changed, 768 insertions(+), 860 deletions(-) delete mode 100644 cpp/src/jit/join_column_accessor.cuh diff --git a/cpp/include/cudf/ast/detail/operator_functor.cuh b/cpp/include/cudf/ast/detail/operator_functor.cuh index 043b2c47746d..5abe32887054 100644 --- a/cpp/include/cudf/ast/detail/operator_functor.cuh +++ b/cpp/include/cudf/ast/detail/operator_functor.cuh @@ -778,12 +778,5 @@ struct operator_functor { } }; -constexpr bool flatten_predicate(possibly_null_value_t value) { return value; } - -constexpr bool flatten_predicate(possibly_null_value_t value) -{ - return value.has_value() && *value; -} - } // namespace ast::detail } // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/ast/expressions.hpp b/cpp/include/cudf/ast/expressions.hpp index 71590f593a01..504679bd9788 100644 --- a/cpp/include/cudf/ast/expressions.hpp +++ b/cpp/include/cudf/ast/expressions.hpp @@ -83,7 +83,7 @@ struct expression { * @param visitor The `row_ir::ast_converter` converting this expression tree * @return The IR node representing this expression */ - virtual std::unique_ptr accept( + [[nodiscard]] virtual std::unique_ptr accept( cudf::detail::row_ir::ast_converter& visitor) const = 0; /** diff --git a/cpp/include/cudf/ast/jit_expressions.hpp b/cpp/include/cudf/ast/jit_expressions.hpp index f051618fe8f9..044301df4af6 100644 --- a/cpp/include/cudf/ast/jit_expressions.hpp +++ b/cpp/include/cudf/ast/jit_expressions.hpp @@ -7,8 +7,8 @@ #include namespace CUDF_EXPORT cudf { - namespace ast { + /** * @addtogroup expressions * @{ @@ -20,4 +20,4 @@ namespace jit { } } // namespace ast -} // namespace CUDF_EXPORT cudf \ No newline at end of file +} // namespace CUDF_EXPORT cudf diff --git a/cpp/include/cudf/operators/casts.cuh b/cpp/include/cudf/operators/casts.cuh index a4363ac44e6a..23b4faeeb30d 100644 --- a/cpp/include/cudf/operators/casts.cuh +++ b/cpp/include/cudf/operators/casts.cuh @@ -4,6 +4,7 @@ */ #pragma once +#include #include namespace CUDF_EXPORT cudf { @@ -188,14 +189,21 @@ __device__ inline errc cast_to_u64(optional* out, optional const* a } return errc::OK; } - template + requires(std::is_integral_v || std::is_floating_point_v) __device__ inline errc cast_to_f32(float* out, T const* a) { *out = static_cast(*a); return errc::OK; } +template +__device__ inline errc cast_to_f32(float* out, decimal const* a) +{ + *out = convert_fixed_to_floating(*a); + return errc::OK; +} + template __device__ inline errc cast_to_f32(optional* out, optional const* a) { @@ -210,12 +218,20 @@ __device__ inline errc cast_to_f32(optional* out, optional const* a) } template + requires(std::is_integral_v || std::is_floating_point_v) __device__ inline errc cast_to_f64(double* out, T const* a) { *out = static_cast(*a); return errc::OK; } +template +__device__ inline errc cast_to_f64(double* out, decimal const* a) +{ + *out = convert_fixed_to_floating(*a); + return errc::OK; +} + template __device__ inline errc cast_to_f64(optional* out, optional const* a) { @@ -241,6 +257,8 @@ __device__ inline errc decimal_cast(decimal* out, decimal const* a) } // namespace detail +// TODO: CAST_TO_DEC32 for int & float + template __device__ inline errc cast_to_dec32(numeric::decimal32* out, decimal const* a) { diff --git a/cpp/include/cudf/operators/logic.cuh b/cpp/include/cudf/operators/logic.cuh index c674e4285434..f81917e83e18 100644 --- a/cpp/include/cudf/operators/logic.cuh +++ b/cpp/include/cudf/operators/logic.cuh @@ -10,14 +10,14 @@ namespace CUDF_EXPORT cudf { namespace ops { template -__device__ inline errc null_logical_and(T* out, T const* a, T const* b) +__device__ inline errc null_logical_and(bool* out, T const* a, T const* b) { *out = (*a && *b); return errc::OK; } template -__device__ inline errc null_logical_and(optional* out, +__device__ inline errc null_logical_and(optional* out, optional const* a, optional const* b) { @@ -38,14 +38,14 @@ __device__ inline errc null_logical_and(optional* out, } template -__device__ inline errc null_logical_or(T* out, T const* a, T const* b) +__device__ inline errc null_logical_or(bool* out, T const* a, T const* b) { *out = (*a || *b); return errc::OK; } template -__device__ inline errc null_logical_or(optional* out, optional const* a, optional const* b) +__device__ inline errc null_logical_or(optional* out, optional const* a, optional const* b) { if (a->has_value() && b->has_value()) { bool r; @@ -64,17 +64,17 @@ __device__ inline errc null_logical_or(optional* out, optional const* a, o } template -__device__ inline errc logical_and(T* out, T const* a, T const* b) +__device__ inline errc logical_and(bool* out, T const* a, T const* b) { *out = (*a && *b); return errc::OK; } template -__device__ inline errc logical_and(optional* out, optional const* a, optional const* b) +__device__ inline errc logical_and(optional* out, optional const* a, optional const* b) { if (a->has_value() && b->has_value()) { - T r; + bool r; logical_and(&r, &a->value(), &b->value()); *out = r; } else { @@ -84,17 +84,17 @@ __device__ inline errc logical_and(optional* out, optional const* a, optio } template -__device__ inline errc logical_or(T* out, T const* a, T const* b) +__device__ inline errc logical_or(bool* out, T const* a, T const* b) { *out = (*a || *b); return errc::OK; } template -__device__ inline errc logical_or(optional* out, optional const* a, optional const* b) +__device__ inline errc logical_or(optional* out, optional const* a, optional const* b) { if (a->has_value() && b->has_value()) { - T r; + bool r; logical_or(&r, &a->value(), &b->value()); *out = r; } else { @@ -104,17 +104,17 @@ __device__ inline errc logical_or(optional* out, optional const* a, option } template -__device__ inline errc logical_not(T* out, T const* a) +__device__ inline errc logical_not(bool* out, T const* a) { *out = !(*a); return errc::OK; } template -__device__ inline errc logical_not(optional* out, optional const* a) +__device__ inline errc logical_not(optional* out, optional const* a) { if (a->has_value()) { - T r; + bool r; logical_not(&r, &a->value()); *out = r; } else { diff --git a/cpp/include/cudf/operators/op_traits.hpp b/cpp/include/cudf/operators/op_traits.hpp index 1948ef038e97..8d3cdfaa8471 100644 --- a/cpp/include/cudf/operators/op_traits.hpp +++ b/cpp/include/cudf/operators/op_traits.hpp @@ -6,6 +6,7 @@ #pragma once #include +#include #include namespace cudf::detail::row_ir { @@ -37,12 +38,12 @@ enum [[nodiscard]] type : uint64_t { DURATION_MICROSECONDS = 0x400000, DURATION_NANOSECONDS = 0x800000, STRING = 0x1000000, - INTEGERS = INT8 | INT16 | INT32 | INT64 | UINT8 | UINT16 | UINT32 | UINT64, SIGNED_INTEGERS = INT8 | INT16 | INT32 | INT64, UNSIGNED_INTEGERS = UINT8 | UINT16 | UINT32 | UINT64, + INTEGERS = SIGNED_INTEGERS | UNSIGNED_INTEGERS, FLOATS = FLOAT32 | FLOAT64, DECIMALS = DECIMAL32 | DECIMAL64 | DECIMAL128, - ARITHMETIC = SIGNED_INTEGERS | UNSIGNED_INTEGERS | FLOATS | DECIMALS, + ARITHMETIC = INTEGERS | FLOATS | DECIMALS, SIGNED_ARITHMETIC = SIGNED_INTEGERS | FLOATS | DECIMALS, ALL = 0x0FFFFFFF, ARG_MASK = 0x10000000, @@ -107,8 +108,8 @@ enum class [[nodiscard]] null_output : uint8_t { case opcode::BIT_INVERT: return "bit_invert"; case opcode::BIT_OR: return "bit_or"; case opcode::BIT_XOR: return "bit_xor"; - case opcode::SHIFT_LEFT: return "shift_left"; - case opcode::SHIFT_RIGHT: return "shift_right"; + case opcode::BIT_SHIFT_LEFT: return "bit_shift_left"; + case opcode::BIT_SHIFT_RIGHT: return "bit_shift_right"; case opcode::CAST_TO_B8: return "cast_to_b8"; case opcode::CAST_TO_I8: return "cast_to_i8"; case opcode::CAST_TO_I16: return "cast_to_i16"; @@ -219,13 +220,12 @@ enum class [[nodiscard]] null_output : uint8_t { } } -[[nodiscard]] inline int32_t get_output_decimal_scale(opcode op, - std::span arg_scales, - std::optional output_scale) +[[nodiscard]] inline int32_t op_rescale(opcode op, + std::span arg_scales, + std::optional target_scale) { - // TODO: finish up switch (op) { - case opcode::GET_INPUT: + case opcode::GET_INPUT: return 0; case opcode::SET_OUTPUT: case opcode::IDENTITY: case opcode::COALESCE: @@ -283,8 +283,8 @@ enum class [[nodiscard]] null_output : uint8_t { case opcode::BIT_INVERT: case opcode::BIT_OR: case opcode::BIT_XOR: - case opcode::SHIFT_LEFT: - case opcode::SHIFT_RIGHT: + case opcode::BIT_SHIFT_LEFT: + case opcode::BIT_SHIFT_RIGHT: case opcode::CBRT: case opcode::CEIL: case opcode::FLOOR: @@ -311,7 +311,7 @@ enum class [[nodiscard]] null_output : uint8_t { case opcode::MUL: case opcode::ANSI_MUL: case opcode::ANSI_TRY_MUL: return arg_scales[0] + arg_scales[1]; - case opcode::RESCALE: return output_scale.value(); + case opcode::RESCALE: return target_scale.value(); default: CUDF_UNREACHABLE("Invalid opcode"); } } @@ -326,7 +326,6 @@ enum class [[nodiscard]] null_output : uint8_t { */ [[nodiscard]] inline op_type get_op_typing(opcode op) { - // TODO: finish up switch (op) { case opcode::GET_INPUT: return {type::INPUT, {}}; case opcode::SET_OUTPUT: return {type::NONE, {type::ALL}}; @@ -341,8 +340,9 @@ enum class [[nodiscard]] null_output : uint8_t { case opcode::ANSI_NEG: case opcode::ANSI_TRY_NEG: case opcode::ANSI_TRY_ABS: return {type::ARG0, {type::ARITHMETIC}}; - case opcode::FLOOR_DIV: - case opcode::TRUE_DIV: return {type::ARG0, {type{type::FLOATS | type::INTEGERS}, type::ARG0}}; + case opcode::FLOOR_DIV: return {type::ARG0, {type{type::FLOATS | type::INTEGERS}, type::ARG0}}; + case opcode::TRUE_DIV: + return {type::FLOAT64, {type{type::FLOATS | type::INTEGERS}, type::ARG0}}; case opcode::ADD: case opcode::DIV: case opcode::MOD: @@ -365,19 +365,19 @@ enum class [[nodiscard]] null_output : uint8_t { case opcode::BIT_INVERT: case opcode::BIT_OR: case opcode::BIT_XOR: - case opcode::SHIFT_LEFT: - case opcode::SHIFT_RIGHT: return {type::ARG0, {type::INTEGERS, type::ARG0}}; - case opcode::CAST_TO_B8: return {type::BOOL8, {type{type::INTEGERS | type::FLOATS}}}; - case opcode::CAST_TO_I8: return {type::INT8, {type{type::INTEGERS | type::FLOATS}}}; - case opcode::CAST_TO_I16: return {type::INT16, {type{type::INTEGERS | type::FLOATS}}}; - case opcode::CAST_TO_I32: return {type::INT32, {type{type::INTEGERS | type::FLOATS}}}; - case opcode::CAST_TO_I64: return {type::INT64, {type{type::INTEGERS | type::FLOATS}}}; - case opcode::CAST_TO_U8: return {type::UINT8, {type{type::INTEGERS | type::FLOATS}}}; - case opcode::CAST_TO_U16: return {type::UINT16, {type{type::INTEGERS | type::FLOATS}}}; - case opcode::CAST_TO_U32: return {type::UINT32, {type{type::INTEGERS | type::FLOATS}}}; - case opcode::CAST_TO_U64: return {type::UINT64, {type{type::INTEGERS | type::FLOATS}}}; - case opcode::CAST_TO_F32: return {type::FLOAT32, {type{type::INTEGERS | type::FLOATS}}}; - case opcode::CAST_TO_F64: return {type::FLOAT64, {type{type::INTEGERS | type::FLOATS}}}; + case opcode::BIT_SHIFT_LEFT: + case opcode::BIT_SHIFT_RIGHT: return {type::ARG0, {type::INTEGERS, type::ARG0}}; + case opcode::CAST_TO_B8: return {type::BOOL8, {type{type::ARITHMETIC | type::BOOL8}}}; + case opcode::CAST_TO_I8: return {type::INT8, {type{type::ARITHMETIC | type::BOOL8}}}; + case opcode::CAST_TO_I16: return {type::INT16, {type{type::ARITHMETIC | type::BOOL8}}}; + case opcode::CAST_TO_I32: return {type::INT32, {type{type::ARITHMETIC | type::BOOL8}}}; + case opcode::CAST_TO_I64: return {type::INT64, {type{type::ARITHMETIC | type::BOOL8}}}; + case opcode::CAST_TO_U8: return {type::UINT8, {type{type::ARITHMETIC | type::BOOL8}}}; + case opcode::CAST_TO_U16: return {type::UINT16, {type{type::ARITHMETIC | type::BOOL8}}}; + case opcode::CAST_TO_U32: return {type::UINT32, {type{type::ARITHMETIC | type::BOOL8}}}; + case opcode::CAST_TO_U64: return {type::UINT64, {type{type::ARITHMETIC | type::BOOL8}}}; + case opcode::CAST_TO_F32: return {type::FLOAT32, {type{type::ARITHMETIC | type::BOOL8}}}; + case opcode::CAST_TO_F64: return {type::FLOAT64, {type{type::ARITHMETIC | type::BOOL8}}}; case opcode::CAST_TO_DEC32: return {type::DECIMAL32, {type::DECIMALS}}; case opcode::CAST_TO_DEC64: return {type::DECIMAL64, {type::DECIMALS}}; case opcode::CAST_TO_DEC128: return {type::DECIMAL128, {type::DECIMALS}}; @@ -386,14 +386,15 @@ enum class [[nodiscard]] null_output : uint8_t { case opcode::GREATER: case opcode::GREATER_EQUAL: case opcode::LESS: - case opcode::LESS_EQUAL: return {type::BOOL8, {type::ALL, type::ARG0}}; + case opcode::LESS_EQUAL: case opcode::NOT_EQUAL: case opcode::NULL_EQUAL: return {type::BOOL8, {type::ALL, type::ARG0}}; case opcode::NULL_LOGICAL_AND: case opcode::NULL_LOGICAL_OR: case opcode::LOGICAL_AND: - case opcode::LOGICAL_OR: return {type::BOOL8, {type::BOOL8, type::ARG0}}; - case opcode::LOGICAL_NOT: return {type::ARG0, {type::BOOL8}}; + case opcode::LOGICAL_OR: + return {type::BOOL8, {type{type::ARITHMETIC | type::BOOL8}, type::ARG0}}; + case opcode::LOGICAL_NOT: return {type::BOOL8, {type{type::ARITHMETIC | type::BOOL8}}}; case opcode::IF_ELSE: return {type::ARG0, {type::ALL, type::ARG0, type::BOOL8}}; case opcode::CBRT: case opcode::CEIL: diff --git a/cpp/include/cudf/operators/opcodes.hpp b/cpp/include/cudf/operators/opcodes.hpp index 36c4a6eb3cca..b0e9c9cfcf3e 100644 --- a/cpp/include/cudf/operators/opcodes.hpp +++ b/cpp/include/cudf/operators/opcodes.hpp @@ -62,8 +62,8 @@ enum class [[nodiscard]] opcode : int32_t { BIT_INVERT, BIT_OR, BIT_XOR, - SHIFT_LEFT, - SHIFT_RIGHT, + BIT_SHIFT_LEFT, + BIT_SHIFT_RIGHT, /// Type conversion operators CAST_TO_B8, diff --git a/cpp/src/jit/column_accessor.cuh b/cpp/src/jit/column_accessor.cuh index 303e8cb029f3..ef1536cb409d 100644 --- a/cpp/src/jit/column_accessor.cuh +++ b/cpp/src/jit/column_accessor.cuh @@ -16,12 +16,13 @@ namespace cudf { namespace jit { -template +template struct column_accessor { - static constexpr int32_t index = Index; - using column_type = Column; - using element_type = Element; - using optional_element_type = cuda::std::optional; + static constexpr int32_t index = Index; + static constexpr int32_t table_index = TableIndex; + using column_type = Column; + using element_type = Element; + using optional_element_type = cuda::std::optional; static constexpr bool as_scalar = AsScalar; diff --git a/cpp/src/jit/join_column_accessor.cuh b/cpp/src/jit/join_column_accessor.cuh deleted file mode 100644 index e78a9ec1991c..000000000000 --- a/cpp/src/jit/join_column_accessor.cuh +++ /dev/null @@ -1,129 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once -#include -#include - -#include -#include - -namespace cudf { -namespace jit { - -// Join-specific accessor for indexed table access. -// Receives both left and right table pointers plus both row indices, -// and selects the appropriate table based on the Side template parameter. -enum class join_side : bool { LEFT, RIGHT }; - -template -struct join_column_accessor { - using type = T; - static constexpr int32_t index = Index; - - static __device__ T element(cudf::column_device_view_core const* left_tables, - cudf::column_device_view_core const* right_tables, - cudf::size_type left_row_idx, - cudf::size_type right_row_idx, - cudf::size_type /* thread_idx */) - { - if constexpr (Side == join_side::LEFT) { - return left_tables[index].template element(left_row_idx); - } else { - return right_tables[index].template element(right_row_idx); - } - } - - static __device__ bool is_null(cudf::column_device_view_core const* left_tables, - cudf::column_device_view_core const* right_tables, - cudf::size_type left_row_idx, - cudf::size_type right_row_idx, - cudf::size_type /* thread_idx */) - { - if constexpr (Side == join_side::LEFT) { - return left_tables[index].is_null(left_row_idx); - } else { - return right_tables[index].is_null(right_row_idx); - } - } - - static __device__ bool is_valid(cudf::column_device_view_core const* left_tables, - cudf::column_device_view_core const* right_tables, - cudf::size_type left_row_idx, - cudf::size_type right_row_idx, - cudf::size_type /* thread_idx */) - { - if constexpr (Side == join_side::LEFT) { - return left_tables[index].is_valid(left_row_idx); - } else { - return right_tables[index].is_valid(right_row_idx); - } - } - - static __device__ cuda::std::optional nullable_element( - cudf::column_device_view_core const* left_tables, - cudf::column_device_view_core const* right_tables, - cudf::size_type left_row_idx, - cudf::size_type right_row_idx, - cudf::size_type thread_idx) - { - if (is_null(left_tables, right_tables, left_row_idx, right_row_idx, thread_idx)) { - return cuda::std::nullopt; - } - return element(left_tables, right_tables, left_row_idx, right_row_idx, thread_idx); - } -}; - -// Join-specific accessor for scalar (literal) values. -// Scalar columns are appended to the left table's device views. -// Always reads at row 0 since scalar columns have size 1. -template -struct join_scalar_accessor { - using type = T; - static constexpr int32_t index = Index; - - static __device__ T element(cudf::column_device_view_core const* left_tables, - cudf::column_device_view_core const*, - cudf::size_type, - cudf::size_type, - cudf::size_type) - { - return left_tables[index].template element(0); - } - - static __device__ bool is_null(cudf::column_device_view_core const* left_tables, - cudf::column_device_view_core const*, - cudf::size_type, - cudf::size_type, - cudf::size_type) - { - return left_tables[index].is_null(0); - } - - static __device__ bool is_valid(cudf::column_device_view_core const* left_tables, - cudf::column_device_view_core const*, - cudf::size_type, - cudf::size_type, - cudf::size_type) - { - return left_tables[index].is_valid(0); - } - - static __device__ cuda::std::optional nullable_element( - cudf::column_device_view_core const* left_tables, - cudf::column_device_view_core const* right_tables, - cudf::size_type left_row_idx, - cudf::size_type right_row_idx, - cudf::size_type thread_idx) - { - if (is_null(left_tables, right_tables, left_row_idx, right_row_idx, thread_idx)) { - return cuda::std::nullopt; - } - return element(left_tables, right_tables, left_row_idx, right_row_idx, thread_idx); - } -}; - -} // namespace jit -} // namespace cudf diff --git a/cpp/src/jit/row_ir.cpp b/cpp/src/jit/row_ir.cpp index 02435c479df8..3f41044b81ac 100644 --- a/cpp/src/jit/row_ir.cpp +++ b/cpp/src/jit/row_ir.cpp @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include @@ -20,41 +19,28 @@ namespace cudf::detail::row_ir { -// Resolve the table for a column input spec, preferring left_table/right_table for join cases, -// falling back to args.table for the single-table case. -table_view const& resolve_table(ast_column_input_spec const& in, ast_args const& args) +int32_t instance_context::add_output() { - if (in.table == ast::table_reference::LEFT) { - return args.left_table.num_columns() > 0 ? args.left_table : args.table; - } - return args.right_table; -} - -void instance_context::add_input_var(ast_column_input_spec const& in, ast_args const& args) -{ - auto id = std::format("in_{}", input_vars_.size()); - auto type = resolve_table(in, args).column(in.column).type(); - input_vars_.emplace_back(std::move(id), type); -} - -void instance_context::add_input_var(ast_scalar_input_spec const& in, - [[maybe_unused]] ast_args const& args) -{ - auto id = std::format("in_{}", input_vars_.size()); - auto type = in.scalar_column->type(); - input_vars_.emplace_back(std::move(id), type); + auto id = static_cast(output_vars_.size()); + auto id_str = std::format("out_{}", id); + output_vars_.emplace_back(std::move(id_str)); + return id; } -void instance_context::add_output_var() +int32_t instance_context::add_input(input in) { - auto id = std::format("out_{}", output_vars_.size()); - output_vars_.emplace_back(std::move(id)); -} + auto id = static_cast(inputs_.size()); + auto id_str = std::format("in_{}", id); -int32_t instance_context::add_ast_input(ast_input_spec in) -{ - auto id = static_cast(input_specs_.size()); - input_specs_.emplace_back(std::move(in)); + data_type type{type_id::EMPTY, 0}; + if (auto* col = std::get_if(&in)) { + type = col->column.type(); + } else { + auto& scalar = std::get(in); + type = scalar.scalar_column->type(); + } + inputs_.emplace_back(std::move(in)); + input_vars_.emplace_back(std::move(id_str), type); return id; } @@ -67,17 +53,11 @@ bool instance_context::has_nulls() const { return has_nulls_; } void instance_context::set_has_nulls(bool has_nulls) { has_nulls_ = has_nulls; } -std::span instance_context::get_input_specs() const { return input_specs_; } - -std::span instance_context::get_inputs() const { return input_vars_; } +std::span instance_context::get_inputs() const { return inputs_; } -std::span instance_context::get_outputs() const { return output_vars_; } +std::span instance_context::get_input_vars() const { return input_vars_; } -int32_t instance_context::add_constant(cudf::scalar const& value) -{ - auto scalar_column = make_column_from_scalar(value, 1, stream_, mr_); - return add_ast_input(ast_scalar_input_spec{.scalar_column = std::move(scalar_column)}); -} +std::span instance_context::get_output_vars() const { return output_vars_; } node::node(opcode op, std::optional target_scale, std::vector> args) : op_{op}, target_scale_{target_scale}, args_{std::move(args)} @@ -102,6 +82,11 @@ node::node(output_reference reference, std::unique_ptr arg) args_.emplace_back(std::move(arg)); } +node::node(output_reference reference, node arg) + : node{reference, std::make_unique(std::move(arg))} +{ +} + std::string_view node::get_id() const { return id_; } data_type node::get_type() const { return type_; } @@ -264,19 +249,20 @@ std::string to_cuda_type(cudf::data_type type, bool nullable) return nullable ? std::format("cuda::std::optional<{}>", name) : name; } -// TODO: implicit casts; what level should this be handled at? AST, IR, or codegen?; AST certainly, -// IR maps to functions with specific type signatures data_type get_return_type(opcode op, std::span args, std::optional target_scale) { std::vector arg_types; + std::vector arg_scales; for (auto& type : args) { arg_types.emplace_back(as_typing(type)); + arg_scales.emplace_back(type.scale()); } auto op_type_match = get_op_typing(op); + auto rescaled = op_rescale(op, arg_scales, target_scale); for (size_t i = 0; i < args.size(); ++i) { auto required_type = op_type_match.args[i]; @@ -286,39 +272,40 @@ data_type get_return_type(opcode op, auto src_index = static_cast(required_type & ~type::ARG_MASK); CUDF_EXPECTS( src_index < i, - std::format("Invalid type match rule for operator `{}` at argument {}", get_op_name(op), i), + std::format( + "Invalid type match rule for operator `{}` at argument #{}", get_op_name(op), i), std::runtime_error); CUDF_EXPECTS(args[i].id() == args[src_index].id(), - std::format("Argument {} of operator `{}` does not match type of argument " - "`{}`. Got `{}`, expected `{}`", + std::format("Argument #{} of operator `{}` does not match type of argument " + "#{}. Got `{}`, expected `{}`", i, get_op_name(op), src_index, type_to_name(args[i]), type_to_name(args[src_index]))); } else { - CUDF_EXPECTS((arg_type & required_type) != 0, - std::format("Argument {} of operator `{}` does not match expected types. Got {}", - i, - get_op_name(op), - type_to_name(args[i]))); + CUDF_EXPECTS( + (arg_type & required_type) != 0, + std::format("Argument #{} of operator `{}` does not match expected types. Got {}", + i, + get_op_name(op), + type_to_name(args[i]))); } } if ((op_type_match.output & type::ARG_MASK) != type::NONE) { auto arg_index = static_cast(op_type_match.output & ~type::ARG_MASK); - auto type = args[arg_index]; - if (target_scale.has_value()) { - type = data_type{type.id(), numeric::scale_type{target_scale.value()}}; - } - return type; + auto type = args[arg_index].id(); + auto scale = numeric::scale_type{is_fixed_point(data_type{type}) ? rescaled : 0}; + return data_type{type, scale}; } else { CUDF_EXPECTS( op_type_match.output != type::NONE && (op_type_match.output & type::DECIMALS) == type::NONE, std::format("Invalid type match rule for operator `{}` return type", get_op_name(op)), std::runtime_error); - return data_type{as_type_id(op_type_match.output), - numeric::scale_type{target_scale.value_or(0)}}; + auto type = as_type_id(op_type_match.output); + auto scale = numeric::scale_type{is_fixed_point(data_type{type}) ? rescaled : 0}; + return data_type{type, scale}; } } @@ -332,7 +319,7 @@ void node::instantiate(instance_context& ctx) switch (op_) { case opcode::GET_INPUT: { - type_ = ctx.get_inputs()[std::get(reference_).index].type; + type_ = ctx.get_input_vars()[std::get(reference_).index].type; } break; case opcode::SET_OUTPUT: { type_ = args_[0]->get_type(); @@ -344,8 +331,8 @@ void node::instantiate(instance_context& ctx) } if (op_ == opcode::RESCALE) { - scale_reference_ = scalar_refernce{ - ctx.add_constant(cudf::numeric_scalar{target_scale_.value_or(0)})}; + scale_reference_ = + input_reference{ctx.add_input(cudf::numeric_scalar{target_scale_.value_or(0)})}; } type_ = get_return_type(op_, arg_types, target_scale_); @@ -370,7 +357,7 @@ void node::emit_code(instance_context& instance, target_info const& info, code_s )***", type, id_, - instance.get_inputs()[std::get(reference_).index].id)); + instance.get_input_vars()[std::get(reference_).index].id)); } break; case opcode::SET_OUTPUT: { @@ -381,7 +368,7 @@ void node::emit_code(instance_context& instance, target_info const& info, code_s type, id_, args_[0]->get_id(), - instance.get_outputs()[std::get(reference_).index].id, + instance.get_output_vars()[std::get(reference_).index].id, id_)); } break; @@ -397,8 +384,8 @@ void node::emit_code(instance_context& instance, target_info const& info, code_s }); if (op_ == opcode::RESCALE) { - args_str = - std::format("{}, &{}", args_str, instance.get_inputs()[scale_reference_.index].id); + args_str = std::format( + "{}, &{}", args_str, instance.get_input_vars()[scale_reference_.index].id); } bool fallible = get_op_is_fallible(op_); @@ -417,8 +404,8 @@ cudf::ops::{}(&{}, {}); } else { sink.emit(std::format( R"***({} {}; -if(cudf::ops::errc e = cudf::ops::{}(&{}, {}); e != cudf::ops::errc::SUCCESS) {{ - return e; +if(cudf::ops::errc e = cudf::ops::{}(&{}, {}); e != cudf::ops::errc::OK) {{ +return e; }} )***", type, @@ -439,16 +426,26 @@ if(cudf::ops::errc e = cudf::ops::{}(&{}, {}); e != cudf::ops::errc::SUCCESS) {{ std::unique_ptr ast_converter::add_ir_node(ast::literal const& expr) { - auto index = instance_.add_ast_input( - ast_scalar_input_spec{make_column_from_scalar(expr.get_scalar(), 1, stream_, mr_)}); - return std::make_unique(input_reference{index}); + auto id = instance_.add_input(expr.get_scalar()); + return std::make_unique(input_reference{id}); } std::unique_ptr ast_converter::add_ir_node(ast::column_reference const& expr) { - auto index = instance_.add_ast_input( - ast_column_input_spec{expr.get_table_source(), expr.get_column_index()}); - return std::make_unique(input_reference{index}); + // resolve the table for a column input spec, preferring left_table/right_table for join cases, + // falling back to args.table for the single-table case. + auto resolve = [&](ast::table_reference ref) { + CUDF_EXPECTS(ref == ast::table_reference::LEFT || ref == ast::table_reference::RIGHT, + "Invalid table reference in column expression"); + return ref == ast::table_reference::LEFT ? left_table_ : right_table_; + }; + + auto table = resolve(expr.get_table_source()); + auto id = instance_.add_input( + column_input{.column = table.column(expr.get_column_index()), + .table_source = (expr.get_table_source() == ast::table_reference::LEFT ? 0 : 1), + .column_index = static_cast(expr.get_column_index())}); + return std::make_unique(input_reference{id}); } std::unique_ptr ast_converter::add_ir_node(ast::operation const& expr) @@ -467,54 +464,23 @@ std::unique_ptr ast_converter::add_ir_node(ast::detail::predicate row_ir::opcode::PREDICATE, std::nullopt, expr.get_operand().accept(*this)); } -template -decltype(auto) dispatch_input_spec(ast_input_spec const& in, Fn&& fn, Args&&... args) -{ - if (std::holds_alternative(in)) { - return fn(std::get(in), std::forward(args)...); - } else if (std::holds_alternative(in)) { - return fn(std::get(in), std::forward(args)...); - } else { - CUDF_FAIL("Unsupported input type"); - } -} +bool is_nullable(scalar_input const& in) { return in.scalar_column->view().nullable(); } -std::variant get_column_view(ast_column_input_spec const& spec, - ast_args const& args) -{ - return resolve_table(spec, args).column(spec.column); -} - -std::variant get_column_view(ast_scalar_input_spec const& spec, - ast_args const& args) -{ - return scalar_column_view{spec.scalar_column->view()}; -} +bool is_nullable(column_input const& in) { return in.column.nullable(); } std::tuple ast_converter::generate_code( - target target_id, ast::expression const& expr, ast_args const& args) + target target_id, ast::expression const& expr, std::string_view function_name) { - output_irs_.emplace_back(std::make_unique(output_reference{0}, expr.accept(*this))); + // add 1 auto-deduced output variable + [[maybe_unused]] auto output_id = instance_.add_output(); - // resolve the flattened input references into IR input variables - for (auto& input : instance_.input_specs_) { - dispatch_input_spec(input, [&](auto&... args) { instance_.add_input_var(args...); }, args); - } + output_irs_.emplace_back(std::make_unique(output_reference{0}, expr.accept(*this))); - bool has_nullable_inputs = std::any_of( - instance_.input_specs_.begin(), instance_.input_specs_.end(), [&](auto const& input) { - return dispatch_input_spec( - input, - [](auto&... args) { - auto col = get_column_view(args...); - return std::visit([](auto& view) { return view.nullable(); }, col); - }, - args); + bool has_nullable_inputs = + std::any_of(instance_.inputs_.begin(), instance_.inputs_.end(), [&](auto& in) { + return std::visit([](auto& c) { return is_nullable(c); }, in); }); - // add 1 auto-deduced output variable - instance_.add_output_var(); - auto is_null_aware = std::any_of( output_irs_.cbegin(), output_irs_.cend(), [](auto& ir) { return ir->is_null_aware(); }) @@ -578,63 +544,84 @@ std::tuple ast_converter::gen }(); code_sink sink; - sink.emit("__device__ cudf::ops::errc expression("); + sink.emit(std::format("__device__ cudf::ops::errc {}(", function_name)); sink.emit(args_decl); sink.emit(")\n{\n"); for (auto& ir : output_irs_) { ir->emit_code(instance_, target, sink); } sink.emit("return cudf::ops::errc::OK;\n}"); + return {sink.get_code(), is_null_aware, null_policy, is_fallible}; +} + +std::variant get_column_view(scalar_input const& in) +{ + return scalar_column_view{in.scalar_column->view()}; +} - return {std::string{sink.get_code()}, is_null_aware, null_policy, is_fallible}; +std::variant get_column_view(column_input const& in) +{ + return column_view{in.column}; } // Due to the AST expression tree structure, we can't generate the IR without the target // tables transform_args ast_converter::compute_column(target target_id, ast::expression const& expr, - ast_args const& args, + table_view const& left_table, + table_view const& right_table, + std::string_view function_name, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - ast_converter converter{stream, mr}; + ast_converter converter{stream, mr, left_table, right_table}; // TODO(lamarrr): consider deduplicating ast expression's input column references. See // TransformTest/1.DeeplyNestedArithmeticLogicalExpression for reference auto [code, is_null_aware, output_nullability, is_fallible] = - converter.generate_code(target_id, expr, args); - + converter.generate_code(target_id, expr, function_name); std::vector> inputs; std::vector> scalar_columns; + std::vector> table_sources; + std::vector> column_indices; + + for (auto& input : converter.instance_.inputs_) { + if (std::holds_alternative(input)) { + auto& col = std::get(input); + table_sources.emplace_back(col.table_source); + column_indices.emplace_back(col.column_index); + } else { + table_sources.emplace_back(std::nullopt); + column_indices.emplace_back(std::nullopt); + } - for (auto& input : converter.instance_.input_specs_) { - auto column_view = - dispatch_input_spec(input, [](auto&... args) { return get_column_view(args...); }, args); - inputs.emplace_back(column_view); + auto view = std::visit([](auto& in) { return get_column_view(in); }, input); + inputs.emplace_back(view); - if (std::holds_alternative(input)) { - auto& scalar_input = std::get(input); - scalar_columns.emplace_back(std::move(scalar_input.scalar_column)); + if (std::holds_alternative(input)) { + auto& scalar = std::get(input); + scalar_columns.emplace_back(std::move(scalar.scalar_column)); } } auto& out = converter.output_irs_[0]; auto output_column_type = out->get_type(); - + auto output = transform_output{.type = output_column_type, .nullability = output_nullability}; + auto row_size = std::max({left_table.num_rows(), right_table.num_rows()}); auto result = - transform_args{.scalar_columns = std::move(scalar_columns), - .inputs = inputs, - .udf = std::move(code), - .output_type = output_column_type, - .source_type = cudf::udf_source_type::CUDA, - .user_data = std::nullopt, - .is_null_aware = is_null_aware, - .null_policy = output_nullability, - .row_size = args.table.num_rows(), - .error_mode = is_fallible ? ops::error_mode::ANY_ROW : ops::error_mode::IGNORE, - .input_specs = std::move(converter.instance_.input_specs_)}; - + transform_args{.scalar_columns = std::move(scalar_columns), + .input_table_sources = std::move(table_sources), + .input_column_indices = std::move(column_indices), + .udf = std::move(code), + .source_type = cudf::udf_source_type::CUDA, + .is_null_aware = is_null_aware, + .user_data = std::nullopt, + .inputs = inputs, + .outputs{output}, + .string_offsets{}, + .row_size = row_size, + .error_mode = is_fallible ? ops::error_mode::ANY_ROW : ops::error_mode::IGNORE}; if (get_context().dump_codegen()) { std::cout << "Generated code for transform: " << result.udf << std::endl; } @@ -642,38 +629,25 @@ transform_args ast_converter::compute_column(target target_id, return result; } -filter_args ast_converter::filter(target target_id, - ast::expression const& expr, - ast_args const& args, - table_view const& filter_table, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +transform_args ast_converter::filter(target target_id, + ast::expression const& expr, + table_view const& left_table, + table_view const& right_table, + std::string_view function_name, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { - auto filter = ast::detail::predicate{expr}; - auto transform = compute_column(target_id, filter, args, stream, mr); + auto filter = ast::detail::predicate{expr}; + auto transform = + compute_column(target_id, filter, left_table, right_table, function_name, stream, mr); - CUDF_EXPECTS(transform.output_type.id() == type_id::BOOL8, + CUDF_EXPECTS(transform.outputs.size() == 1, + "Filter expression must have exactly one output column."); + CUDF_EXPECTS(transform.outputs[0].type.id() == type_id::BOOL8, "Filter expression must return a boolean type.", std::invalid_argument); - std::vector filter_columns; - std::transform(filter_table.begin(), - filter_table.end(), - std::back_inserter(filter_columns), - [](auto const& col) { return col; }); - - auto result = filter_args{.scalar_columns = std::move(transform.scalar_columns), - .inputs = std::move(transform.inputs), - .filter_columns = std::move(filter_columns), - .udf = std::move(transform.udf), - .source_type = transform.source_type, - .user_data = transform.user_data, - .is_null_aware = transform.is_null_aware, - .predicate_nullability = transform.null_policy, - .error_mode = transform.error_mode, - .input_specs = std::move(transform.input_specs)}; - - return result; + return transform; } } // namespace cudf::detail::row_ir diff --git a/cpp/src/jit/row_ir.hpp b/cpp/src/jit/row_ir.hpp index b0a27888b513..21fd46dcce4f 100644 --- a/cpp/src/jit/row_ir.hpp +++ b/cpp/src/jit/row_ir.hpp @@ -6,9 +6,11 @@ #pragma once #include #include +#include #include #include #include +#include #include #include #include @@ -18,7 +20,6 @@ #include #include -#include #include #include #include @@ -66,34 +67,36 @@ struct target_info { target id = target::CUDA; ///< The target identifier }; -/** - * @brief A specification of an input column to the AST - */ -struct ast_column_input_spec { - ast::table_reference table = {}; ///< The table reference (LEFT or RIGHT) - int32_t column = 0; ///< The column index in the referenced table +struct scalar_input { + std::unique_ptr scalar_column = + nullptr; ///< The scalar value represented as a column with a single element }; -/** - * @brief A specification of an input scalar to the AST - */ -struct ast_scalar_input_spec { - std::unique_ptr scalar_column = nullptr; ///< The broadcasted column, a column of size 1 +struct column_input { + column_view column = {}; ///< The column input + std::optional table_source = std::nullopt; + std::optional column_index = std::nullopt; }; -/** - * @brief The AST input column arguments used to resolve the column expressions - */ -struct ast_args { - table_view table = {}; ///< The table view containing the columns (single-table case) - table_view left_table = {}; ///< The left table for join predicates - table_view right_table = {}; ///< The right table for join predicates -}; +using input = std::variant; /** - * @brief An input specification for the AST + * @brief The arguments needed to invoke a `cudf::transform` */ -using ast_input_spec = std::variant; +struct [[nodiscard]] transform_args { + std::vector> scalar_columns = {}; + std::vector> input_table_sources = {}; + std::vector> input_column_indices = {}; + std::string udf = {}; + udf_source_type source_type = cudf::udf_source_type::CUDA; + null_aware is_null_aware = null_aware::NO; + std::optional user_data = std::nullopt; + std::vector inputs = {}; + std::vector outputs = {}; + std::vector> string_offsets = {}; + std::optional row_size = std::nullopt; + ops::error_mode error_mode = ops::error_mode::IGNORE; +}; /** * @brief The context within which the IR is instantiated. @@ -105,7 +108,7 @@ struct [[nodiscard]] instance_context { int32_t num_tmp_vars_ = 0; ///< The number of temporary variables generated std::string tmp_prefix_ = "tmp_"; ///< The prefix for temporary variable identifiers bool has_nulls_ = false; ///< If expressions involve null values - std::vector input_specs_; ///< The input specs for the AST + std::vector inputs_; ///< The inputs for the IR std::vector input_vars_; ///< The input variables for the IR std::vector output_vars_; ///< The output variables for the IR rmm::cuda_stream_view @@ -113,17 +116,9 @@ struct [[nodiscard]] instance_context { rmm::device_async_resource_ref mr_; ///< The device memory resource for any device memory allocation during IR generation - private: - void add_input_var(ast_column_input_spec const& in, ast_args const& args); - - void add_input_var(ast_scalar_input_spec const& in, ast_args const& args); - - void add_output_var(); - - [[nodiscard]] int32_t add_ast_input(ast_input_spec in); - public: friend struct ast_converter; + friend struct node; instance_context(rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) : stream_(stream), mr_(mr) @@ -140,6 +135,21 @@ struct [[nodiscard]] instance_context { ~instance_context() = default; ///< Destructor + [[nodiscard]] int32_t add_output(); + + [[nodiscard]] int32_t add_input(input in); + + [[nodiscard]] int32_t add_input(scalar const& scalar) + { + return add_input( + scalar_input{.scalar_column = make_column_from_scalar(scalar, 1, stream_, mr_)}); + } + + [[nodiscard]] int32_t add_input(column_view const& column) + { + return add_input(column_input{.column = column}); + } + /** * @brief Generate a globally unique temporary variable identifier * @return A unique temporary variable identifier @@ -158,29 +168,22 @@ struct [[nodiscard]] instance_context { void set_has_nulls(bool has_nulls); /** - * @brief Get the input specifications for the AST - * @return A span of AST input specifications + * @brief Get the input values for the IR + * @return A span of input values for the IR */ - [[nodiscard]] std::span get_input_specs() const; + [[nodiscard]] std::span get_inputs() const; /** * @brief Get the input variables for the IR * @return A span of input variable information */ - [[nodiscard]] std::span get_inputs() const; + [[nodiscard]] std::span get_input_vars() const; /** * @brief Get the output variables for the IR * @return A span of output variable information */ - [[nodiscard]] std::span get_outputs() const; - - /** - * @brief Add a constant scalar value to the IR - * @param value The scalar value to add - * @return The identifier of the constant variable - */ - [[nodiscard]] int32_t add_constant(cudf::scalar const& value); + [[nodiscard]] std::span get_output_vars() const; }; struct [[nodiscard]] code_sink { @@ -190,7 +193,7 @@ struct [[nodiscard]] code_sink { public: void emit(std::string_view code) { code_ += code; } - [[nodiscard]] std::string_view get_code() const { return code_; } + [[nodiscard]] std::string const& get_code() const { return code_; } }; struct [[nodiscard]] input_reference { @@ -201,10 +204,6 @@ struct [[nodiscard]] output_reference { int32_t index = 0; ///< The index of the output variable }; -struct [[nodiscard]] scalar_refernce { - int32_t index = 0; ///< The index of the scalar variable -}; - struct [[nodiscard]] node { private: std::variant reference_ = @@ -216,7 +215,7 @@ struct [[nodiscard]] node { data_type type_ = {}; ///< The resolved type information of the IR node std::string id_ = {}; ///< The identifier of the IR node - scalar_refernce + input_reference scale_reference_; ///< The index of the scale variable for decimal rescaling if applicable /** @@ -288,6 +287,13 @@ struct [[nodiscard]] node { */ node(output_reference reference, std::unique_ptr arg); + /** + * @brief Construct a new output reference IR node + * @param output The index of the output variable + * @param arg The argument node that produces the value to be set to the output variable + */ + node(output_reference reference, node arg); + node(node const& other) = delete; node(node&& other) = default; ///< Move constructor node& operator=(node const& other) = delete; @@ -362,45 +368,6 @@ struct [[nodiscard]] node { void emit_code(instance_context& ctx, target_info const& info, code_sink& sink) const; }; -/** - * @brief The arguments needed to invoke a `cudf::transform` - */ -struct [[nodiscard]] transform_args { - std::vector> scalar_columns = - {}; ///< The scalar columns created during the expression conversion - std::vector> inputs = - {}; ///< The input columns to the transform UDF - std::string udf = {}; ///< The user-defined function to apply - data_type output_type = data_type{type_id::EMPTY}; ///< The output type of the transform - cudf::udf_source_type source_type = cudf::udf_source_type::CUDA; ///< The source type of the UDF - std::optional user_data = std::nullopt; ///< User data to pass to the transform - null_aware is_null_aware = null_aware::NO; ///< Whether the transform is null-aware - output_nullability null_policy = output_nullability::PRESERVE; ///< Null-transformation policy - std::optional row_size = std::nullopt; ///< The row size of the transform operation - ops::error_mode error_mode = - ops::error_mode::IGNORE; ///< The error handling mode for the transform - std::vector input_specs = {}; ///< The input specs (table ref + column index) -}; - -/** - * @brief The arguments needed to invoke a `cudf::filter` - */ -struct [[nodiscard]] filter_args { - std::vector> scalar_columns = - {}; ///< The scalar columns created during the expression conversion - std::vector> inputs = - {}; ///< The input columns to the transform UDF - std::vector filter_columns = {}; ///< The input columns to the filter - std::string udf = {}; ///< The user-defined function to apply as a predicate - cudf::udf_source_type source_type = cudf::udf_source_type::CUDA; ///< The source type of the UDF - std::optional user_data = std::nullopt; ///< User data to pass to the filter - null_aware is_null_aware = null_aware::NO; ///< Whether the filter is null-aware - output_nullability predicate_nullability = - output_nullability::PRESERVE; ///< Null-transformation policy for the predicate output - ops::error_mode error_mode = ops::error_mode::IGNORE; ///< The error handling mode for the filter - std::vector input_specs = {}; ///< The input specs (table ref + column index) -}; - /** * @brief AST Converter is a class for converting AST expressions to codegen targets, ie. CUDA. */ @@ -412,6 +379,8 @@ struct [[nodiscard]] ast_converter { rmm::device_async_resource_ref mr_; ///< Device memory resource used to allocate the returned table's device memory instance_context instance_; ///< The instance context used during the IR generation + table_view left_table_; ///< The left input table for the expression + table_view right_table_; ///< The right input table for the expression public: /** @@ -419,8 +388,15 @@ struct [[nodiscard]] ast_converter { * @param stream CUDA stream used for device memory operations and kernel launches. * @param mr Device memory resource used to allocate the returned table's device memory */ - ast_converter(rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) - : stream_(std::move(stream)), mr_(std::move(mr)), instance_(stream_, mr_) + ast_converter(rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr, + table_view left_table, + table_view right_table) + : stream_(std::move(stream)), + mr_(std::move(mr)), + instance_(stream_, mr_), + left_table_(std::move(left_table)), + right_table_(std::move(right_table)) { } @@ -449,21 +425,26 @@ struct [[nodiscard]] ast_converter { [[nodiscard]] std::unique_ptr add_ir_node(ast::detail::predicate const& expr); [[nodiscard]] std::tuple generate_code( - target target, ast::expression const& expr, ast_args const& args); + target target, ast::expression const& expr, std::string_view function_name); public: /** * @brief Convert an AST `compute_column` expression to a `cudf::transform` * @param target The target for which the IR is generated * @param expr The AST expression to convert - * @param args The arguments needed to resolve the AST expression + * @param left_table The left input table for the expression + * @param right_table The right input table for the expression + * @param table The input table for the expression + * @param function_name The name of the generated function * @param stream CUDA stream used for device memory operations and kernel launches. * @param mr Device memory resource used to allocate the returned table's device memory * @return The result of the conversion, containing the transform arguments and scalar columns */ static transform_args compute_column(target target, ast::expression const& expr, - ast_args const& args, + table_view const& left_table, + table_view const& right_table, + std::string_view function_name, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr); @@ -471,18 +452,21 @@ struct [[nodiscard]] ast_converter { * @brief Convert an AST `filter` expression to a `cudf::filter` * @param target The target for which the IR is generated * @param expr The AST expression to convert - * @param args The arguments needed to resolve the AST expression - * @param filter_table The table to be filtered + * @param left_table The left input table for the expression + * @param right_table The right input table for the expression + * @param table The input table for the expression + * @param function_name The name of the generated function * @param stream CUDA stream used for device memory operations and kernel launches. * @param mr Device memory resource used to allocate the returned table's device memory * @return The result of the conversion, containing the filter arguments and scalar columns */ - static filter_args filter(target target, - ast::expression const& expr, - ast_args const& args, - table_view const& filter_table, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr); + static transform_args filter(target target, + ast::expression const& expr, + table_view const& left_table, + table_view const& right_table, + std::string_view function_name, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr); }; } // namespace row_ir diff --git a/cpp/src/join/filter_join_indices_jit.cu b/cpp/src/join/filter_join_indices_jit.cu index d7a2565df341..d841f46fb5bd 100644 --- a/cpp/src/join/filter_join_indices_jit.cu +++ b/cpp/src/join/filter_join_indices_jit.cu @@ -48,33 +48,35 @@ namespace detail { namespace { -// Build template parameters for JIT kernel -jitify2::StringVec build_join_filter_template_params(std::vector const& left_columns, - std::vector const& right_columns, - bool has_user_data, - null_aware is_null_aware) +jitify2::StringVec build_join_filter_template_params( + std::span inputs, + std::span> table_sources, + null_aware is_null_aware) { jitify2::StringVec template_params; - - template_params.emplace_back(jitify2::reflection::reflect(has_user_data)); + template_params.emplace_back(jitify2::reflection::reflect(false)); // has_user_data = false template_params.emplace_back(jitify2::reflection::reflect(is_null_aware)); - // Add left column accessors - for (std::size_t i = 0; i < left_columns.size(); ++i) { - auto const& col = left_columns[i]; - std::string type_name = cudf::type_to_name(col.type()); - template_params.emplace_back( - jitify2::reflection::Template("cudf::jit::join_column_accessor") - .instantiate(type_name, std::to_string(i), "cudf::jit::join_side::LEFT")); - } - - // Add right column accessors - for (std::size_t i = 0; i < right_columns.size(); ++i) { - auto const& col = right_columns[i]; - std::string type_name = cudf::type_to_name(col.type()); - template_params.emplace_back( - jitify2::reflection::Template("cudf::jit::join_column_accessor") - .instantiate(type_name, std::to_string(i), "cudf::jit::join_side::RIGHT")); + for (size_t i = 0; i < inputs.size(); ++i) { + auto const& input = inputs[i]; + if (auto* col = std::get_if(&input)) { + auto element = cudf::type_to_name(col->type()); + template_params.emplace_back( + jitify2::reflection::Template("cudf::jit::column_accessor") + .instantiate( + i, "cudf::column_device_view_core", element, false, table_sources[i].value())); + } else { + auto& scalar = std::get(input); + auto element = cudf::type_to_name(scalar.as_column_view().type()); + template_params.emplace_back( + jitify2::reflection::Template("cudf::jit::column_accessor") + .instantiate(i, + "cudf::column_device_view_core", + element, + true, + 0 // scalars dont belong to a table, so just use 0 as placeholder + )); + } } return template_params; @@ -82,8 +84,8 @@ jitify2::StringVec build_join_filter_template_params(std::vector co // Build the JIT kernel for join filtering jitify2::ConfiguredKernel build_join_filter_kernel(std::string const& predicate_code, - std::vector const& left_columns, - std::vector const& right_columns, + std::span inputs, + std::span> table_sources, bool is_ptx, bool has_user_data, null_aware is_null_aware, @@ -92,24 +94,28 @@ jitify2::ConfiguredKernel build_join_filter_kernel(std::string const& predicate_ { CUDF_FUNC_RANGE(); + std::vector ptx_output_types{"bool"}; + std::vector ptx_input_types; + + for (auto const& input : inputs) { + if (auto* col = std::get_if(&input)) { + ptx_input_types.push_back(cudf::type_to_name(col->type())); + } else { + auto& scalar = std::get(input); + ptx_input_types.push_back(cudf::type_to_name(scalar.type())); + } + } + // Parse predicate code auto const cuda_source = is_ptx ? cudf::jit::parse_single_function_ptx( predicate_code, "GENERIC_JOIN_FILTER_OP", - [&] { - std::vector left_types, right_types; - for (auto const& col : left_columns) - left_types.push_back(cudf::type_to_name(col.type())); - for (auto const& col : right_columns) - right_types.push_back(cudf::type_to_name(col.type())); - return cudf::jit::build_ptx_params(left_types, right_types, has_user_data); - }()) + cudf::jit::build_ptx_params(ptx_output_types, ptx_input_types, has_user_data)) : cudf::jit::parse_single_function_cuda(predicate_code, "GENERIC_JOIN_FILTER_OP"); // Build template parameters and kernel name - auto template_args = - build_join_filter_template_params(left_columns, right_columns, has_user_data, is_null_aware); + auto template_args = build_join_filter_template_params(inputs, table_sources, is_null_aware); auto kernel_name = jitify2::reflection::Template("cudf::join::jit::filter_join_kernel").instantiate(template_args); @@ -122,44 +128,45 @@ jitify2::ConfiguredKernel build_join_filter_kernel(std::string const& predicate_ // Launch the JIT kernel for join filtering void launch_join_filter_kernel(jitify2::ConfiguredKernel& kernel, - cudf::table_view const& left, - cudf::table_view const& right, cudf::device_span left_indices, cudf::device_span right_indices, + std::span inputs, bool* predicate_results, std::optional user_data, - std::vector const& extra_left_cols, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); // Create device views of tables - std::vector left_cols(left.begin(), left.end()); - left_cols.insert(left_cols.end(), extra_left_cols.begin(), extra_left_cols.end()); - std::vector right_cols(right.begin(), right.end()); + std::vector column_views; + for (auto const& input : inputs) { + if (auto* col = std::get_if(&input)) { + column_views.push_back(*col); + } else { + auto& scalar = std::get(input); + column_views.push_back(scalar.as_column_view()); + } + } - auto [left_handles, left_device_views] = - cudf::jit::column_views_to_device(left_cols, stream, mr); - auto [right_handles, right_device_views] = - cudf::jit::column_views_to_device(right_cols, stream, mr); + auto [handles, device_views] = + cudf::jit::column_views_to_device(column_views, stream, mr); // Set up kernel parameters - use JIT-compatible span type - cudf::jit::device_span left_span{left_indices.data(), left_indices.size()}; - cudf::jit::device_span right_span{right_indices.data(), - right_indices.size()}; - cudf::column_device_view_core const* left_tables_ptr = left_device_views.data(); - cudf::column_device_view_core const* right_tables_ptr = right_device_views.data(); - void* user_data_ptr = user_data.value_or(nullptr); - - std::array args{&left_span, - &right_span, - &left_tables_ptr, - &right_tables_ptr, - &predicate_results, - &user_data_ptr}; - - kernel->launch_raw(args.data()); + cudf::size_type num_rows = left_indices.size(); + cudf::size_type const* left_indices_ptr = left_indices.data(); + cudf::size_type const* right_indices_ptr = right_indices.data(); + cudf::column_device_view_core const* columns_ptr = device_views.data(); + void* user_data_ptr = user_data.value_or(nullptr); + + void* args[]{&num_rows, + &left_indices_ptr, + &right_indices_ptr, + &columns_ptr, + &predicate_results, + &user_data_ptr}; + + kernel->launch_raw(args); } // Same join semantics handling as the AST version @@ -348,43 +355,6 @@ apply_join_semantics(cudf::table_view const& left, } } -// Build template parameters from AST input specs (preserves expression input order) -jitify2::StringVec build_join_filter_template_params_from_specs( - std::vector const& input_specs, - cudf::table_view const& left, - cudf::table_view const& right, - null_aware is_null_aware) -{ - jitify2::StringVec template_params; - template_params.emplace_back(jitify2::reflection::reflect(false)); // has_user_data = false - template_params.emplace_back(jitify2::reflection::reflect(is_null_aware)); - - // Scalar columns are appended to the left table's device views, - // starting at index left.num_columns(). - auto scalar_index = left.num_columns(); - - for (auto const& spec : input_specs) { - if (std::holds_alternative(spec)) { - auto const& col_spec = std::get(spec); - auto const& table = col_spec.table == ast::table_reference::LEFT ? left : right; - auto const side_str = col_spec.table == ast::table_reference::LEFT - ? "cudf::jit::join_side::LEFT" - : "cudf::jit::join_side::RIGHT"; - auto type_name = cudf::type_to_name(table.column(col_spec.column).type()); - template_params.emplace_back( - jitify2::reflection::Template("cudf::jit::join_column_accessor") - .instantiate(type_name, std::to_string(col_spec.column), side_str)); - } else if (std::holds_alternative(spec)) { - auto const& scalar_spec = std::get(spec); - auto type_name = cudf::type_to_name(scalar_spec.ref.get().type()); - template_params.emplace_back(jitify2::reflection::Template("cudf::jit::join_scalar_accessor") - .instantiate(type_name, std::to_string(scalar_index++))); - } - } - - return template_params; -} - void validate_column_types(cudf::table_view const& table, char const* side) { for (auto const& col : table) { @@ -433,12 +403,20 @@ filter_join_indices_jit(cudf::table_view const& left, if (left_indices.empty()) { return make_empty_result(); } // Compile JIT kernel - std::vector left_cols(left.begin(), left.end()); - std::vector right_cols(right.begin(), right.end()); + std::vector inputs; + std::vector> table_sources; + for (auto const& col : left) { + inputs.emplace_back(col); + table_sources.emplace_back(0); + } + for (auto const& col : right) { + inputs.emplace_back(col); + table_sources.emplace_back(1); + } auto kernel = build_join_filter_kernel(predicate_code, - left_cols, - right_cols, + inputs, + table_sources, is_ptx, false, // has_user_data = false for now null_aware::NO, @@ -450,13 +428,11 @@ filter_join_indices_jit(cudf::table_view const& left, // Launch kernel launch_join_filter_kernel(kernel, - left, - right, left_indices, right_indices, + inputs, predicate_results.data(), std::nullopt, // no user data for now - {}, stream, mr); @@ -496,13 +472,11 @@ filter_join_indices_jit(cudf::table_view const& left, } // Convert AST predicate to JIT code - row_ir::ast_args ast_args{.left_table = left, .right_table = right}; auto filter_result = row_ir::ast_converter::filter( - row_ir::target::CUDA, predicate, ast_args, table_view{}, stream, mr); + row_ir::target::CUDA, predicate, left, right, "filter_operation", stream, mr); - // Build template params matching the AST input order - auto template_args = build_join_filter_template_params_from_specs( - filter_result.input_specs, left, right, filter_result.is_null_aware); + auto template_args = build_join_filter_template_params( + filter_result.inputs, filter_result.input_table_sources, filter_result.is_null_aware); auto const cuda_source = cudf::jit::parse_single_function_cuda(filter_result.udf, "GENERIC_JOIN_FILTER_OP"); @@ -513,23 +487,14 @@ filter_join_indices_jit(cudf::table_view const& left, cudf::jit::get_udf_kernel(*join_jit_filter_join_kernel_cu_jit, kernel_name, cuda_source); auto configured_kernel = kernel->configure_1d_max_occupancy(0, 0, nullptr, stream.value()); - // Collect scalar columns to append to left device views so join_scalar_accessor - // can read them at indices >= left.num_columns(). - std::vector scalar_cols; - for (auto const& col : filter_result.scalar_columns) { - scalar_cols.push_back(col->view()); - } - // Allocate and compute predicate results auto predicate_results = rmm::device_uvector(left_indices.size(), stream); launch_join_filter_kernel(configured_kernel, - left, - right, left_indices, right_indices, + filter_result.inputs, predicate_results.data(), std::nullopt, - scalar_cols, stream, mr); diff --git a/cpp/src/join/jit/filter_join_kernel.cu b/cpp/src/join/jit/filter_join_kernel.cu index c8b07bc7a625..61f26d261067 100644 --- a/cpp/src/join/jit/filter_join_kernel.cu +++ b/cpp/src/join/jit/filter_join_kernel.cu @@ -10,8 +10,9 @@ #include #include +#include -#include +#include #include #include @@ -31,62 +32,64 @@ namespace cudf::join::jit { // This must match the definition in cudf/join/join.hpp constexpr cudf::size_type JoinNoMatch = cuda::std::numeric_limits::min(); -template -CUDF_KERNEL void filter_join_kernel(cudf::jit::device_span left_indices, - cudf::jit::device_span right_indices, - cudf::column_device_view_core const* left_tables, - cudf::column_device_view_core const* right_tables, - bool* predicate_results, - void* user_data) +template +__device__ void execute_predicate_op(void* user_data, + size_type row_index, + cuda::std::tuple args) +{ + if constexpr (has_user_data) { + cuda::std::apply([&](auto&&... args) { GENERIC_JOIN_FILTER_OP(user_data, row_index, args...); }, + args); + } else { + cuda::std::apply([&](auto&&... args) { GENERIC_JOIN_FILTER_OP(args...); }, args); + } +} + +template +CUDF_KERNEL void filter_join_kernel(cudf::size_type num_rows, + cudf::size_type const* __restrict__ left_indices, + cudf::size_type const* __restrict__ right_indices, + cudf::column_device_view_core const* __restrict__ columns, + bool* __restrict__ predicate_results, + void* __restrict__ user_data) { auto const start = cudf::detail::grid_1d::global_thread_id(); auto const stride = cudf::detail::grid_1d::grid_stride(); - auto const size = left_indices.size(); - - for (auto i = start; i < size; i += stride) { - auto const left_idx = left_indices[i]; - auto const right_idx = right_indices[i]; + for (auto i = start; i < num_rows; i += stride) { // Skip if either index is JoinNoMatch - if (left_idx == JoinNoMatch || right_idx == JoinNoMatch) { + if (left_indices[i] == JoinNoMatch || right_indices[i] == JoinNoMatch) { predicate_results[i] = false; continue; } + cudf::size_type const* indices[] = {left_indices, right_indices}; + // Each accessor receives both tables and both indices, and internally selects // the appropriate table based on whether it's a left or right accessor. if constexpr (is_null_aware == null_aware::YES) { // Null-aware path: pass optional inputs, get optional result cuda::std::optional result{false}; - if constexpr (has_user_data) { - GENERIC_JOIN_FILTER_OP( - user_data, - i, - &result, - InputAccessors::nullable_element(left_tables, right_tables, left_idx, right_idx, i)...); - } else { - GENERIC_JOIN_FILTER_OP( - &result, - InputAccessors::nullable_element(left_tables, right_tables, left_idx, right_idx, i)...); - } + auto inputs = Accessors::map([&]() { + return cuda::std::tuple{A::nullable_element(columns, indices[A::table_index][i])...}; + }); + execute_predicate_op( + user_data, i, cuda::std::tuple_cat(cuda::std::tuple{&result}, inputs)); predicate_results[i] = result.has_value() && result.value(); } else { // Non-null-aware path: if any input is null, predicate is false - if ((InputAccessors::is_null(left_tables, right_tables, left_idx, right_idx, i) || ...)) { + auto any_null = Accessors::map( + [&]() { return (A::is_null(columns, indices[A::table_index][i]) || ...); }); + if (any_null) { predicate_results[i] = false; continue; } bool result = false; - if constexpr (has_user_data) { - GENERIC_JOIN_FILTER_OP( - user_data, - i, - &result, - InputAccessors::element(left_tables, right_tables, left_idx, right_idx, i)...); - } else { - GENERIC_JOIN_FILTER_OP( - &result, InputAccessors::element(left_tables, right_tables, left_idx, right_idx, i)...); - } + auto inputs = Accessors::map([&]() { + return cuda::std::tuple{A::element(columns, indices[A::table_index][i])...}; + }); + execute_predicate_op( + user_data, i, cuda::std::tuple_cat(cuda::std::tuple{&result}, inputs)); predicate_results[i] = result; } } diff --git a/cpp/src/join/jit/filter_join_kernel.cuh b/cpp/src/join/jit/filter_join_kernel.cuh index 5a9215e810a8..f5d086d215cc 100644 --- a/cpp/src/join/jit/filter_join_kernel.cuh +++ b/cpp/src/join/jit/filter_join_kernel.cuh @@ -17,20 +17,21 @@ namespace cudf::join::jit { * * @tparam has_user_data Whether the predicate function requires user data * @tparam is_null_aware Whether the expression needs input validity as part of its computation - * @tparam InputAccessors Variadic template for input column accessors + * @tparam Accessors type list of accessors for columns used in the predicate * @param left_indices Device span of left table indices * @param right_indices Device span of right table indices - * @param left_tables Device view of left table columns - * @param right_tables Device view of right table columns + * @param left_table Device view of left table columns + * @param right_table Device view of right table columns + * @param scalars Device view of scalar values used in the predicate * @param predicate_results Output array for predicate evaluation results * @param user_data Optional user data for predicate function */ -template -CUDF_KERNEL void filter_join_kernel(cudf::jit::device_span left_indices, - cudf::jit::device_span right_indices, - cudf::column_device_view_core const* left_tables, - cudf::column_device_view_core const* right_tables, - bool* predicate_results, - void* user_data); +template +CUDF_KERNEL void filter_join_kernel(cudf::size_type num_rows, + cudf::size_type const* __restrict__ left_indices, + cudf::size_type const* __restrict__ right_indices, + cudf::column_device_view_core const* __restrict__ columns, + bool* __restrict__ predicate_results, + void* __restrict__ user_data); } // namespace cudf::join::jit diff --git a/cpp/src/stream_compaction/filter/filter.cu b/cpp/src/stream_compaction/filter/filter.cu index 407fe0c4d3b7..e2f8b43a3fbd 100644 --- a/cpp/src/stream_compaction/filter/filter.cu +++ b/cpp/src/stream_compaction/filter/filter.cu @@ -22,41 +22,42 @@ namespace cudf { namespace detail { -std::vector> filter( - std::span const> predicate_inputs, - std::string const& predicate_udf, - std::vector const& filter_columns, - cudf::udf_source_type source_type, - std::optional user_data, - null_aware is_null_aware, - output_nullability predicate_nullability, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +std::unique_ptr
filter(std::string const& predicate_udf, + cudf::udf_source_type source_type, + null_aware is_null_aware, + std::optional user_data, + std::span predicate_inputs, + table_view const& filter_table, + ops::error_mode error_mode, + output_nullability predicate_nullability, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { - CUDF_EXPECTS(!filter_columns.empty(), + CUDF_EXPECTS(filter_table.num_columns() > 0, "At least one column must be provided to filter.", std::invalid_argument); - auto row_size = filter_columns[0].size(); - CUDF_EXPECTS(std::all_of(filter_columns.begin(), - filter_columns.end(), + auto row_size = filter_table.num_rows(); + CUDF_EXPECTS(std::all_of(filter_table.begin(), + filter_table.end(), [&](auto const& col) { return col.size() == row_size; }), "All columns to filter must have the same number of rows.", std::invalid_argument); - auto predicate = cudf::transform_extended(predicate_inputs, - predicate_udf, - data_type{type_id::BOOL8}, - source_type, - user_data, - is_null_aware, - row_size, - predicate_nullability, - stream, - mr); - - return apply_mask( - cudf::table_view{filter_columns}, predicate->view(), mask_type::RETENTION, stream, mr) - ->release(); + transform_output outputs[] = {transform_output{data_type{type_id::BOOL8}, predicate_nullability}}; + + auto result = cudf::multi_transform(predicate_udf, + source_type, + is_null_aware, + user_data, + predicate_inputs, + outputs, + {}, + filter_table.num_rows(), + error_mode, + stream, + mr); + + return apply_mask(filter_table, result->get_column(0), mask_type::RETENTION, stream, mr); } } // namespace detail @@ -67,19 +68,24 @@ std::unique_ptr
filter(table_view const& predicate_table, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - cudf::detail::row_ir::ast_args ast_args{.table = predicate_table}; - auto args = cudf::detail::row_ir::ast_converter::filter( - cudf::detail::row_ir::target::CUDA, predicate_expr, ast_args, filter_table, stream, mr); - - return std::make_unique
(cudf::detail::filter(args.inputs, - args.udf, - args.filter_columns, - args.source_type, - args.user_data, - args.is_null_aware, - args.predicate_nullability, - stream, - mr)); + auto args = cudf::detail::row_ir::ast_converter::filter(cudf::detail::row_ir::target::CUDA, + predicate_expr, + predicate_table, + {}, + "filter_operation", + stream, + mr); + + return detail::filter(args.udf, + args.source_type, + args.is_null_aware, + args.user_data, + args.inputs, + filter_table, + args.error_mode, + args.outputs[0].nullability, + stream, + mr); } std::vector> filter_extended( @@ -94,12 +100,13 @@ std::vector> filter_extended( rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::filter(predicate_inputs, - predicate_udf, - filter_columns, + return detail::filter(predicate_udf, source_type, - user_data, is_null_aware, + user_data, + predicate_inputs, + table_view{filter_columns}, + ops::error_mode::IGNORE, predicate_nullability, stream, mr); @@ -130,12 +137,13 @@ std::vector> filter(std::vector const& pred } } - return detail::filter(inputs, - predicate_udf, - filter_columns, + return detail::filter(predicate_udf, is_ptx ? cudf::udf_source_type::PTX : cudf::udf_source_type::CUDA, - user_data, is_null_aware, + user_data, + inputs, + table_view{filter_columns}, + ops::error_mode::IGNORE, predicate_nullability, stream, mr); diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index 35a698c82208..e1942ff0a3a9 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -248,7 +248,7 @@ auto reflect(udf_source_type source_type, auto element = std::visit([](auto& c) { return reflect_input_element(c); }, in); bool as_scalar = std::holds_alternative(in); auto accessor = jitify2::reflection::Template("cudf::jit::column_accessor") - .instantiate(i, column, element, as_scalar); + .instantiate(i, column, element, as_scalar, 0); in_types.push_back(accessor); } @@ -260,7 +260,7 @@ auto reflect(udf_source_type source_type, auto element = std::visit([](auto& c) { return reflect_output_element(c); }, out); bool as_scalar = false; // never scalar auto accessor = jitify2::reflection::Template("cudf::jit::column_accessor") - .instantiate(i, column, element, as_scalar); + .instantiate(i, column, element, as_scalar, 0); out_types.push_back(accessor); } @@ -857,11 +857,14 @@ std::unique_ptr
execute_transform(std::string const& udf, case ops::error_mode::IGNORE: { } break; case ops::error_mode::ANY_ROW: { - auto sink = d_error_sink->value(stream).any_error(); - auto err = sink->error(); - CUDF_EXPECTS(err == ops::errc::OK, - std::format("Error `{}` in transform UDF", ops::to_string(err)), - std::runtime_error); + auto error = d_error_sink->value(stream).any_error(); + switch (error) { + case ops::errc::OK: break; + case ops::errc::OVERFLOW: CUDF_FAIL("Overflow error in transform UDF", std::overflow_error); + case ops::errc::DIVISION_BY_ZERO: + CUDF_FAIL("Division by zero error in transform UDF", std::overflow_error); + default: CUDF_FAIL("Unknown error in transform UDF", std::runtime_error); + } } break; } @@ -870,18 +873,17 @@ std::unique_ptr
execute_transform(std::string const& udf, } // namespace -std::unique_ptr
multi_transform_extended( - std::string const& udf, - udf_source_type source_type, - null_aware is_null_aware, - std::optional user_data, - std::span inputs, - std::span outputs, - std::vector>&& string_offsets, - std::optional row_size, - ops::error_mode error_handling_mode, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +std::unique_ptr
multi_transform(std::string const& udf, + udf_source_type source_type, + null_aware is_null_aware, + std::optional user_data, + std::span inputs, + std::span outputs, + std::vector>&& string_offsets, + std::optional row_size, + ops::error_mode error_handling_mode, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); perform_checks(source_type, is_null_aware, row_size, inputs, outputs, string_offsets); @@ -910,9 +912,18 @@ std::unique_ptr transform_extended(std::span inpu rmm::device_async_resource_ref mr) { transform_output outputs[] = {{.type = output_type, .nullability = null_policy}}; - auto table = multi_transform( - udf, source_type, is_null_aware, user_data, inputs, outputs, {}, row_size, stream, mr); - auto cols = table->release(); + auto table = multi_transform(udf, + source_type, + is_null_aware, + user_data, + inputs, + outputs, + {}, + row_size, + ops::error_mode::IGNORE, + stream, + mr); + auto cols = table->release(); return std::move(cols[0]); } @@ -958,19 +969,21 @@ std::unique_ptr compute_column_jit(table_view const& table, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - detail::row_ir::ast_args ast_args{.table = table}; auto args = detail::row_ir::ast_converter::compute_column( - detail::row_ir::target::CUDA, expr, ast_args, stream, mr); - return transform_extended(args.inputs, - args.udf, - args.output_type, - args.source_type, - args.user_data, - args.is_null_aware, - args.row_size, - args.null_policy, - stream, - mr); + detail::row_ir::target::CUDA, expr, table, {}, "compute_operation", stream, mr); + auto result = multi_transform(args.udf, + args.source_type, + args.is_null_aware, + args.user_data, + args.inputs, + args.outputs, + std::move(args.string_offsets), + args.row_size, + args.error_handling_mode, + stream, + mr); + auto cols = result->release(); + return std::move(cols[0]); } } // namespace cudf diff --git a/cpp/tests/jit/row_ir.cpp b/cpp/tests/jit/row_ir.cpp index c69daecb0a11..3cf9c37546f1 100644 --- a/cpp/tests/jit/row_ir.cpp +++ b/cpp/tests/jit/row_ir.cpp @@ -20,37 +20,49 @@ namespace row_ir = cudf::detail::row_ir; -struct RowIRCudaCodeGenTest : public ::testing::Test {}; +struct RowIRCudaCodeGenTest : public ::testing::Test { + std::unique_ptr f32 = + cudf::test::fixed_width_column_wrapper({1.0f, 2.0f, 3.0f}).release(); + std::unique_ptr f64 = + cudf::test::fixed_width_column_wrapper({1.0, 2.0, 3.0}).release(); + std::unique_ptr d32 = + cudf::test::fixed_point_column_wrapper({1, 2, 3}, numeric::scale_type{2}).release(); + std::unique_ptr i32 = + cudf::test::fixed_width_column_wrapper({1, 2, 3}).release(); + std::unique_ptr b8 = + cudf::test::fixed_width_column_wrapper({true, false, true}).release(); + cudf::table_view table = cudf::table_view({*f32, *f64, *d32, *i32}); +}; TEST_F(RowIRCudaCodeGenTest, GetInput) { row_ir::target_info target_info{row_ir::target::CUDA}; - row_ir::var_info inputs[] = {{"in_0", {cudf::data_type{cudf::type_id::INT32}}}, - {"in_1", {cudf::data_type{cudf::type_id::FLOAT32}}}}; - - row_ir::instance_info info{inputs, {}}; - { - row_ir::instance_context ctx{}; + row_ir::instance_context ctx{cudf::get_default_stream(), + cudf::get_current_device_resource_ref()}; + [[maybe_unused]] auto in0 = ctx.add_input(*i32); row_ir::code_sink sink; row_ir::node get_input_0{row_ir::input_reference{0}}; - get_input_0.instantiate(ctx, info); - get_input_0.emit_code(ctx, target_info, info, sink); + get_input_0.instantiate(ctx); + get_input_0.emit_code(ctx, target_info, sink); - auto expected_code = "int32_t tmp_0 = in_0;"; + auto expected_code = "int32_t tmp_0 = in_0;\n"; EXPECT_EQ(sink.get_code(), expected_code); } { - row_ir::instance_context ctx{}; + row_ir::instance_context ctx{cudf::get_default_stream(), + cudf::get_current_device_resource_ref()}; row_ir::code_sink sink; + [[maybe_unused]] auto in0 = ctx.add_input(*i32); + [[maybe_unused]] auto in1 = ctx.add_input(*f32); row_ir::node get_input_1{row_ir::input_reference{1}}; - get_input_1.instantiate(ctx, info); - get_input_1.emit_code(ctx, target_info, info, sink); + get_input_1.instantiate(ctx); + get_input_1.emit_code(ctx, target_info, sink); - auto expected_null_code = "float tmp_0 = in_1;"; + auto expected_null_code = "float tmp_0 = in_1;\n"; EXPECT_EQ(sink.get_code(), expected_null_code); } @@ -60,41 +72,47 @@ TEST_F(RowIRCudaCodeGenTest, SetOutput) { row_ir::target_info target_info{row_ir::target::CUDA}; - row_ir::var_info inputs[] = {{"in_0", {cudf::data_type{cudf::type_id::INT32}}}, - {"in_1", {cudf::data_type{cudf::type_id::FLOAT32}}}}; - - row_ir::untyped_var_info outputs[] = {{"out_0"}, {"out_1"}}; - - row_ir::instance_info info{inputs, outputs}; - { - row_ir::instance_context ctx{}; + row_ir::instance_context ctx{cudf::get_default_stream(), + cudf::get_current_device_resource_ref()}; + + [[maybe_unused]] auto in0 = ctx.add_input(*i32); + [[maybe_unused]] auto in1 = ctx.add_input(*f32); + [[maybe_unused]] auto out0 = ctx.add_output(); + [[maybe_unused]] auto out1 = ctx.add_output(); row_ir::code_sink sink; row_ir::node set_output_0{row_ir::output_reference{0}, - std::make_unique(row_ir::input_reference{0})}; - set_output_0.instantiate(ctx, info); - set_output_0.emit_code(ctx, target_info, info, sink); + row_ir::node{row_ir::input_reference{0}}}; + set_output_0.instantiate(ctx); + set_output_0.emit_code(ctx, target_info, sink); auto expected_code = - R"***(int32_t tmp_0 = in_0; -int32_t tmp_1 = tmp_0; -*out_0 = tmp_1;)***"; + R"***(int32_t tmp_1 = in_0; +int32_t tmp_0 = tmp_1; +*out_0 = tmp_0; +)***"; EXPECT_EQ(sink.get_code(), expected_code); } { - row_ir::instance_context ctx{}; + row_ir::instance_context ctx{cudf::get_default_stream(), + cudf::get_current_device_resource_ref()}; row_ir::code_sink sink; + [[maybe_unused]] auto in0 = ctx.add_input(*i32); + [[maybe_unused]] auto in1 = ctx.add_input(*f32); + [[maybe_unused]] auto out0 = ctx.add_output(); + [[maybe_unused]] auto out1 = ctx.add_output(); row_ir::node set_output_1{row_ir::output_reference{1}, - std::make_unique(row_ir::input_reference{1})}; - set_output_1.instantiate(ctx, info); - set_output_1.emit_code(ctx, target_info, info, sink); + row_ir::node{row_ir::input_reference{1}}}; + set_output_1.instantiate(ctx); + set_output_1.emit_code(ctx, target_info, sink); auto expected_code = - R"***(float tmp_0 = in_1; -float tmp_1 = tmp_0; -*out_1 = tmp_1;)***"; + R"***(float tmp_1 = in_1; +float tmp_0 = tmp_1; +*out_1 = tmp_0; +)***"; EXPECT_EQ(sink.get_code(), expected_code); } @@ -104,39 +122,43 @@ TEST_F(RowIRCudaCodeGenTest, UnaryOperation) { row_ir::target_info target_info{row_ir::target::CUDA}; - row_ir::var_info inputs[] = {{"in_0", {cudf::data_type{cudf::type_id::INT32}}}, - {"in_1", {cudf::data_type{cudf::type_id::DECIMAL32}}}}; - - row_ir::untyped_var_info outputs[] = {{"out_0"}, {"out_1"}}; - - row_ir::instance_info info{inputs, outputs}; - { - row_ir::instance_context ctx{}; + row_ir::instance_context ctx{cudf::get_default_stream(), + cudf::get_current_device_resource_ref()}; + [[maybe_unused]] auto in0 = ctx.add_input(*i32); + [[maybe_unused]] auto in1 = ctx.add_input(*f32); + row_ir::code_sink sink; - row_ir::node op{row_ir::opcode::IDENTITY, row_ir::node{row_ir::input_reference{0}}}; - op.instantiate(ctx, info); - op.emit_code(ctx, target_info, info, sink); + row_ir::node op{ + row_ir::opcode::IDENTITY, std::nullopt, row_ir::node{row_ir::input_reference{0}}}; + op.instantiate(ctx); + op.emit_code(ctx, target_info, sink); auto expected_code = - R"***(int32_t tmp_0 = in_0; -int32_t tmp_1; -cudf::ops::identity(&tmp_1, &tmp_0);)***"; + R"***(int32_t tmp_1 = in_0; +int32_t tmp_0; +cudf::ops::identity(&tmp_0, &tmp_1); +)***"; EXPECT_EQ(sink.get_code(), expected_code); } { - row_ir::instance_context ctx{}; + row_ir::instance_context ctx{cudf::get_default_stream(), + cudf::get_current_device_resource_ref()}; + [[maybe_unused]] auto in0 = ctx.add_input(*i32); + [[maybe_unused]] auto in1 = ctx.add_input(*d32); row_ir::code_sink sink; - row_ir::node op{row_ir::opcode::IDENTITY, row_ir::node{row_ir::input_reference{1}}}; - op.instantiate(ctx, info); - op.emit_code(ctx, target_info, info, sink); + row_ir::node op{ + row_ir::opcode::IDENTITY, std::nullopt, row_ir::node{row_ir::input_reference{1}}}; + op.instantiate(ctx); + op.emit_code(ctx, target_info, sink); auto expected_null_code = - R"***(numeric::decimal32 tmp_0 = in_1; -numeric::decimal32 tmp_1; -cudf::ops::identity(&tmp_1, &tmp_0);)***"; + R"***(numeric::decimal32 tmp_1 = in_1; +numeric::decimal32 tmp_0; +cudf::ops::identity(&tmp_0, &tmp_1); +)***"; EXPECT_EQ(sink.get_code(), expected_null_code); } @@ -146,45 +168,48 @@ TEST_F(RowIRCudaCodeGenTest, BinaryOperation) { row_ir::target_info target_info{row_ir::target::CUDA}; - row_ir::var_info inputs[] = {{"in_0", {cudf::data_type{cudf::type_id::INT32}}}, - {"in_1", {cudf::data_type{cudf::type_id::DECIMAL32}}}}; - - row_ir::untyped_var_info outputs[] = {{"out_0"}, {"out_1"}}; - - row_ir::instance_info info{inputs, outputs}; - { - row_ir::instance_context ctx{}; + row_ir::instance_context ctx{cudf::get_default_stream(), + cudf::get_current_device_resource_ref()}; + [[maybe_unused]] auto in0 = ctx.add_input(*i32); + [[maybe_unused]] auto in1 = ctx.add_input(*d32); row_ir::code_sink sink; row_ir::node op{row_ir::opcode::ADD, + std::nullopt, row_ir::node{row_ir::input_reference{0}}, row_ir::node{row_ir::input_reference{0}}}; - op.instantiate(ctx, info); - op.emit_code(ctx, target_info, info, sink); + op.instantiate(ctx); + op.emit_code(ctx, target_info, sink); auto expected_code = - R"***(int32_t tmp_0 = in_0; -int32_t tmp_1 = in_0; -int32_t tmp_2; -cudf::ops::add(&tmp_2, &tmp_0, &tmp_1);)***"; + R"***(int32_t tmp_1 = in_0; +int32_t tmp_2 = in_0; +int32_t tmp_0; +cudf::ops::add(&tmp_0, &tmp_1, &tmp_2); +)***"; EXPECT_EQ(sink.get_code(), expected_code); } { - row_ir::instance_context ctx{}; + row_ir::instance_context ctx{cudf::get_default_stream(), + cudf::get_current_device_resource_ref()}; + [[maybe_unused]] auto in0 = ctx.add_input(*i32); + [[maybe_unused]] auto in1 = ctx.add_input(*d32); row_ir::code_sink sink; row_ir::node op{row_ir::opcode::ADD, + std::nullopt, row_ir::node{row_ir::input_reference{1}}, row_ir::node{row_ir::input_reference{1}}}; - op.instantiate(ctx, info); - op.emit_code(ctx, target_info, info, sink); + op.instantiate(ctx); + op.emit_code(ctx, target_info, sink); auto expected_null_code = - R"***(numeric::decimal32 tmp_0 = in_1; -numeric::decimal32 tmp_1 = in_1; -numeric::decimal32 tmp_2; -cudf::ops::add(&tmp_2, &tmp_0, &tmp_1);)***"; + R"***(numeric::decimal32 tmp_1 = in_1; +numeric::decimal32 tmp_2 = in_1; +numeric::decimal32 tmp_0; +cudf::ops::add(&tmp_0, &tmp_1, &tmp_2); +)***"; EXPECT_EQ(sink.get_code(), expected_null_code); } @@ -194,59 +219,55 @@ TEST_F(RowIRCudaCodeGenTest, VectorLengthOperation) { row_ir::target_info target_info{row_ir::target::CUDA}; - row_ir::var_info inputs[] = { - {"in_0", {cudf::data_type{cudf::type_id::FLOAT64}}}, - {"in_1", {cudf::data_type{cudf::type_id::FLOAT64}}}, - {"in_2", {cudf::data_type{cudf::type_id::FLOAT64}}}, - {"in_3", {cudf::data_type{cudf::type_id::FLOAT64}}}, - }; - - row_ir::untyped_var_info outputs[] = {{"out_0"}, {"out_1"}}; - - row_ir::instance_info info{inputs, outputs}; - auto length_operation = [&](int32_t input0, int32_t input1, int32_t output) { // This function generates the IR for the vector length operation: // length(v) = sqrt(x^2 + y^2) // where v = (x, y) and v is a 2D vector. auto x2 = row_ir::node(row_ir::opcode::MUL, + std::nullopt, row_ir::node{row_ir::input_reference{input0}}, row_ir::node{row_ir::input_reference{input0}}); auto y2 = row_ir::node(row_ir::opcode::MUL, + std::nullopt, row_ir::node{row_ir::input_reference{input1}}, row_ir::node{row_ir::input_reference{input1}}); - auto sum = row_ir::node(row_ir::opcode::ADD, std::move(x2), std::move(y2)); + auto sum = row_ir::node(row_ir::opcode::ADD, std::nullopt, std::move(x2), std::move(y2)); - auto length = row_ir::node(row_ir::opcode::SQRT, std::move(sum)); + auto length = row_ir::node(row_ir::opcode::SQRT, std::nullopt, std::move(sum)); - return std::make_unique(row_ir::opcode::SET_OUTPUT, std::move(length)); + return row_ir::node(row_ir::output_reference{0}, std::move(length)); }; { - row_ir::instance_context ctx{}; + row_ir::instance_context ctx{cudf::get_default_stream(), + cudf::get_current_device_resource_ref()}; + [[maybe_unused]] auto in0 = ctx.add_input(*f64); + [[maybe_unused]] auto in1 = ctx.add_input(*f64); + [[maybe_unused]] auto out0 = ctx.add_output(); row_ir::code_sink sink; auto expr_ir = length_operation(0, 1, 0); - expr_ir->instantiate(ctx, info); - expr_ir->emit_code(ctx, target_info, info, sink); + expr_ir.instantiate(ctx); + expr_ir.emit_code(ctx, target_info, sink); auto expected_code = - R"***(double tmp_0 = in_0; -double tmp_1 = in_0; -double tmp_2; -cudf::ops::mul(&tmp_2, &tmp_0, &tmp_1); -double tmp_3 = in_1; -double tmp_4 = in_1; -double tmp_5; -cudf::ops::mul(&tmp_5, &tmp_3, &tmp_4); + R"***(double tmp_4 = in_0; +double tmp_5 = in_0; +double tmp_3; +cudf::ops::mul(&tmp_3, &tmp_4, &tmp_5); +double tmp_7 = in_1; +double tmp_8 = in_1; double tmp_6; -cudf::ops::add(&tmp_6, &tmp_2, &tmp_5); -double tmp_7; -cudf::ops::sqrt(&tmp_7, &tmp_6); -double tmp_8 = tmp_7; -*out_0 = tmp_8;)***"; +cudf::ops::mul(&tmp_6, &tmp_7, &tmp_8); +double tmp_2; +cudf::ops::add(&tmp_2, &tmp_3, &tmp_6); +double tmp_1; +cudf::ops::sqrt(&tmp_1, &tmp_2); +double tmp_0 = tmp_1; +*out_0 = tmp_0; +)***"; EXPECT_EQ(sink.get_code(), expected_code); } @@ -268,12 +289,12 @@ TEST_F(RowIRCudaCodeGenTest, AstConversionBasic) auto expected = cudf::test::fixed_width_column_wrapper(expected_iter, expected_iter + column->size()); - row_ir::ast_args args{.table = cudf::table_view{{column->view()}}}; - auto transform_args = row_ir::ast_converter::compute_column(row_ir::target::CUDA, add_op, - args, + cudf::table_view{{*column}}, + cudf::table_view{}, + "expression", cudf::get_default_stream(), cudf::get_current_device_resource_ref()); @@ -281,8 +302,9 @@ TEST_F(RowIRCudaCodeGenTest, AstConversionBasic) ASSERT_EQ(transform_args.scalar_columns[0]->view().size(), 1); EXPECT_EQ(transform_args.source_type, cudf::udf_source_type::CUDA); EXPECT_EQ(transform_args.is_null_aware, cudf::null_aware::NO); - EXPECT_EQ(transform_args.null_policy, cudf::output_nullability::ALL_VALID); - EXPECT_EQ(transform_args.output_type, cudf::data_type{cudf::type_id::INT32}); + EXPECT_EQ(transform_args.outputs.size(), 1); + EXPECT_EQ(transform_args.outputs[0].nullability, cudf::output_nullability::ALL_VALID); + EXPECT_EQ(transform_args.outputs[0].type, cudf::data_type{cudf::type_id::INT32}); ASSERT_EQ(transform_args.inputs.size(), 2); /// The first input should be a scalar value of 42 @@ -298,74 +320,128 @@ TEST_F(RowIRCudaCodeGenTest, AstConversionBasic) EXPECT_EQ(std::get(transform_args.inputs[1]).null_count(), column->null_count()); - auto expected_udf = R"***( -__device__ void expression(int32_t* out_0, int32_t in_0, int32_t in_1) + auto expected_udf = + R"***(__device__ cudf::ops::errc expression(int32_t* out_0, int32_t in_0, int32_t in_1) { -int32_t tmp_0 = in_0; -int32_t tmp_1 = in_1; -int32_t tmp_2; -cudf::ops::add(&tmp_2, &tmp_0, &tmp_1); -int32_t tmp_3 = tmp_2; -*out_0 = tmp_3; - -return; -} -)***"; +int32_t tmp_2 = in_0; +int32_t tmp_3 = in_1; +int32_t tmp_1; +cudf::ops::add(&tmp_1, &tmp_2, &tmp_3); +int32_t tmp_0 = tmp_1; +*out_0 = tmp_0; +return cudf::ops::errc::OK; +})***"; EXPECT_EQ(transform_args.udf, expected_udf); - auto result = cudf::transform_extended(transform_args.inputs, - transform_args.udf, - transform_args.output_type, - transform_args.source_type, - transform_args.user_data, - transform_args.is_null_aware, - transform_args.row_size, - transform_args.null_policy); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view()); + auto result = cudf::multi_transform(transform_args.udf, + transform_args.source_type, + transform_args.is_null_aware, + transform_args.user_data, + transform_args.inputs, + transform_args.outputs, + std::move(transform_args.string_offsets), + transform_args.row_size, + transform_args.error_mode); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->get_column(0).view()); } TEST_F(RowIRCudaCodeGenTest, FilterPredicate) { row_ir::target_info target_info{row_ir::target::CUDA}; - row_ir::var_info inputs[] = {{"in_0", {cudf::data_type{cudf::type_id::BOOL8}}}}; + { + row_ir::instance_context ctx{cudf::get_default_stream(), + cudf::get_current_device_resource_ref()}; + [[maybe_unused]] auto in0 = ctx.add_input(*b8); + row_ir::code_sink sink; + row_ir::node filter_predicate( + row_ir::opcode::PREDICATE, std::nullopt, row_ir::node{row_ir::input_reference{0}}); + filter_predicate.instantiate(ctx); + filter_predicate.emit_code(ctx, target_info, sink); + + auto expected_code = R"***(bool tmp_1 = in_0; +bool tmp_0; +cudf::ops::predicate(&tmp_0, &tmp_1); +)***"; - row_ir::instance_info info{inputs, {}}; + EXPECT_EQ(sink.get_code(), expected_code); + } { - row_ir::instance_context ctx{}; + row_ir::instance_context ctx{cudf::get_default_stream(), + cudf::get_current_device_resource_ref()}; + [[maybe_unused]] auto in0 = ctx.add_input(*b8); row_ir::code_sink sink; - row_ir::node filter_predicate(row_ir::opcode::PREDICATE, - std::make_unique(row_ir::input_reference{0})); - filter_predicate.instantiate(ctx, info); - filter_predicate.emit_code(ctx, target_info, info, sink); - - auto expected_code = R"***(bool tmp_0 = in_0; -bool tmp_1; -cudf::ops::predicate(&tmp_1, &tmp_0); + row_ir::node filter_predicate( + row_ir::opcode::PREDICATE, std::nullopt, row_ir::node{row_ir::input_reference{0}}); + ctx.set_has_nulls(true); + filter_predicate.instantiate(ctx); + filter_predicate.emit_code(ctx, target_info, sink); + + auto expected_code = R"***(cuda::std::optional tmp_1 = in_0; +cuda::std::optional tmp_0; +cudf::ops::predicate(&tmp_0, &tmp_1); )***"; EXPECT_EQ(sink.get_code(), expected_code); } +} + +TEST_F(RowIRCudaCodeGenTest, FallibleBinaryOperation) +{ + row_ir::target_info target_info{row_ir::target::CUDA}; { - row_ir::instance_context ctx{}; + row_ir::instance_context ctx{cudf::get_default_stream(), + cudf::get_current_device_resource_ref()}; + [[maybe_unused]] auto in0 = ctx.add_input(*i32); + [[maybe_unused]] auto in1 = ctx.add_input(*d32); row_ir::code_sink sink; - row_ir::node filter_predicate(row_ir::opcode::PREDICATE, - std::make_unique(row_ir::input_reference{0})); - ctx.set_has_nulls(true); - filter_predicate.instantiate(ctx, info); - filter_predicate.emit_code(ctx, target_info, info, sink); + row_ir::node op{row_ir::opcode::ANSI_ADD, + std::nullopt, + row_ir::node{row_ir::input_reference{0}}, + row_ir::node{row_ir::input_reference{0}}}; + op.instantiate(ctx); + op.emit_code(ctx, target_info, sink); - auto expected_code = R"***(cuda::std::optional tmp_0 = in_0; -cuda::std::optional tmp_1; -cudf::ops::predicate(&tmp_1, &tmp_0); + auto expected_code = + R"***(int32_t tmp_1 = in_0; +int32_t tmp_2 = in_0; +int32_t tmp_0; +if(cudf::ops::errc e = cudf::ops::ansi_add(&tmp_0, &tmp_1, &tmp_2); e != cudf::ops::errc::OK) { +return e; +} )***"; EXPECT_EQ(sink.get_code(), expected_code); } + + { + row_ir::instance_context ctx{cudf::get_default_stream(), + cudf::get_current_device_resource_ref()}; + [[maybe_unused]] auto in0 = ctx.add_input(*i32); + [[maybe_unused]] auto in1 = ctx.add_input(*d32); + row_ir::code_sink sink; + row_ir::node op{row_ir::opcode::ANSI_ADD, + std::nullopt, + row_ir::node{row_ir::input_reference{1}}, + row_ir::node{row_ir::input_reference{1}}}; + op.instantiate(ctx); + op.emit_code(ctx, target_info, sink); + + auto expected_null_code = + R"***(numeric::decimal32 tmp_1 = in_1; +numeric::decimal32 tmp_2 = in_1; +numeric::decimal32 tmp_0; +if(cudf::ops::errc e = cudf::ops::ansi_add(&tmp_0, &tmp_1, &tmp_2); e != cudf::ops::errc::OK) { +return e; +} +)***"; + + EXPECT_EQ(sink.get_code(), expected_null_code); + } } CUDF_TEST_PROGRAM_MAIN() From dc47b85b349d49cf20808b6dcd34140f485b4b6e Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Fri, 1 May 2026 22:37:06 +0000 Subject: [PATCH 08/15] Implement JIT expressions and operations for ANSI SQL compatibility --- cpp/CMakeLists.txt | 1 + cpp/include/cudf/ast/jit_expressions.hpp | 161 +++++++++++- cpp/src/ast/jit_expressions.cpp | 231 ++++++++++++++++++ cpp/src/jit/row_ir.cpp | 10 + cpp/src/jit/row_ir.hpp | 12 +- cpp/src/stream_compaction/filter/filter.cu | 42 ++-- cpp/src/transform/transform.cu | 2 +- .../integration/unary_transform_test.cpp | 56 +++++ 8 files changed, 485 insertions(+), 30 deletions(-) create mode 100644 cpp/src/ast/jit_expressions.cpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 5a0b2f95e830..7a043d91e698 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -337,6 +337,7 @@ add_library( src/aggregation/result_cache.cpp src/ast/expression_parser.cpp src/ast/expressions.cpp + src/ast/jit_expressions.cpp src/ast/operators.cpp src/binaryop/binaryop.cpp src/binaryop/compiled/ATan2.cu diff --git a/cpp/include/cudf/ast/jit_expressions.hpp b/cpp/include/cudf/ast/jit_expressions.hpp index 044301df4af6..c96455201e72 100644 --- a/cpp/include/cudf/ast/jit_expressions.hpp +++ b/cpp/include/cudf/ast/jit_expressions.hpp @@ -5,6 +5,7 @@ #pragma once #include +#include namespace CUDF_EXPORT cudf { namespace ast { @@ -16,8 +17,166 @@ namespace ast { */ namespace jit { +namespace detail { -} +struct operation : public ast::expression { + /** +* @brief Construct a new operation object. +* @param op The opcode for this operation +* @param args The arguments for this operation + */ + operation(cudf::detail::row_ir::opcode op, + std::vector> args) + : op_{op}, args_{std::move(args)} + { + } + + /** + * @brief Construct a new operation object with a target scale (for rescale and precision check + * operations). + * @param op The opcode for this operation + * @param args The arguments for this operation + * @param target_scale The target scale for this operation (only applicable for rescale and precision check operations) + */ + operation(cudf::detail::row_ir::opcode op, + std::vector> args, + int32_t target_scale) + : op_{op}, args_{std::move(args)}, target_scale_{target_scale} + { + } + + operation(operation const&) = default; //< Copy constructor + operation(operation&&) = default; //< Move constructor + operation& operator=(operation const&) = default; //< Copy assignment + operation& operator=(operation&&) = default; //< Move assignment + ~operation() override = default; //< Destructor + + /** + * @brief Get the opcode. + * + * @return The opcode + */ + [[nodiscard]] cudf::detail::row_ir::opcode get_opcode() const { return op_; } + + /** + * @brief Get the operands. + * + * @return Vector of operands + */ + [[nodiscard]] std::span const> get_arguments() const + { + return args_; + } + + /** + * @brief Get the target scale for rescale and precision check operations. + * + * @return The target scale if applicable, std::nullopt otherwise + */ + [[nodiscard]] std::optional get_target_scale() const { return target_scale_; } + + /** + * @copydoc expression::accept + */ + cudf::size_type accept(cudf::ast::detail::expression_parser& visitor) const override; + + /** + * @copydoc expression::accept + */ + std::reference_wrapper accept( + cudf::ast::detail::expression_transformer& visitor) const override; + + [[nodiscard]] bool may_evaluate_null(table_view const& left, + table_view const& right, + rmm::cuda_stream_view stream) const override; + + /** + * @copydoc expression::accept + */ + [[nodiscard]] std::unique_ptr accept( + cudf::detail::row_ir::ast_converter& visitor) const override; + + private: + cudf::detail::row_ir::opcode op_; + std::vector> args_; + std::optional target_scale_ = std::nullopt; +}; + +} // namespace detail + +expression const& nullify_if(ast::tree& tree, expression const& condition); + +expression const& coalesce(ast::tree& tree, expression const& a, expression const& b); + +expression const& predicate(ast::tree& tree, expression const& condition); + +expression const& ansi_add(ast::tree& tree, expression const& a, expression const& b); + +expression const& ansi_sub(ast::tree& tree, expression const& a, expression const& b); + +expression const& ansi_mul(ast::tree& tree, expression const& a, expression const& b); + +expression const& ansi_div(ast::tree& tree, expression const& a, expression const& b); + +expression const& ansi_mod(ast::tree& tree, expression const& a, expression const& b); + +expression const& ansi_abs(ast::tree& tree, expression const& a); + +expression const& ansi_neg(ast::tree& tree, expression const& a); + +expression const& ansi_precision_check(ast::tree& tree, expression const& a, int32_t precision); + +expression const& ansi_try_add(ast::tree& tree, expression const& a, expression const& b); + +expression const& ansi_try_sub(ast::tree& tree, expression const& a, expression const& b); + +expression const& ansi_try_mul(ast::tree& tree, expression const& a, expression const& b); + +expression const& ansi_try_div(ast::tree& tree, expression const& a, expression const& b); + +expression const& ansi_try_mod(ast::tree& tree, expression const& a, expression const& b); + +expression const& ansi_try_abs(ast::tree& tree, expression const& a); + +expression const& ansi_try_neg(ast::tree& tree, expression const& a); + +expression const& ansi_try_precision_check(ast::tree& tree, expression const& a, int32_t precision); + +expression const& bit_shift_left(ast::tree& tree, expression const& a, expression const& b); + +expression const& bit_shift_right(ast::tree& tree, expression const& a, expression const& b); + +expression const& cast_to_b8(ast::tree& tree, expression const& a); + +expression const& cast_to_i8(ast::tree& tree, expression const& a); + +expression const& cast_to_i16(ast::tree& tree, expression const& a); + +expression const& cast_to_i32(ast::tree& tree, expression const& a); + +expression const& cast_to_i64(ast::tree& tree, expression const& a); + +expression const& cast_to_u8(ast::tree& tree, expression const& a); + +expression const& cast_to_u16(ast::tree& tree, expression const& a); + +expression const& cast_to_u32(ast::tree& tree, expression const& a); + +expression const& cast_to_u64(ast::tree& tree, expression const& a); + +expression const& cast_to_f32(ast::tree& tree, expression const& a); + +expression const& cast_to_f64(ast::tree& tree, expression const& a); + +expression const& cast_to_dec32(ast::tree& tree, expression const& a); + +expression const& cast_to_dec64(ast::tree& tree, expression const& a); + +expression const& cast_to_dec128(ast::tree& tree, expression const& a); + +expression const& rescale(ast::tree& tree, expression const& a, int32_t new_scale); + +} // namespace jit } // namespace ast } // namespace CUDF_EXPORT cudf diff --git a/cpp/src/ast/jit_expressions.cpp b/cpp/src/ast/jit_expressions.cpp new file mode 100644 index 000000000000..7571b0ce3688 --- /dev/null +++ b/cpp/src/ast/jit_expressions.cpp @@ -0,0 +1,231 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#include "cudf/ast/jit_expressions.hpp" + +#include "jit/row_ir.hpp" + +namespace cudf { +namespace ast { + +namespace jit::detail { + +cudf::size_type operation::accept(cudf::ast::detail::expression_parser& visitor) const +{ + CUDF_FAIL("predicate is an internal expression and should not be visited by expression_parser", + std::invalid_argument); +} + +std::reference_wrapper operation::accept( + cudf::ast::detail::expression_transformer& visitor) const +{ + CUDF_FAIL( + "predicate is an internal expression and should not be visited by " + "expression_transformer", + std::invalid_argument); +} + +bool operation::may_evaluate_null(table_view const& left, + table_view const& right, + rmm::cuda_stream_view stream) const +{ + CUDF_FAIL("predicate is an internal expression and should not be evaluated directly", + std::invalid_argument); +} + +std::unique_ptr operation::accept( + cudf::detail::row_ir::ast_converter& converter) const +{ + return converter.add_ir_node(*this); +} + +} // namespace jit::detail + +expression const& jit::nullify_if(ast::tree& tree, expression const& condition) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::NULLIFY_IF, {condition})); +} + +expression const& jit::coalesce(ast::tree& tree, expression const& a, expression const& b) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::COALESCE, {a, b})); +} + +expression const& jit::predicate(ast::tree& tree, expression const& condition) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::PREDICATE, {condition})); +} + +expression const& jit::ansi_add(ast::tree& tree, expression const& a, expression const& b) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::ANSI_ADD, {a, b})); +} + +expression const& jit::ansi_sub(ast::tree& tree, expression const& a, expression const& b) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::ANSI_SUB, {a, b})); +} + +expression const& jit::ansi_mul(ast::tree& tree, expression const& a, expression const& b) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::ANSI_MUL, {a, b})); +} + +expression const& jit::ansi_div(ast::tree& tree, expression const& a, expression const& b) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::ANSI_DIV, {a, b})); +} + +expression const& jit::ansi_mod(ast::tree& tree, expression const& a, expression const& b) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::ANSI_MOD, {a, b})); +} + +expression const& jit::ansi_abs(ast::tree& tree, expression const& a) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::ANSI_ABS, {a})); +} + +expression const& jit::ansi_neg(ast::tree& tree, expression const& a) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::ANSI_NEG, {a})); +} + +expression const& jit::ansi_precision_check(ast::tree& tree, expression const& a, int32_t precision) +{ + // TODO: actually insert a precision + return tree.push( + detail::operation(cudf::detail::row_ir::opcode::ANSI_PRECISION_CHECK, {a}, precision)); +} + +expression const& jit::ansi_try_add(ast::tree& tree, expression const& a, expression const& b) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::ANSI_TRY_ADD, {a, b})); +} + +expression const& jit::ansi_try_sub(ast::tree& tree, expression const& a, expression const& b) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::ANSI_TRY_SUB, {a, b})); +} + +expression const& jit::ansi_try_mul(ast::tree& tree, expression const& a, expression const& b) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::ANSI_TRY_MUL, {a, b})); +} + +expression const& jit::ansi_try_div(ast::tree& tree, expression const& a, expression const& b) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::ANSI_TRY_DIV, {a, b})); +} + +expression const& jit::ansi_try_mod(ast::tree& tree, expression const& a, expression const& b) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::ANSI_TRY_MOD, {a, b})); +} + +expression const& jit::ansi_try_abs(ast::tree& tree, expression const& a) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::ANSI_TRY_ABS, {a})); +} + +expression const& jit::ansi_try_neg(ast::tree& tree, expression const& a) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::ANSI_TRY_NEG, {a})); +} + +expression const& jit::ansi_try_precision_check(ast::tree& tree, + expression const& a, + int32_t precision) +{ + return tree.push( + detail::operation(cudf::detail::row_ir::opcode::ANSI_TRY_PRECISION_CHECK, {a}, precision)); +} + +expression const& jit::bit_shift_left(ast::tree& tree, expression const& a, expression const& b) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::BIT_SHIFT_LEFT, {a, b})); +} + +expression const& jit::bit_shift_right(ast::tree& tree, expression const& a, expression const& b) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::BIT_SHIFT_RIGHT, {a, b})); +} + +expression const& jit::cast_to_b8(ast::tree& tree, expression const& a) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::CAST_TO_B8, {a})); +} + +expression const& jit::cast_to_i8(ast::tree& tree, expression const& a) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::CAST_TO_I8, {a})); +} + +expression const& jit::cast_to_i16(ast::tree& tree, expression const& a) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::CAST_TO_I16, {a})); +} + +expression const& jit::cast_to_i32(ast::tree& tree, expression const& a) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::CAST_TO_I32, {a})); +} + +expression const& jit::cast_to_i64(ast::tree& tree, expression const& a) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::CAST_TO_I64, {a})); +} + +expression const& jit::cast_to_u8(ast::tree& tree, expression const& a) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::CAST_TO_U8, {a})); +} + +expression const& jit::cast_to_u16(ast::tree& tree, expression const& a) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::CAST_TO_U16, {a})); +} + +expression const& jit::cast_to_u32(ast::tree& tree, expression const& a) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::CAST_TO_U32, {a})); +} + +expression const& jit::cast_to_u64(ast::tree& tree, expression const& a) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::CAST_TO_U64, {a})); +} + +expression const& jit::cast_to_f32(ast::tree& tree, expression const& a) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::CAST_TO_F32, {a})); +} + +expression const& jit::cast_to_f64(ast::tree& tree, expression const& a) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::CAST_TO_F64, {a})); +} + +expression const& jit::cast_to_dec32(ast::tree& tree, expression const& a) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::CAST_TO_DEC32, {a})); +} + +expression const& jit::cast_to_dec64(ast::tree& tree, expression const& a) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::CAST_TO_DEC64, {a})); +} + +expression const& jit::cast_to_dec128(ast::tree& tree, expression const& a) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::CAST_TO_DEC128, {a})); +} + +expression const& jit::rescale(ast::tree& tree, expression const& a, int32_t target_scale) +{ + return tree.push(detail::operation(cudf::detail::row_ir::opcode::RESCALE, {a}, target_scale)); +} + +} // namespace ast +} // namespace cudf diff --git a/cpp/src/jit/row_ir.cpp b/cpp/src/jit/row_ir.cpp index 3f41044b81ac..99caae038513 100644 --- a/cpp/src/jit/row_ir.cpp +++ b/cpp/src/jit/row_ir.cpp @@ -464,6 +464,16 @@ std::unique_ptr ast_converter::add_ir_node(ast::detail::predicate row_ir::opcode::PREDICATE, std::nullopt, expr.get_operand().accept(*this)); } +std::unique_ptr ast_converter::add_ir_node(ast::jit::detail::operation const& expr) +{ + std::vector> args; + for (auto &arg : expr.get_arguments()) { + args.emplace_back(arg.get().accept(*this)); + } + return std::make_unique( + expr.get_opcode(), expr.get_target_scale(), std::move(args)); +} + bool is_nullable(scalar_input const& in) { return in.scalar_column->view().nullable(); } bool is_nullable(column_input const& in) { return in.column.nullable(); } diff --git a/cpp/src/jit/row_ir.hpp b/cpp/src/jit/row_ir.hpp index 21fd46dcce4f..1686faed997b 100644 --- a/cpp/src/jit/row_ir.hpp +++ b/cpp/src/jit/row_ir.hpp @@ -6,6 +6,7 @@ #pragma once #include #include +#include #include #include #include @@ -409,13 +410,7 @@ struct [[nodiscard]] ast_converter { ~ast_converter() = default; ///< Destructor - private: - friend class ast::literal; - friend class ast::column_reference; - friend class ast::operation; - friend class ast::column_name_reference; - friend class ast::detail::predicate; - + public: [[nodiscard]] std::unique_ptr add_ir_node(ast::literal const& expr); [[nodiscard]] std::unique_ptr add_ir_node(ast::column_reference const& expr); @@ -424,10 +419,11 @@ struct [[nodiscard]] ast_converter { [[nodiscard]] std::unique_ptr add_ir_node(ast::detail::predicate const& expr); + [[nodiscard]] std::unique_ptr add_ir_node(ast::jit::detail::operation const& expr); + [[nodiscard]] std::tuple generate_code( target target, ast::expression const& expr, std::string_view function_name); - public: /** * @brief Convert an AST `compute_column` expression to a `cudf::transform` * @param target The target for which the IR is generated diff --git a/cpp/src/stream_compaction/filter/filter.cu b/cpp/src/stream_compaction/filter/filter.cu index e2f8b43a3fbd..f7eea83b7af9 100644 --- a/cpp/src/stream_compaction/filter/filter.cu +++ b/cpp/src/stream_compaction/filter/filter.cu @@ -100,16 +100,17 @@ std::vector> filter_extended( rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::filter(predicate_udf, - source_type, - is_null_aware, - user_data, - predicate_inputs, - table_view{filter_columns}, - ops::error_mode::IGNORE, - predicate_nullability, - stream, - mr); + auto table = detail::filter(predicate_udf, + source_type, + is_null_aware, + user_data, + predicate_inputs, + table_view{filter_columns}, + ops::error_mode::IGNORE, + predicate_nullability, + stream, + mr); + return table->release(); } std::vector> filter(std::vector const& predicate_columns, @@ -137,16 +138,17 @@ std::vector> filter(std::vector const& pred } } - return detail::filter(predicate_udf, - is_ptx ? cudf::udf_source_type::PTX : cudf::udf_source_type::CUDA, - is_null_aware, - user_data, - inputs, - table_view{filter_columns}, - ops::error_mode::IGNORE, - predicate_nullability, - stream, - mr); + auto table = detail::filter(predicate_udf, + is_ptx ? cudf::udf_source_type::PTX : cudf::udf_source_type::CUDA, + is_null_aware, + user_data, + inputs, + table_view{filter_columns}, + ops::error_mode::IGNORE, + predicate_nullability, + stream, + mr); + return table->release(); } } // namespace cudf diff --git a/cpp/src/transform/transform.cu b/cpp/src/transform/transform.cu index e1942ff0a3a9..a8886eb8273e 100644 --- a/cpp/src/transform/transform.cu +++ b/cpp/src/transform/transform.cu @@ -979,7 +979,7 @@ std::unique_ptr compute_column_jit(table_view const& table, args.outputs, std::move(args.string_offsets), args.row_size, - args.error_handling_mode, + args.error_mode, stream, mr); auto cols = result->release(); diff --git a/cpp/tests/transform/integration/unary_transform_test.cpp b/cpp/tests/transform/integration/unary_transform_test.cpp index 9c342a8f06c7..cdbc50aa8dac 100644 --- a/cpp/tests/transform/integration/unary_transform_test.cpp +++ b/cpp/tests/transform/integration/unary_transform_test.cpp @@ -200,6 +200,62 @@ __device__ inline void fdsf ( test_udf(ptx, op, data_init, 0, cudf::udf_source_type::PTX); } +TEST_F(UnaryOperationIntegrationTest, Transform_ErrorHandling) +{ + // c = a*a*a*a + std::string const cuda = + R"***( +__device__ cudf::ops::errc expression ( + int* C, + int a, + int b +) +{ + return cudf::ops::ansi_div(C, &a, &b); +} +)***"; + + cudf::test::fixed_width_column_wrapper a{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + cudf::test::fixed_width_column_wrapper b_fail{1, 2, 3, 4, 5, 6, 0, 8, 9, 10}; + cudf::test::fixed_width_column_wrapper b{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + cudf::test::fixed_width_column_wrapper expected{1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; + + { + cudf::transform_input inputs[] = {a, b}; + cudf::transform_output outputs[] = { + {cudf::data_type(cudf::type_id::INT32), cudf::output_nullability::ALL_VALID}}; + std::unique_ptr result; + EXPECT_NO_THROW(result = cudf::multi_transform(cuda, + cudf::udf_source_type::CUDA, + cudf::null_aware::NO, + std::nullopt, + inputs, + outputs, + {}, + std::nullopt, + cudf::ops::error_mode::ANY_ROW)); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->get_column(0), expected); + } + + { + cudf::transform_input inputs[] = {a, b_fail}; + cudf::transform_output outputs[] = { + {cudf::data_type(cudf::type_id::INT32), cudf::output_nullability::ALL_VALID}}; + std::unique_ptr result; + EXPECT_THROW(result = cudf::multi_transform(cuda, + cudf::udf_source_type::CUDA, + cudf::null_aware::NO, + std::nullopt, + inputs, + outputs, + {}, + std::nullopt, + cudf::ops::error_mode::ANY_ROW), + std::overflow_error); + } +} + TEST_F(UnaryOperationIntegrationTest, Transform_INT32_INT32) { // c = a * a - a From 3a1c7cafb50f7fdc967ee69209027ce1ca14e48a Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Sun, 3 May 2026 01:07:11 +0000 Subject: [PATCH 09/15] Enhance JIT Expression Handling and Add Tests - Mark `expression` struct with [[nodiscard]] to prevent unintended discards. - Update `operation` constructors and methods for improved clarity and consistency. - Introduce new JIT expressions: `nullify_if`, `coalesce`, `predicate`, and ANSI-compliant arithmetic operations (add, sub, mul, div) with error handling. - Implement tests for JIT expressions including `nullify_if`, `coalesce`, and ANSI arithmetic operations for both integer and decimal types. - Ensure proper handling of overflow scenarios in arithmetic operations. - Refactor null awareness and validity checks in the row IR to improve robustness. - Add additional casting and rescaling expressions with corresponding tests. --- cpp/include/cudf/ast/expressions.hpp | 2 +- cpp/include/cudf/ast/jit_expressions.hpp | 294 +++++++++++++++++- .../cudf/operators/ansi_arithmetic.cuh | 26 +- cpp/src/ast/jit_expressions.cpp | 15 +- cpp/src/jit/row_ir.cpp | 56 ++-- cpp/tests/ast/transform_tests.cpp | 274 ++++++++++++++++ 6 files changed, 613 insertions(+), 54 deletions(-) diff --git a/cpp/include/cudf/ast/expressions.hpp b/cpp/include/cudf/ast/expressions.hpp index 504679bd9788..22ace86028e0 100644 --- a/cpp/include/cudf/ast/expressions.hpp +++ b/cpp/include/cudf/ast/expressions.hpp @@ -59,7 +59,7 @@ class expression_transformer; * This class is a part of a "visitor" pattern with the `expression_parser` class. * Expressions inheriting from this class can accept parsers as visitors. */ -struct expression { +struct [[nodiscard]] expression { /** * @brief Accepts a visitor class. * diff --git a/cpp/include/cudf/ast/jit_expressions.hpp b/cpp/include/cudf/ast/jit_expressions.hpp index c96455201e72..23a5f87ba3db 100644 --- a/cpp/include/cudf/ast/jit_expressions.hpp +++ b/cpp/include/cudf/ast/jit_expressions.hpp @@ -21,10 +21,10 @@ namespace detail { struct operation : public ast::expression { /** -* @brief Construct a new operation object. -* @param op The opcode for this operation -* @param args The arguments for this operation - */ + * @brief Construct a new operation object. + * @param op The opcode for this operation + * @param args The arguments for this operation + */ operation(cudf::detail::row_ir::opcode op, std::vector> args) : op_{op}, args_{std::move(args)} @@ -32,11 +32,12 @@ struct operation : public ast::expression { } /** - * @brief Construct a new operation object with a target scale (for rescale and precision check - * operations). - * @param op The opcode for this operation - * @param args The arguments for this operation - * @param target_scale The target scale for this operation (only applicable for rescale and precision check operations) + * @brief Construct a new operation object with a target scale (for rescale and precision check + * operations). + * @param op The opcode for this operation + * @param args The arguments for this operation + * @param target_scale The target scale for this operation (only applicable for rescale and + * precision check operations) */ operation(cudf::detail::row_ir::opcode op, std::vector> args, @@ -45,11 +46,11 @@ struct operation : public ast::expression { { } - operation(operation const&) = default; //< Copy constructor - operation(operation&&) = default; //< Move constructor - operation& operator=(operation const&) = default; //< Copy assignment - operation& operator=(operation&&) = default; //< Move assignment - ~operation() override = default; //< Destructor + operation(operation const&) = default; //< Copy constructor + operation(operation&&) = default; //< Move constructor + operation& operator=(operation const&) = default; //< Copy assignment + operation& operator=(operation&&) = default; //< Move assignment + ~operation() override = default; //< Destructor /** * @brief Get the opcode. @@ -104,76 +105,335 @@ struct operation : public ast::expression { } // namespace detail -expression const& nullify_if(ast::tree& tree, expression const& condition); +/** + * @brief Creates an expression that evaluates to `NULL` if the condition is true, and the value of + * `a` otherwise. + * @param tree The expression tree to which this expression will be added + * @param a The expression to nullify if the condition is true + * @param condition The condition under which to nullify the value + * @return An expression representing the nullified value + */ +expression const& nullify_if(ast::tree& tree, expression const& a, expression const& condition); +/** + * @brief Creates an expression that evaluates to the first non-null value among its arguments. + * @param tree The expression tree to which this expression will be added + * @param a The first expression to coalesce + * @param b The second expression to coalesce + * @return An expression representing the coalesced value + */ expression const& coalesce(ast::tree& tree, expression const& a, expression const& b); +/** + * @brief Creates an expression that evaluates to `true` if the condition is true and not null, and + * `false` otherwise. This is used to implement predicates in the JIT. + * @param tree The expression tree to which this expression will be added + * @param condition The condition to evaluate as a predicate + * @return An expression representing the result of the predicate + */ expression const& predicate(ast::tree& tree, expression const& condition); +/** + * @brief Creates an expression that performs ANSI-compliant addition of `a` and `b`, which throws + * an error on overflow. + * @param tree The expression tree to which this expression will be added + * @param a The first addend + * @param b The second addend + * @return An expression representing the result of the addition + */ expression const& ansi_add(ast::tree& tree, expression const& a, expression const& b); +/** + * @brief Creates an expression that performs ANSI-compliant subtraction of `a` and `b`, which + * throws an error on overflow. + * @param tree The expression tree to which this expression will be added + * @param a The minuend + * @param b The subtrahend + * @return An expression representing the result of the subtraction + */ expression const& ansi_sub(ast::tree& tree, expression const& a, expression const& b); +/** + * @brief Creates an expression that performs ANSI-compliant multiplication of `a` and `b`, which + * throws an error on overflow. + * @param tree The expression tree to which this expression will be added + * @param a The first factor + * @param b The second factor + * @return An expression representing the result of the multiplication + */ expression const& ansi_mul(ast::tree& tree, expression const& a, expression const& b); +/** + * @brief Creates an expression that performs ANSI-compliant division of `a` by `b`, which throws an + * error on division by zero. + * @param tree The expression tree to which this expression will be added + * @param a The dividend + * @param b The divisor + * @return An expression representing the result of the division + */ expression const& ansi_div(ast::tree& tree, expression const& a, expression const& b); +/** + * @brief Creates an expression that performs ANSI-compliant modulus of `a` by `b`, which throws an + * error on division by zero. + * @param tree The expression tree to which this expression will be added + * @param a The value to be divided + * @param b The divisor + * @return An expression representing the result of the modulus operation + */ expression const& ansi_mod(ast::tree& tree, expression const& a, expression const& b); +/** + * @brief Creates an expression that performs ANSI-compliant absolute value of `a`, which throws an + * error on overflow. + * @param tree The expression tree to which this expression will be added + * @param a The value for which to compute the absolute value + * @return An expression representing the absolute value + */ expression const& ansi_abs(ast::tree& tree, expression const& a); +/** + * @brief Creates an expression that performs ANSI-compliant negation of `a`, which throws an error + * on overflow. + * @param tree The expression tree to which this expression will be added + * @param a The value to negate + * @return An expression representing the negated value + */ expression const& ansi_neg(ast::tree& tree, expression const& a); -expression const& ansi_precision_check(ast::tree& tree, expression const& a, int32_t precision); +/** + * @brief Creates an expression that performs an ANSI-compliant precision check on `a` with the + * given precision, which throws an error if the value of `a` exceeds the specified precision. + * @param tree The expression tree to which this expression will be added + * @param a The value for which to perform the precision check + * @param precision The precision to check against + * @return An expression representing the result of the precision check + */ +expression const& ansi_precision_check(ast::tree& tree, + expression const& a, + expression const& precision); +/** + * @brief Creates an expression that performs ANSI-compliant addition of `a` and `b`, which returns + * `NULL` on overflow instead of throwing an error. + * @param tree The expression tree to which this expression will be added + * @param a The first addend + * @param b The second addend + * @return An expression representing the result of the addition, or `NULL` if overflow occurs + */ expression const& ansi_try_add(ast::tree& tree, expression const& a, expression const& b); +/** + * @brief Creates an expression that performs ANSI-compliant subtraction of `a` and `b`, which + * returns `NULL` on overflow instead of throwing an error. + * @param tree The expression tree to which this expression will be added + * @param a The minuend + * @param b The subtrahend + * @return An expression representing the result of the subtraction, or `NULL` if overflow occurs + */ expression const& ansi_try_sub(ast::tree& tree, expression const& a, expression const& b); +/** + * @brief Creates an expression that performs ANSI-compliant multiplication of `a` and `b`, which + * returns `NULL` on overflow instead of throwing an error. + * @param tree The expression tree to which this expression will be added + * @param a The first factor + * @param b The second factor + * @return An expression representing the result of the multiplication, or `NULL` if overflow occurs + */ expression const& ansi_try_mul(ast::tree& tree, expression const& a, expression const& b); +/** + * @brief Creates an expression that performs ANSI-compliant division of `a` by `b`, which returns + * `NULL` on division by zero instead of throwing an error. + * @param tree The expression tree to which this expression will be added + * @param a The dividend + * @param b The divisor + * @return An expression representing the result of the division, or `NULL` if division by zero + * occurs + */ expression const& ansi_try_div(ast::tree& tree, expression const& a, expression const& b); +/** + * @brief Creates an expression that performs ANSI-compliant modulus of `a` by `b`, which returns + * `NULL` on division by zero instead of throwing an error. + * @param tree The expression tree to which this expression will be added + * @param a The value to be divided + * @param b The divisor + * @return An expression representing the result of the modulus operation, or `NULL` if division by + * zero occurs + */ expression const& ansi_try_mod(ast::tree& tree, expression const& a, expression const& b); +/** + * @brief Creates an expression that performs ANSI-compliant absolute value of `a`, which returns + * `NULL` on overflow instead of throwing an error. + * @param tree The expression tree to which this expression will be added + * @param a The value for which to compute the absolute value + * @return An expression representing the absolute value, or `NULL` if overflow occurs + */ expression const& ansi_try_abs(ast::tree& tree, expression const& a); +/** + * @brief Creates an expression that performs ANSI-compliant negation of `a`, which returns `NULL` + * on overflow instead of throwing an error. + * @param tree The expression tree to which this expression will be added + * @param a The value to negate + * @return An expression representing the negated value, or `NULL` if overflow occurs + */ expression const& ansi_try_neg(ast::tree& tree, expression const& a); -expression const& ansi_try_precision_check(ast::tree& tree, expression const& a, int32_t precision); +/** + * @brief Creates an expression that performs an ANSI-compliant precision check on `a` with the + * given precision, which returns `NULL` if the value of `a` exceeds the specified precision instead + * of throwing an error. + * @param tree The expression tree to which this expression will be added + * @param a The value for which to perform the precision check + * @param precision The precision to check against + * @return An expression representing the result of the precision check, or `NULL` if the value + * exceeds the specified precision + */ +expression const& ansi_try_precision_check(ast::tree& tree, + expression const& a, + expression const& precision); +/** + * @brief Creates an expression that performs a bitwise left shift of `a` by `b`. + * @param tree The expression tree to which this expression will be added + * @param a The value to shift + * @param b The number of bits by which to shift + * @return An expression representing the result of the bitwise left shift + */ expression const& bit_shift_left(ast::tree& tree, expression const& a, expression const& b); +/** + * @brief Creates an expression that performs a bitwise right shift of `a` by `b`. + * @param tree The expression tree to which this expression will be added + * @param a The value to shift + * @param b The number of bits by which to shift + * @return An expression representing the result of the bitwise right shift + */ expression const& bit_shift_right(ast::tree& tree, expression const& a, expression const& b); +/** + * @brief Creates an expression that casts `a` to a boolean type. + * @param tree The expression tree to which this expression will be added + * @param a The value to cast + * @return An expression representing the result of the cast + */ expression const& cast_to_b8(ast::tree& tree, expression const& a); +/** + * @brief Creates an expression that casts `a` to a 8-bit signed integer type. + * @param tree The expression tree to which this expression will be added + * @param a The value to cast + * @return An expression representing the result of the cast + */ expression const& cast_to_i8(ast::tree& tree, expression const& a); +/** + * @brief Creates an expression that casts `a` to a 16-bit signed integer type. + * @param tree The expression tree to which this expression will be added + * @param a The value to cast + * @return An expression representing the result of the cast + */ expression const& cast_to_i16(ast::tree& tree, expression const& a); +/** + * @brief Creates an expression that casts `a` to a 32-bit signed integer type. + * @param tree The expression tree to which this expression will be added + * @param a The value to cast + * @return An expression representing the result of the cast + */ expression const& cast_to_i32(ast::tree& tree, expression const& a); +/** + * @brief Creates an expression that casts `a` to a 64-bit signed integer type. + * @param tree The expression tree to which this expression will be added + * @param a The value to cast + * @return An expression representing the result of the cast + */ expression const& cast_to_i64(ast::tree& tree, expression const& a); +/** + * @brief Creates an expression that casts `a` to an 8-bit unsigned integer type. + * @param tree The expression tree to which this expression will be added + * @param a The value to cast + * @return An expression representing the result of the cast + */ expression const& cast_to_u8(ast::tree& tree, expression const& a); +/** + * @brief Creates an expression that casts `a` to a 16-bit unsigned integer type. + * @param tree The expression tree to which this expression will be added + * @param a The value to cast + * @return An expression representing the result of the cast + */ expression const& cast_to_u16(ast::tree& tree, expression const& a); +/** + * @brief Creates an expression that casts `a` to a 32-bit unsigned integer type. + * @param tree The expression tree to which this expression will be added + * @param a The value to cast + * @return An expression representing the result of the cast + */ expression const& cast_to_u32(ast::tree& tree, expression const& a); +/** + * @brief Creates an expression that casts `a` to a 64-bit unsigned integer type. + * @param tree The expression tree to which this expression will be added + * @param a The value to cast + * @return An expression representing the result of the cast + */ expression const& cast_to_u64(ast::tree& tree, expression const& a); +/** + * @brief Creates an expression that casts `a` to a 32-bit floating point type. + * @param tree The expression tree to which this expression will be added + * @param a The value to cast + * @return An expression representing the result of the cast + */ expression const& cast_to_f32(ast::tree& tree, expression const& a); +/** + * @brief Creates an expression that casts `a` to a 64-bit floating point type. + * @param tree The expression tree to which this expression will be added + * @param a The value to cast + * @return An expression representing the result of the cast + */ expression const& cast_to_f64(ast::tree& tree, expression const& a); +/** + * @brief Creates an expression that casts `a` to a 32-bit decimal type. + * @param tree The expression tree to which this expression will be added + * @param a The value to cast + * @return An expression representing the result of the cast + */ expression const& cast_to_dec32(ast::tree& tree, expression const& a); +/** + * @brief Creates an expression that casts `a` to a 64-bit decimal type. + * @param tree The expression tree to which this expression will be added + * @param a The value to cast + * @return An expression representing the result of the cast + */ expression const& cast_to_dec64(ast::tree& tree, expression const& a); +/** + * @brief Creates an expression that casts `a` to a 128-bit decimal type. + * @param tree The expression tree to which this expression will be added + * @param a The value to cast + * @return An expression representing the result of the cast + */ expression const& cast_to_dec128(ast::tree& tree, expression const& a); +/** + * @brief Creates an expression that rescales a decimal expression `a` to a new scale `new_scale`. + * @param tree The expression tree to which this expression will be added + * @param a The decimal expression to rescale + * @param new_scale The new scale to which to rescale the decimal expression + * @return An expression representing the rescaled decimal value + */ expression const& rescale(ast::tree& tree, expression const& a, int32_t new_scale); } // namespace jit diff --git a/cpp/include/cudf/operators/ansi_arithmetic.cuh b/cpp/include/cudf/operators/ansi_arithmetic.cuh index 9bfac77021b1..fc084afa6948 100644 --- a/cpp/include/cudf/operators/ansi_arithmetic.cuh +++ b/cpp/include/cudf/operators/ansi_arithmetic.cuh @@ -96,13 +96,13 @@ template __device__ inline errc ansi_add(decimal* out, decimal const* a, decimal const* b) { auto scale = cuda::std::min(a->scale(), b->scale()); - auto sum = a->rescaled(scale).value() + b->rescaled(scale).value(); - if (numeric::addition_overflow(a->rescaled(scale).value(), b->rescaled(scale).value())) { + if (numeric::addition_overflow(a->rescaled(scale).value(), b->rescaled(scale).value())) { return errc::OVERFLOW; } - *out = decimal{numeric::scaled_integer{sum, scale}}; + *out = decimal{numeric::scaled_integer{ + a->rescaled(scale).value() + b->rescaled(scale).value(), numeric::scale_type{scale}}}; return errc::OK; } @@ -159,13 +159,13 @@ template __device__ inline errc ansi_sub(decimal* out, decimal const* a, decimal const* b) { auto scale = cuda::std::min(a->scale(), b->scale()); - auto sum = a->rescaled(scale).value() - b->rescaled(scale).value(); - if (numeric::subtraction_overflow(a->rescaled(scale).value(), b->rescaled(scale).value())) { + if (numeric::subtraction_overflow(a->rescaled(scale).value(), b->rescaled(scale).value())) { return errc::OVERFLOW; } - *out = decimal{numeric::scaled_integer{sum, scale}}; + *out = decimal{numeric::scaled_integer{ + a->rescaled(scale).value() - b->rescaled(scale).value(), numeric::scale_type{scale}}}; return errc::OK; } @@ -221,9 +221,10 @@ __device__ inline errc ansi_mul(T* out, T const* a, T const* b) template __device__ inline errc ansi_mul(decimal* out, decimal const* a, decimal const* b) { - if (numeric::multiplication_overflow(a->value(), b->value())) { return errc::OVERFLOW; } + if (numeric::multiplication_overflow(a->value(), b->value())) { return errc::OVERFLOW; } - *out = decimal{numeric::scaled_integer{a->value() * b->value(), a->scale() + b->scale()}}; + *out = decimal{numeric::scaled_integer{a->value() * b->value(), + numeric::scale_type{a->scale() + b->scale()}}}; return errc::OK; } @@ -280,9 +281,12 @@ __device__ inline errc ansi_div(T* out, T const* a, T const* b) template __device__ inline errc ansi_div(decimal* out, decimal const* a, decimal const* b) { - if (numeric::division_overflow(a->value(), b->value())) { return errc::OVERFLOW; } + if (numeric::division_overflow(a->value(), b->value()) || b->value() == 0) { + return errc::OVERFLOW; + } - *out = decimal{numeric::scaled_integer{a->value() / b->value(), a->scale() - b->scale()}}; + *out = decimal{numeric::scaled_integer{a->value() / b->value(), + numeric::scale_type{a->scale() - b->scale()}}}; return errc::OK; } @@ -428,7 +432,7 @@ __device__ inline errc ansi_neg(decimal* out, decimal const* a) { if (a->value() == cuda::std::numeric_limits::min()) { return errc::OVERFLOW; } auto rep = -a->value(); - *out = decimal{numeric::scaled_integer{rep, a->scale()}}; + *out = decimal{numeric::scaled_integer{rep, numeric::scale_type{a->scale()}}}; return errc::OK; } diff --git a/cpp/src/ast/jit_expressions.cpp b/cpp/src/ast/jit_expressions.cpp index 7571b0ce3688..3911b1ee6c8a 100644 --- a/cpp/src/ast/jit_expressions.cpp +++ b/cpp/src/ast/jit_expressions.cpp @@ -42,9 +42,9 @@ std::unique_ptr operation::accept( } // namespace jit::detail -expression const& jit::nullify_if(ast::tree& tree, expression const& condition) +expression const& jit::nullify_if(ast::tree& tree, expression const& a, expression const& condition) { - return tree.push(detail::operation(cudf::detail::row_ir::opcode::NULLIFY_IF, {condition})); + return tree.push(detail::operation(cudf::detail::row_ir::opcode::NULLIFY_IF, {a, condition})); } expression const& jit::coalesce(ast::tree& tree, expression const& a, expression const& b) @@ -92,11 +92,12 @@ expression const& jit::ansi_neg(ast::tree& tree, expression const& a) return tree.push(detail::operation(cudf::detail::row_ir::opcode::ANSI_NEG, {a})); } -expression const& jit::ansi_precision_check(ast::tree& tree, expression const& a, int32_t precision) +expression const& jit::ansi_precision_check(ast::tree& tree, + expression const& a, + expression const& precision) { - // TODO: actually insert a precision return tree.push( - detail::operation(cudf::detail::row_ir::opcode::ANSI_PRECISION_CHECK, {a}, precision)); + detail::operation(cudf::detail::row_ir::opcode::ANSI_PRECISION_CHECK, {a, precision})); } expression const& jit::ansi_try_add(ast::tree& tree, expression const& a, expression const& b) @@ -136,10 +137,10 @@ expression const& jit::ansi_try_neg(ast::tree& tree, expression const& a) expression const& jit::ansi_try_precision_check(ast::tree& tree, expression const& a, - int32_t precision) + expression const& precision) { return tree.push( - detail::operation(cudf::detail::row_ir::opcode::ANSI_TRY_PRECISION_CHECK, {a}, precision)); + detail::operation(cudf::detail::row_ir::opcode::ANSI_TRY_PRECISION_CHECK, {a, precision})); } expression const& jit::bit_shift_left(ast::tree& tree, expression const& a, expression const& b) diff --git a/cpp/src/jit/row_ir.cpp b/cpp/src/jit/row_ir.cpp index 99caae038513..bc4863592bf5 100644 --- a/cpp/src/jit/row_ir.cpp +++ b/cpp/src/jit/row_ir.cpp @@ -99,23 +99,44 @@ std::span const> node::get_args() const { return args_; } bool node::is_null_aware() const { - return get_op_null_output(op_) == - null_output::ALWAYS_NULLABLE || // to emit nulls for always-nullable operators, we need - // to mark them as null-aware - get_op_requires_nulls(op_) || - std::any_of(args_.begin(), args_.end(), [](auto& a) { return a->is_null_aware(); }); + if (op_ == opcode::GET_INPUT) { return false; } + + // to emit nulls for always-nullable operators, we need to mark them as null-aware + if (get_op_null_output(op_) == null_output::ALWAYS_NULLABLE) { return true; } + + if (get_op_requires_nulls(op_)) { return true; } + + CUDF_EXPECTS(!args_.empty(), + "Unexpectedly found an operator node with no arguments. All operator nodes should " + "have at least one argument."); + + return std::any_of(args_.begin(), args_.end(), [](auto& a) { return a->is_null_aware(); }); } bool node::is_always_valid() const { - return get_op_null_output(op_) == null_output::ALWAYS_VALID || - std::all_of(args_.begin(), args_.end(), [](auto& a) { return a->is_always_valid(); }); + if (op_ == opcode::GET_INPUT) { return false; } + + if (get_op_null_output(op_) == null_output::ALWAYS_VALID) { return true; } + + CUDF_EXPECTS(!args_.empty(), + "Unexpectedly found an operator node with no arguments. All operator nodes should " + "have at least one argument."); + + return std::all_of(args_.begin(), args_.end(), [](auto& a) { return a->is_always_valid(); }); } bool node::is_fallible() const { - return get_op_is_fallible(op_) || - std::any_of(args_.begin(), args_.end(), [](auto& a) { return a->is_fallible(); }); + if (op_ == opcode::GET_INPUT) { return false; } + + if (get_op_is_fallible(op_)) { return true; } + + CUDF_EXPECTS(!args_.empty(), + "Unexpectedly found an operator node with no arguments. All operator nodes should " + "have at least one argument."); + + return std::any_of(args_.begin(), args_.end(), [](auto& a) { return a->is_fallible(); }); } row_ir::type as_typing(data_type type) @@ -467,7 +488,7 @@ std::unique_ptr ast_converter::add_ir_node(ast::detail::predicate std::unique_ptr ast_converter::add_ir_node(ast::jit::detail::operation const& expr) { std::vector> args; - for (auto &arg : expr.get_arguments()) { + for (auto& arg : expr.get_arguments()) { args.emplace_back(arg.get().accept(*this)); } return std::make_unique( @@ -491,23 +512,21 @@ std::tuple ast_converter::gen return std::visit([](auto& c) { return is_nullable(c); }, in); }); - auto is_null_aware = - std::any_of( - output_irs_.cbegin(), output_irs_.cend(), [](auto& ir) { return ir->is_null_aware(); }) - ? null_aware::YES - : null_aware::NO; + bool is_null_aware = std::any_of( + output_irs_.cbegin(), output_irs_.cend(), [](auto& ir) { return ir->is_null_aware(); }); bool output_is_always_valid = std::all_of( output_irs_.cbegin(), output_irs_.cend(), [](auto& ir) { return ir->is_always_valid(); }); bool may_evaluate_null = !output_is_always_valid || has_nullable_inputs; + auto null_policy = may_evaluate_null ? output_nullability::PRESERVE : output_nullability::ALL_VALID; auto is_fallible = std::any_of( output_irs_.cbegin(), output_irs_.cend(), [](auto& ir) { return ir->is_fallible(); }); - instance_.set_has_nulls(is_null_aware == null_aware::YES); + instance_.set_has_nulls(is_null_aware); // instantiate the IR nodes for (auto& ir : output_irs_) { @@ -561,7 +580,8 @@ std::tuple ast_converter::gen ir->emit_code(instance_, target, sink); } sink.emit("return cudf::ops::errc::OK;\n}"); - return {sink.get_code(), is_null_aware, null_policy, is_fallible}; + return { + sink.get_code(), is_null_aware ? null_aware::YES : null_aware::NO, null_policy, is_fallible}; } std::variant get_column_view(scalar_input const& in) @@ -633,7 +653,7 @@ transform_args ast_converter::compute_column(target target_id, .row_size = row_size, .error_mode = is_fallible ? ops::error_mode::ANY_ROW : ops::error_mode::IGNORE}; if (get_context().dump_codegen()) { - std::cout << "Generated code for transform: " << result.udf << std::endl; + std::cout << "Generated code for transform: \n" << result.udf << std::endl; } return result; diff --git a/cpp/tests/ast/transform_tests.cpp b/cpp/tests/ast/transform_tests.cpp index 06da7f8a858a..068c672aa269 100644 --- a/cpp/tests/ast/transform_tests.cpp +++ b/cpp/tests/ast/transform_tests.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -1391,4 +1392,277 @@ TYPED_TEST(TransformTest, NonDefaultStream) CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); } +struct JITExpressionTest : public cudf::test::BaseFixture {}; + +TEST_F(JITExpressionTest, NullifyIf) +{ + auto a = column_wrapper{3, 20, 1, 50, 0, 20}; + auto condition = column_wrapper{false, true, false, true, false, true}; + auto expected = column_wrapper{{3, 0, 1, 0, 0, 0}, {1, 0, 1, 0, 1, 0}}; + auto table = cudf::table_view{{a, condition}}; + auto tree = cudf::ast::tree{}; + auto a_ref = cudf::ast::column_reference(0); + auto condition_ref = cudf::ast::column_reference(1); + auto& nullify_if = cudf::ast::jit::nullify_if(tree, a_ref, condition_ref); + auto result = cudf::compute_column_jit(table, nullify_if); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); +} + +TEST_F(JITExpressionTest, Coalesce) +{ + auto a = column_wrapper{{1, 3, 5, 7, 9, 11}, {1, 0, 0, 1, 0, 0}}; + auto b = column_wrapper{{2, 4, 6, 8, 10, 12}, {1, 1, 1, 0, 1, 0}}; + auto expected = column_wrapper{{1, 4, 6, 7, 10, 0}, {1, 1, 1, 1, 1, 0}}; + auto table = cudf::table_view{{a, b}}; + auto tree = cudf::ast::tree{}; + auto a_ref = cudf::ast::column_reference(0); + auto b_ref = cudf::ast::column_reference(1); + auto& coalesce = cudf::ast::jit::coalesce(tree, a_ref, b_ref); + auto result = cudf::compute_column_jit(table, coalesce); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); +} + +TEST_F(JITExpressionTest, Predicate) {} + +TEST_F(JITExpressionTest, AnsiAdd_Int) +{ + constexpr auto I32_MAX = std::numeric_limits::max(); + auto a = column_wrapper{3, 20, 1, 50}; + auto b_success = column_wrapper{10, 7, 20, 0}; + auto b_fail = column_wrapper{10, I32_MAX, 20, 0}; + auto expected = column_wrapper{13, 27, 21, 50}; + auto table = cudf::table_view{{a, b_success, b_fail}}; + auto tree = cudf::ast::tree{}; + auto a_ref = cudf::ast::column_reference(0); + auto b_success_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto& add_success = cudf::ast::jit::ansi_add(tree, a_ref, b_success_ref); + auto& add_fail = cudf::ast::jit::ansi_add(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, add_success); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, add_fail), std::overflow_error); +} + +TEST_F(JITExpressionTest, AnsiAdd_Decimal) +{ + constexpr auto I32_MAX = std::numeric_limits::max(); + auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 1, 50}, numeric::scale_type{0}}; + auto b_success = + cudf::test::fixed_point_column_wrapper{{10, 7, 20, 0}, numeric::scale_type{0}}; + auto b_fail = + cudf::test::fixed_point_column_wrapper{{10, I32_MAX, 20, 0}, numeric::scale_type{0}}; + auto expected = + cudf::test::fixed_point_column_wrapper{{13, 27, 21, 50}, numeric::scale_type{0}}; + auto table = cudf::table_view{{a, b_success, b_fail}}; + auto tree = cudf::ast::tree{}; + auto a_ref = cudf::ast::column_reference(0); + auto b_success_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto& add_success = cudf::ast::jit::ansi_add(tree, a_ref, b_success_ref); + auto& add_fail = cudf::ast::jit::ansi_add(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, add_success); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, add_fail), std::overflow_error); +} + +TEST_F(JITExpressionTest, AnsiSub) +{ + constexpr auto I32_MIN = std::numeric_limits::min(); + auto a = column_wrapper{3, 20, 1, 50}; + auto b_success = column_wrapper{10, 7, 20, 0}; + auto b_fail = column_wrapper{10, I32_MIN, 20, 0}; + auto expected = column_wrapper{-7, 13, -19, 50}; + auto table = cudf::table_view{{a, b_success, b_fail}}; + auto tree = cudf::ast::tree{}; + auto a_ref = cudf::ast::column_reference(0); + auto b_success_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto& sub_success = cudf::ast::jit::ansi_sub(tree, a_ref, b_success_ref); + auto& sub_fail = cudf::ast::jit::ansi_sub(tree, a_ref, b_fail_ref); + + auto result = cudf::compute_column_jit(table, sub_success); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, sub_fail), std::overflow_error); +} + +TEST_F(JITExpressionTest, AnsiSub_Decimal) +{ + constexpr auto I32_MIN = std::numeric_limits::min(); + auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 1, 50}, numeric::scale_type{0}}; + auto b_success = + cudf::test::fixed_point_column_wrapper{{10, 7, 20, 0}, numeric::scale_type{0}}; + auto b_fail = + cudf::test::fixed_point_column_wrapper{{10, I32_MIN, 20, 0}, numeric::scale_type{0}}; + auto expected = + cudf::test::fixed_point_column_wrapper{{-7, 13, -19, 50}, numeric::scale_type{0}}; + auto table = cudf::table_view{{a, b_success, b_fail}}; + auto tree = cudf::ast::tree{}; + auto a_ref = cudf::ast::column_reference(0); + auto b_success_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto& sub_success = cudf::ast::jit::ansi_sub(tree, a_ref, b_success_ref); + auto& sub_fail = cudf::ast::jit::ansi_sub(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, sub_success); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, sub_fail), std::overflow_error); +} + +TEST_F(JITExpressionTest, AnsiMul) +{ + constexpr auto I32_MAX = std::numeric_limits::max(); + auto a = column_wrapper{3, 20, 2, 50}; + auto b_success = column_wrapper{10, 7, 1, 0}; + auto b_fail = column_wrapper{10, I32_MAX, 1, 0}; + auto expected = column_wrapper{30, 140, 2, 0}; + auto table = cudf::table_view{{a, b_success, b_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto b_success_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto tree = cudf::ast::tree{}; + auto& mul_success = cudf::ast::jit::ansi_mul(tree, a_ref, b_success_ref); + auto& mul_fail = cudf::ast::jit::ansi_mul(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, mul_success); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, mul_fail), std::overflow_error); +} + +TEST_F(JITExpressionTest, AnsiMul_Decimal) +{ + constexpr auto I32_MAX = std::numeric_limits::max(); + auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 2, 50}, numeric::scale_type{0}}; + auto b_success = + cudf::test::fixed_point_column_wrapper{{10, 7, 1, 0}, numeric::scale_type{0}}; + auto b_fail = + cudf::test::fixed_point_column_wrapper{{10, I32_MAX, 1, 0}, numeric::scale_type{0}}; + auto expected = + cudf::test::fixed_point_column_wrapper{{30, 140, 2, 0}, numeric::scale_type{0}}; + auto table = cudf::table_view{{a, b_success, b_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto b_success_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto tree = cudf::ast::tree{}; + auto& mul_success = cudf::ast::jit::ansi_mul(tree, a_ref, b_success_ref); + auto& mul_fail = cudf::ast::jit::ansi_mul(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, mul_success); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, mul_fail), std::overflow_error); +} + +TEST_F(JITExpressionTest, AnsiDiv) +{ + auto a = column_wrapper{3, 20, 1, 50}; + auto b_success = column_wrapper{10, 7, 2, 1}; + auto b_fail = column_wrapper{10, 1, 20, 0}; + auto expected = column_wrapper{0, 2, 0, 50}; + auto table = cudf::table_view{{a, b_success, b_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto b_success_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto tree = cudf::ast::tree{}; + auto& div_success = cudf::ast::jit::ansi_div(tree, a_ref, b_success_ref); + auto& div_fail = cudf::ast::jit::ansi_div(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, div_success); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, div_fail), std::overflow_error); +} + +TEST_F(JITExpressionTest, AnsiDiv_Decimal) +{ + auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 1, 50}, numeric::scale_type{0}}; + auto b_success = + cudf::test::fixed_point_column_wrapper{{10, 7, 2, 1}, numeric::scale_type{0}}; + auto b_fail = + cudf::test::fixed_point_column_wrapper{{10, 1, 20, 0}, numeric::scale_type{0}}; + auto expected = + cudf::test::fixed_point_column_wrapper{{0, 2, 0, 50}, numeric::scale_type{0}}; + auto table = cudf::table_view{{a, b_success, b_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto b_success_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto tree = cudf::ast::tree{}; + auto& div_success = cudf::ast::jit::ansi_div(tree, a_ref, b_success_ref); + auto& div_fail = cudf::ast::jit::ansi_div(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, div_success); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, div_fail), std::overflow_error); +} + +TEST_F(JITExpressionTest, AnsiMod) {} + +TEST_F(JITExpressionTest, AnsiAbs) {} + +TEST_F(JITExpressionTest, AnsiNeg) {} + +TEST_F(JITExpressionTest, AnsiPrecisionCheck) {} + +TEST_F(JITExpressionTest, AnsiTryAdd) {} + +TEST_F(JITExpressionTest, AnsiTrySub) {} + +TEST_F(JITExpressionTest, AnsiTryMul) {} + +TEST_F(JITExpressionTest, AnsiTryDiv) {} + +TEST_F(JITExpressionTest, AnsiTryMod) {} + +TEST_F(JITExpressionTest, AnsiTryAbs) {} + +TEST_F(JITExpressionTest, AnsiTryNeg) {} + +TEST_F(JITExpressionTest, AnsiTryPrecisionCheck) {} + +TEST_F(JITExpressionTest, BitShiftLeft) {} + +TEST_F(JITExpressionTest, BitShiftRight) {} + +TEST_F(JITExpressionTest, CastToBool) {} + +TEST_F(JITExpressionTest, CastToI8) {} + +TEST_F(JITExpressionTest, CastToI16) {} + +TEST_F(JITExpressionTest, CastToI32) {} + +TEST_F(JITExpressionTest, CastToI64) {} + +TEST_F(JITExpressionTest, CastToU8) {} + +TEST_F(JITExpressionTest, CastToU16) {} + +TEST_F(JITExpressionTest, CastToU32) {} + +TEST_F(JITExpressionTest, CastToU64) {} + +TEST_F(JITExpressionTest, CastToF32) {} + +TEST_F(JITExpressionTest, CastToF64) {} + +TEST_F(JITExpressionTest, CastToDec32) {} + +TEST_F(JITExpressionTest, CastToDec64) {} + +TEST_F(JITExpressionTest, CastToDec128) {} + +TEST_F(JITExpressionTest, Rescale) {} + +// TODO: TEST ANSI op fusion + CUDF_TEST_PROGRAM_MAIN() From e90392dcad1da4d62d80599fff8c1d8c966dc1cb Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 4 May 2026 16:54:09 +0000 Subject: [PATCH 10/15] add more tests and bug fixes --- .../cudf/operators/ansi_arithmetic.cuh | 2 + cpp/tests/ast/transform_tests.cpp | 287 +++++++++++++----- 2 files changed, 210 insertions(+), 79 deletions(-) diff --git a/cpp/include/cudf/operators/ansi_arithmetic.cuh b/cpp/include/cudf/operators/ansi_arithmetic.cuh index fc084afa6948..356ad393fa13 100644 --- a/cpp/include/cudf/operators/ansi_arithmetic.cuh +++ b/cpp/include/cudf/operators/ansi_arithmetic.cuh @@ -328,12 +328,14 @@ __device__ inline errc ansi_mod(T* out, T const* a, T const* b) __device__ inline errc ansi_mod(float* out, float const* a, float const* b) { + if (*b == 0) { return errc::DIVISION_BY_ZERO; } *out = (*a) - (*b) * ::floorf((*a) / (*b)); return errc::OK; } __device__ inline errc ansi_mod(double* out, double const* a, double const* b) { + if (*b == 0) { return errc::DIVISION_BY_ZERO; } *out = (*a) - (*b) * ::floor((*a) / (*b)); return errc::OK; } diff --git a/cpp/tests/ast/transform_tests.cpp b/cpp/tests/ast/transform_tests.cpp index 068c672aa269..d07a7a17671e 100644 --- a/cpp/tests/ast/transform_tests.cpp +++ b/cpp/tests/ast/transform_tests.cpp @@ -1426,45 +1426,51 @@ TEST_F(JITExpressionTest, Coalesce) TEST_F(JITExpressionTest, Predicate) {} +// TODO: try variants TEST_F(JITExpressionTest, AnsiAdd_Int) { constexpr auto I32_MAX = std::numeric_limits::max(); auto a = column_wrapper{3, 20, 1, 50}; - auto b_success = column_wrapper{10, 7, 20, 0}; + auto b = column_wrapper{10, 7, 20, 0}; auto b_fail = column_wrapper{10, I32_MAX, 20, 0}; auto expected = column_wrapper{13, 27, 21, 50}; - auto table = cudf::table_view{{a, b_success, b_fail}}; + auto expected_null = column_wrapper{{13, 0, 21, 50}, {1, 0, 1, 1}}; + auto table = cudf::table_view{{a, b, b_fail}}; auto tree = cudf::ast::tree{}; auto a_ref = cudf::ast::column_reference(0); - auto b_success_ref = cudf::ast::column_reference(1); + auto b_ref = cudf::ast::column_reference(1); auto b_fail_ref = cudf::ast::column_reference(2); - auto& add_success = cudf::ast::jit::ansi_add(tree, a_ref, b_success_ref); + auto& add = cudf::ast::jit::ansi_add(tree, a_ref, b_ref); auto& add_fail = cudf::ast::jit::ansi_add(tree, a_ref, b_fail_ref); - auto result = cudf::compute_column_jit(table, add_success); + auto& try_add_fail = cudf::ast::jit::ansi_try_add(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, add); + auto result_null = cudf::compute_column_jit(table, try_add_fail); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); EXPECT_THROW(result = cudf::compute_column_jit(table, add_fail), std::overflow_error); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_null, result_null->view(), verbosity); } +// TODO: parametrizie on decimal types and float types TEST_F(JITExpressionTest, AnsiAdd_Decimal) { constexpr auto I32_MAX = std::numeric_limits::max(); auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 1, 50}, numeric::scale_type{0}}; - auto b_success = - cudf::test::fixed_point_column_wrapper{{10, 7, 20, 0}, numeric::scale_type{0}}; + auto b = cudf::test::fixed_point_column_wrapper{{10, 7, 20, 0}, numeric::scale_type{0}}; auto b_fail = cudf::test::fixed_point_column_wrapper{{10, I32_MAX, 20, 0}, numeric::scale_type{0}}; auto expected = cudf::test::fixed_point_column_wrapper{{13, 27, 21, 50}, numeric::scale_type{0}}; - auto table = cudf::table_view{{a, b_success, b_fail}}; - auto tree = cudf::ast::tree{}; - auto a_ref = cudf::ast::column_reference(0); - auto b_success_ref = cudf::ast::column_reference(1); - auto b_fail_ref = cudf::ast::column_reference(2); - auto& add_success = cudf::ast::jit::ansi_add(tree, a_ref, b_success_ref); - auto& add_fail = cudf::ast::jit::ansi_add(tree, a_ref, b_fail_ref); - auto result = cudf::compute_column_jit(table, add_success); + auto table = cudf::table_view{{a, b, b_fail}}; + auto tree = cudf::ast::tree{}; + auto a_ref = cudf::ast::column_reference(0); + auto b_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto& add = cudf::ast::jit::ansi_add(tree, a_ref, b_ref); + auto& add_fail = cudf::ast::jit::ansi_add(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, add); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); @@ -1475,18 +1481,18 @@ TEST_F(JITExpressionTest, AnsiSub) { constexpr auto I32_MIN = std::numeric_limits::min(); auto a = column_wrapper{3, 20, 1, 50}; - auto b_success = column_wrapper{10, 7, 20, 0}; + auto b = column_wrapper{10, 7, 20, 0}; auto b_fail = column_wrapper{10, I32_MIN, 20, 0}; auto expected = column_wrapper{-7, 13, -19, 50}; - auto table = cudf::table_view{{a, b_success, b_fail}}; + auto table = cudf::table_view{{a, b, b_fail}}; auto tree = cudf::ast::tree{}; auto a_ref = cudf::ast::column_reference(0); - auto b_success_ref = cudf::ast::column_reference(1); + auto b_ref = cudf::ast::column_reference(1); auto b_fail_ref = cudf::ast::column_reference(2); - auto& sub_success = cudf::ast::jit::ansi_sub(tree, a_ref, b_success_ref); + auto& sub = cudf::ast::jit::ansi_sub(tree, a_ref, b_ref); auto& sub_fail = cudf::ast::jit::ansi_sub(tree, a_ref, b_fail_ref); - auto result = cudf::compute_column_jit(table, sub_success); + auto result = cudf::compute_column_jit(table, sub); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); @@ -1497,20 +1503,19 @@ TEST_F(JITExpressionTest, AnsiSub_Decimal) { constexpr auto I32_MIN = std::numeric_limits::min(); auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 1, 50}, numeric::scale_type{0}}; - auto b_success = - cudf::test::fixed_point_column_wrapper{{10, 7, 20, 0}, numeric::scale_type{0}}; + auto b = cudf::test::fixed_point_column_wrapper{{10, 7, 20, 0}, numeric::scale_type{0}}; auto b_fail = cudf::test::fixed_point_column_wrapper{{10, I32_MIN, 20, 0}, numeric::scale_type{0}}; auto expected = cudf::test::fixed_point_column_wrapper{{-7, 13, -19, 50}, numeric::scale_type{0}}; - auto table = cudf::table_view{{a, b_success, b_fail}}; - auto tree = cudf::ast::tree{}; - auto a_ref = cudf::ast::column_reference(0); - auto b_success_ref = cudf::ast::column_reference(1); - auto b_fail_ref = cudf::ast::column_reference(2); - auto& sub_success = cudf::ast::jit::ansi_sub(tree, a_ref, b_success_ref); - auto& sub_fail = cudf::ast::jit::ansi_sub(tree, a_ref, b_fail_ref); - auto result = cudf::compute_column_jit(table, sub_success); + auto table = cudf::table_view{{a, b, b_fail}}; + auto tree = cudf::ast::tree{}; + auto a_ref = cudf::ast::column_reference(0); + auto b_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto& sub = cudf::ast::jit::ansi_sub(tree, a_ref, b_ref); + auto& sub_fail = cudf::ast::jit::ansi_sub(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, sub); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); @@ -1521,17 +1526,17 @@ TEST_F(JITExpressionTest, AnsiMul) { constexpr auto I32_MAX = std::numeric_limits::max(); auto a = column_wrapper{3, 20, 2, 50}; - auto b_success = column_wrapper{10, 7, 1, 0}; + auto b = column_wrapper{10, 7, 1, 0}; auto b_fail = column_wrapper{10, I32_MAX, 1, 0}; auto expected = column_wrapper{30, 140, 2, 0}; - auto table = cudf::table_view{{a, b_success, b_fail}}; + auto table = cudf::table_view{{a, b, b_fail}}; auto a_ref = cudf::ast::column_reference(0); - auto b_success_ref = cudf::ast::column_reference(1); + auto b_ref = cudf::ast::column_reference(1); auto b_fail_ref = cudf::ast::column_reference(2); auto tree = cudf::ast::tree{}; - auto& mul_success = cudf::ast::jit::ansi_mul(tree, a_ref, b_success_ref); + auto& mul = cudf::ast::jit::ansi_mul(tree, a_ref, b_ref); auto& mul_fail = cudf::ast::jit::ansi_mul(tree, a_ref, b_fail_ref); - auto result = cudf::compute_column_jit(table, mul_success); + auto result = cudf::compute_column_jit(table, mul); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); @@ -1542,20 +1547,19 @@ TEST_F(JITExpressionTest, AnsiMul_Decimal) { constexpr auto I32_MAX = std::numeric_limits::max(); auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 2, 50}, numeric::scale_type{0}}; - auto b_success = - cudf::test::fixed_point_column_wrapper{{10, 7, 1, 0}, numeric::scale_type{0}}; + auto b = cudf::test::fixed_point_column_wrapper{{10, 7, 1, 0}, numeric::scale_type{0}}; auto b_fail = cudf::test::fixed_point_column_wrapper{{10, I32_MAX, 1, 0}, numeric::scale_type{0}}; auto expected = cudf::test::fixed_point_column_wrapper{{30, 140, 2, 0}, numeric::scale_type{0}}; - auto table = cudf::table_view{{a, b_success, b_fail}}; - auto a_ref = cudf::ast::column_reference(0); - auto b_success_ref = cudf::ast::column_reference(1); - auto b_fail_ref = cudf::ast::column_reference(2); - auto tree = cudf::ast::tree{}; - auto& mul_success = cudf::ast::jit::ansi_mul(tree, a_ref, b_success_ref); - auto& mul_fail = cudf::ast::jit::ansi_mul(tree, a_ref, b_fail_ref); - auto result = cudf::compute_column_jit(table, mul_success); + auto table = cudf::table_view{{a, b, b_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto b_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto tree = cudf::ast::tree{}; + auto& mul = cudf::ast::jit::ansi_mul(tree, a_ref, b_ref); + auto& mul_fail = cudf::ast::jit::ansi_mul(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, mul); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); @@ -1564,18 +1568,18 @@ TEST_F(JITExpressionTest, AnsiMul_Decimal) TEST_F(JITExpressionTest, AnsiDiv) { - auto a = column_wrapper{3, 20, 1, 50}; - auto b_success = column_wrapper{10, 7, 2, 1}; - auto b_fail = column_wrapper{10, 1, 20, 0}; - auto expected = column_wrapper{0, 2, 0, 50}; - auto table = cudf::table_view{{a, b_success, b_fail}}; - auto a_ref = cudf::ast::column_reference(0); - auto b_success_ref = cudf::ast::column_reference(1); - auto b_fail_ref = cudf::ast::column_reference(2); - auto tree = cudf::ast::tree{}; - auto& div_success = cudf::ast::jit::ansi_div(tree, a_ref, b_success_ref); - auto& div_fail = cudf::ast::jit::ansi_div(tree, a_ref, b_fail_ref); - auto result = cudf::compute_column_jit(table, div_success); + auto a = column_wrapper{3, 20, 1, 50}; + auto b = column_wrapper{10, 7, 2, 1}; + auto b_fail = column_wrapper{10, 1, 20, 0}; + auto expected = column_wrapper{0, 2, 0, 50}; + auto table = cudf::table_view{{a, b, b_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto b_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto tree = cudf::ast::tree{}; + auto& div = cudf::ast::jit::ansi_div(tree, a_ref, b_ref); + auto& div_fail = cudf::ast::jit::ansi_div(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, div); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); @@ -1585,49 +1589,174 @@ TEST_F(JITExpressionTest, AnsiDiv) TEST_F(JITExpressionTest, AnsiDiv_Decimal) { auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 1, 50}, numeric::scale_type{0}}; - auto b_success = - cudf::test::fixed_point_column_wrapper{{10, 7, 2, 1}, numeric::scale_type{0}}; + auto b = cudf::test::fixed_point_column_wrapper{{10, 7, 2, 1}, numeric::scale_type{0}}; auto b_fail = cudf::test::fixed_point_column_wrapper{{10, 1, 20, 0}, numeric::scale_type{0}}; auto expected = cudf::test::fixed_point_column_wrapper{{0, 2, 0, 50}, numeric::scale_type{0}}; - auto table = cudf::table_view{{a, b_success, b_fail}}; - auto a_ref = cudf::ast::column_reference(0); - auto b_success_ref = cudf::ast::column_reference(1); - auto b_fail_ref = cudf::ast::column_reference(2); - auto tree = cudf::ast::tree{}; - auto& div_success = cudf::ast::jit::ansi_div(tree, a_ref, b_success_ref); - auto& div_fail = cudf::ast::jit::ansi_div(tree, a_ref, b_fail_ref); - auto result = cudf::compute_column_jit(table, div_success); + auto table = cudf::table_view{{a, b, b_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto b_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto tree = cudf::ast::tree{}; + auto& div = cudf::ast::jit::ansi_div(tree, a_ref, b_ref); + auto& div_fail = cudf::ast::jit::ansi_div(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, div); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); EXPECT_THROW(result = cudf::compute_column_jit(table, div_fail), std::overflow_error); } -TEST_F(JITExpressionTest, AnsiMod) {} +TEST_F(JITExpressionTest, AnsiMod) +{ + auto a = column_wrapper{3, 20, 1, 50}; + auto b = column_wrapper{10, 7, 2, 1}; + auto b_fail = column_wrapper{10, 1, 20, 0}; + auto expected = column_wrapper{10, 6, 2, 0}; + auto table = cudf::table_view{{a, b, b_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto b_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto tree = cudf::ast::tree{}; + auto& mod = cudf::ast::jit::ansi_mod(tree, a_ref, b_ref); + auto& mod_fail = cudf::ast::jit::ansi_mod(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, mod); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, mod_fail), std::overflow_error); +} + +TEST_F(JITExpressionTest, AnsiMod_Decimal) +{ + auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 1, 50}, numeric::scale_type{0}}; + auto b = cudf::test::fixed_point_column_wrapper{{10, 7, 2, 1}, numeric::scale_type{0}}; + auto b_fail = + cudf::test::fixed_point_column_wrapper{{10, 1, 20, 0}, numeric::scale_type{0}}; + auto expected = + cudf::test::fixed_point_column_wrapper{{10, 6, 2, 0}, numeric::scale_type{0}}; + auto table = cudf::table_view{{a, b, b_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto b_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto tree = cudf::ast::tree{}; + auto& mod = cudf::ast::jit::ansi_mod(tree, a_ref, b_ref); + auto& mod_fail = cudf::ast::jit::ansi_mod(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, mod); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, mod_fail), std::overflow_error); +} + +TEST_F(JITExpressionTest, AnsiAbs) +{ + constexpr auto I32_MIN = std::numeric_limits::min(); + constexpr auto I32_MAX = std::numeric_limits::max(); + auto a = column_wrapper{3, -20, 1, -50, I32_MAX, I32_MIN + 1, 0}; + auto a_fail = column_wrapper{3, -20, 1, -50, I32_MIN, 0}; + auto expected = column_wrapper{3, 20, 1, 50, I32_MAX, std::abs(I32_MIN + 1), 0}; + auto table = cudf::table_view{{a, a_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto a_fail_ref = cudf::ast::column_reference(1); + auto tree = cudf::ast::tree{}; + auto& abs = cudf::ast::jit::ansi_abs(tree, a_ref); + auto& abs_fail = cudf::ast::jit::ansi_abs(tree, a_fail_ref); + auto result = cudf::compute_column_jit(table, abs); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, abs_fail), std::overflow_error); +} + +TEST_F(JITExpressionTest, AnsiAbs_Decimal) +{ + constexpr auto I32_MIN = std::numeric_limits::min(); + constexpr auto I32_MAX = std::numeric_limits::max(); + auto a = cudf::test::fixed_point_column_wrapper{ + {3, -20, 1, -50, I32_MAX, I32_MIN + 1, 0}, numeric::scale_type{0}}; + auto a_fail = cudf::test::fixed_point_column_wrapper{{3, -20, 1, -50, I32_MIN, 0}, + numeric::scale_type{0}}; + auto expected = cudf::test::fixed_point_column_wrapper{ + {3, 20, 1, 50, I32_MAX, std::abs(I32_MIN + 1), 0}, numeric::scale_type{0}}; + auto table = cudf::table_view{{a, a_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto a_fail_ref = cudf::ast::column_reference(1); + auto tree = cudf::ast::tree{}; + auto& abs = cudf::ast::jit::ansi_abs(tree, a_ref); + auto& abs_fail = cudf::ast::jit::ansi_abs(tree, a_fail_ref); + auto result = cudf::compute_column_jit(table, abs); -TEST_F(JITExpressionTest, AnsiAbs) {} + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); -TEST_F(JITExpressionTest, AnsiNeg) {} + EXPECT_THROW(result = cudf::compute_column_jit(table, abs_fail), std::overflow_error); +} -TEST_F(JITExpressionTest, AnsiPrecisionCheck) {} +TEST_F(JITExpressionTest, AnsiNeg) +{ + constexpr auto I32_MIN = std::numeric_limits::min(); + constexpr auto I32_MAX = std::numeric_limits::max(); + auto a = column_wrapper{3, -20, 1, -50, I32_MAX, -I32_MAX, 0}; + auto a_fail = column_wrapper{3, -20, 1, -50, I32_MIN, 0}; + auto expected = column_wrapper{-3, 20, -1, 50, -I32_MAX, I32_MAX, 0}; + auto table = cudf::table_view{{a, a_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto a_fail_ref = cudf::ast::column_reference(1); + auto tree = cudf::ast::tree{}; + auto& neg = cudf::ast::jit::ansi_neg(tree, a_ref); + auto& neg_fail = cudf::ast::jit::ansi_neg(tree, a_fail_ref); + auto result = cudf::compute_column_jit(table, neg); -TEST_F(JITExpressionTest, AnsiTryAdd) {} + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); -TEST_F(JITExpressionTest, AnsiTrySub) {} + EXPECT_THROW(result = cudf::compute_column_jit(table, neg_fail), std::overflow_error); +} -TEST_F(JITExpressionTest, AnsiTryMul) {} +TEST_F(JITExpressionTest, AnsiNeg_Decimal) +{ + constexpr auto I32_MIN = std::numeric_limits::min(); + constexpr auto I32_MAX = std::numeric_limits::max(); + auto a = cudf::test::fixed_point_column_wrapper{{3, -20, 1, -50, I32_MAX, -I32_MAX, 0}, + numeric::scale_type{0}}; + auto a_fail = cudf::test::fixed_point_column_wrapper{{3, -20, 1, -50, I32_MIN, 0}, + numeric::scale_type{0}}; + auto expected = cudf::test::fixed_point_column_wrapper{ + {-3, 20, -1, 50, -I32_MAX, I32_MAX, 0}, numeric::scale_type{0}}; + auto table = cudf::table_view{{a, a_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto a_fail_ref = cudf::ast::column_reference(1); + auto tree = cudf::ast::tree{}; + auto& neg = cudf::ast::jit::ansi_neg(tree, a_ref); + auto& neg_fail = cudf::ast::jit::ansi_neg(tree, a_fail_ref); + auto result = cudf::compute_column_jit(table, neg); -TEST_F(JITExpressionTest, AnsiTryDiv) {} + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); -TEST_F(JITExpressionTest, AnsiTryMod) {} + EXPECT_THROW(result = cudf::compute_column_jit(table, neg_fail), std::overflow_error); +} -TEST_F(JITExpressionTest, AnsiTryAbs) {} +TEST_F(JITExpressionTest, AnsiPrecisionCheck) +{ + auto a = + cudf::test::fixed_point_column_wrapper{{3, 200, 250, 200}, numeric::scale_type{0}}; + auto a_fail = + cudf::test::fixed_point_column_wrapper{{3, 200, 250, 20000}, numeric::scale_type{0}}; + auto expected = + cudf::test::fixed_point_column_wrapper{{3, 200, 250, 200}, numeric::scale_type{0}}; + auto max_precision = cudf::numeric_scalar(3); + auto table = cudf::table_view{{a}}; + auto a_ref = cudf::ast::column_reference(0); + auto tree = cudf::ast::tree{}; + auto precision = cudf::ast::literal(max_precision); + auto& precision_check = cudf::ast::jit::ansi_precision_check(tree, a_ref, precision); + auto result = cudf::compute_column_jit(table, precision_check); -TEST_F(JITExpressionTest, AnsiTryNeg) {} + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); -TEST_F(JITExpressionTest, AnsiTryPrecisionCheck) {} + EXPECT_THROW(result = cudf::compute_column_jit(cudf::table_view{{a_fail}}, precision_check), + std::overflow_error); +} TEST_F(JITExpressionTest, BitShiftLeft) {} From 02878c3f28cf4734896527c7f9d313d04f4dbf7d Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 4 May 2026 19:58:38 +0000 Subject: [PATCH 11/15] bug fixes and more tests --- .../cudf/operators/ansi_arithmetic.cuh | 3 +- cpp/include/cudf/operators/bitwise.cuh | 12 +- cpp/include/cudf/operators/casts.cuh | 10 +- cpp/src/jit/row_ir.cpp | 24 +- cpp/tests/ast/transform_tests.cpp | 297 ++++++++++++++++-- 5 files changed, 302 insertions(+), 44 deletions(-) diff --git a/cpp/include/cudf/operators/ansi_arithmetic.cuh b/cpp/include/cudf/operators/ansi_arithmetic.cuh index 356ad393fa13..367b51c901e8 100644 --- a/cpp/include/cudf/operators/ansi_arithmetic.cuh +++ b/cpp/include/cudf/operators/ansi_arithmetic.cuh @@ -400,7 +400,8 @@ template __device__ inline errc ansi_abs(decimal* out, decimal const* a) { if (a->value() == cuda::std::numeric_limits::min()) { return errc::OVERFLOW; } - out->value() = (a->value() < 0) ? -a->value() : a->value(); + auto rep = a->value() < 0 ? -a->value() : a->value(); + *out = decimal{numeric::scaled_integer{rep, numeric::scale_type{a->scale()}}}; return errc::OK; } diff --git a/cpp/include/cudf/operators/bitwise.cuh b/cpp/include/cudf/operators/bitwise.cuh index c9b76518bbfe..18943bbd56ab 100644 --- a/cpp/include/cudf/operators/bitwise.cuh +++ b/cpp/include/cudf/operators/bitwise.cuh @@ -90,18 +90,18 @@ __device__ inline errc bit_xor(optional* out, optional const* a, optional< } template -__device__ inline errc shift_left(T* out, T const* a, T const* b) +__device__ inline errc bit_shift_left(T* out, T const* a, T const* b) { *out = (*a << *b); return errc::OK; } template -__device__ inline errc shift_left(optional* out, optional const* a, optional const* b) +__device__ inline errc bit_shift_left(optional* out, optional const* a, optional const* b) { if (a->has_value() && b->has_value()) { T r; - shift_left(&r, &a->value(), &b->value()); + bit_shift_left(&r, &a->value(), &b->value()); *out = r; } else { *out = nullopt; @@ -110,18 +110,18 @@ __device__ inline errc shift_left(optional* out, optional const* a, option } template -__device__ inline errc shift_right(T* out, T const* a, T const* b) +__device__ inline errc bit_shift_right(T* out, T const* a, T const* b) { *out = (*a >> *b); return errc::OK; } template -__device__ inline errc shift_right(optional* out, optional const* a, optional const* b) +__device__ inline errc bit_shift_right(optional* out, optional const* a, optional const* b) { if (a->has_value() && b->has_value()) { T r; - shift_right(&r, &a->value(), &b->value()); + bit_shift_right(&r, &a->value(), &b->value()); *out = r; } else { *out = nullopt; diff --git a/cpp/include/cudf/operators/casts.cuh b/cpp/include/cudf/operators/casts.cuh index 23b4faeeb30d..4b0fd81619a3 100644 --- a/cpp/include/cudf/operators/casts.cuh +++ b/cpp/include/cudf/operators/casts.cuh @@ -247,11 +247,11 @@ __device__ inline errc cast_to_f64(optional* out, optional const* a) namespace detail { -template -__device__ inline errc decimal_cast(decimal* out, decimal const* a) +template +__device__ inline errc decimal_cast(decimal* out, decimal const* a) { - auto rep = static_cast(a->value()); - *out = decimal{numeric::scaled_integer{rep, a->scale()}}; + auto rep = static_cast(a->value()); + *out = decimal{numeric::scaled_integer{rep, a->scale()}}; return errc::OK; } @@ -322,7 +322,7 @@ __device__ inline errc cast_to_dec128(optional* out, template __device__ inline errc rescale(decimal* out, decimal const* a, int32_t const* new_scale) { - *out = a->rescaled(new_scale); + *out = a->rescaled(numeric::scale_type{*new_scale}); return errc::OK; } diff --git a/cpp/src/jit/row_ir.cpp b/cpp/src/jit/row_ir.cpp index bc4863592bf5..cf331c54d0f1 100644 --- a/cpp/src/jit/row_ir.cpp +++ b/cpp/src/jit/row_ir.cpp @@ -64,14 +64,20 @@ node::node(opcode op, std::optional target_scale, std::vector(get_op_arity(op)), - std::format("Invalid number of arguments for operator `{}`. Expected {}, Got {}.", - get_op_name(op), - get_op_arity(op), - args_.size())); - CUDF_EXPECTS(target_scale_.has_value() == (op == opcode::RESCALE), - std::format("Target scale must be provided for RESCALE operator and must be nullopt " - "for other operators.")); + if (op_ != opcode::RESCALE) { + CUDF_EXPECTS(args_.size() == static_cast(get_op_arity(op)), + std::format("Invalid number of arguments for operator `{}`. Expected {}, Got {}.", + get_op_name(op), + get_op_arity(op), + args_.size())); + } else { + CUDF_EXPECTS(args_.size() == 1, + std::format("RESCALE operator expects exactly 1 argument. Got {}.", args_.size())); + CUDF_EXPECTS( + target_scale_.has_value(), + std::format("Target scale must be provided for RESCALE operator and must be nullopt " + "for other operators.")); + } } node::node(input_reference input) : reference_{input}, op_{opcode::GET_INPUT} {} @@ -321,7 +327,7 @@ data_type get_return_type(opcode op, return data_type{type, scale}; } else { CUDF_EXPECTS( - op_type_match.output != type::NONE && (op_type_match.output & type::DECIMALS) == type::NONE, + op_type_match.output != type::NONE, std::format("Invalid type match rule for operator `{}` return type", get_op_name(op)), std::runtime_error); auto type = as_type_id(op_type_match.output); diff --git a/cpp/tests/ast/transform_tests.cpp b/cpp/tests/ast/transform_tests.cpp index d07a7a17671e..df4edc43c672 100644 --- a/cpp/tests/ast/transform_tests.cpp +++ b/cpp/tests/ast/transform_tests.cpp @@ -1613,7 +1613,7 @@ TEST_F(JITExpressionTest, AnsiMod) auto a = column_wrapper{3, 20, 1, 50}; auto b = column_wrapper{10, 7, 2, 1}; auto b_fail = column_wrapper{10, 1, 20, 0}; - auto expected = column_wrapper{10, 6, 2, 0}; + auto expected = column_wrapper{3, 6, 1, 0}; auto table = cudf::table_view{{a, b, b_fail}}; auto a_ref = cudf::ast::column_reference(0); auto b_ref = cudf::ast::column_reference(1); @@ -1635,7 +1635,7 @@ TEST_F(JITExpressionTest, AnsiMod_Decimal) auto b_fail = cudf::test::fixed_point_column_wrapper{{10, 1, 20, 0}, numeric::scale_type{0}}; auto expected = - cudf::test::fixed_point_column_wrapper{{10, 6, 2, 0}, numeric::scale_type{0}}; + cudf::test::fixed_point_column_wrapper{{3, 6, 1, 0}, numeric::scale_type{0}}; auto table = cudf::table_view{{a, b, b_fail}}; auto a_ref = cudf::ast::column_reference(0); auto b_ref = cudf::ast::column_reference(1); @@ -1655,7 +1655,7 @@ TEST_F(JITExpressionTest, AnsiAbs) constexpr auto I32_MIN = std::numeric_limits::min(); constexpr auto I32_MAX = std::numeric_limits::max(); auto a = column_wrapper{3, -20, 1, -50, I32_MAX, I32_MIN + 1, 0}; - auto a_fail = column_wrapper{3, -20, 1, -50, I32_MIN, 0}; + auto a_fail = column_wrapper{3, -20, 1, -50, I32_MIN, 1, 0}; auto expected = column_wrapper{3, 20, 1, 50, I32_MAX, std::abs(I32_MIN + 1), 0}; auto table = cudf::table_view{{a, a_fail}}; auto a_ref = cudf::ast::column_reference(0); @@ -1676,7 +1676,7 @@ TEST_F(JITExpressionTest, AnsiAbs_Decimal) constexpr auto I32_MAX = std::numeric_limits::max(); auto a = cudf::test::fixed_point_column_wrapper{ {3, -20, 1, -50, I32_MAX, I32_MIN + 1, 0}, numeric::scale_type{0}}; - auto a_fail = cudf::test::fixed_point_column_wrapper{{3, -20, 1, -50, I32_MIN, 0}, + auto a_fail = cudf::test::fixed_point_column_wrapper{{3, -20, 1, -50, I32_MIN, 1, 0}, numeric::scale_type{0}}; auto expected = cudf::test::fixed_point_column_wrapper{ {3, 20, 1, 50, I32_MAX, std::abs(I32_MIN + 1), 0}, numeric::scale_type{0}}; @@ -1698,7 +1698,7 @@ TEST_F(JITExpressionTest, AnsiNeg) constexpr auto I32_MIN = std::numeric_limits::min(); constexpr auto I32_MAX = std::numeric_limits::max(); auto a = column_wrapper{3, -20, 1, -50, I32_MAX, -I32_MAX, 0}; - auto a_fail = column_wrapper{3, -20, 1, -50, I32_MIN, 0}; + auto a_fail = column_wrapper{3, -20, 1, -50, I32_MIN, 1, 0}; auto expected = column_wrapper{-3, 20, -1, 50, -I32_MAX, I32_MAX, 0}; auto table = cudf::table_view{{a, a_fail}}; auto a_ref = cudf::ast::column_reference(0); @@ -1719,7 +1719,7 @@ TEST_F(JITExpressionTest, AnsiNeg_Decimal) constexpr auto I32_MAX = std::numeric_limits::max(); auto a = cudf::test::fixed_point_column_wrapper{{3, -20, 1, -50, I32_MAX, -I32_MAX, 0}, numeric::scale_type{0}}; - auto a_fail = cudf::test::fixed_point_column_wrapper{{3, -20, 1, -50, I32_MIN, 0}, + auto a_fail = cudf::test::fixed_point_column_wrapper{{3, -20, 1, -50, I32_MIN, 1, 0}, numeric::scale_type{0}}; auto expected = cudf::test::fixed_point_column_wrapper{ {-3, 20, -1, 50, -I32_MAX, I32_MAX, 0}, numeric::scale_type{0}}; @@ -1758,39 +1758,290 @@ TEST_F(JITExpressionTest, AnsiPrecisionCheck) std::overflow_error); } -TEST_F(JITExpressionTest, BitShiftLeft) {} +TEST_F(JITExpressionTest, BitShiftLeft) +{ + auto a = cudf::test::fixed_width_column_wrapper{ + 0b0011'1111, 0b0001'1111, 0b101111, 0b0000'001100}; + auto expected = + cudf::test::fixed_width_column_wrapper{0b1111'1100, 0b01'11100, 0b1111, 0b110000}; + auto shift = cudf::numeric_scalar(2); + auto table = cudf::table_view{{a}}; + auto a_ref = cudf::ast::column_reference(0); + auto tree = cudf::ast::tree{}; + auto shift_literal = cudf::ast::literal(shift); + auto& shift_left = cudf::ast::jit::bit_shift_left(tree, a_ref, shift_literal); + auto result = cudf::compute_column_jit(table, shift_left); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); +} + +TEST_F(JITExpressionTest, BitShiftRight) +{ + auto a = cudf::test::fixed_width_column_wrapper{0b1111, 0b10111, 0b11100, 0b11110011}; + auto expected = cudf::test::fixed_width_column_wrapper{0b11, 0b101, 0b111, 0b111100}; + auto shift = cudf::numeric_scalar(2); + auto table = cudf::table_view{{a}}; + auto a_ref = cudf::ast::column_reference(0); + auto tree = cudf::ast::tree{}; + auto shift_literal = cudf::ast::literal(shift); + auto& shift_right = cudf::ast::jit::bit_shift_right(tree, a_ref, shift_literal); + auto result = cudf::compute_column_jit(table, shift_right); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); +} + +template +void test_cast() +{ + auto a = cudf::test::fixed_width_column_wrapper{0, 1, 2, 3, 4, 5}; + auto expected = cudf::test::fixed_width_column_wrapper{0, 1, 2, 3, 4, 5}; + auto table = cudf::table_view{{a}}; + auto a_ref = cudf::ast::column_reference(0); + auto tree = cudf::ast::tree{}; + + cudf::ast::expression const* cast = nullptr; + + if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_b8(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_i8(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_i16(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_i32(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_i64(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_u8(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_u16(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_u32(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_u64(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_f32(tree, a_ref); + } else { + static_assert(std::is_same_v); + cast = &cudf::ast::jit::cast_to_f64(tree, a_ref); + } -TEST_F(JITExpressionTest, BitShiftRight) {} + auto result = cudf::compute_column_jit(table, *cast); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); +} + +template +void test_from_decimal_cast() +{ + auto a = cudf::test::fixed_point_column_wrapper{{0, 1, 2, 3, 4, 5}, + numeric::scale_type{0}}; + auto expected = cudf::test::fixed_width_column_wrapper{0, 1, 2, 3, 4, 5}; + auto table = cudf::table_view{{a}}; + auto a_ref = cudf::ast::column_reference(0); + auto tree = cudf::ast::tree{}; + + cudf::ast::expression const* cast = nullptr; + + if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_b8(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_i8(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_i16(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_i32(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_i64(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_u8(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_u16(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_u32(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_u64(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_f32(tree, a_ref); + } else { + static_assert(std::is_same_v); + cast = &cudf::ast::jit::cast_to_f64(tree, a_ref); + } -TEST_F(JITExpressionTest, CastToBool) {} + auto result = cudf::compute_column_jit(table, *cast); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); +} -TEST_F(JITExpressionTest, CastToI8) {} +TEST_F(JITExpressionTest, CastToBool) +{ + test_cast(); + test_cast(); + test_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); +} -TEST_F(JITExpressionTest, CastToI16) {} +TEST_F(JITExpressionTest, CastToI8) +{ + test_cast(); + test_cast(); + test_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); +} -TEST_F(JITExpressionTest, CastToI32) {} +TEST_F(JITExpressionTest, CastToI16) +{ + test_cast(); + test_cast(); + test_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); +} -TEST_F(JITExpressionTest, CastToI64) {} +TEST_F(JITExpressionTest, CastToI32) +{ + test_cast(); + test_cast(); + test_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); +} -TEST_F(JITExpressionTest, CastToU8) {} +TEST_F(JITExpressionTest, CastToI64) +{ + test_cast(); + test_cast(); + test_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); +} -TEST_F(JITExpressionTest, CastToU16) {} +TEST_F(JITExpressionTest, CastToU8) +{ + test_cast(); + test_cast(); + test_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); +} -TEST_F(JITExpressionTest, CastToU32) {} +TEST_F(JITExpressionTest, CastToU16) +{ + test_cast(); + test_cast(); + test_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); +} -TEST_F(JITExpressionTest, CastToU64) {} +TEST_F(JITExpressionTest, CastToU32) +{ + test_cast(); + test_cast(); + test_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); +} -TEST_F(JITExpressionTest, CastToF32) {} +TEST_F(JITExpressionTest, CastToU64) +{ + test_cast(); + test_cast(); + test_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); +} -TEST_F(JITExpressionTest, CastToF64) {} +TEST_F(JITExpressionTest, CastToF32) +{ + test_cast(); + test_cast(); + test_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); +} + +TEST_F(JITExpressionTest, CastToF64) +{ + test_cast(); + test_cast(); + test_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); +} + +template +void test_decimal_cast() +{ + auto a = cudf::test::fixed_point_column_wrapper{{0, 1, 2, 3, 4, 5}, + numeric::scale_type{0}}; + auto expected = cudf::test::fixed_point_column_wrapper{{0, 1, 2, 3, 4, 5}, + numeric::scale_type{0}}; + auto table = cudf::table_view{{a}}; + auto a_ref = cudf::ast::column_reference(0); + auto tree = cudf::ast::tree{}; + + cudf::ast::expression const* cast = nullptr; + + if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_dec32(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_dec64(tree, a_ref); + } else if constexpr (std::is_same_v) { + static_assert(std::is_same_v); + cast = &cudf::ast::jit::cast_to_dec128(tree, a_ref); + } + + auto result = cudf::compute_column_jit(table, *cast); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); +} + +TEST_F(JITExpressionTest, CastToDec32) +{ + test_decimal_cast(); + test_decimal_cast(); + test_decimal_cast(); +} -TEST_F(JITExpressionTest, CastToDec32) {} +TEST_F(JITExpressionTest, CastToDec64) +{ + test_decimal_cast(); + test_decimal_cast(); + test_decimal_cast(); +} -TEST_F(JITExpressionTest, CastToDec64) {} +TEST_F(JITExpressionTest, CastToDec128) +{ + test_decimal_cast(); + test_decimal_cast(); + test_decimal_cast(); +} -TEST_F(JITExpressionTest, CastToDec128) {} +TEST_F(JITExpressionTest, Rescale) +{ + auto a = cudf::test::fixed_point_column_wrapper{{123, 1234, 12345, 123456, 1234567}, + numeric::scale_type{0}}; + auto expected = cudf::test::fixed_point_column_wrapper{ + {12300, 123400, 1234500, 12345600, 123456700}, numeric::scale_type{-2}}; + auto table = cudf::table_view{{a}}; + auto a_ref = cudf::ast::column_reference(0); + auto tree = cudf::ast::tree{}; + auto& rescaled = cudf::ast::jit::rescale(tree, a_ref, 2); + auto result = cudf::compute_column_jit(table, rescaled); -TEST_F(JITExpressionTest, Rescale) {} + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); +} // TODO: TEST ANSI op fusion From 8292fc483b41a2849a21763e968cf459c0082bd9 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 4 May 2026 20:30:06 +0000 Subject: [PATCH 12/15] bug fixes and more tests --- cpp/tests/ast/transform_tests.cpp | 138 +++++++----------------------- 1 file changed, 30 insertions(+), 108 deletions(-) diff --git a/cpp/tests/ast/transform_tests.cpp b/cpp/tests/ast/transform_tests.cpp index df4edc43c672..b602b8058ac9 100644 --- a/cpp/tests/ast/transform_tests.cpp +++ b/cpp/tests/ast/transform_tests.cpp @@ -1760,10 +1760,9 @@ TEST_F(JITExpressionTest, AnsiPrecisionCheck) TEST_F(JITExpressionTest, BitShiftLeft) { - auto a = cudf::test::fixed_width_column_wrapper{ - 0b0011'1111, 0b0001'1111, 0b101111, 0b0000'001100}; + auto a = cudf::test::fixed_width_column_wrapper{0b111111, 0b111110, 0b101111, 0b1100}; auto expected = - cudf::test::fixed_width_column_wrapper{0b1111'1100, 0b01'11100, 0b1111, 0b110000}; + cudf::test::fixed_width_column_wrapper{0b11111100, 0b11111000, 0b10111100, 0b110000}; auto shift = cudf::numeric_scalar(2); auto table = cudf::table_view{{a}}; auto a_ref = cudf::ast::column_reference(0); @@ -1871,114 +1870,37 @@ void test_from_decimal_cast() CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); } -TEST_F(JITExpressionTest, CastToBool) +template +void test_cast_to() { - test_cast(); - test_cast(); - test_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); + test_cast(); + test_cast(); + test_cast(); + test_cast(); + test_cast(); + test_cast(); + test_cast(); + test_cast(); + test_cast(); + test_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); } -TEST_F(JITExpressionTest, CastToI8) +TEST_F(JITExpressionTest, Cast) { - test_cast(); - test_cast(); - test_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); -} - -TEST_F(JITExpressionTest, CastToI16) -{ - test_cast(); - test_cast(); - test_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); -} - -TEST_F(JITExpressionTest, CastToI32) -{ - test_cast(); - test_cast(); - test_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); -} - -TEST_F(JITExpressionTest, CastToI64) -{ - test_cast(); - test_cast(); - test_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); -} - -TEST_F(JITExpressionTest, CastToU8) -{ - test_cast(); - test_cast(); - test_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); -} - -TEST_F(JITExpressionTest, CastToU16) -{ - test_cast(); - test_cast(); - test_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); -} - -TEST_F(JITExpressionTest, CastToU32) -{ - test_cast(); - test_cast(); - test_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); -} - -TEST_F(JITExpressionTest, CastToU64) -{ - test_cast(); - test_cast(); - test_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); -} - -TEST_F(JITExpressionTest, CastToF32) -{ - test_cast(); - test_cast(); - test_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); -} - -TEST_F(JITExpressionTest, CastToF64) -{ - test_cast(); - test_cast(); - test_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); + test_cast_to(); + test_cast_to(); + test_cast_to(); + test_cast_to(); + test_cast_to(); + test_cast_to(); + test_cast_to(); + test_cast_to(); + test_cast_to(); + test_cast_to(); + test_cast_to(); } template @@ -2037,7 +1959,7 @@ TEST_F(JITExpressionTest, Rescale) auto table = cudf::table_view{{a}}; auto a_ref = cudf::ast::column_reference(0); auto tree = cudf::ast::tree{}; - auto& rescaled = cudf::ast::jit::rescale(tree, a_ref, 2); + auto& rescaled = cudf::ast::jit::rescale(tree, a_ref, -2); auto result = cudf::compute_column_jit(table, rescaled); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); From 923bd2c0fea6b1d48454515b05d4c3a4b203b5dd Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 5 May 2026 00:40:32 +0000 Subject: [PATCH 13/15] bug fixes and more tests --- cpp/include/cudf/ast/jit_expressions.hpp | 2 +- .../cudf/operators/ansi_arithmetic.cuh | 4 +- cpp/include/cudf/operators/logic.cuh | 4 +- cpp/tests/CMakeLists.txt | 4 + cpp/tests/ast/jit_ast_tests.cpp | 708 ++++++++++++++++++ cpp/tests/ast/transform_tests.cpp | 576 -------------- cpp/tests/jit/row_ir.cpp | 2 +- 7 files changed, 719 insertions(+), 581 deletions(-) create mode 100644 cpp/tests/ast/jit_ast_tests.cpp diff --git a/cpp/include/cudf/ast/jit_expressions.hpp b/cpp/include/cudf/ast/jit_expressions.hpp index 23a5f87ba3db..37ea00912537 100644 --- a/cpp/include/cudf/ast/jit_expressions.hpp +++ b/cpp/include/cudf/ast/jit_expressions.hpp @@ -113,7 +113,7 @@ struct operation : public ast::expression { * @param condition The condition under which to nullify the value * @return An expression representing the nullified value */ -expression const& nullify_if(ast::tree& tree, expression const& a, expression const& condition); +expression const& nullify_if(ast::tree& tree, expression const& a, expression const& condition); /** * @brief Creates an expression that evaluates to the first non-null value among its arguments. diff --git a/cpp/include/cudf/operators/ansi_arithmetic.cuh b/cpp/include/cudf/operators/ansi_arithmetic.cuh index 367b51c901e8..2288f099a1da 100644 --- a/cpp/include/cudf/operators/ansi_arithmetic.cuh +++ b/cpp/include/cudf/operators/ansi_arithmetic.cuh @@ -401,7 +401,7 @@ __device__ inline errc ansi_abs(decimal* out, decimal const* a) { if (a->value() == cuda::std::numeric_limits::min()) { return errc::OVERFLOW; } auto rep = a->value() < 0 ? -a->value() : a->value(); - *out = decimal{numeric::scaled_integer{rep, numeric::scale_type{a->scale()}}}; + *out = decimal{numeric::scaled_integer{rep, numeric::scale_type{a->scale()}}}; return errc::OK; } @@ -568,7 +568,7 @@ __device__ inline errc ansi_try_abs(optional* out, optional const* a) { if (a->has_value()) { T r; - if (errc e = ansi_abs(&r, a); e != errc::OK) { + if (errc e = ansi_abs(&r, &a->value()); e != errc::OK) { *out = nullopt; } else { *out = r; diff --git a/cpp/include/cudf/operators/logic.cuh b/cpp/include/cudf/operators/logic.cuh index f81917e83e18..fbbc8f8148e5 100644 --- a/cpp/include/cudf/operators/logic.cuh +++ b/cpp/include/cudf/operators/logic.cuh @@ -45,7 +45,9 @@ __device__ inline errc null_logical_or(bool* out, T const* a, T const* b) } template -__device__ inline errc null_logical_or(optional* out, optional const* a, optional const* b) +__device__ inline errc null_logical_or(optional* out, + optional const* a, + optional const* b) { if (a->has_value() && b->has_value()) { bool r; diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 68cde65c57bb..9945490e0120 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -681,6 +681,10 @@ ConfigureTest(ENCODE_TEST encode/encode_tests.cpp) # * ast tests ------------------------------------------------------------------------------------- ConfigureTest(AST_TEST ast/transform_tests.cpp ast/ast_tree_tests.cpp) +# ################################################################################################## +# * jit-ast tests ------------------------------------------------------------------------------------- +ConfigureTest(JIT_AST_TEST ast/jit_ast_tests.cpp) + # ################################################################################################## # * lists tests ---------------------------------------------------------------------------------- ConfigureTest( diff --git a/cpp/tests/ast/jit_ast_tests.cpp b/cpp/tests/ast/jit_ast_tests.cpp new file mode 100644 index 000000000000..0c8a1ea61f94 --- /dev/null +++ b/cpp/tests/ast/jit_ast_tests.cpp @@ -0,0 +1,708 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include + +template +using column_wrapper = cudf::test::fixed_width_column_wrapper; + +constexpr cudf::test::debug_output_level verbosity{cudf::test::debug_output_level::ALL_ERRORS}; + +struct JITExpressionTest : public cudf::test::BaseFixture {}; + +TEST_F(JITExpressionTest, NullifyIf) +{ + auto a = column_wrapper{3, 20, 1, 50, 0, 20}; + auto condition = column_wrapper{false, true, false, true, false, true}; + auto expected = column_wrapper{{3, 0, 1, 0, 0, 0}, {1, 0, 1, 0, 1, 0}}; + auto table = cudf::table_view{{a, condition}}; + auto tree = cudf::ast::tree{}; + auto a_ref = cudf::ast::column_reference(0); + auto condition_ref = cudf::ast::column_reference(1); + auto& nullify_if = cudf::ast::jit::nullify_if(tree, a_ref, condition_ref); + auto result = cudf::compute_column_jit(table, nullify_if); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); +} + +TEST_F(JITExpressionTest, Coalesce) +{ + auto a = column_wrapper{{1, 3, 5, 7, 9, 11}, {1, 0, 0, 1, 0, 0}}; + auto b = column_wrapper{{2, 4, 6, 8, 10, 12}, {1, 1, 1, 0, 1, 0}}; + auto expected = column_wrapper{{1, 4, 6, 7, 10, 0}, {1, 1, 1, 1, 1, 0}}; + auto table = cudf::table_view{{a, b}}; + auto tree = cudf::ast::tree{}; + auto a_ref = cudf::ast::column_reference(0); + auto b_ref = cudf::ast::column_reference(1); + auto& coalesce = cudf::ast::jit::coalesce(tree, a_ref, b_ref); + auto result = cudf::compute_column_jit(table, coalesce); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); +} + +// TODO: parameterize on decimal types and float types + +TEST_F(JITExpressionTest, AnsiAdd_Int) +{ + constexpr auto I32_MAX = std::numeric_limits::max(); + auto a = column_wrapper{3, 20, 1, 50}; + auto b = column_wrapper{10, 7, 20, 0}; + auto b_fail = column_wrapper{10, I32_MAX, 20, 0}; + auto expected = column_wrapper{13, 27, 21, 50}; + auto expected_fail = column_wrapper{{13, 0, 21, 50}, {1, 0, 1, 1}}; + auto table = cudf::table_view{{a, b, b_fail}}; + auto tree = cudf::ast::tree{}; + auto a_ref = cudf::ast::column_reference(0); + auto b_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto& add = cudf::ast::jit::ansi_add(tree, a_ref, b_ref); + auto& add_fail = cudf::ast::jit::ansi_add(tree, a_ref, b_fail_ref); + auto& try_add_fail = cudf::ast::jit::ansi_try_add(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, add); + auto result_fail = cudf::compute_column_jit(table, try_add_fail); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, add_fail), std::overflow_error); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); +} + +TEST_F(JITExpressionTest, AnsiAdd_Decimal) +{ + constexpr auto I32_MAX = std::numeric_limits::max(); + auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 1, 50}, numeric::scale_type{0}}; + auto b = cudf::test::fixed_point_column_wrapper{{10, 7, 20, 0}, numeric::scale_type{0}}; + auto b_fail = + cudf::test::fixed_point_column_wrapper{{10, I32_MAX, 20, 0}, numeric::scale_type{0}}; + auto expected = + cudf::test::fixed_point_column_wrapper{{13, 27, 21, 50}, numeric::scale_type{0}}; + auto expected_fail = cudf::test::fixed_point_column_wrapper{ + {13, 0, 21, 50}, {1, 0, 1, 1}, numeric::scale_type{0}}; + auto table = cudf::table_view{{a, b, b_fail}}; + auto tree = cudf::ast::tree{}; + auto a_ref = cudf::ast::column_reference(0); + auto b_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto& add = cudf::ast::jit::ansi_add(tree, a_ref, b_ref); + auto& add_fail = cudf::ast::jit::ansi_add(tree, a_ref, b_fail_ref); + auto& try_add_fail = cudf::ast::jit::ansi_try_add(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, add); + auto result_fail = cudf::compute_column_jit(table, try_add_fail); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, add_fail), std::overflow_error); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); +} + +TEST_F(JITExpressionTest, AnsiSub) +{ + constexpr auto I32_MIN = std::numeric_limits::min(); + auto a = column_wrapper{3, 20, 1, 50}; + auto b = column_wrapper{10, 7, 20, 0}; + auto b_fail = column_wrapper{10, I32_MIN, 20, 0}; + auto expected = column_wrapper{-7, 13, -19, 50}; + auto expected_fail = column_wrapper{{-7, 0, -19, 50}, {1, 0, 1, 1}}; + auto table = cudf::table_view{{a, b, b_fail}}; + auto tree = cudf::ast::tree{}; + auto a_ref = cudf::ast::column_reference(0); + auto b_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto& sub = cudf::ast::jit::ansi_sub(tree, a_ref, b_ref); + auto& sub_fail = cudf::ast::jit::ansi_sub(tree, a_ref, b_fail_ref); + auto& try_sub_fail = cudf::ast::jit::ansi_try_sub(tree, a_ref, b_fail_ref); + + auto result = cudf::compute_column_jit(table, sub); + auto result_fail = cudf::compute_column_jit(table, try_sub_fail); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, sub_fail), std::overflow_error); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); +} + +TEST_F(JITExpressionTest, AnsiSub_Decimal) +{ + constexpr auto I32_MIN = std::numeric_limits::min(); + auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 1, 50}, numeric::scale_type{0}}; + auto b = cudf::test::fixed_point_column_wrapper{{10, 7, 20, 0}, numeric::scale_type{0}}; + auto b_fail = + cudf::test::fixed_point_column_wrapper{{10, I32_MIN, 20, 0}, numeric::scale_type{0}}; + auto expected = + cudf::test::fixed_point_column_wrapper{{-7, 13, -19, 50}, numeric::scale_type{0}}; + auto expected_fail = cudf::test::fixed_point_column_wrapper{ + {-7, 0, -19, 50}, {1, 0, 1, 1}, numeric::scale_type{0}}; + auto table = cudf::table_view{{a, b, b_fail}}; + auto tree = cudf::ast::tree{}; + auto a_ref = cudf::ast::column_reference(0); + auto b_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto& sub = cudf::ast::jit::ansi_sub(tree, a_ref, b_ref); + auto& sub_fail = cudf::ast::jit::ansi_sub(tree, a_ref, b_fail_ref); + auto& try_sub_fail = cudf::ast::jit::ansi_try_sub(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, sub); + auto result_fail = cudf::compute_column_jit(table, try_sub_fail); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, sub_fail), std::overflow_error); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); +} + +TEST_F(JITExpressionTest, AnsiMul) +{ + constexpr auto I32_MAX = std::numeric_limits::max(); + auto a = column_wrapper{3, 20, 2, 50}; + auto b = column_wrapper{10, 7, 1, 0}; + auto b_fail = column_wrapper{10, I32_MAX, 1, 0}; + auto expected = column_wrapper{30, 140, 2, 0}; + auto expected_fail = column_wrapper{{30, 0, 2, 0}, {1, 0, 1, 1}}; + auto table = cudf::table_view{{a, b, b_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto b_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto tree = cudf::ast::tree{}; + auto& mul = cudf::ast::jit::ansi_mul(tree, a_ref, b_ref); + auto& mul_fail = cudf::ast::jit::ansi_mul(tree, a_ref, b_fail_ref); + auto& try_mul_fail = cudf::ast::jit::ansi_try_mul(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, mul); + auto result_fail = cudf::compute_column_jit(table, try_mul_fail); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, mul_fail), std::overflow_error); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); +} + +TEST_F(JITExpressionTest, AnsiMul_Decimal) +{ + constexpr auto I32_MAX = std::numeric_limits::max(); + auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 2, 50}, numeric::scale_type{0}}; + auto b = cudf::test::fixed_point_column_wrapper{{10, 7, 1, 0}, numeric::scale_type{0}}; + auto b_fail = + cudf::test::fixed_point_column_wrapper{{10, I32_MAX, 1, 0}, numeric::scale_type{0}}; + auto expected = + cudf::test::fixed_point_column_wrapper{{30, 140, 2, 0}, numeric::scale_type{0}}; + auto expected_fail = cudf::test::fixed_point_column_wrapper{ + {30, 0, 2, 0}, {1, 0, 1, 1}, numeric::scale_type{0}}; + auto table = cudf::table_view{{a, b, b_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto b_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto tree = cudf::ast::tree{}; + auto& mul = cudf::ast::jit::ansi_mul(tree, a_ref, b_ref); + auto& mul_fail = cudf::ast::jit::ansi_mul(tree, a_ref, b_fail_ref); + auto& try_mul_fail = cudf::ast::jit::ansi_try_mul(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, mul); + auto result_fail = cudf::compute_column_jit(table, try_mul_fail); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, mul_fail), std::overflow_error); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); +} + +TEST_F(JITExpressionTest, AnsiDiv) +{ + auto a = column_wrapper{3, 20, 1, 50}; + auto b = column_wrapper{10, 7, 2, 1}; + auto b_fail = column_wrapper{10, 1, 20, 0}; + auto expected = column_wrapper{0, 2, 0, 50}; + auto expected_fail = column_wrapper{{0, 20, 0, 50}, {1, 1, 1, 0}}; + auto table = cudf::table_view{{a, b, b_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto b_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto tree = cudf::ast::tree{}; + auto& div = cudf::ast::jit::ansi_div(tree, a_ref, b_ref); + auto& div_fail = cudf::ast::jit::ansi_div(tree, a_ref, b_fail_ref); + auto& try_div_fail = cudf::ast::jit::ansi_try_div(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, div); + auto result_fail = cudf::compute_column_jit(table, try_div_fail); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, div_fail), std::overflow_error); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); +} + +TEST_F(JITExpressionTest, AnsiDiv_Decimal) +{ + auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 1, 50}, numeric::scale_type{0}}; + auto b = cudf::test::fixed_point_column_wrapper{{10, 7, 2, 1}, numeric::scale_type{0}}; + auto b_fail = + cudf::test::fixed_point_column_wrapper{{10, 1, 20, 0}, numeric::scale_type{0}}; + auto expected = + cudf::test::fixed_point_column_wrapper{{0, 2, 0, 50}, numeric::scale_type{0}}; + auto expected_fail = cudf::test::fixed_point_column_wrapper{ + {0, 20, 0, 50}, {1, 1, 1, 0}, numeric::scale_type{0}}; + auto table = cudf::table_view{{a, b, b_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto b_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto tree = cudf::ast::tree{}; + auto& div = cudf::ast::jit::ansi_div(tree, a_ref, b_ref); + auto& div_fail = cudf::ast::jit::ansi_div(tree, a_ref, b_fail_ref); + auto& try_div_fail = cudf::ast::jit::ansi_try_div(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, div); + auto result_fail = cudf::compute_column_jit(table, try_div_fail); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, div_fail), std::overflow_error); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); +} + +TEST_F(JITExpressionTest, AnsiMod) +{ + auto a = column_wrapper{3, 20, 1, 50}; + auto b = column_wrapper{10, 7, 2, 1}; + auto b_fail = column_wrapper{10, 1, 20, 0}; + auto expected = column_wrapper{3, 6, 1, 0}; + auto expected_fail = column_wrapper{{3, 0, 1, 0}, {1, 1, 1, 0}}; + auto table = cudf::table_view{{a, b, b_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto b_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto tree = cudf::ast::tree{}; + auto& mod = cudf::ast::jit::ansi_mod(tree, a_ref, b_ref); + auto& mod_fail = cudf::ast::jit::ansi_mod(tree, a_ref, b_fail_ref); + auto& try_mod_fail = cudf::ast::jit::ansi_try_mod(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, mod); + auto result_fail = cudf::compute_column_jit(table, try_mod_fail); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, mod_fail), std::overflow_error); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); +} + +TEST_F(JITExpressionTest, AnsiMod_Decimal) +{ + auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 1, 50}, numeric::scale_type{0}}; + auto b = cudf::test::fixed_point_column_wrapper{{10, 7, 2, 1}, numeric::scale_type{0}}; + auto b_fail = + cudf::test::fixed_point_column_wrapper{{10, 1, 20, 0}, numeric::scale_type{0}}; + auto expected = + cudf::test::fixed_point_column_wrapper{{3, 6, 1, 0}, numeric::scale_type{0}}; + auto expected_fail = cudf::test::fixed_point_column_wrapper{ + {3, 0, 1, 0}, {1, 1, 1, 0}, numeric::scale_type{0}}; + auto table = cudf::table_view{{a, b, b_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto b_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto tree = cudf::ast::tree{}; + auto& mod = cudf::ast::jit::ansi_mod(tree, a_ref, b_ref); + auto& mod_fail = cudf::ast::jit::ansi_mod(tree, a_ref, b_fail_ref); + auto& try_mod_fail = cudf::ast::jit::ansi_try_mod(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, mod); + auto result_fail = cudf::compute_column_jit(table, try_mod_fail); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, mod_fail), std::overflow_error); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); +} + +TEST_F(JITExpressionTest, AnsiAbs) +{ + constexpr auto I32_MIN = std::numeric_limits::min(); + constexpr auto I32_MAX = std::numeric_limits::max(); + auto a = column_wrapper{3, -20, 1, -50, I32_MAX, I32_MIN + 1, 0}; + auto a_fail = column_wrapper{3, -20, 1, -50, I32_MIN, 1, 0}; + auto expected = column_wrapper{3, 20, 1, 50, I32_MAX, std::abs(I32_MIN + 1), 0}; + auto expected_fail = column_wrapper{{3, 20, 1, 50, 0, 1, 0}, {1, 1, 1, 1, 0, 1, 1}}; + auto table = cudf::table_view{{a, a_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto a_fail_ref = cudf::ast::column_reference(1); + auto tree = cudf::ast::tree{}; + auto& abs = cudf::ast::jit::ansi_abs(tree, a_ref); + auto& abs_fail = cudf::ast::jit::ansi_abs(tree, a_fail_ref); + auto& try_abs_fail = cudf::ast::jit::ansi_try_abs(tree, a_fail_ref); + auto result = cudf::compute_column_jit(table, abs); + auto result_fail = cudf::compute_column_jit(table, try_abs_fail); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, abs_fail), std::overflow_error); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); +} + +TEST_F(JITExpressionTest, AnsiAbs_Decimal) +{ + constexpr auto I32_MIN = std::numeric_limits::min(); + constexpr auto I32_MAX = std::numeric_limits::max(); + auto a = cudf::test::fixed_point_column_wrapper{ + {3, -20, 1, -50, I32_MAX, I32_MIN + 1, 0}, numeric::scale_type{0}}; + auto a_fail = cudf::test::fixed_point_column_wrapper{{3, -20, 1, -50, I32_MIN, 1, 0}, + numeric::scale_type{0}}; + auto expected = cudf::test::fixed_point_column_wrapper{ + {3, 20, 1, 50, I32_MAX, std::abs(I32_MIN + 1), 0}, numeric::scale_type{0}}; + auto expected_fail = cudf::test::fixed_point_column_wrapper{ + {3, 20, 1, 50, 0, 1, 0}, {1, 1, 1, 1, 0, 1, 1}, numeric::scale_type{0}}; + auto table = cudf::table_view{{a, a_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto a_fail_ref = cudf::ast::column_reference(1); + auto tree = cudf::ast::tree{}; + auto& abs = cudf::ast::jit::ansi_abs(tree, a_ref); + auto& abs_fail = cudf::ast::jit::ansi_abs(tree, a_fail_ref); + auto& try_abs_fail = cudf::ast::jit::ansi_try_abs(tree, a_fail_ref); + auto result = cudf::compute_column_jit(table, abs); + auto result_fail = cudf::compute_column_jit(table, try_abs_fail); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, abs_fail), std::overflow_error); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); +} + +TEST_F(JITExpressionTest, AnsiNeg) +{ + constexpr auto I32_MIN = std::numeric_limits::min(); + constexpr auto I32_MAX = std::numeric_limits::max(); + auto a = column_wrapper{3, -20, 1, -50, I32_MAX, -I32_MAX, 0}; + auto a_fail = column_wrapper{3, -20, 1, -50, I32_MIN, 1, 0}; + auto expected = column_wrapper{-3, 20, -1, 50, -I32_MAX, I32_MAX, 0}; + auto expected_fail = column_wrapper{{-3, 20, -1, 50, 0, -1, 0}, {1, 1, 1, 1, 0, 1, 1}}; + auto table = cudf::table_view{{a, a_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto a_fail_ref = cudf::ast::column_reference(1); + auto tree = cudf::ast::tree{}; + auto& neg = cudf::ast::jit::ansi_neg(tree, a_ref); + auto& neg_fail = cudf::ast::jit::ansi_neg(tree, a_fail_ref); + auto& try_neg_fail = cudf::ast::jit::ansi_try_neg(tree, a_fail_ref); + auto result = cudf::compute_column_jit(table, neg); + auto result_fail = cudf::compute_column_jit(table, try_neg_fail); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, neg_fail), std::overflow_error); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); +} + +TEST_F(JITExpressionTest, AnsiNeg_Decimal) +{ + constexpr auto I32_MIN = std::numeric_limits::min(); + constexpr auto I32_MAX = std::numeric_limits::max(); + auto a = cudf::test::fixed_point_column_wrapper{{3, -20, 1, -50, I32_MAX, -I32_MAX, 0}, + numeric::scale_type{0}}; + auto a_fail = cudf::test::fixed_point_column_wrapper{{3, -20, 1, -50, I32_MIN, 1, 0}, + numeric::scale_type{0}}; + auto expected = cudf::test::fixed_point_column_wrapper{ + {-3, 20, -1, 50, -I32_MAX, I32_MAX, 0}, numeric::scale_type{0}}; + auto expected_fail = cudf::test::fixed_point_column_wrapper{ + {-3, 20, -1, 50, 0, -1, 0}, {1, 1, 1, 1, 0, 1, 1}, numeric::scale_type{0}}; + auto table = cudf::table_view{{a, a_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto a_fail_ref = cudf::ast::column_reference(1); + auto tree = cudf::ast::tree{}; + auto& neg = cudf::ast::jit::ansi_neg(tree, a_ref); + auto& neg_fail = cudf::ast::jit::ansi_neg(tree, a_fail_ref); + auto& try_neg_fail = cudf::ast::jit::ansi_try_neg(tree, a_fail_ref); + auto result = cudf::compute_column_jit(table, neg); + auto result_fail = cudf::compute_column_jit(table, try_neg_fail); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(table, neg_fail), std::overflow_error); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); +} + +// TODO: some device malloc failure happens in this test, need to investigate and re-enable +TEST_F(JITExpressionTest, DISABLED_AnsiPrecisionCheck) +{ + auto a = + cudf::test::fixed_point_column_wrapper{{3, 200, 250, 200}, numeric::scale_type{0}}; + auto a_fail = + cudf::test::fixed_point_column_wrapper{{3, 200, 250, 20000}, numeric::scale_type{0}}; + auto expected = + cudf::test::fixed_point_column_wrapper{{3, 200, 250, 200}, numeric::scale_type{0}}; + auto expected_fail = cudf::test::fixed_point_column_wrapper{ + {3, 200, 250, 200}, {1, 1, 1, 0}, numeric::scale_type{0}}; + auto max_precision = cudf::numeric_scalar(3); + auto table = cudf::table_view{{a}}; + auto a_ref = cudf::ast::column_reference(0); + auto tree = cudf::ast::tree{}; + auto precision = cudf::ast::literal(max_precision); + auto& precision_check = cudf::ast::jit::ansi_precision_check(tree, a_ref, precision); + auto& try_precision_check = cudf::ast::jit::ansi_try_precision_check(tree, a_ref, precision); + auto result = cudf::compute_column_jit(table, precision_check); + auto result_fail = cudf::compute_column_jit(table, try_precision_check); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + + EXPECT_THROW(result = cudf::compute_column_jit(cudf::table_view{{a_fail}}, precision_check), + std::overflow_error); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); +} + +TEST_F(JITExpressionTest, BitShiftLeft) +{ + auto a = cudf::test::fixed_width_column_wrapper{0b111111, 0b111110, 0b101111, 0b1100}; + auto expected = + cudf::test::fixed_width_column_wrapper{0b11111100, 0b11111000, 0b10111100, 0b110000}; + auto shift = cudf::numeric_scalar(2); + auto table = cudf::table_view{{a}}; + auto a_ref = cudf::ast::column_reference(0); + auto tree = cudf::ast::tree{}; + auto shift_literal = cudf::ast::literal(shift); + auto& shift_left = cudf::ast::jit::bit_shift_left(tree, a_ref, shift_literal); + auto result = cudf::compute_column_jit(table, shift_left); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); +} + +TEST_F(JITExpressionTest, BitShiftRight) +{ + auto a = cudf::test::fixed_width_column_wrapper{0b1111, 0b10111, 0b11100, 0b11110011}; + auto expected = cudf::test::fixed_width_column_wrapper{0b11, 0b101, 0b111, 0b111100}; + auto shift = cudf::numeric_scalar(2); + auto table = cudf::table_view{{a}}; + auto a_ref = cudf::ast::column_reference(0); + auto tree = cudf::ast::tree{}; + auto shift_literal = cudf::ast::literal(shift); + auto& shift_right = cudf::ast::jit::bit_shift_right(tree, a_ref, shift_literal); + auto result = cudf::compute_column_jit(table, shift_right); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); +} + +template +void test_cast() +{ + auto a = cudf::test::fixed_width_column_wrapper{0, 1, 2, 3, 4, 5}; + auto expected = cudf::test::fixed_width_column_wrapper{0, 1, 2, 3, 4, 5}; + auto table = cudf::table_view{{a}}; + auto a_ref = cudf::ast::column_reference(0); + auto tree = cudf::ast::tree{}; + + cudf::ast::expression const* cast = nullptr; + + if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_b8(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_i8(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_i16(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_i32(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_i64(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_u8(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_u16(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_u32(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_u64(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_f32(tree, a_ref); + } else { + static_assert(std::is_same_v); + cast = &cudf::ast::jit::cast_to_f64(tree, a_ref); + } + + auto result = cudf::compute_column_jit(table, *cast); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); +} + +template +void test_from_decimal_cast() +{ + auto a = cudf::test::fixed_point_column_wrapper{{0, 1, 2, 3, 4, 5}, + numeric::scale_type{0}}; + auto expected = cudf::test::fixed_width_column_wrapper{0, 1, 2, 3, 4, 5}; + auto table = cudf::table_view{{a}}; + auto a_ref = cudf::ast::column_reference(0); + auto tree = cudf::ast::tree{}; + + cudf::ast::expression const* cast = nullptr; + + if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_b8(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_i8(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_i16(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_i32(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_i64(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_u8(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_u16(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_u32(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_u64(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_f32(tree, a_ref); + } else { + static_assert(std::is_same_v); + cast = &cudf::ast::jit::cast_to_f64(tree, a_ref); + } + + auto result = cudf::compute_column_jit(table, *cast); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); +} + +template +void test_cast_to() +{ + test_cast(); + test_cast(); + test_cast(); + test_cast(); + test_cast(); + test_cast(); + test_cast(); + test_cast(); + test_cast(); + test_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); + test_from_decimal_cast(); +} + +TEST_F(JITExpressionTest, Cast) +{ + test_cast_to(); + test_cast_to(); + test_cast_to(); + test_cast_to(); + test_cast_to(); + test_cast_to(); + test_cast_to(); + test_cast_to(); + test_cast_to(); + test_cast_to(); + test_cast_to(); +} + +template +void test_decimal_cast() +{ + auto a = cudf::test::fixed_point_column_wrapper{{0, 1, 2, 3, 4, 5}, + numeric::scale_type{0}}; + auto expected = cudf::test::fixed_point_column_wrapper{{0, 1, 2, 3, 4, 5}, + numeric::scale_type{0}}; + auto table = cudf::table_view{{a}}; + auto a_ref = cudf::ast::column_reference(0); + auto tree = cudf::ast::tree{}; + + cudf::ast::expression const* cast = nullptr; + + if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_dec32(tree, a_ref); + } else if constexpr (std::is_same_v) { + cast = &cudf::ast::jit::cast_to_dec64(tree, a_ref); + } else if constexpr (std::is_same_v) { + static_assert(std::is_same_v); + cast = &cudf::ast::jit::cast_to_dec128(tree, a_ref); + } + + auto result = cudf::compute_column_jit(table, *cast); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); +} + +TEST_F(JITExpressionTest, CastToDec32) +{ + test_decimal_cast(); + test_decimal_cast(); + test_decimal_cast(); +} + +TEST_F(JITExpressionTest, CastToDec64) +{ + test_decimal_cast(); + test_decimal_cast(); + test_decimal_cast(); +} + +TEST_F(JITExpressionTest, CastToDec128) +{ + test_decimal_cast(); + test_decimal_cast(); + test_decimal_cast(); +} + +TEST_F(JITExpressionTest, Rescale) +{ + auto a = cudf::test::fixed_point_column_wrapper{{123, 1234, 12345, 123456, 1234567}, + numeric::scale_type{0}}; + auto expected = cudf::test::fixed_point_column_wrapper{ + {12300, 123400, 1234500, 12345600, 123456700}, numeric::scale_type{-2}}; + auto table = cudf::table_view{{a}}; + auto a_ref = cudf::ast::column_reference(0); + auto tree = cudf::ast::tree{}; + auto& rescaled = cudf::ast::jit::rescale(tree, a_ref, -2); + auto result = cudf::compute_column_jit(table, rescaled); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); +} + +TEST_F(JITExpressionTest, AnsiFused) +{ + constexpr auto I32_MAX = std::numeric_limits::max(); + auto a = column_wrapper{1, 3, 20, 1, 50, 10}; + auto b = column_wrapper{1, 10, 7, 20, I32_MAX, 2}; + auto c = column_wrapper{1, 5, 4, I32_MAX, 2, 5}; + auto d = column_wrapper{0, 1, 0, 0, 1, 5}; + auto expected = column_wrapper{{0, 65, 0, 0, 0, 12}, {0, 1, 0, 0, 0, 1}}; + auto table = cudf::table_view{{a, b, c, d}}; + auto tree = cudf::ast::tree{}; + auto a_ref = cudf::ast::column_reference(0); + auto b_ref = cudf::ast::column_reference(1); + auto c_ref = cudf::ast::column_reference(2); + auto d_ref = cudf::ast::column_reference(3); + auto& add = cudf::ast::jit::ansi_try_add(tree, a_ref, b_ref); + auto& mul = cudf::ast::jit::ansi_try_mul(tree, add, c_ref); + auto& div = cudf::ast::jit::ansi_try_div(tree, mul, d_ref); + auto result = cudf::compute_column_jit(table, div); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); +} + +CUDF_TEST_PROGRAM_MAIN() diff --git a/cpp/tests/ast/transform_tests.cpp b/cpp/tests/ast/transform_tests.cpp index b602b8058ac9..06da7f8a858a 100644 --- a/cpp/tests/ast/transform_tests.cpp +++ b/cpp/tests/ast/transform_tests.cpp @@ -11,7 +11,6 @@ #include #include -#include #include #include #include @@ -1392,579 +1391,4 @@ TYPED_TEST(TransformTest, NonDefaultStream) CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); } -struct JITExpressionTest : public cudf::test::BaseFixture {}; - -TEST_F(JITExpressionTest, NullifyIf) -{ - auto a = column_wrapper{3, 20, 1, 50, 0, 20}; - auto condition = column_wrapper{false, true, false, true, false, true}; - auto expected = column_wrapper{{3, 0, 1, 0, 0, 0}, {1, 0, 1, 0, 1, 0}}; - auto table = cudf::table_view{{a, condition}}; - auto tree = cudf::ast::tree{}; - auto a_ref = cudf::ast::column_reference(0); - auto condition_ref = cudf::ast::column_reference(1); - auto& nullify_if = cudf::ast::jit::nullify_if(tree, a_ref, condition_ref); - auto result = cudf::compute_column_jit(table, nullify_if); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); -} - -TEST_F(JITExpressionTest, Coalesce) -{ - auto a = column_wrapper{{1, 3, 5, 7, 9, 11}, {1, 0, 0, 1, 0, 0}}; - auto b = column_wrapper{{2, 4, 6, 8, 10, 12}, {1, 1, 1, 0, 1, 0}}; - auto expected = column_wrapper{{1, 4, 6, 7, 10, 0}, {1, 1, 1, 1, 1, 0}}; - auto table = cudf::table_view{{a, b}}; - auto tree = cudf::ast::tree{}; - auto a_ref = cudf::ast::column_reference(0); - auto b_ref = cudf::ast::column_reference(1); - auto& coalesce = cudf::ast::jit::coalesce(tree, a_ref, b_ref); - auto result = cudf::compute_column_jit(table, coalesce); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); -} - -TEST_F(JITExpressionTest, Predicate) {} - -// TODO: try variants -TEST_F(JITExpressionTest, AnsiAdd_Int) -{ - constexpr auto I32_MAX = std::numeric_limits::max(); - auto a = column_wrapper{3, 20, 1, 50}; - auto b = column_wrapper{10, 7, 20, 0}; - auto b_fail = column_wrapper{10, I32_MAX, 20, 0}; - auto expected = column_wrapper{13, 27, 21, 50}; - auto expected_null = column_wrapper{{13, 0, 21, 50}, {1, 0, 1, 1}}; - auto table = cudf::table_view{{a, b, b_fail}}; - auto tree = cudf::ast::tree{}; - auto a_ref = cudf::ast::column_reference(0); - auto b_ref = cudf::ast::column_reference(1); - auto b_fail_ref = cudf::ast::column_reference(2); - auto& add = cudf::ast::jit::ansi_add(tree, a_ref, b_ref); - auto& add_fail = cudf::ast::jit::ansi_add(tree, a_ref, b_fail_ref); - auto& try_add_fail = cudf::ast::jit::ansi_try_add(tree, a_ref, b_fail_ref); - auto result = cudf::compute_column_jit(table, add); - auto result_null = cudf::compute_column_jit(table, try_add_fail); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); - - EXPECT_THROW(result = cudf::compute_column_jit(table, add_fail), std::overflow_error); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_null, result_null->view(), verbosity); -} - -// TODO: parametrizie on decimal types and float types -TEST_F(JITExpressionTest, AnsiAdd_Decimal) -{ - constexpr auto I32_MAX = std::numeric_limits::max(); - auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 1, 50}, numeric::scale_type{0}}; - auto b = cudf::test::fixed_point_column_wrapper{{10, 7, 20, 0}, numeric::scale_type{0}}; - auto b_fail = - cudf::test::fixed_point_column_wrapper{{10, I32_MAX, 20, 0}, numeric::scale_type{0}}; - auto expected = - cudf::test::fixed_point_column_wrapper{{13, 27, 21, 50}, numeric::scale_type{0}}; - auto table = cudf::table_view{{a, b, b_fail}}; - auto tree = cudf::ast::tree{}; - auto a_ref = cudf::ast::column_reference(0); - auto b_ref = cudf::ast::column_reference(1); - auto b_fail_ref = cudf::ast::column_reference(2); - auto& add = cudf::ast::jit::ansi_add(tree, a_ref, b_ref); - auto& add_fail = cudf::ast::jit::ansi_add(tree, a_ref, b_fail_ref); - auto result = cudf::compute_column_jit(table, add); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); - - EXPECT_THROW(result = cudf::compute_column_jit(table, add_fail), std::overflow_error); -} - -TEST_F(JITExpressionTest, AnsiSub) -{ - constexpr auto I32_MIN = std::numeric_limits::min(); - auto a = column_wrapper{3, 20, 1, 50}; - auto b = column_wrapper{10, 7, 20, 0}; - auto b_fail = column_wrapper{10, I32_MIN, 20, 0}; - auto expected = column_wrapper{-7, 13, -19, 50}; - auto table = cudf::table_view{{a, b, b_fail}}; - auto tree = cudf::ast::tree{}; - auto a_ref = cudf::ast::column_reference(0); - auto b_ref = cudf::ast::column_reference(1); - auto b_fail_ref = cudf::ast::column_reference(2); - auto& sub = cudf::ast::jit::ansi_sub(tree, a_ref, b_ref); - auto& sub_fail = cudf::ast::jit::ansi_sub(tree, a_ref, b_fail_ref); - - auto result = cudf::compute_column_jit(table, sub); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); - - EXPECT_THROW(result = cudf::compute_column_jit(table, sub_fail), std::overflow_error); -} - -TEST_F(JITExpressionTest, AnsiSub_Decimal) -{ - constexpr auto I32_MIN = std::numeric_limits::min(); - auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 1, 50}, numeric::scale_type{0}}; - auto b = cudf::test::fixed_point_column_wrapper{{10, 7, 20, 0}, numeric::scale_type{0}}; - auto b_fail = - cudf::test::fixed_point_column_wrapper{{10, I32_MIN, 20, 0}, numeric::scale_type{0}}; - auto expected = - cudf::test::fixed_point_column_wrapper{{-7, 13, -19, 50}, numeric::scale_type{0}}; - auto table = cudf::table_view{{a, b, b_fail}}; - auto tree = cudf::ast::tree{}; - auto a_ref = cudf::ast::column_reference(0); - auto b_ref = cudf::ast::column_reference(1); - auto b_fail_ref = cudf::ast::column_reference(2); - auto& sub = cudf::ast::jit::ansi_sub(tree, a_ref, b_ref); - auto& sub_fail = cudf::ast::jit::ansi_sub(tree, a_ref, b_fail_ref); - auto result = cudf::compute_column_jit(table, sub); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); - - EXPECT_THROW(result = cudf::compute_column_jit(table, sub_fail), std::overflow_error); -} - -TEST_F(JITExpressionTest, AnsiMul) -{ - constexpr auto I32_MAX = std::numeric_limits::max(); - auto a = column_wrapper{3, 20, 2, 50}; - auto b = column_wrapper{10, 7, 1, 0}; - auto b_fail = column_wrapper{10, I32_MAX, 1, 0}; - auto expected = column_wrapper{30, 140, 2, 0}; - auto table = cudf::table_view{{a, b, b_fail}}; - auto a_ref = cudf::ast::column_reference(0); - auto b_ref = cudf::ast::column_reference(1); - auto b_fail_ref = cudf::ast::column_reference(2); - auto tree = cudf::ast::tree{}; - auto& mul = cudf::ast::jit::ansi_mul(tree, a_ref, b_ref); - auto& mul_fail = cudf::ast::jit::ansi_mul(tree, a_ref, b_fail_ref); - auto result = cudf::compute_column_jit(table, mul); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); - - EXPECT_THROW(result = cudf::compute_column_jit(table, mul_fail), std::overflow_error); -} - -TEST_F(JITExpressionTest, AnsiMul_Decimal) -{ - constexpr auto I32_MAX = std::numeric_limits::max(); - auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 2, 50}, numeric::scale_type{0}}; - auto b = cudf::test::fixed_point_column_wrapper{{10, 7, 1, 0}, numeric::scale_type{0}}; - auto b_fail = - cudf::test::fixed_point_column_wrapper{{10, I32_MAX, 1, 0}, numeric::scale_type{0}}; - auto expected = - cudf::test::fixed_point_column_wrapper{{30, 140, 2, 0}, numeric::scale_type{0}}; - auto table = cudf::table_view{{a, b, b_fail}}; - auto a_ref = cudf::ast::column_reference(0); - auto b_ref = cudf::ast::column_reference(1); - auto b_fail_ref = cudf::ast::column_reference(2); - auto tree = cudf::ast::tree{}; - auto& mul = cudf::ast::jit::ansi_mul(tree, a_ref, b_ref); - auto& mul_fail = cudf::ast::jit::ansi_mul(tree, a_ref, b_fail_ref); - auto result = cudf::compute_column_jit(table, mul); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); - - EXPECT_THROW(result = cudf::compute_column_jit(table, mul_fail), std::overflow_error); -} - -TEST_F(JITExpressionTest, AnsiDiv) -{ - auto a = column_wrapper{3, 20, 1, 50}; - auto b = column_wrapper{10, 7, 2, 1}; - auto b_fail = column_wrapper{10, 1, 20, 0}; - auto expected = column_wrapper{0, 2, 0, 50}; - auto table = cudf::table_view{{a, b, b_fail}}; - auto a_ref = cudf::ast::column_reference(0); - auto b_ref = cudf::ast::column_reference(1); - auto b_fail_ref = cudf::ast::column_reference(2); - auto tree = cudf::ast::tree{}; - auto& div = cudf::ast::jit::ansi_div(tree, a_ref, b_ref); - auto& div_fail = cudf::ast::jit::ansi_div(tree, a_ref, b_fail_ref); - auto result = cudf::compute_column_jit(table, div); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); - - EXPECT_THROW(result = cudf::compute_column_jit(table, div_fail), std::overflow_error); -} - -TEST_F(JITExpressionTest, AnsiDiv_Decimal) -{ - auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 1, 50}, numeric::scale_type{0}}; - auto b = cudf::test::fixed_point_column_wrapper{{10, 7, 2, 1}, numeric::scale_type{0}}; - auto b_fail = - cudf::test::fixed_point_column_wrapper{{10, 1, 20, 0}, numeric::scale_type{0}}; - auto expected = - cudf::test::fixed_point_column_wrapper{{0, 2, 0, 50}, numeric::scale_type{0}}; - auto table = cudf::table_view{{a, b, b_fail}}; - auto a_ref = cudf::ast::column_reference(0); - auto b_ref = cudf::ast::column_reference(1); - auto b_fail_ref = cudf::ast::column_reference(2); - auto tree = cudf::ast::tree{}; - auto& div = cudf::ast::jit::ansi_div(tree, a_ref, b_ref); - auto& div_fail = cudf::ast::jit::ansi_div(tree, a_ref, b_fail_ref); - auto result = cudf::compute_column_jit(table, div); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); - - EXPECT_THROW(result = cudf::compute_column_jit(table, div_fail), std::overflow_error); -} - -TEST_F(JITExpressionTest, AnsiMod) -{ - auto a = column_wrapper{3, 20, 1, 50}; - auto b = column_wrapper{10, 7, 2, 1}; - auto b_fail = column_wrapper{10, 1, 20, 0}; - auto expected = column_wrapper{3, 6, 1, 0}; - auto table = cudf::table_view{{a, b, b_fail}}; - auto a_ref = cudf::ast::column_reference(0); - auto b_ref = cudf::ast::column_reference(1); - auto b_fail_ref = cudf::ast::column_reference(2); - auto tree = cudf::ast::tree{}; - auto& mod = cudf::ast::jit::ansi_mod(tree, a_ref, b_ref); - auto& mod_fail = cudf::ast::jit::ansi_mod(tree, a_ref, b_fail_ref); - auto result = cudf::compute_column_jit(table, mod); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); - - EXPECT_THROW(result = cudf::compute_column_jit(table, mod_fail), std::overflow_error); -} - -TEST_F(JITExpressionTest, AnsiMod_Decimal) -{ - auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 1, 50}, numeric::scale_type{0}}; - auto b = cudf::test::fixed_point_column_wrapper{{10, 7, 2, 1}, numeric::scale_type{0}}; - auto b_fail = - cudf::test::fixed_point_column_wrapper{{10, 1, 20, 0}, numeric::scale_type{0}}; - auto expected = - cudf::test::fixed_point_column_wrapper{{3, 6, 1, 0}, numeric::scale_type{0}}; - auto table = cudf::table_view{{a, b, b_fail}}; - auto a_ref = cudf::ast::column_reference(0); - auto b_ref = cudf::ast::column_reference(1); - auto b_fail_ref = cudf::ast::column_reference(2); - auto tree = cudf::ast::tree{}; - auto& mod = cudf::ast::jit::ansi_mod(tree, a_ref, b_ref); - auto& mod_fail = cudf::ast::jit::ansi_mod(tree, a_ref, b_fail_ref); - auto result = cudf::compute_column_jit(table, mod); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); - - EXPECT_THROW(result = cudf::compute_column_jit(table, mod_fail), std::overflow_error); -} - -TEST_F(JITExpressionTest, AnsiAbs) -{ - constexpr auto I32_MIN = std::numeric_limits::min(); - constexpr auto I32_MAX = std::numeric_limits::max(); - auto a = column_wrapper{3, -20, 1, -50, I32_MAX, I32_MIN + 1, 0}; - auto a_fail = column_wrapper{3, -20, 1, -50, I32_MIN, 1, 0}; - auto expected = column_wrapper{3, 20, 1, 50, I32_MAX, std::abs(I32_MIN + 1), 0}; - auto table = cudf::table_view{{a, a_fail}}; - auto a_ref = cudf::ast::column_reference(0); - auto a_fail_ref = cudf::ast::column_reference(1); - auto tree = cudf::ast::tree{}; - auto& abs = cudf::ast::jit::ansi_abs(tree, a_ref); - auto& abs_fail = cudf::ast::jit::ansi_abs(tree, a_fail_ref); - auto result = cudf::compute_column_jit(table, abs); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); - - EXPECT_THROW(result = cudf::compute_column_jit(table, abs_fail), std::overflow_error); -} - -TEST_F(JITExpressionTest, AnsiAbs_Decimal) -{ - constexpr auto I32_MIN = std::numeric_limits::min(); - constexpr auto I32_MAX = std::numeric_limits::max(); - auto a = cudf::test::fixed_point_column_wrapper{ - {3, -20, 1, -50, I32_MAX, I32_MIN + 1, 0}, numeric::scale_type{0}}; - auto a_fail = cudf::test::fixed_point_column_wrapper{{3, -20, 1, -50, I32_MIN, 1, 0}, - numeric::scale_type{0}}; - auto expected = cudf::test::fixed_point_column_wrapper{ - {3, 20, 1, 50, I32_MAX, std::abs(I32_MIN + 1), 0}, numeric::scale_type{0}}; - auto table = cudf::table_view{{a, a_fail}}; - auto a_ref = cudf::ast::column_reference(0); - auto a_fail_ref = cudf::ast::column_reference(1); - auto tree = cudf::ast::tree{}; - auto& abs = cudf::ast::jit::ansi_abs(tree, a_ref); - auto& abs_fail = cudf::ast::jit::ansi_abs(tree, a_fail_ref); - auto result = cudf::compute_column_jit(table, abs); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); - - EXPECT_THROW(result = cudf::compute_column_jit(table, abs_fail), std::overflow_error); -} - -TEST_F(JITExpressionTest, AnsiNeg) -{ - constexpr auto I32_MIN = std::numeric_limits::min(); - constexpr auto I32_MAX = std::numeric_limits::max(); - auto a = column_wrapper{3, -20, 1, -50, I32_MAX, -I32_MAX, 0}; - auto a_fail = column_wrapper{3, -20, 1, -50, I32_MIN, 1, 0}; - auto expected = column_wrapper{-3, 20, -1, 50, -I32_MAX, I32_MAX, 0}; - auto table = cudf::table_view{{a, a_fail}}; - auto a_ref = cudf::ast::column_reference(0); - auto a_fail_ref = cudf::ast::column_reference(1); - auto tree = cudf::ast::tree{}; - auto& neg = cudf::ast::jit::ansi_neg(tree, a_ref); - auto& neg_fail = cudf::ast::jit::ansi_neg(tree, a_fail_ref); - auto result = cudf::compute_column_jit(table, neg); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); - - EXPECT_THROW(result = cudf::compute_column_jit(table, neg_fail), std::overflow_error); -} - -TEST_F(JITExpressionTest, AnsiNeg_Decimal) -{ - constexpr auto I32_MIN = std::numeric_limits::min(); - constexpr auto I32_MAX = std::numeric_limits::max(); - auto a = cudf::test::fixed_point_column_wrapper{{3, -20, 1, -50, I32_MAX, -I32_MAX, 0}, - numeric::scale_type{0}}; - auto a_fail = cudf::test::fixed_point_column_wrapper{{3, -20, 1, -50, I32_MIN, 1, 0}, - numeric::scale_type{0}}; - auto expected = cudf::test::fixed_point_column_wrapper{ - {-3, 20, -1, 50, -I32_MAX, I32_MAX, 0}, numeric::scale_type{0}}; - auto table = cudf::table_view{{a, a_fail}}; - auto a_ref = cudf::ast::column_reference(0); - auto a_fail_ref = cudf::ast::column_reference(1); - auto tree = cudf::ast::tree{}; - auto& neg = cudf::ast::jit::ansi_neg(tree, a_ref); - auto& neg_fail = cudf::ast::jit::ansi_neg(tree, a_fail_ref); - auto result = cudf::compute_column_jit(table, neg); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); - - EXPECT_THROW(result = cudf::compute_column_jit(table, neg_fail), std::overflow_error); -} - -TEST_F(JITExpressionTest, AnsiPrecisionCheck) -{ - auto a = - cudf::test::fixed_point_column_wrapper{{3, 200, 250, 200}, numeric::scale_type{0}}; - auto a_fail = - cudf::test::fixed_point_column_wrapper{{3, 200, 250, 20000}, numeric::scale_type{0}}; - auto expected = - cudf::test::fixed_point_column_wrapper{{3, 200, 250, 200}, numeric::scale_type{0}}; - auto max_precision = cudf::numeric_scalar(3); - auto table = cudf::table_view{{a}}; - auto a_ref = cudf::ast::column_reference(0); - auto tree = cudf::ast::tree{}; - auto precision = cudf::ast::literal(max_precision); - auto& precision_check = cudf::ast::jit::ansi_precision_check(tree, a_ref, precision); - auto result = cudf::compute_column_jit(table, precision_check); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); - - EXPECT_THROW(result = cudf::compute_column_jit(cudf::table_view{{a_fail}}, precision_check), - std::overflow_error); -} - -TEST_F(JITExpressionTest, BitShiftLeft) -{ - auto a = cudf::test::fixed_width_column_wrapper{0b111111, 0b111110, 0b101111, 0b1100}; - auto expected = - cudf::test::fixed_width_column_wrapper{0b11111100, 0b11111000, 0b10111100, 0b110000}; - auto shift = cudf::numeric_scalar(2); - auto table = cudf::table_view{{a}}; - auto a_ref = cudf::ast::column_reference(0); - auto tree = cudf::ast::tree{}; - auto shift_literal = cudf::ast::literal(shift); - auto& shift_left = cudf::ast::jit::bit_shift_left(tree, a_ref, shift_literal); - auto result = cudf::compute_column_jit(table, shift_left); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); -} - -TEST_F(JITExpressionTest, BitShiftRight) -{ - auto a = cudf::test::fixed_width_column_wrapper{0b1111, 0b10111, 0b11100, 0b11110011}; - auto expected = cudf::test::fixed_width_column_wrapper{0b11, 0b101, 0b111, 0b111100}; - auto shift = cudf::numeric_scalar(2); - auto table = cudf::table_view{{a}}; - auto a_ref = cudf::ast::column_reference(0); - auto tree = cudf::ast::tree{}; - auto shift_literal = cudf::ast::literal(shift); - auto& shift_right = cudf::ast::jit::bit_shift_right(tree, a_ref, shift_literal); - auto result = cudf::compute_column_jit(table, shift_right); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); -} - -template -void test_cast() -{ - auto a = cudf::test::fixed_width_column_wrapper{0, 1, 2, 3, 4, 5}; - auto expected = cudf::test::fixed_width_column_wrapper{0, 1, 2, 3, 4, 5}; - auto table = cudf::table_view{{a}}; - auto a_ref = cudf::ast::column_reference(0); - auto tree = cudf::ast::tree{}; - - cudf::ast::expression const* cast = nullptr; - - if constexpr (std::is_same_v) { - cast = &cudf::ast::jit::cast_to_b8(tree, a_ref); - } else if constexpr (std::is_same_v) { - cast = &cudf::ast::jit::cast_to_i8(tree, a_ref); - } else if constexpr (std::is_same_v) { - cast = &cudf::ast::jit::cast_to_i16(tree, a_ref); - } else if constexpr (std::is_same_v) { - cast = &cudf::ast::jit::cast_to_i32(tree, a_ref); - } else if constexpr (std::is_same_v) { - cast = &cudf::ast::jit::cast_to_i64(tree, a_ref); - } else if constexpr (std::is_same_v) { - cast = &cudf::ast::jit::cast_to_u8(tree, a_ref); - } else if constexpr (std::is_same_v) { - cast = &cudf::ast::jit::cast_to_u16(tree, a_ref); - } else if constexpr (std::is_same_v) { - cast = &cudf::ast::jit::cast_to_u32(tree, a_ref); - } else if constexpr (std::is_same_v) { - cast = &cudf::ast::jit::cast_to_u64(tree, a_ref); - } else if constexpr (std::is_same_v) { - cast = &cudf::ast::jit::cast_to_f32(tree, a_ref); - } else { - static_assert(std::is_same_v); - cast = &cudf::ast::jit::cast_to_f64(tree, a_ref); - } - - auto result = cudf::compute_column_jit(table, *cast); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); -} - -template -void test_from_decimal_cast() -{ - auto a = cudf::test::fixed_point_column_wrapper{{0, 1, 2, 3, 4, 5}, - numeric::scale_type{0}}; - auto expected = cudf::test::fixed_width_column_wrapper{0, 1, 2, 3, 4, 5}; - auto table = cudf::table_view{{a}}; - auto a_ref = cudf::ast::column_reference(0); - auto tree = cudf::ast::tree{}; - - cudf::ast::expression const* cast = nullptr; - - if constexpr (std::is_same_v) { - cast = &cudf::ast::jit::cast_to_b8(tree, a_ref); - } else if constexpr (std::is_same_v) { - cast = &cudf::ast::jit::cast_to_i8(tree, a_ref); - } else if constexpr (std::is_same_v) { - cast = &cudf::ast::jit::cast_to_i16(tree, a_ref); - } else if constexpr (std::is_same_v) { - cast = &cudf::ast::jit::cast_to_i32(tree, a_ref); - } else if constexpr (std::is_same_v) { - cast = &cudf::ast::jit::cast_to_i64(tree, a_ref); - } else if constexpr (std::is_same_v) { - cast = &cudf::ast::jit::cast_to_u8(tree, a_ref); - } else if constexpr (std::is_same_v) { - cast = &cudf::ast::jit::cast_to_u16(tree, a_ref); - } else if constexpr (std::is_same_v) { - cast = &cudf::ast::jit::cast_to_u32(tree, a_ref); - } else if constexpr (std::is_same_v) { - cast = &cudf::ast::jit::cast_to_u64(tree, a_ref); - } else if constexpr (std::is_same_v) { - cast = &cudf::ast::jit::cast_to_f32(tree, a_ref); - } else { - static_assert(std::is_same_v); - cast = &cudf::ast::jit::cast_to_f64(tree, a_ref); - } - - auto result = cudf::compute_column_jit(table, *cast); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); -} - -template -void test_cast_to() -{ - test_cast(); - test_cast(); - test_cast(); - test_cast(); - test_cast(); - test_cast(); - test_cast(); - test_cast(); - test_cast(); - test_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); - test_from_decimal_cast(); -} - -TEST_F(JITExpressionTest, Cast) -{ - test_cast_to(); - test_cast_to(); - test_cast_to(); - test_cast_to(); - test_cast_to(); - test_cast_to(); - test_cast_to(); - test_cast_to(); - test_cast_to(); - test_cast_to(); - test_cast_to(); -} - -template -void test_decimal_cast() -{ - auto a = cudf::test::fixed_point_column_wrapper{{0, 1, 2, 3, 4, 5}, - numeric::scale_type{0}}; - auto expected = cudf::test::fixed_point_column_wrapper{{0, 1, 2, 3, 4, 5}, - numeric::scale_type{0}}; - auto table = cudf::table_view{{a}}; - auto a_ref = cudf::ast::column_reference(0); - auto tree = cudf::ast::tree{}; - - cudf::ast::expression const* cast = nullptr; - - if constexpr (std::is_same_v) { - cast = &cudf::ast::jit::cast_to_dec32(tree, a_ref); - } else if constexpr (std::is_same_v) { - cast = &cudf::ast::jit::cast_to_dec64(tree, a_ref); - } else if constexpr (std::is_same_v) { - static_assert(std::is_same_v); - cast = &cudf::ast::jit::cast_to_dec128(tree, a_ref); - } - - auto result = cudf::compute_column_jit(table, *cast); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); -} - -TEST_F(JITExpressionTest, CastToDec32) -{ - test_decimal_cast(); - test_decimal_cast(); - test_decimal_cast(); -} - -TEST_F(JITExpressionTest, CastToDec64) -{ - test_decimal_cast(); - test_decimal_cast(); - test_decimal_cast(); -} - -TEST_F(JITExpressionTest, CastToDec128) -{ - test_decimal_cast(); - test_decimal_cast(); - test_decimal_cast(); -} - -TEST_F(JITExpressionTest, Rescale) -{ - auto a = cudf::test::fixed_point_column_wrapper{{123, 1234, 12345, 123456, 1234567}, - numeric::scale_type{0}}; - auto expected = cudf::test::fixed_point_column_wrapper{ - {12300, 123400, 1234500, 12345600, 123456700}, numeric::scale_type{-2}}; - auto table = cudf::table_view{{a}}; - auto a_ref = cudf::ast::column_reference(0); - auto tree = cudf::ast::tree{}; - auto& rescaled = cudf::ast::jit::rescale(tree, a_ref, -2); - auto result = cudf::compute_column_jit(table, rescaled); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); -} - -// TODO: TEST ANSI op fusion - CUDF_TEST_PROGRAM_MAIN() diff --git a/cpp/tests/jit/row_ir.cpp b/cpp/tests/jit/row_ir.cpp index 3cf9c37546f1..16eaddec081b 100644 --- a/cpp/tests/jit/row_ir.cpp +++ b/cpp/tests/jit/row_ir.cpp @@ -303,7 +303,7 @@ TEST_F(RowIRCudaCodeGenTest, AstConversionBasic) EXPECT_EQ(transform_args.source_type, cudf::udf_source_type::CUDA); EXPECT_EQ(transform_args.is_null_aware, cudf::null_aware::NO); EXPECT_EQ(transform_args.outputs.size(), 1); - EXPECT_EQ(transform_args.outputs[0].nullability, cudf::output_nullability::ALL_VALID); + EXPECT_EQ(transform_args.outputs[0].nullability, cudf::output_nullability::PRESERVE); EXPECT_EQ(transform_args.outputs[0].type, cudf::data_type{cudf::type_id::INT32}); ASSERT_EQ(transform_args.inputs.size(), 2); From 0423a7d4e998b6e78e0a1861876a999048603dcc Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 5 May 2026 01:34:09 +0000 Subject: [PATCH 14/15] bug fixes and more tests --- .../cudf/operators/ansi_arithmetic.cuh | 22 ++++++++++++++----- cpp/tests/ast/jit_ast_tests.cpp | 22 +++++++++---------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/cpp/include/cudf/operators/ansi_arithmetic.cuh b/cpp/include/cudf/operators/ansi_arithmetic.cuh index 2288f099a1da..ab7c1889d5f4 100644 --- a/cpp/include/cudf/operators/ansi_arithmetic.cuh +++ b/cpp/include/cudf/operators/ansi_arithmetic.cuh @@ -462,9 +462,12 @@ __device__ inline errc ansi_precision_check(decimal* out, { if (*precision <= 0) { return errc::OVERFLOW; } - if (a->value() == cuda::std::numeric_limits::min()) { return errc::OVERFLOW; } + auto value = a->value(); + if (value == cuda::std::numeric_limits::min()) { return errc::OVERFLOW; } + + auto abs_value = value < 0 ? -value : value; - if (::abs(a->value()) >= detail::ipow10(static_cast(*precision))) { return errc::OVERFLOW; } + if (abs_value >= detail::ipow10(static_cast(*precision))) { return errc::OVERFLOW; } *out = *a; return errc::OK; @@ -476,7 +479,14 @@ __device__ inline errc ansi_precision_check(optional* out, optional const* precision) { if (a->has_value()) { - return ansi_precision_check(&out->value(), &a->value(), &precision->value()); + T r; + if (errc e = ansi_precision_check(&r, &a->value(), &precision->value()); e != errc::OK) { + *out = nullopt; + return e; + } else { + *out = r; + return errc::OK; + } } else { *out = nullopt; return errc::OK; @@ -601,11 +611,11 @@ __device__ inline errc ansi_try_precision_check(optional>* out, optional const* precision) { if (a->has_value() && precision->has_value()) { - if (errc e = ansi_precision_check(&out->value(), &a->value(), &precision->value()); - e != errc::OK) { + decimal r; + if (errc e = ansi_precision_check(&r, &a->value(), &precision->value()); e != errc::OK) { *out = nullopt; } else { - *out = a->value(); + *out = r; } } else { *out = nullopt; diff --git a/cpp/tests/ast/jit_ast_tests.cpp b/cpp/tests/ast/jit_ast_tests.cpp index 0c8a1ea61f94..8f026e077cbb 100644 --- a/cpp/tests/ast/jit_ast_tests.cpp +++ b/cpp/tests/ast/jit_ast_tests.cpp @@ -447,8 +447,7 @@ TEST_F(JITExpressionTest, AnsiNeg_Decimal) CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); } -// TODO: some device malloc failure happens in this test, need to investigate and re-enable -TEST_F(JITExpressionTest, DISABLED_AnsiPrecisionCheck) +TEST_F(JITExpressionTest, AnsiPrecisionCheck) { auto a = cudf::test::fixed_point_column_wrapper{{3, 200, 250, 200}, numeric::scale_type{0}}; @@ -458,20 +457,21 @@ TEST_F(JITExpressionTest, DISABLED_AnsiPrecisionCheck) cudf::test::fixed_point_column_wrapper{{3, 200, 250, 200}, numeric::scale_type{0}}; auto expected_fail = cudf::test::fixed_point_column_wrapper{ {3, 200, 250, 200}, {1, 1, 1, 0}, numeric::scale_type{0}}; - auto max_precision = cudf::numeric_scalar(3); - auto table = cudf::table_view{{a}}; - auto a_ref = cudf::ast::column_reference(0); - auto tree = cudf::ast::tree{}; - auto precision = cudf::ast::literal(max_precision); - auto& precision_check = cudf::ast::jit::ansi_precision_check(tree, a_ref, precision); - auto& try_precision_check = cudf::ast::jit::ansi_try_precision_check(tree, a_ref, precision); + auto max_precision = cudf::numeric_scalar(3); + auto table = cudf::table_view{{a, a_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto a_fail_ref = cudf::ast::column_reference(1); + auto tree = cudf::ast::tree{}; + auto precision = cudf::ast::literal(max_precision); + auto& precision_check = cudf::ast::jit::ansi_precision_check(tree, a_ref, precision); + auto& precision_check_fail = cudf::ast::jit::ansi_precision_check(tree, a_fail_ref, precision); + auto& try_precision_check = cudf::ast::jit::ansi_try_precision_check(tree, a_fail_ref, precision); auto result = cudf::compute_column_jit(table, precision_check); auto result_fail = cudf::compute_column_jit(table, try_precision_check); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); - EXPECT_THROW(result = cudf::compute_column_jit(cudf::table_view{{a_fail}}, precision_check), - std::overflow_error); + EXPECT_THROW(result = cudf::compute_column_jit(table, precision_check_fail), std::overflow_error); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); } From 59f7f0bb60d1fb342f400a7fc46ff9a0b166c989 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 5 May 2026 15:06:01 +0000 Subject: [PATCH 15/15] parameterize tests --- cpp/tests/ast/jit_ast_tests.cpp | 485 ++++++++++++++++---------------- 1 file changed, 242 insertions(+), 243 deletions(-) diff --git a/cpp/tests/ast/jit_ast_tests.cpp b/cpp/tests/ast/jit_ast_tests.cpp index 8f026e077cbb..cb3ddbde7718 100644 --- a/cpp/tests/ast/jit_ast_tests.cpp +++ b/cpp/tests/ast/jit_ast_tests.cpp @@ -28,17 +28,38 @@ #include #include +constexpr cudf::test::debug_output_level VERBOSITY{cudf::test::debug_output_level::ALL_ERRORS}; + template using column_wrapper = cudf::test::fixed_width_column_wrapper; -constexpr cudf::test::debug_output_level verbosity{cudf::test::debug_output_level::ALL_ERRORS}; +template +using decimal_column_wrapper = cudf::test::fixed_point_column_wrapper; struct JITExpressionTest : public cudf::test::BaseFixture {}; +template +struct JITIntegerArithmeticTest : public cudf::test::BaseFixture { + static constexpr T MAX = std::numeric_limits::max(); + static constexpr T MIN = std::numeric_limits::min(); +}; + +template +struct JITSignedIntegerArithmeticTest : public JITIntegerArithmeticTest {}; + +template +struct JITDecimalArithmeticTest : public JITIntegerArithmeticTest {}; + +using SignedIntegralTypesNotBool = cudf::test::Types; + +TYPED_TEST_SUITE(JITIntegerArithmeticTest, cudf::test::IntegralTypesNotBool); +TYPED_TEST_SUITE(JITSignedIntegerArithmeticTest, SignedIntegralTypesNotBool); +TYPED_TEST_SUITE(JITDecimalArithmeticTest, cudf::test::FixedPointTypes); + TEST_F(JITExpressionTest, NullifyIf) { - auto a = column_wrapper{3, 20, 1, 50, 0, 20}; - auto condition = column_wrapper{false, true, false, true, false, true}; + auto a = column_wrapper{{3, 20, 1, 50, 0, 20}}; + auto condition = column_wrapper{{false, true, false, true, false, true}}; auto expected = column_wrapper{{3, 0, 1, 0, 0, 0}, {1, 0, 1, 0, 1, 0}}; auto table = cudf::table_view{{a, condition}}; auto tree = cudf::ast::tree{}; @@ -47,7 +68,7 @@ TEST_F(JITExpressionTest, NullifyIf) auto& nullify_if = cudf::ast::jit::nullify_if(tree, a_ref, condition_ref); auto result = cudf::compute_column_jit(table, nullify_if); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); } TEST_F(JITExpressionTest, Coalesce) @@ -62,48 +83,45 @@ TEST_F(JITExpressionTest, Coalesce) auto& coalesce = cudf::ast::jit::coalesce(tree, a_ref, b_ref); auto result = cudf::compute_column_jit(table, coalesce); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); } -// TODO: parameterize on decimal types and float types - -TEST_F(JITExpressionTest, AnsiAdd_Int) +TYPED_TEST(JITIntegerArithmeticTest, AnsiAdd) { - constexpr auto I32_MAX = std::numeric_limits::max(); - auto a = column_wrapper{3, 20, 1, 50}; - auto b = column_wrapper{10, 7, 20, 0}; - auto b_fail = column_wrapper{10, I32_MAX, 20, 0}; - auto expected = column_wrapper{13, 27, 21, 50}; - auto expected_fail = column_wrapper{{13, 0, 21, 50}, {1, 0, 1, 1}}; - auto table = cudf::table_view{{a, b, b_fail}}; - auto tree = cudf::ast::tree{}; - auto a_ref = cudf::ast::column_reference(0); - auto b_ref = cudf::ast::column_reference(1); - auto b_fail_ref = cudf::ast::column_reference(2); - auto& add = cudf::ast::jit::ansi_add(tree, a_ref, b_ref); - auto& add_fail = cudf::ast::jit::ansi_add(tree, a_ref, b_fail_ref); - auto& try_add_fail = cudf::ast::jit::ansi_try_add(tree, a_ref, b_fail_ref); - auto result = cudf::compute_column_jit(table, add); - auto result_fail = cudf::compute_column_jit(table, try_add_fail); + using T = TypeParam; + auto a = column_wrapper{{3, 20, 1, 50}}; + auto b = column_wrapper{{10, 7, 20, 0}}; + auto b_fail = column_wrapper{{T{10}, this->MAX, T{20}, T{0}}}; + auto expected = column_wrapper{{13, 27, 21, 50}}; + auto expected_fail = column_wrapper{{13, 0, 21, 50}, {1, 0, 1, 1}}; + auto table = cudf::table_view{{a, b, b_fail}}; + auto tree = cudf::ast::tree{}; + auto a_ref = cudf::ast::column_reference(0); + auto b_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto& add = cudf::ast::jit::ansi_add(tree, a_ref, b_ref); + auto& add_fail = cudf::ast::jit::ansi_add(tree, a_ref, b_fail_ref); + auto& try_add_fail = cudf::ast::jit::ansi_try_add(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, add); + auto result_fail = cudf::compute_column_jit(table, try_add_fail); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); EXPECT_THROW(result = cudf::compute_column_jit(table, add_fail), std::overflow_error); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), VERBOSITY); } -TEST_F(JITExpressionTest, AnsiAdd_Decimal) +TYPED_TEST(JITDecimalArithmeticTest, AnsiAdd) { - constexpr auto I32_MAX = std::numeric_limits::max(); - auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 1, 50}, numeric::scale_type{0}}; - auto b = cudf::test::fixed_point_column_wrapper{{10, 7, 20, 0}, numeric::scale_type{0}}; - auto b_fail = - cudf::test::fixed_point_column_wrapper{{10, I32_MAX, 20, 0}, numeric::scale_type{0}}; - auto expected = - cudf::test::fixed_point_column_wrapper{{13, 27, 21, 50}, numeric::scale_type{0}}; - auto expected_fail = cudf::test::fixed_point_column_wrapper{ - {13, 0, 21, 50}, {1, 0, 1, 1}, numeric::scale_type{0}}; + using T = TypeParam; + using R = typename T::rep; + auto a = decimal_column_wrapper{{3, 20, 1, 50}, numeric::scale_type{0}}; + auto b = decimal_column_wrapper{{10, 7, 20, 0}, numeric::scale_type{0}}; + auto b_fail = decimal_column_wrapper{{R{10}, this->MAX, R{20}, R{0}}, numeric::scale_type{0}}; + auto expected = decimal_column_wrapper{{13, 27, 21, 50}, numeric::scale_type{0}}; + auto expected_fail = + decimal_column_wrapper{{13, 0, 21, 50}, {1, 0, 1, 1}, numeric::scale_type{0}}; auto table = cudf::table_view{{a, b, b_fail}}; auto tree = cudf::ast::tree{}; auto a_ref = cudf::ast::column_reference(0); @@ -115,51 +133,51 @@ TEST_F(JITExpressionTest, AnsiAdd_Decimal) auto result = cudf::compute_column_jit(table, add); auto result_fail = cudf::compute_column_jit(table, try_add_fail); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); EXPECT_THROW(result = cudf::compute_column_jit(table, add_fail), std::overflow_error); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), VERBOSITY); } -TEST_F(JITExpressionTest, AnsiSub) +TYPED_TEST(JITSignedIntegerArithmeticTest, AnsiSub) { - constexpr auto I32_MIN = std::numeric_limits::min(); - auto a = column_wrapper{3, 20, 1, 50}; - auto b = column_wrapper{10, 7, 20, 0}; - auto b_fail = column_wrapper{10, I32_MIN, 20, 0}; - auto expected = column_wrapper{-7, 13, -19, 50}; - auto expected_fail = column_wrapper{{-7, 0, -19, 50}, {1, 0, 1, 1}}; - auto table = cudf::table_view{{a, b, b_fail}}; - auto tree = cudf::ast::tree{}; - auto a_ref = cudf::ast::column_reference(0); - auto b_ref = cudf::ast::column_reference(1); - auto b_fail_ref = cudf::ast::column_reference(2); - auto& sub = cudf::ast::jit::ansi_sub(tree, a_ref, b_ref); - auto& sub_fail = cudf::ast::jit::ansi_sub(tree, a_ref, b_fail_ref); - auto& try_sub_fail = cudf::ast::jit::ansi_try_sub(tree, a_ref, b_fail_ref); + using T = TypeParam; + auto a = column_wrapper{{3, 20, 1, 50}}; + auto b = column_wrapper{{10, 7, 20, 0}}; + auto b_fail = column_wrapper{{T{10}, T{this->MIN}, T{20}, T{0}}}; + auto expected = column_wrapper{{-7, 13, -19, 50}}; + auto expected_fail = column_wrapper{{-7, 0, -19, 50}, {1, 0, 1, 1}}; + auto table = cudf::table_view{{a, b, b_fail}}; + auto tree = cudf::ast::tree{}; + auto a_ref = cudf::ast::column_reference(0); + auto b_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto& sub = cudf::ast::jit::ansi_sub(tree, a_ref, b_ref); + auto& sub_fail = cudf::ast::jit::ansi_sub(tree, a_ref, b_fail_ref); + auto& try_sub_fail = cudf::ast::jit::ansi_try_sub(tree, a_ref, b_fail_ref); auto result = cudf::compute_column_jit(table, sub); auto result_fail = cudf::compute_column_jit(table, try_sub_fail); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); EXPECT_THROW(result = cudf::compute_column_jit(table, sub_fail), std::overflow_error); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), VERBOSITY); } -TEST_F(JITExpressionTest, AnsiSub_Decimal) +TYPED_TEST(JITDecimalArithmeticTest, AnsiSub) { - constexpr auto I32_MIN = std::numeric_limits::min(); - auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 1, 50}, numeric::scale_type{0}}; - auto b = cudf::test::fixed_point_column_wrapper{{10, 7, 20, 0}, numeric::scale_type{0}}; + using T = TypeParam; + using R = typename T::rep; + auto a = decimal_column_wrapper{{3, 20, 1, 50}, numeric::scale_type{0}}; + auto b = decimal_column_wrapper{{10, 7, 20, 0}, numeric::scale_type{0}}; auto b_fail = - cudf::test::fixed_point_column_wrapper{{10, I32_MIN, 20, 0}, numeric::scale_type{0}}; - auto expected = - cudf::test::fixed_point_column_wrapper{{-7, 13, -19, 50}, numeric::scale_type{0}}; - auto expected_fail = cudf::test::fixed_point_column_wrapper{ - {-7, 0, -19, 50}, {1, 0, 1, 1}, numeric::scale_type{0}}; + decimal_column_wrapper{{R{10}, R{this->MIN}, R{20}, R{0}}, numeric::scale_type{0}}; + auto expected = decimal_column_wrapper{{-7, 13, -19, 50}, numeric::scale_type{0}}; + auto expected_fail = + decimal_column_wrapper{{-7, 0, -19, 50}, {1, 0, 1, 1}, numeric::scale_type{0}}; auto table = cudf::table_view{{a, b, b_fail}}; auto tree = cudf::ast::tree{}; auto a_ref = cudf::ast::column_reference(0); @@ -171,50 +189,50 @@ TEST_F(JITExpressionTest, AnsiSub_Decimal) auto result = cudf::compute_column_jit(table, sub); auto result_fail = cudf::compute_column_jit(table, try_sub_fail); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); EXPECT_THROW(result = cudf::compute_column_jit(table, sub_fail), std::overflow_error); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), VERBOSITY); } -TEST_F(JITExpressionTest, AnsiMul) +TYPED_TEST(JITIntegerArithmeticTest, AnsiMul) { - constexpr auto I32_MAX = std::numeric_limits::max(); - auto a = column_wrapper{3, 20, 2, 50}; - auto b = column_wrapper{10, 7, 1, 0}; - auto b_fail = column_wrapper{10, I32_MAX, 1, 0}; - auto expected = column_wrapper{30, 140, 2, 0}; - auto expected_fail = column_wrapper{{30, 0, 2, 0}, {1, 0, 1, 1}}; - auto table = cudf::table_view{{a, b, b_fail}}; - auto a_ref = cudf::ast::column_reference(0); - auto b_ref = cudf::ast::column_reference(1); - auto b_fail_ref = cudf::ast::column_reference(2); - auto tree = cudf::ast::tree{}; - auto& mul = cudf::ast::jit::ansi_mul(tree, a_ref, b_ref); - auto& mul_fail = cudf::ast::jit::ansi_mul(tree, a_ref, b_fail_ref); - auto& try_mul_fail = cudf::ast::jit::ansi_try_mul(tree, a_ref, b_fail_ref); - auto result = cudf::compute_column_jit(table, mul); - auto result_fail = cudf::compute_column_jit(table, try_mul_fail); + using T = TypeParam; + auto a = column_wrapper{{3, 20, 2, 50}}; + auto b = column_wrapper{{10, 2, 1, 0}}; + auto b_fail = column_wrapper{{T{10}, T{this->MAX}, T{1}, T{0}}}; + auto expected = column_wrapper{{30, 40, 2, 0}}; + auto expected_fail = column_wrapper{{30, 0, 2, 0}, {1, 0, 1, 1}}; + auto table = cudf::table_view{{a, b, b_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto b_ref = cudf::ast::column_reference(1); + auto b_fail_ref = cudf::ast::column_reference(2); + auto tree = cudf::ast::tree{}; + auto& mul = cudf::ast::jit::ansi_mul(tree, a_ref, b_ref); + auto& mul_fail = cudf::ast::jit::ansi_mul(tree, a_ref, b_fail_ref); + auto& try_mul_fail = cudf::ast::jit::ansi_try_mul(tree, a_ref, b_fail_ref); + auto result = cudf::compute_column_jit(table, mul); + auto result_fail = cudf::compute_column_jit(table, try_mul_fail); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); EXPECT_THROW(result = cudf::compute_column_jit(table, mul_fail), std::overflow_error); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), VERBOSITY); } -TEST_F(JITExpressionTest, AnsiMul_Decimal) +TYPED_TEST(JITDecimalArithmeticTest, AnsiMul) { - constexpr auto I32_MAX = std::numeric_limits::max(); - auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 2, 50}, numeric::scale_type{0}}; - auto b = cudf::test::fixed_point_column_wrapper{{10, 7, 1, 0}, numeric::scale_type{0}}; + using T = TypeParam; + using R = typename T::rep; + auto a = decimal_column_wrapper{{3, 20, 2, 50}, numeric::scale_type{0}}; + auto b = decimal_column_wrapper{{10, 7, 1, 0}, numeric::scale_type{0}}; auto b_fail = - cudf::test::fixed_point_column_wrapper{{10, I32_MAX, 1, 0}, numeric::scale_type{0}}; - auto expected = - cudf::test::fixed_point_column_wrapper{{30, 140, 2, 0}, numeric::scale_type{0}}; - auto expected_fail = cudf::test::fixed_point_column_wrapper{ - {30, 0, 2, 0}, {1, 0, 1, 1}, numeric::scale_type{0}}; + decimal_column_wrapper{{R{10}, R{this->MAX}, R{1}, R{0}}, numeric::scale_type{0}}; + auto expected = decimal_column_wrapper{{30, 140, 2, 0}, numeric::scale_type{0}}; + auto expected_fail = + decimal_column_wrapper{{30, 0, 2, 0}, {1, 0, 1, 1}, numeric::scale_type{0}}; auto table = cudf::table_view{{a, b, b_fail}}; auto a_ref = cudf::ast::column_reference(0); auto b_ref = cudf::ast::column_reference(1); @@ -226,20 +244,21 @@ TEST_F(JITExpressionTest, AnsiMul_Decimal) auto result = cudf::compute_column_jit(table, mul); auto result_fail = cudf::compute_column_jit(table, try_mul_fail); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); EXPECT_THROW(result = cudf::compute_column_jit(table, mul_fail), std::overflow_error); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), VERBOSITY); } -TEST_F(JITExpressionTest, AnsiDiv) +TYPED_TEST(JITIntegerArithmeticTest, AnsiDiv) { - auto a = column_wrapper{3, 20, 1, 50}; - auto b = column_wrapper{10, 7, 2, 1}; - auto b_fail = column_wrapper{10, 1, 20, 0}; - auto expected = column_wrapper{0, 2, 0, 50}; - auto expected_fail = column_wrapper{{0, 20, 0, 50}, {1, 1, 1, 0}}; + using T = TypeParam; + auto a = column_wrapper{{3, 20, 1, 50}}; + auto b = column_wrapper{{10, 7, 2, 1}}; + auto b_fail = column_wrapper{{10, 1, 20, 0}}; + auto expected = column_wrapper{{0, 2, 0, 50}}; + auto expected_fail = column_wrapper{{0, 20, 0, 50}, {1, 1, 1, 0}}; auto table = cudf::table_view{{a, b, b_fail}}; auto a_ref = cudf::ast::column_reference(0); auto b_ref = cudf::ast::column_reference(1); @@ -251,23 +270,22 @@ TEST_F(JITExpressionTest, AnsiDiv) auto result = cudf::compute_column_jit(table, div); auto result_fail = cudf::compute_column_jit(table, try_div_fail); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); EXPECT_THROW(result = cudf::compute_column_jit(table, div_fail), std::overflow_error); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), VERBOSITY); } -TEST_F(JITExpressionTest, AnsiDiv_Decimal) +TYPED_TEST(JITDecimalArithmeticTest, AnsiDiv) { - auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 1, 50}, numeric::scale_type{0}}; - auto b = cudf::test::fixed_point_column_wrapper{{10, 7, 2, 1}, numeric::scale_type{0}}; - auto b_fail = - cudf::test::fixed_point_column_wrapper{{10, 1, 20, 0}, numeric::scale_type{0}}; - auto expected = - cudf::test::fixed_point_column_wrapper{{0, 2, 0, 50}, numeric::scale_type{0}}; - auto expected_fail = cudf::test::fixed_point_column_wrapper{ - {0, 20, 0, 50}, {1, 1, 1, 0}, numeric::scale_type{0}}; + using T = TypeParam; + auto a = decimal_column_wrapper{{3, 20, 1, 50}, numeric::scale_type{0}}; + auto b = decimal_column_wrapper{{10, 7, 2, 1}, numeric::scale_type{0}}; + auto b_fail = decimal_column_wrapper{{10, 1, 20, 0}, numeric::scale_type{0}}; + auto expected = decimal_column_wrapper{{0, 2, 0, 50}, numeric::scale_type{0}}; + auto expected_fail = + decimal_column_wrapper{{0, 20, 0, 50}, {1, 1, 1, 0}, numeric::scale_type{0}}; auto table = cudf::table_view{{a, b, b_fail}}; auto a_ref = cudf::ast::column_reference(0); auto b_ref = cudf::ast::column_reference(1); @@ -279,20 +297,21 @@ TEST_F(JITExpressionTest, AnsiDiv_Decimal) auto result = cudf::compute_column_jit(table, div); auto result_fail = cudf::compute_column_jit(table, try_div_fail); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); EXPECT_THROW(result = cudf::compute_column_jit(table, div_fail), std::overflow_error); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), VERBOSITY); } -TEST_F(JITExpressionTest, AnsiMod) +TYPED_TEST(JITIntegerArithmeticTest, AnsiMod) { - auto a = column_wrapper{3, 20, 1, 50}; - auto b = column_wrapper{10, 7, 2, 1}; - auto b_fail = column_wrapper{10, 1, 20, 0}; - auto expected = column_wrapper{3, 6, 1, 0}; - auto expected_fail = column_wrapper{{3, 0, 1, 0}, {1, 1, 1, 0}}; + using T = TypeParam; + auto a = column_wrapper{{3, 20, 1, 50}}; + auto b = column_wrapper{{10, 7, 2, 1}}; + auto b_fail = column_wrapper{{10, 1, 20, 0}}; + auto expected = column_wrapper{{3, 6, 1, 0}}; + auto expected_fail = column_wrapper{{3, 0, 1, 0}, {1, 1, 1, 0}}; auto table = cudf::table_view{{a, b, b_fail}}; auto a_ref = cudf::ast::column_reference(0); auto b_ref = cudf::ast::column_reference(1); @@ -304,23 +323,22 @@ TEST_F(JITExpressionTest, AnsiMod) auto result = cudf::compute_column_jit(table, mod); auto result_fail = cudf::compute_column_jit(table, try_mod_fail); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); EXPECT_THROW(result = cudf::compute_column_jit(table, mod_fail), std::overflow_error); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), VERBOSITY); } -TEST_F(JITExpressionTest, AnsiMod_Decimal) +TYPED_TEST(JITDecimalArithmeticTest, AnsiMod) { - auto a = cudf::test::fixed_point_column_wrapper{{3, 20, 1, 50}, numeric::scale_type{0}}; - auto b = cudf::test::fixed_point_column_wrapper{{10, 7, 2, 1}, numeric::scale_type{0}}; - auto b_fail = - cudf::test::fixed_point_column_wrapper{{10, 1, 20, 0}, numeric::scale_type{0}}; - auto expected = - cudf::test::fixed_point_column_wrapper{{3, 6, 1, 0}, numeric::scale_type{0}}; - auto expected_fail = cudf::test::fixed_point_column_wrapper{ - {3, 0, 1, 0}, {1, 1, 1, 0}, numeric::scale_type{0}}; + using T = TypeParam; + auto a = decimal_column_wrapper{{3, 20, 1, 50}, numeric::scale_type{0}}; + auto b = decimal_column_wrapper{{10, 7, 2, 1}, numeric::scale_type{0}}; + auto b_fail = decimal_column_wrapper{{10, 1, 20, 0}, numeric::scale_type{0}}; + auto expected = decimal_column_wrapper{{3, 6, 1, 0}, numeric::scale_type{0}}; + auto expected_fail = + decimal_column_wrapper{{3, 0, 1, 0}, {1, 1, 1, 0}, numeric::scale_type{0}}; auto table = cudf::table_view{{a, b, b_fail}}; auto a_ref = cudf::ast::column_reference(0); auto b_ref = cudf::ast::column_reference(1); @@ -332,49 +350,50 @@ TEST_F(JITExpressionTest, AnsiMod_Decimal) auto result = cudf::compute_column_jit(table, mod); auto result_fail = cudf::compute_column_jit(table, try_mod_fail); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); EXPECT_THROW(result = cudf::compute_column_jit(table, mod_fail), std::overflow_error); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), VERBOSITY); } -TEST_F(JITExpressionTest, AnsiAbs) +TYPED_TEST(JITSignedIntegerArithmeticTest, AnsiAbs) { - constexpr auto I32_MIN = std::numeric_limits::min(); - constexpr auto I32_MAX = std::numeric_limits::max(); - auto a = column_wrapper{3, -20, 1, -50, I32_MAX, I32_MIN + 1, 0}; - auto a_fail = column_wrapper{3, -20, 1, -50, I32_MIN, 1, 0}; - auto expected = column_wrapper{3, 20, 1, 50, I32_MAX, std::abs(I32_MIN + 1), 0}; - auto expected_fail = column_wrapper{{3, 20, 1, 50, 0, 1, 0}, {1, 1, 1, 1, 0, 1, 1}}; - auto table = cudf::table_view{{a, a_fail}}; - auto a_ref = cudf::ast::column_reference(0); - auto a_fail_ref = cudf::ast::column_reference(1); - auto tree = cudf::ast::tree{}; - auto& abs = cudf::ast::jit::ansi_abs(tree, a_ref); - auto& abs_fail = cudf::ast::jit::ansi_abs(tree, a_fail_ref); - auto& try_abs_fail = cudf::ast::jit::ansi_try_abs(tree, a_fail_ref); - auto result = cudf::compute_column_jit(table, abs); - auto result_fail = cudf::compute_column_jit(table, try_abs_fail); + using T = TypeParam; + auto a = column_wrapper{{T{3}, T{-20}, T{1}, T{-50}, this->MAX, T{this->MIN + 1}, T{0}}}; + auto a_fail = column_wrapper{{T{3}, T{-20}, T{1}, T{-50}, this->MIN, T{1}, T{0}}}; + auto expected = + column_wrapper{{T{3}, T{20}, T{1}, T{50}, this->MAX, T{std::abs(this->MIN + 1)}, T{0}}}; + auto expected_fail = column_wrapper{{3, 20, 1, 50, 0, 1, 0}, {1, 1, 1, 1, 0, 1, 1}}; + auto table = cudf::table_view{{a, a_fail}}; + auto a_ref = cudf::ast::column_reference(0); + auto a_fail_ref = cudf::ast::column_reference(1); + auto tree = cudf::ast::tree{}; + auto& abs = cudf::ast::jit::ansi_abs(tree, a_ref); + auto& abs_fail = cudf::ast::jit::ansi_abs(tree, a_fail_ref); + auto& try_abs_fail = cudf::ast::jit::ansi_try_abs(tree, a_fail_ref); + auto result = cudf::compute_column_jit(table, abs); + auto result_fail = cudf::compute_column_jit(table, try_abs_fail); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); EXPECT_THROW(result = cudf::compute_column_jit(table, abs_fail), std::overflow_error); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), VERBOSITY); } -TEST_F(JITExpressionTest, AnsiAbs_Decimal) +TYPED_TEST(JITDecimalArithmeticTest, AnsiAbs) { - constexpr auto I32_MIN = std::numeric_limits::min(); - constexpr auto I32_MAX = std::numeric_limits::max(); - auto a = cudf::test::fixed_point_column_wrapper{ - {3, -20, 1, -50, I32_MAX, I32_MIN + 1, 0}, numeric::scale_type{0}}; - auto a_fail = cudf::test::fixed_point_column_wrapper{{3, -20, 1, -50, I32_MIN, 1, 0}, - numeric::scale_type{0}}; - auto expected = cudf::test::fixed_point_column_wrapper{ - {3, 20, 1, 50, I32_MAX, std::abs(I32_MIN + 1), 0}, numeric::scale_type{0}}; - auto expected_fail = cudf::test::fixed_point_column_wrapper{ + using T = TypeParam; + using R = typename T::rep; + auto a = decimal_column_wrapper{ + {R{3}, R{-20}, R{1}, R{-50}, this->MAX, R{this->MIN + 1}, R{0}}, numeric::scale_type{0}}; + auto a_fail = decimal_column_wrapper{{R{3}, R{-20}, R{1}, R{-50}, this->MIN, R{1}, R{0}}, + numeric::scale_type{0}}; + auto expected = decimal_column_wrapper{ + {R{3}, R{20}, R{1}, R{50}, this->MAX, R{std::abs(this->MIN + 1)}, R{0}}, + numeric::scale_type{0}}; + auto expected_fail = decimal_column_wrapper{ {3, 20, 1, 50, 0, 1, 0}, {1, 1, 1, 1, 0, 1, 1}, numeric::scale_type{0}}; auto table = cudf::table_view{{a, a_fail}}; auto a_ref = cudf::ast::column_reference(0); @@ -386,21 +405,20 @@ TEST_F(JITExpressionTest, AnsiAbs_Decimal) auto result = cudf::compute_column_jit(table, abs); auto result_fail = cudf::compute_column_jit(table, try_abs_fail); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); EXPECT_THROW(result = cudf::compute_column_jit(table, abs_fail), std::overflow_error); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), VERBOSITY); } -TEST_F(JITExpressionTest, AnsiNeg) +TYPED_TEST(JITSignedIntegerArithmeticTest, AnsiNeg) { - constexpr auto I32_MIN = std::numeric_limits::min(); - constexpr auto I32_MAX = std::numeric_limits::max(); - auto a = column_wrapper{3, -20, 1, -50, I32_MAX, -I32_MAX, 0}; - auto a_fail = column_wrapper{3, -20, 1, -50, I32_MIN, 1, 0}; - auto expected = column_wrapper{-3, 20, -1, 50, -I32_MAX, I32_MAX, 0}; - auto expected_fail = column_wrapper{{-3, 20, -1, 50, 0, -1, 0}, {1, 1, 1, 1, 0, 1, 1}}; + using T = TypeParam; + auto a = column_wrapper{{T{3}, T{-20}, T{1}, T{-50}, this->MAX, T{-this->MAX}, T{0}}}; + auto a_fail = column_wrapper{{T{3}, T{-20}, T{1}, T{-50}, this->MIN, T{1}, T{0}}}; + auto expected = column_wrapper{{T{-3}, T{20}, T{-1}, T{50}, T{-this->MAX}, this->MAX, T{0}}}; + auto expected_fail = column_wrapper{{-3, 20, -1, 50, 0, -1, 0}, {1, 1, 1, 1, 0, 1, 1}}; auto table = cudf::table_view{{a, a_fail}}; auto a_ref = cudf::ast::column_reference(0); auto a_fail_ref = cudf::ast::column_reference(1); @@ -411,24 +429,24 @@ TEST_F(JITExpressionTest, AnsiNeg) auto result = cudf::compute_column_jit(table, neg); auto result_fail = cudf::compute_column_jit(table, try_neg_fail); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); EXPECT_THROW(result = cudf::compute_column_jit(table, neg_fail), std::overflow_error); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), VERBOSITY); } -TEST_F(JITExpressionTest, AnsiNeg_Decimal) +TYPED_TEST(JITDecimalArithmeticTest, AnsiNeg) { - constexpr auto I32_MIN = std::numeric_limits::min(); - constexpr auto I32_MAX = std::numeric_limits::max(); - auto a = cudf::test::fixed_point_column_wrapper{{3, -20, 1, -50, I32_MAX, -I32_MAX, 0}, - numeric::scale_type{0}}; - auto a_fail = cudf::test::fixed_point_column_wrapper{{3, -20, 1, -50, I32_MIN, 1, 0}, - numeric::scale_type{0}}; - auto expected = cudf::test::fixed_point_column_wrapper{ - {-3, 20, -1, 50, -I32_MAX, I32_MAX, 0}, numeric::scale_type{0}}; - auto expected_fail = cudf::test::fixed_point_column_wrapper{ + using T = TypeParam; + using R = typename T::rep; + auto a = decimal_column_wrapper{{R{3}, R{-20}, R{1}, R{-50}, this->MAX, R{-this->MAX}, R{0}}, + numeric::scale_type{0}}; + auto a_fail = decimal_column_wrapper{{R{3}, R{-20}, R{1}, R{-50}, this->MIN, R{1}, R{0}}, + numeric::scale_type{0}}; + auto expected = decimal_column_wrapper{ + {R{-3}, R{20}, R{-1}, R{50}, R{-this->MAX}, this->MAX, R{0}}, numeric::scale_type{0}}; + auto expected_fail = decimal_column_wrapper{ {-3, 20, -1, 50, 0, -1, 0}, {1, 1, 1, 1, 0, 1, 1}, numeric::scale_type{0}}; auto table = cudf::table_view{{a, a_fail}}; auto a_ref = cudf::ast::column_reference(0); @@ -440,23 +458,21 @@ TEST_F(JITExpressionTest, AnsiNeg_Decimal) auto result = cudf::compute_column_jit(table, neg); auto result_fail = cudf::compute_column_jit(table, try_neg_fail); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); EXPECT_THROW(result = cudf::compute_column_jit(table, neg_fail), std::overflow_error); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), VERBOSITY); } -TEST_F(JITExpressionTest, AnsiPrecisionCheck) +TYPED_TEST(JITDecimalArithmeticTest, AnsiPrecisionCheck) { - auto a = - cudf::test::fixed_point_column_wrapper{{3, 200, 250, 200}, numeric::scale_type{0}}; - auto a_fail = - cudf::test::fixed_point_column_wrapper{{3, 200, 250, 20000}, numeric::scale_type{0}}; - auto expected = - cudf::test::fixed_point_column_wrapper{{3, 200, 250, 200}, numeric::scale_type{0}}; - auto expected_fail = cudf::test::fixed_point_column_wrapper{ - {3, 200, 250, 200}, {1, 1, 1, 0}, numeric::scale_type{0}}; + using T = TypeParam; + auto a = decimal_column_wrapper{{3, 200, 250, 200}, numeric::scale_type{0}}; + auto a_fail = decimal_column_wrapper{{3, 200, 250, 20000}, numeric::scale_type{0}}; + auto expected = decimal_column_wrapper{{3, 200, 250, 200}, numeric::scale_type{0}}; + auto expected_fail = + decimal_column_wrapper{{3, 200, 250, 200}, {1, 1, 1, 0}, numeric::scale_type{0}}; auto max_precision = cudf::numeric_scalar(3); auto table = cudf::table_view{{a, a_fail}}; auto a_ref = cudf::ast::column_reference(0); @@ -469,18 +485,17 @@ TEST_F(JITExpressionTest, AnsiPrecisionCheck) auto result = cudf::compute_column_jit(table, precision_check); auto result_fail = cudf::compute_column_jit(table, try_precision_check); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); EXPECT_THROW(result = cudf::compute_column_jit(table, precision_check_fail), std::overflow_error); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_fail, result_fail->view(), VERBOSITY); } TEST_F(JITExpressionTest, BitShiftLeft) { - auto a = cudf::test::fixed_width_column_wrapper{0b111111, 0b111110, 0b101111, 0b1100}; - auto expected = - cudf::test::fixed_width_column_wrapper{0b11111100, 0b11111000, 0b10111100, 0b110000}; + auto a = column_wrapper{0b111111, 0b111110, 0b101111, 0b1100}; + auto expected = column_wrapper{0b11111100, 0b11111000, 0b10111100, 0b110000}; auto shift = cudf::numeric_scalar(2); auto table = cudf::table_view{{a}}; auto a_ref = cudf::ast::column_reference(0); @@ -489,29 +504,29 @@ TEST_F(JITExpressionTest, BitShiftLeft) auto& shift_left = cudf::ast::jit::bit_shift_left(tree, a_ref, shift_literal); auto result = cudf::compute_column_jit(table, shift_left); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); } TEST_F(JITExpressionTest, BitShiftRight) { - auto a = cudf::test::fixed_width_column_wrapper{0b1111, 0b10111, 0b11100, 0b11110011}; - auto expected = cudf::test::fixed_width_column_wrapper{0b11, 0b101, 0b111, 0b111100}; - auto shift = cudf::numeric_scalar(2); - auto table = cudf::table_view{{a}}; - auto a_ref = cudf::ast::column_reference(0); - auto tree = cudf::ast::tree{}; + auto a = column_wrapper{0b1111, 0b10111, 0b11100, 0b11110011}; + auto expected = column_wrapper{0b11, 0b101, 0b111, 0b111100}; + auto shift = cudf::numeric_scalar(2); + auto table = cudf::table_view{{a}}; + auto a_ref = cudf::ast::column_reference(0); + auto tree = cudf::ast::tree{}; auto shift_literal = cudf::ast::literal(shift); auto& shift_right = cudf::ast::jit::bit_shift_right(tree, a_ref, shift_literal); auto result = cudf::compute_column_jit(table, shift_right); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); } template void test_cast() { - auto a = cudf::test::fixed_width_column_wrapper{0, 1, 2, 3, 4, 5}; - auto expected = cudf::test::fixed_width_column_wrapper{0, 1, 2, 3, 4, 5}; + auto a = column_wrapper{{0, 1, 2, 3, 4, 5}}; + auto expected = column_wrapper{{0, 1, 2, 3, 4, 5}}; auto table = cudf::table_view{{a}}; auto a_ref = cudf::ast::column_reference(0); auto tree = cudf::ast::tree{}; @@ -544,15 +559,14 @@ void test_cast() } auto result = cudf::compute_column_jit(table, *cast); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); } template void test_from_decimal_cast() { - auto a = cudf::test::fixed_point_column_wrapper{{0, 1, 2, 3, 4, 5}, - numeric::scale_type{0}}; - auto expected = cudf::test::fixed_width_column_wrapper{0, 1, 2, 3, 4, 5}; + auto a = decimal_column_wrapper{{0, 1, 2, 3, 4, 5}, numeric::scale_type{0}}; + auto expected = column_wrapper{0, 1, 2, 3, 4, 5}; auto table = cudf::table_view{{a}}; auto a_ref = cudf::ast::column_reference(0); auto tree = cudf::ast::tree{}; @@ -585,7 +599,7 @@ void test_from_decimal_cast() } auto result = cudf::compute_column_jit(table, *cast); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); } template @@ -624,10 +638,8 @@ TEST_F(JITExpressionTest, Cast) template void test_decimal_cast() { - auto a = cudf::test::fixed_point_column_wrapper{{0, 1, 2, 3, 4, 5}, - numeric::scale_type{0}}; - auto expected = cudf::test::fixed_point_column_wrapper{{0, 1, 2, 3, 4, 5}, - numeric::scale_type{0}}; + auto a = decimal_column_wrapper{{0, 1, 2, 3, 4, 5}, numeric::scale_type{0}}; + auto expected = decimal_column_wrapper{{0, 1, 2, 3, 4, 5}, numeric::scale_type{0}}; auto table = cudf::table_view{{a}}; auto a_ref = cudf::ast::column_reference(0); auto tree = cudf::ast::tree{}; @@ -644,28 +656,15 @@ void test_decimal_cast() } auto result = cudf::compute_column_jit(table, *cast); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); -} - -TEST_F(JITExpressionTest, CastToDec32) -{ - test_decimal_cast(); - test_decimal_cast(); - test_decimal_cast(); -} - -TEST_F(JITExpressionTest, CastToDec64) -{ - test_decimal_cast(); - test_decimal_cast(); - test_decimal_cast(); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); } -TEST_F(JITExpressionTest, CastToDec128) +TYPED_TEST(JITDecimalArithmeticTest, CastTo) { - test_decimal_cast(); - test_decimal_cast(); - test_decimal_cast(); + using T = TypeParam; + test_decimal_cast(); + test_decimal_cast(); + test_decimal_cast(); } TEST_F(JITExpressionTest, Rescale) @@ -680,16 +679,16 @@ TEST_F(JITExpressionTest, Rescale) auto& rescaled = cudf::ast::jit::rescale(tree, a_ref, -2); auto result = cudf::compute_column_jit(table, rescaled); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); } TEST_F(JITExpressionTest, AnsiFused) { constexpr auto I32_MAX = std::numeric_limits::max(); - auto a = column_wrapper{1, 3, 20, 1, 50, 10}; - auto b = column_wrapper{1, 10, 7, 20, I32_MAX, 2}; - auto c = column_wrapper{1, 5, 4, I32_MAX, 2, 5}; - auto d = column_wrapper{0, 1, 0, 0, 1, 5}; + auto a = column_wrapper{{1, 3, 20, 1, 50, 10}}; + auto b = column_wrapper{{1, 10, 7, 20, I32_MAX, 2}}; + auto c = column_wrapper{{1, 5, 4, I32_MAX, 2, 5}}; + auto d = column_wrapper{{0, 1, 0, 0, 1, 5}}; auto expected = column_wrapper{{0, 65, 0, 0, 0, 12}, {0, 1, 0, 0, 0, 1}}; auto table = cudf::table_view{{a, b, c, d}}; auto tree = cudf::ast::tree{}; @@ -702,7 +701,7 @@ TEST_F(JITExpressionTest, AnsiFused) auto& div = cudf::ast::jit::ansi_try_div(tree, mul, d_ref); auto result = cudf::compute_column_jit(table, div); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), VERBOSITY); } CUDF_TEST_PROGRAM_MAIN()